1//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/ModuleDepCollector.h"
10
11#include "clang/Basic/MakeSupport.h"
12#include "clang/DependencyScanning/DependencyScanningWorker.h"
13#include "clang/Frontend/CompilerInstance.h"
14#include "clang/Lex/Preprocessor.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/Support/BLAKE3.h"
17#include <optional>
18
19using namespace clang;
20using namespace dependencies;
21
22void ModuleDeps::forEachFileDep(llvm::function_ref<void(StringRef)> Cb) const {
23 SmallString<0> PathBuf;
24 PathBuf.reserve(N: 256);
25 for (StringRef FileDep : FileDeps) {
26 auto ResolvedFileDep =
27 ASTReader::ResolveImportedPath(Buf&: PathBuf, Path: FileDep, Prefix: FileDepsBaseDir);
28 Cb(*ResolvedFileDep);
29 }
30}
31
32const std::vector<std::string> &ModuleDeps::getBuildArguments() const {
33 // FIXME: this operation is not thread safe and is expected to be called
34 // on a single thread. Otherwise it should be protected with a lock.
35 assert(!std::holds_alternative<std::monostate>(BuildInfo) &&
36 "Using uninitialized ModuleDeps");
37 if (const auto *CI = std::get_if<CowCompilerInvocation>(ptr: &BuildInfo))
38 BuildInfo = CI->getCC1CommandLine();
39 return std::get<std::vector<std::string>>(v&: BuildInfo);
40}
41
42void PrebuiltModuleASTAttrs::updateDependentsNotInStableDirs(
43 PrebuiltModulesAttrsMap &PrebuiltModulesMap) {
44 setInStableDir();
45 for (const auto Dep : ModuleFileDependents) {
46 if (!PrebuiltModulesMap[Dep].isInStableDir())
47 return;
48 PrebuiltModulesMap[Dep].updateDependentsNotInStableDirs(PrebuiltModulesMap);
49 }
50}
51
52static void
53optimizeHeaderSearchOpts(HeaderSearchOptions &Opts, ASTReader &Reader,
54 const serialization::ModuleFile &MF,
55 const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
56 ScanningOptimizations OptimizeArgs) {
57 if (any(Val: OptimizeArgs & ScanningOptimizations::HeaderSearch)) {
58 // Only preserve search paths that were used during the dependency scan.
59 std::vector<HeaderSearchOptions::Entry> Entries;
60 std::swap(x&: Opts.UserEntries, y&: Entries);
61
62 llvm::BitVector SearchPathUsage(Entries.size());
63 llvm::DenseSet<const serialization::ModuleFile *> Visited;
64 std::function<void(const serialization::ModuleFile *)> VisitMF =
65 [&](const serialization::ModuleFile *MF) {
66 SearchPathUsage |= MF->SearchPathUsage;
67 Visited.insert(V: MF);
68 for (const serialization::ModuleFile *Import : MF->Imports)
69 if (!Visited.contains(V: Import))
70 VisitMF(Import);
71 };
72 VisitMF(&MF);
73
74 if (SearchPathUsage.size() != Entries.size())
75 llvm::report_fatal_error(
76 reason: "Inconsistent search path options between modules detected");
77
78 for (auto Idx : SearchPathUsage.set_bits())
79 Opts.UserEntries.push_back(x: std::move(Entries[Idx]));
80 }
81 if (any(Val: OptimizeArgs & ScanningOptimizations::VFS)) {
82 std::vector<std::string> VFSOverlayFiles;
83 std::swap(x&: Opts.VFSOverlayFiles, y&: VFSOverlayFiles);
84
85 llvm::BitVector VFSUsage(VFSOverlayFiles.size());
86 llvm::DenseSet<const serialization::ModuleFile *> Visited;
87 std::function<void(const serialization::ModuleFile *)> VisitMF =
88 [&](const serialization::ModuleFile *MF) {
89 Visited.insert(V: MF);
90 if (MF->Kind == serialization::MK_ImplicitModule) {
91 VFSUsage |= MF->VFSUsage;
92 // We only need to recurse into implicit modules. Other module types
93 // will have the correct set of VFSs for anything they depend on.
94 for (const serialization::ModuleFile *Import : MF->Imports)
95 if (!Visited.contains(V: Import))
96 VisitMF(Import);
97 } else {
98 // This is not an implicitly built module, so it may have different
99 // VFS options. Fall back to a string comparison instead.
100 auto PrebuiltModulePropIt =
101 PrebuiltModulesASTMap.find(Key: MF->FileName);
102 if (PrebuiltModulePropIt == PrebuiltModulesASTMap.end())
103 return;
104 for (std::size_t I = 0, E = VFSOverlayFiles.size(); I != E; ++I) {
105 if (PrebuiltModulePropIt->second.getVFS().contains(
106 key: VFSOverlayFiles[I]))
107 VFSUsage[I] = true;
108 }
109 }
110 };
111 VisitMF(&MF);
112
113 if (VFSUsage.size() != VFSOverlayFiles.size())
114 llvm::report_fatal_error(
115 reason: "Inconsistent -ivfsoverlay options between modules detected");
116
117 for (auto Idx : VFSUsage.set_bits())
118 Opts.VFSOverlayFiles.push_back(x: std::move(VFSOverlayFiles[Idx]));
119 }
120}
121
122static void optimizeDiagnosticOpts(DiagnosticOptions &Opts,
123 bool IsSystemModule) {
124 // If this is not a system module or -Wsystem-headers was passed, don't
125 // optimize.
126 if (!IsSystemModule)
127 return;
128 bool Wsystem_headers = false;
129 for (StringRef Opt : Opts.Warnings) {
130 bool isPositive = !Opt.consume_front(Prefix: "no-");
131 if (Opt == "system-headers")
132 Wsystem_headers = isPositive;
133 }
134 if (Wsystem_headers)
135 return;
136
137 // Remove all warning flags. System modules suppress most, but not all,
138 // warnings.
139 Opts.Warnings.clear();
140 Opts.UndefPrefixes.clear();
141 Opts.Remarks.clear();
142}
143
144static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {
145 BuildInvocation.getMutFileSystemOpts().WorkingDir.clear();
146 BuildInvocation.getMutCodeGenOpts().DebugCompilationDir.clear();
147 BuildInvocation.getMutCodeGenOpts().CoverageCompilationDir.clear();
148}
149
150static std::vector<std::string> splitString(std::string S, char Separator) {
151 SmallVector<StringRef> Segments;
152 StringRef(S).split(A&: Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
153 std::vector<std::string> Result;
154 Result.reserve(n: Segments.size());
155 for (StringRef Segment : Segments)
156 Result.push_back(x: Segment.str());
157 return Result;
158}
159
160void ModuleDepCollector::addOutputPaths(CowCompilerInvocation &CI,
161 ModuleDeps &Deps) {
162 CI.getMutFrontendOpts().OutputFile =
163 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::ModuleFile);
164 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
165 CI.getMutDiagnosticOpts().DiagnosticSerializationFile =
166 Controller.lookupModuleOutput(
167 MD: Deps, Kind: ModuleOutputKind::DiagnosticSerializationFile);
168 if (!CI.getDependencyOutputOpts().OutputFile.empty()) {
169 CI.getMutDependencyOutputOpts().OutputFile =
170 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::DependencyFile);
171 CI.getMutDependencyOutputOpts().Targets =
172 splitString(S: Controller.lookupModuleOutput(
173 MD: Deps, Kind: ModuleOutputKind::DependencyTargets),
174 Separator: '\0');
175 if (!CI.getDependencyOutputOpts().OutputFile.empty() &&
176 CI.getDependencyOutputOpts().Targets.empty()) {
177 // Fallback to -o as dependency target, as in the driver.
178 SmallString<128> Target;
179 quoteMakeTarget(Target: CI.getFrontendOpts().OutputFile, Res&: Target);
180 CI.getMutDependencyOutputOpts().Targets.push_back(x: std::string(Target));
181 }
182 }
183}
184
185void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
186 const LangOptions &LangOpts,
187 CodeGenOptions &CGOpts) {
188 // TODO: Figure out better way to set options to their default value.
189 if (ProgramAction == frontend::GenerateModule) {
190 CGOpts.MainFileName.clear();
191 CGOpts.DwarfDebugFlags.clear();
192 }
193 if (ProgramAction == frontend::GeneratePCH ||
194 (ProgramAction == frontend::GenerateModule && !LangOpts.ModulesCodegen)) {
195 CGOpts.DebugCompilationDir.clear();
196 CGOpts.CoverageCompilationDir.clear();
197 CGOpts.CoverageDataFile.clear();
198 CGOpts.CoverageNotesFile.clear();
199 CGOpts.ProfileInstrumentUsePath.clear();
200 CGOpts.SampleProfileFile.clear();
201 CGOpts.ProfileRemappingFile.clear();
202 }
203}
204
205bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,
206 const StringRef Input) {
207 using namespace llvm::sys;
208
209 if (!path::is_absolute(path: Input))
210 return false;
211
212 auto PathStartsWith = [](StringRef Prefix, StringRef Path) {
213 auto PrefixIt = path::begin(path: Prefix), PrefixEnd = path::end(path: Prefix);
214 for (auto PathIt = path::begin(path: Path), PathEnd = path::end(path: Path);
215 PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {
216 if (*PrefixIt != *PathIt)
217 return false;
218 }
219 return PrefixIt == PrefixEnd;
220 };
221
222 return any_of(Range: Directories, P: [&](StringRef Dir) {
223 return !Dir.empty() && PathStartsWith(Dir, Input);
224 });
225}
226
227bool dependencies::areOptionsInStableDir(const ArrayRef<StringRef> Directories,
228 const HeaderSearchOptions &HSOpts) {
229 assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&
230 "Sysroots differ between module dependencies and current TU");
231
232 assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&
233 "ResourceDirs differ between module dependencies and current TU");
234
235 for (const auto &Entry : HSOpts.UserEntries) {
236 if (!Entry.IgnoreSysRoot)
237 continue;
238 if (!isPathInStableDir(Directories, Input: Entry.Path))
239 return false;
240 }
241
242 for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {
243 if (!isPathInStableDir(Directories, Input: SysPrefix.Prefix))
244 return false;
245 }
246
247 return true;
248}
249
250static CowCompilerInvocation
251makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
252 CI.resetNonModularOptions();
253 CI.clearImplicitModuleBuildOptions();
254
255 // The scanner takes care to avoid passing non-affecting module maps to the
256 // explicit compiles. No need to do extra work just to find out there are no
257 // module map files to prune.
258 CI.getHeaderSearchOpts().ModulesPruneNonAffectingModuleMaps = false;
259
260 // Remove options incompatible with explicit module build or are likely to
261 // differ between identical modules discovered from different translation
262 // units.
263 CI.getFrontendOpts().Inputs.clear();
264 CI.getFrontendOpts().OutputFile.clear();
265 CI.getFrontendOpts().GenReducedBMI = false;
266 CI.getFrontendOpts().ModuleOutputPath.clear();
267 CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = false;
268 CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = false;
269 // LLVM options are not going to affect the AST
270 CI.getFrontendOpts().LLVMArgs.clear();
271
272 resetBenignCodeGenOptions(ProgramAction: frontend::GenerateModule, LangOpts: CI.getLangOpts(),
273 CGOpts&: CI.getCodeGenOpts());
274
275 // Map output paths that affect behaviour to "-" so their existence is in the
276 // context hash. The final path will be computed in addOutputPaths.
277 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
278 CI.getDiagnosticOpts().DiagnosticSerializationFile = "-";
279 if (!CI.getDependencyOutputOpts().OutputFile.empty())
280 CI.getDependencyOutputOpts().OutputFile = "-";
281 CI.getDependencyOutputOpts().Targets.clear();
282
283 CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
284 CI.getLangOpts().ModuleName.clear();
285
286 // Remove any macro definitions that are explicitly ignored.
287 if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {
288 llvm::erase_if(
289 C&: CI.getPreprocessorOpts().Macros,
290 P: [&CI](const std::pair<std::string, bool> &Def) {
291 StringRef MacroDef = Def.first;
292 return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(
293 key: llvm::CachedHashString(MacroDef.split(Separator: '=').first));
294 });
295 // Remove the now unused option.
296 CI.getHeaderSearchOpts().ModulesIgnoreMacros.clear();
297 }
298
299 return CI;
300}
301
302CowCompilerInvocation
303ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs(
304 const ModuleDeps &Deps,
305 llvm::function_ref<void(CowCompilerInvocation &)> Optimize) const {
306 CowCompilerInvocation CI = CommonInvocation;
307
308 CI.getMutLangOpts().ModuleName = Deps.ID.ModuleName;
309 CI.getMutFrontendOpts().IsSystemModule = Deps.IsSystem;
310
311 // Inputs
312 InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),
313 InputKind::Format::ModuleMap);
314 CI.getMutFrontendOpts().Inputs.emplace_back(Args: Deps.ClangModuleMapFile,
315 Args&: ModuleMapInputKind);
316
317 auto CurrentModuleMapEntry =
318 ScanInstance.getFileManager().getOptionalFileRef(Filename: Deps.ClangModuleMapFile);
319 assert(CurrentModuleMapEntry && "module map file entry not found");
320
321 // Remove directly passed modulemap files. They will get added back if they
322 // were actually used.
323 CI.getMutFrontendOpts().ModuleMapFiles.clear();
324
325 auto DepModuleMapFiles = collectModuleMapFiles(ClangModuleDeps: Deps.ClangModuleDeps);
326 for (StringRef ModuleMapFile : Deps.ModuleMapFileDeps) {
327 // TODO: Track these as `FileEntryRef` to simplify the equality check below.
328 auto ModuleMapEntry =
329 ScanInstance.getFileManager().getOptionalFileRef(Filename: ModuleMapFile);
330 assert(ModuleMapEntry && "module map file entry not found");
331
332 // Don't report module maps describing eagerly-loaded dependency. This
333 // information will be deserialized from the PCM.
334 // TODO: Verify this works fine when modulemap for module A is eagerly
335 // loaded from A.pcm, and module map passed on the command line contains
336 // definition of a submodule: "explicit module A.Private { ... }".
337 if (Service.shouldEagerLoadModules() &&
338 DepModuleMapFiles.contains(V: *ModuleMapEntry))
339 continue;
340
341 // Don't report module map file of the current module unless it also
342 // describes a dependency (for symmetry).
343 if (*ModuleMapEntry == *CurrentModuleMapEntry &&
344 !DepModuleMapFiles.contains(V: *ModuleMapEntry))
345 continue;
346
347 CI.getMutFrontendOpts().ModuleMapFiles.emplace_back(args&: ModuleMapFile);
348 }
349
350 // Report the prebuilt modules this module uses.
351 for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
352 CI.getMutFrontendOpts().ModuleFiles.push_back(x: PrebuiltModule.PCMFile);
353
354 // Add module file inputs from dependencies.
355 addModuleFiles(CI, ClangModuleDeps: Deps.ClangModuleDeps);
356
357 if (!CI.getDiagnosticOpts().SystemHeaderWarningsModules.empty()) {
358 // Apply -Wsystem-headers-in-module for the current module.
359 if (llvm::is_contained(Range: CI.getDiagnosticOpts().SystemHeaderWarningsModules,
360 Element: Deps.ID.ModuleName))
361 CI.getMutDiagnosticOpts().Warnings.push_back(x: "system-headers");
362 // Remove the now unused option(s).
363 CI.getMutDiagnosticOpts().SystemHeaderWarningsModules.clear();
364 }
365
366 Optimize(CI);
367
368 return CI;
369}
370
371llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(
372 ArrayRef<ModuleID> ClangModuleDeps) const {
373 llvm::DenseSet<const FileEntry *> ModuleMapFiles;
374 for (const ModuleID &MID : ClangModuleDeps) {
375 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
376 assert(MD && "Inconsistent dependency info");
377 // TODO: Track ClangModuleMapFile as `FileEntryRef`.
378 auto FE = ScanInstance.getFileManager().getOptionalFileRef(
379 Filename: MD->ClangModuleMapFile);
380 assert(FE && "Missing module map file that was previously found");
381 ModuleMapFiles.insert(V: *FE);
382 }
383 return ModuleMapFiles;
384}
385
386void ModuleDepCollector::addModuleMapFiles(
387 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
388 if (Service.shouldEagerLoadModules())
389 return; // Only pcm is needed for eager load.
390
391 for (const ModuleID &MID : ClangModuleDeps) {
392 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
393 assert(MD && "Inconsistent dependency info");
394 CI.getFrontendOpts().ModuleMapFiles.push_back(x: MD->ClangModuleMapFile);
395 }
396}
397
398void ModuleDepCollector::addModuleFiles(
399 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
400 for (const ModuleID &MID : ClangModuleDeps) {
401 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
402 std::string PCMPath =
403 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
404
405 if (Service.shouldEagerLoadModules())
406 CI.getFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
407 else
408 CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
409 x: {MID.ModuleName, std::move(PCMPath)});
410 }
411}
412
413void ModuleDepCollector::addModuleFiles(
414 CowCompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
415 for (const ModuleID &MID : ClangModuleDeps) {
416 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
417 std::string PCMPath =
418 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
419
420 if (Service.shouldEagerLoadModules())
421 CI.getMutFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
422 else
423 CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert(
424 x: {MID.ModuleName, std::move(PCMPath)});
425 }
426}
427
428static bool needsModules(FrontendInputFile FIF) {
429 switch (FIF.getKind().getLanguage()) {
430 case Language::Unknown:
431 case Language::Asm:
432 case Language::LLVM_IR:
433 return false;
434 default:
435 return true;
436 }
437}
438
439void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
440 CI.clearImplicitModuleBuildOptions();
441 resetBenignCodeGenOptions(ProgramAction: CI.getFrontendOpts().ProgramAction,
442 LangOpts: CI.getLangOpts(), CGOpts&: CI.getCodeGenOpts());
443
444 if (llvm::any_of(Range&: CI.getFrontendOpts().Inputs, P: needsModules)) {
445 Preprocessor &PP = ScanInstance.getPreprocessor();
446 if (Module *CurrentModule = PP.getCurrentModuleImplementation())
447 if (OptionalFileEntryRef CurrentModuleMap =
448 PP.getHeaderSearchInfo()
449 .getModuleMap()
450 .getModuleMapFileForUniquing(M: CurrentModule))
451 CI.getFrontendOpts().ModuleMapFiles.emplace_back(
452 args: CurrentModuleMap->getNameAsRequested());
453
454 SmallVector<ModuleID> DirectDeps;
455 for (const auto &KV : ModularDeps)
456 if (DirectModularDeps.contains(key: KV.first))
457 DirectDeps.push_back(Elt: KV.second->ID);
458
459 // TODO: Report module maps the same way it's done for modular dependencies.
460 addModuleMapFiles(CI, ClangModuleDeps: DirectDeps);
461
462 addModuleFiles(CI, ClangModuleDeps: DirectDeps);
463
464 for (const auto &KV : DirectPrebuiltModularDeps)
465 CI.getFrontendOpts().ModuleFiles.push_back(x: KV.second.PCMFile);
466 }
467}
468
469static bool isSafeToIgnoreCWD(const CowCompilerInvocation &CI) {
470 // Check if the command line input uses relative paths.
471 // It is not safe to ignore the current working directory if any of the
472 // command line inputs use relative paths.
473 bool AnyRelative = false;
474 CI.visitPaths(Callback: [&](StringRef Path) {
475 assert(!AnyRelative && "Continuing path visitation despite returning true");
476 AnyRelative |= !Path.empty() && !llvm::sys::path::is_absolute(path: Path);
477 return AnyRelative;
478 });
479 return !AnyRelative;
480}
481
482static std::string getModuleContextHash(const ModuleDeps &MD,
483 const CowCompilerInvocation &CI,
484 bool EagerLoadModules, bool IgnoreCWD,
485 llvm::vfs::FileSystem &VFS) {
486 llvm::HashBuilder<llvm::TruncatedBLAKE3<16>, llvm::endianness::native>
487 HashBuilder;
488
489 // Hash the compiler version and serialization version to ensure the module
490 // will be readable.
491 HashBuilder.add(Value: getClangFullRepositoryVersion());
492 HashBuilder.add(Args: serialization::VERSION_MAJOR, Args: serialization::VERSION_MINOR);
493 llvm::ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();
494 if (CWD && !IgnoreCWD)
495 HashBuilder.add(Value: *CWD);
496
497 // Hash the BuildInvocation without any input files.
498 SmallString<0> ArgVec;
499 ArgVec.reserve(N: 4096);
500 CI.generateCC1CommandLine(Consumer: [&](const Twine &Arg) {
501 Arg.toVector(Out&: ArgVec);
502 ArgVec.push_back(Elt: '\0');
503 });
504 HashBuilder.add(Value: ArgVec);
505
506 // Hash the module dependencies. These paths may differ even if the invocation
507 // is identical if they depend on the contents of the files in the TU -- for
508 // example, case-insensitive paths to modulemap files. Usually such a case
509 // would indicate a missed optimization to canonicalize, but it may be
510 // difficult to canonicalize all cases when there is a VFS.
511 for (const auto &ID : MD.ClangModuleDeps) {
512 HashBuilder.add(Value: ID.ModuleName);
513 HashBuilder.add(Value: ID.ContextHash);
514 }
515
516 HashBuilder.add(Value: EagerLoadModules);
517
518 llvm::BLAKE3Result<16> Hash = HashBuilder.final();
519 std::array<uint64_t, 2> Words;
520 static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");
521 std::memcpy(dest: Words.data(), src: Hash.data(), n: sizeof(Hash));
522 return toString(I: llvm::APInt(sizeof(Words) * 8, Words), Radix: 36, /*Signed=*/false);
523}
524
525void ModuleDepCollector::associateWithContextHash(
526 const CowCompilerInvocation &CI, bool IgnoreCWD, ModuleDeps &Deps) {
527 Deps.ID.ContextHash =
528 getModuleContextHash(MD: Deps, CI, EagerLoadModules: Service.shouldEagerLoadModules(),
529 IgnoreCWD, VFS&: ScanInstance.getVirtualFileSystem());
530 bool Inserted = ModuleDepsByID.insert(KV: {Deps.ID, &Deps}).second;
531 (void)Inserted;
532 assert(Inserted && "duplicate module mapping");
533}
534
535void ModuleDepCollectorPP::LexedFileChanged(FileID FID,
536 LexedFileChangeReason Reason,
537 SrcMgr::CharacteristicKind FileType,
538 FileID PrevFID,
539 SourceLocation Loc) {
540 if (Reason != LexedFileChangeReason::EnterFile)
541 return;
542
543 SourceManager &SM = MDC.ScanInstance.getSourceManager();
544
545 // Dependency generation really does want to go all the way to the
546 // file entry for a source location to find out what is depended on.
547 // We do not want #line markers to affect dependency generation!
548 if (std::optional<StringRef> Filename = SM.getNonBuiltinFilenameForID(FID))
549 MDC.addFileDep(Path: llvm::sys::path::remove_leading_dotslash(path: *Filename));
550}
551
552void ModuleDepCollectorPP::InclusionDirective(
553 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
554 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
555 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
556 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
557 if (!File && !ModuleImported) {
558 // This is a non-modular include that HeaderSearch failed to find. Add it
559 // here as `FileChanged` will never see it.
560 MDC.addFileDep(Path: FileName);
561 }
562 handleImport(Imported: SuggestedModule);
563}
564
565void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
566 ModuleIdPath Path,
567 const Module *Imported) {
568 auto &PP = MDC.ScanInstance.getPreprocessor();
569 if (PP.getLangOpts().CPlusPlusModules && PP.isImportingCXXNamedModules()) {
570 P1689ModuleInfo RequiredModule;
571 RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
572 RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
573 MDC.RequiredStdCXXModules.push_back(x: std::move(RequiredModule));
574 return;
575 }
576
577 handleImport(Imported);
578}
579
580void ModuleDepCollectorPP::handleImport(const Module *Imported) {
581 if (!Imported)
582 return;
583
584 const Module *TopLevelModule = Imported->getTopLevelModule();
585
586 if (MDC.isPrebuiltModule(M: TopLevelModule))
587 MDC.DirectPrebuiltModularDeps.insert(
588 KV: {TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
589 else {
590 MDC.DirectModularDeps.insert(X: TopLevelModule);
591 MDC.DirectImports.insert(Ptr: Imported);
592 }
593}
594
595void ModuleDepCollectorPP::EndOfMainFile() {
596 FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
597 MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
598 .getFileEntryRefForID(FID: MainFileID)
599 ->getName());
600
601 auto &PP = MDC.ScanInstance.getPreprocessor();
602 if (PP.isInNamedModule()) {
603 P1689ModuleInfo ProvidedModule;
604 ProvidedModule.ModuleName = PP.getNamedModuleName();
605 ProvidedModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
606 ProvidedModule.IsStdCXXModuleInterface = PP.isInNamedInterfaceUnit();
607 // Don't put implementation (non partition) unit as Provide.
608 // Put the module as required instead. Since the implementation
609 // unit will import the primary module implicitly.
610 if (PP.isInImplementationUnit())
611 MDC.RequiredStdCXXModules.push_back(x: ProvidedModule);
612 else
613 MDC.ProvidedStdCXXModule = ProvidedModule;
614 }
615
616 if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
617 MDC.addFileDep(Path: MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
618
619 for (StringRef VFS : MDC.ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
620 MDC.addFileDep(Path: VFS);
621
622 if (Module *CurrentModule = PP.getCurrentModuleImplementation()) {
623 if (OptionalFileEntryRef CurrentModuleMap =
624 PP.getHeaderSearchInfo().getModuleMap().getModuleMapFileForUniquing(
625 M: CurrentModule))
626 MDC.addFileDep(Path: CurrentModuleMap->getName());
627 }
628
629 for (const Module *M :
630 MDC.ScanInstance.getPreprocessor().getAffectingClangModules())
631 if (!MDC.isPrebuiltModule(M))
632 MDC.DirectModularDeps.insert(X: M);
633
634 MDC.addVisibleModules();
635
636 for (const Module *M : MDC.DirectModularDeps)
637 handleTopLevelModule(M);
638
639 MDC.Consumer.handleContextHash(
640 Hash: MDC.ScanInstance.getInvocation().computeContextHash());
641
642 MDC.Consumer.handleDependencyOutputOpts(Opts: *MDC.Opts);
643
644 MDC.Consumer.handleProvidedAndRequiredStdCXXModules(
645 Provided: MDC.ProvidedStdCXXModule, Requires: MDC.RequiredStdCXXModules);
646
647 for (auto &&I : MDC.ModularDeps)
648 MDC.Consumer.handleModuleDependency(MD: *I.second);
649
650 for (const Module *M : MDC.DirectModularDeps) {
651 auto It = MDC.ModularDeps.find(Key: M);
652 // Only report direct dependencies that were successfully handled.
653 if (It != MDC.ModularDeps.end())
654 MDC.Consumer.handleDirectModuleDependency(MD: It->second->ID);
655 }
656
657 for (auto &&I : MDC.VisibleModules)
658 MDC.Consumer.handleVisibleModule(ModuleName: std::string(I.getKey()));
659
660 for (auto &&I : MDC.FileDeps)
661 MDC.Consumer.handleFileDependency(Filename: I);
662
663 for (auto &&I : MDC.DirectPrebuiltModularDeps)
664 MDC.Consumer.handlePrebuiltModuleDependency(PMD: I.second);
665}
666
667std::optional<ModuleID>
668ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
669 assert(M == M->getTopLevelModule() && "Expected top level module!");
670
671 // A top-level module might not be actually imported as a module when
672 // -fmodule-name is used to compile a translation unit that imports this
673 // module. In that case it can be skipped. The appropriate header
674 // dependencies will still be reported as expected.
675 if (!M->getASTFile())
676 return {};
677
678 // If this module has been handled already, just return its ID.
679 if (auto ModI = MDC.ModularDeps.find(Key: M); ModI != MDC.ModularDeps.end())
680 return ModI->second->ID;
681
682 auto OwnedMD = std::make_unique<ModuleDeps>();
683 ModuleDeps &MD = *OwnedMD;
684
685 MD.ID.ModuleName = M->getFullModuleName();
686 MD.IsSystem = M->IsSystem;
687
688 // Start off with the assumption that this module is shareable when there
689 // are stable directories. As more dependencies are discovered, check if those
690 // come from the provided directories.
691 MD.IsInStableDirectories = !MDC.StableDirs.empty();
692
693 // For modules which use export_as link name, the linked product that of the
694 // corresponding export_as-named module.
695 if (!M->UseExportAsModuleLinkName)
696 MD.LinkLibraries = M->LinkLibraries;
697
698 ModuleMap &ModMapInfo =
699 MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();
700
701 if (auto ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M)) {
702 SmallString<128> Path = ModuleMap->getNameAsRequested();
703 ModMapInfo.canonicalizeModuleMapPath(Path);
704 MD.ClangModuleMapFile = std::string(Path);
705 }
706
707 serialization::ModuleFile *MF =
708 MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
709 File: *M->getASTFile());
710 MD.FileDepsBaseDir = MF->BaseDirectory;
711 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
712 MF&: *MF, /*IncludeSystem=*/true,
713 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
714 // The __inferred_module.map file is an insignificant implementation
715 // detail of implicitly-built modules. The PCM will also report the
716 // actual on-disk module map file that allowed inferring the module,
717 // which is what we need for building the module explicitly
718 // Let's ignore this file.
719 if (IFI.UnresolvedImportedFilename.ends_with(Suffix: "__inferred_module.map"))
720 return;
721 MDC.addFileDep(MD, Path: IFI.UnresolvedImportedFilename);
722 });
723
724 llvm::DenseSet<const Module *> SeenDeps;
725 addAllSubmodulePrebuiltDeps(M, MD, SeenSubmodules&: SeenDeps);
726 addAllSubmoduleDeps(M, MD, AddedModules&: SeenDeps);
727 addAllAffectingClangModules(M, MD, AddedModules&: SeenDeps);
728
729 SmallString<0> PathBuf;
730 PathBuf.reserve(N: 256);
731 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
732 MF&: *MF, /*IncludeSystem=*/true,
733 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
734 if (MD.IsInStableDirectories) {
735 auto FullFilePath = ASTReader::ResolveImportedPath(
736 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilename, Prefix: MF->BaseDirectory);
737 MD.IsInStableDirectories =
738 isPathInStableDir(Directories: MDC.StableDirs, Input: *FullFilePath);
739 }
740 if (!(IFI.TopLevel && IFI.ModuleMap))
741 return;
742 if (IFI.UnresolvedImportedFilenameAsRequested.ends_with(
743 Suffix: "__inferred_module.map"))
744 return;
745 auto ResolvedFilenameAsRequested = ASTReader::ResolveImportedPath(
746 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilenameAsRequested,
747 Prefix: MF->BaseDirectory);
748 MD.ModuleMapFileDeps.emplace_back(args: *ResolvedFilenameAsRequested);
749 });
750
751 bool IgnoreCWD = false;
752 CowCompilerInvocation CI =
753 MDC.getInvocationAdjustedForModuleBuildWithoutOutputs(
754 Deps: MD, Optimize: [&](CowCompilerInvocation &BuildInvocation) {
755 if (any(Val: MDC.Service.getOptimizeArgs() &
756 (ScanningOptimizations::HeaderSearch |
757 ScanningOptimizations::VFS)))
758 optimizeHeaderSearchOpts(Opts&: BuildInvocation.getMutHeaderSearchOpts(),
759 Reader&: *MDC.ScanInstance.getASTReader(), MF: *MF,
760 PrebuiltModulesASTMap: MDC.PrebuiltModulesASTMap,
761 OptimizeArgs: MDC.Service.getOptimizeArgs());
762
763 if (any(Val: MDC.Service.getOptimizeArgs() &
764 ScanningOptimizations::SystemWarnings))
765 optimizeDiagnosticOpts(
766 Opts&: BuildInvocation.getMutDiagnosticOpts(),
767 IsSystemModule: BuildInvocation.getFrontendOpts().IsSystemModule);
768
769 IgnoreCWD = any(Val: MDC.Service.getOptimizeArgs() &
770 ScanningOptimizations::IgnoreCWD) &&
771 isSafeToIgnoreCWD(CI: BuildInvocation);
772 if (IgnoreCWD) {
773 llvm::ErrorOr<std::string> CWD =
774 MDC.ScanInstance.getVirtualFileSystem()
775 .getCurrentWorkingDirectory();
776 if (CWD)
777 optimizeCWD(BuildInvocation, CWD: *CWD);
778 }
779 });
780
781 // Check provided input paths from the invocation for determining
782 // IsInStableDirectories.
783 if (MD.IsInStableDirectories)
784 MD.IsInStableDirectories =
785 areOptionsInStableDir(Directories: MDC.StableDirs, HSOpts: CI.getHeaderSearchOpts());
786
787 MDC.associateWithContextHash(CI, IgnoreCWD, Deps&: MD);
788
789 // Finish the compiler invocation. Requires dependencies and the context hash.
790 MDC.addOutputPaths(CI, Deps&: MD);
791
792 MD.BuildInfo = std::move(CI);
793
794 MDC.ModularDeps.insert(KV: {M, std::move(OwnedMD)});
795
796 return MD.ID;
797}
798
799static void forEachSubmoduleSorted(const Module *M,
800 llvm::function_ref<void(const Module *)> F) {
801 // Submodule order depends on order of header includes for inferred submodules
802 // we don't care about the exact order, so sort so that it's consistent across
803 // TUs to improve sharing.
804 SmallVector<const Module *> Submodules(M->submodules());
805 llvm::stable_sort(Range&: Submodules, C: [](const Module *A, const Module *B) {
806 return A->Name < B->Name;
807 });
808 for (const Module *SubM : Submodules)
809 F(SubM);
810}
811
812void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
813 const Module *M, ModuleDeps &MD,
814 llvm::DenseSet<const Module *> &SeenSubmodules) {
815 addModulePrebuiltDeps(M, MD, SeenSubmodules);
816
817 forEachSubmoduleSorted(M, F: [&](const Module *SubM) {
818 addAllSubmodulePrebuiltDeps(M: SubM, MD, SeenSubmodules);
819 });
820}
821
822void ModuleDepCollectorPP::addModulePrebuiltDeps(
823 const Module *M, ModuleDeps &MD,
824 llvm::DenseSet<const Module *> &SeenSubmodules) {
825 for (const Module *Import : M->Imports)
826 if (Import->getTopLevelModule() != M->getTopLevelModule())
827 if (MDC.isPrebuiltModule(M: Import->getTopLevelModule()))
828 if (SeenSubmodules.insert(V: Import->getTopLevelModule()).second) {
829 MD.PrebuiltModuleDeps.emplace_back(args: Import->getTopLevelModule());
830 if (MD.IsInStableDirectories) {
831 auto PrebuiltModulePropIt = MDC.PrebuiltModulesASTMap.find(
832 Key: MD.PrebuiltModuleDeps.back().PCMFile);
833 MD.IsInStableDirectories =
834 (PrebuiltModulePropIt != MDC.PrebuiltModulesASTMap.end()) &&
835 PrebuiltModulePropIt->second.isInStableDir();
836 }
837 }
838}
839
840void ModuleDepCollectorPP::addAllSubmoduleDeps(
841 const Module *M, ModuleDeps &MD,
842 llvm::DenseSet<const Module *> &AddedModules) {
843 addModuleDep(M, MD, AddedModules);
844
845 forEachSubmoduleSorted(M, F: [&](const Module *SubM) {
846 addAllSubmoduleDeps(M: SubM, MD, AddedModules);
847 });
848}
849
850void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
851 ModuleDeps &MD) {
852 MD.ClangModuleDeps.push_back(x: std::move(ID));
853 if (MD.IsInStableDirectories)
854 MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;
855}
856
857void ModuleDepCollectorPP::addModuleDep(
858 const Module *M, ModuleDeps &MD,
859 llvm::DenseSet<const Module *> &AddedModules) {
860 for (const Module *Import : M->Imports) {
861 if (Import->getTopLevelModule() != M->getTopLevelModule() &&
862 !MDC.isPrebuiltModule(M: Import)) {
863 if (auto ImportID = handleTopLevelModule(M: Import->getTopLevelModule()))
864 if (AddedModules.insert(V: Import->getTopLevelModule()).second)
865 addOneModuleDep(M: Import->getTopLevelModule(), ID: *ImportID, MD);
866 }
867 }
868}
869
870void ModuleDepCollectorPP::addAllAffectingClangModules(
871 const Module *M, ModuleDeps &MD,
872 llvm::DenseSet<const Module *> &AddedModules) {
873 addAffectingClangModule(M, MD, AddedModules);
874
875 for (const Module *SubM : M->submodules())
876 addAllAffectingClangModules(M: SubM, MD, AddedModules);
877}
878
879void ModuleDepCollectorPP::addAffectingClangModule(
880 const Module *M, ModuleDeps &MD,
881 llvm::DenseSet<const Module *> &AddedModules) {
882 for (const Module *Affecting : M->AffectingClangModules) {
883 assert(Affecting == Affecting->getTopLevelModule() &&
884 "Not quite import not top-level module");
885 if (Affecting != M->getTopLevelModule() &&
886 !MDC.isPrebuiltModule(M: Affecting)) {
887 if (auto ImportID = handleTopLevelModule(M: Affecting))
888 if (AddedModules.insert(V: Affecting).second)
889 addOneModuleDep(M: Affecting, ID: *ImportID, MD);
890 }
891 }
892}
893
894ModuleDepCollector::ModuleDepCollector(
895 DependencyScanningService &Service,
896 std::unique_ptr<DependencyOutputOptions> Opts,
897 CompilerInstance &ScanInstance, DependencyConsumer &C,
898 DependencyActionController &Controller, CompilerInvocation OriginalCI,
899 const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
900 const ArrayRef<StringRef> StableDirs)
901 : Service(Service), ScanInstance(ScanInstance), Consumer(C),
902 Controller(Controller),
903 PrebuiltModulesASTMap(std::move(PrebuiltModulesASTMap)),
904 StableDirs(StableDirs), Opts(std::move(Opts)),
905 CommonInvocation(
906 makeCommonInvocationForModuleBuild(CI: std::move(OriginalCI))) {}
907
908void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
909 auto CollectorPP = std::make_unique<ModuleDepCollectorPP>(args&: *this);
910 CollectorPPPtr = CollectorPP.get();
911 PP.addPPCallbacks(C: std::move(CollectorPP));
912}
913
914void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
915
916bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
917 std::string Name(M->getTopLevelModuleName());
918 const auto &PrebuiltModuleFiles =
919 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;
920 auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(x: Name);
921 if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
922 return false;
923 assert("Prebuilt module came from the expected AST file" &&
924 PrebuiltModuleFileIt->second == M->getASTFile()->getName());
925 return true;
926}
927
928void ModuleDepCollector::addVisibleModules() {
929 llvm::DenseSet<const Module *> ImportedModules;
930 auto InsertVisibleModules = [&](const Module *M) {
931 if (ImportedModules.contains(V: M))
932 return;
933
934 VisibleModules.insert(key: M->getTopLevelModuleName());
935 SmallVector<Module *> Stack;
936 M->getExportedModules(Exported&: Stack);
937 while (!Stack.empty()) {
938 const Module *CurrModule = Stack.pop_back_val();
939 if (ImportedModules.contains(V: CurrModule))
940 continue;
941 ImportedModules.insert(V: CurrModule);
942 VisibleModules.insert(key: CurrModule->getTopLevelModuleName());
943 CurrModule->getExportedModules(Exported&: Stack);
944 }
945 };
946
947 for (const Module *Import : DirectImports)
948 InsertVisibleModules(Import);
949}
950
951static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path,
952 SmallVectorImpl<char> &Storage) {
953 if (llvm::sys::path::is_absolute(path: Path) &&
954 !llvm::sys::path::is_style_windows(S: llvm::sys::path::Style::native))
955 return Path;
956 Storage.assign(in_start: Path.begin(), in_end: Path.end());
957 CI.getFileManager().makeAbsolutePath(Path&: Storage);
958 llvm::sys::path::make_preferred(path&: Storage);
959 return StringRef(Storage.data(), Storage.size());
960}
961
962void ModuleDepCollector::addFileDep(StringRef Path) {
963 if (Service.getFormat() == ScanningOutputFormat::P1689) {
964 // Within P1689 format, we don't want all the paths to be absolute path
965 // since it may violate the traditional make style dependencies info.
966 FileDeps.emplace_back(args&: Path);
967 return;
968 }
969
970 llvm::SmallString<256> Storage;
971 Path = makeAbsoluteAndPreferred(CI&: ScanInstance, Path, Storage);
972 FileDeps.emplace_back(args&: Path);
973}
974
975void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) {
976 MD.FileDeps.emplace_back(args&: Path);
977}
978