1//===- ModuleManager.cpp - Module Manager ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ModuleManager class, which manages a set of loaded
10// modules for the ASTReader.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Serialization/ModuleManager.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/LLVM.h"
17#include "clang/Lex/HeaderSearch.h"
18#include "clang/Lex/ModuleMap.h"
19#include "clang/Serialization/GlobalModuleIndex.h"
20#include "clang/Serialization/InMemoryModuleCache.h"
21#include "clang/Serialization/ModuleCache.h"
22#include "clang/Serialization/ModuleFile.h"
23#include "clang/Serialization/PCHContainerOperations.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/Support/DOTGraphTraits.h"
31#include "llvm/Support/ErrorOr.h"
32#include "llvm/Support/GraphWriter.h"
33#include "llvm/Support/MemoryBuffer.h"
34#include "llvm/Support/VirtualFileSystem.h"
35#include <cassert>
36#include <memory>
37#include <string>
38#include <system_error>
39
40using namespace clang;
41using namespace serialization;
42
43ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const {
44 auto Entry = FileMgr.getOptionalFileRef(Filename: Name, /*OpenFile=*/false,
45 /*CacheFailure=*/false);
46 if (Entry)
47 return lookup(File: *Entry);
48
49 return nullptr;
50}
51
52ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
53 if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
54 if (OptionalFileEntryRef File = Mod->getASTFile())
55 return lookup(File: *File);
56
57 return nullptr;
58}
59
60ModuleFile *ModuleManager::lookup(const FileEntry *File) const {
61 return Modules.lookup(Val: File);
62}
63
64std::unique_ptr<llvm::MemoryBuffer>
65ModuleManager::lookupBuffer(StringRef Name) {
66 auto Entry = FileMgr.getOptionalFileRef(Filename: Name, /*OpenFile=*/false,
67 /*CacheFailure=*/false);
68 if (!Entry)
69 return nullptr;
70 return std::move(InMemoryBuffers[*Entry]);
71}
72
73static bool checkSignature(ASTFileSignature Signature,
74 ASTFileSignature ExpectedSignature,
75 std::string &ErrorStr) {
76 if (!ExpectedSignature || Signature == ExpectedSignature)
77 return false;
78
79 ErrorStr =
80 Signature ? "signature mismatch" : "could not read module signature";
81 return true;
82}
83
84static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
85 SourceLocation ImportLoc) {
86 if (ImportedBy) {
87 MF.ImportedBy.insert(X: ImportedBy);
88 ImportedBy->Imports.insert(X: &MF);
89 } else {
90 if (!MF.DirectlyImported)
91 MF.ImportLoc = ImportLoc;
92
93 MF.DirectlyImported = true;
94 }
95}
96
97ModuleManager::AddModuleResult
98ModuleManager::addModule(StringRef FileName, ModuleKind Type,
99 SourceLocation ImportLoc, ModuleFile *ImportedBy,
100 unsigned Generation,
101 off_t ExpectedSize, time_t ExpectedModTime,
102 ASTFileSignature ExpectedSignature,
103 ASTFileSignatureReader ReadSignature,
104 ModuleFile *&Module,
105 std::string &ErrorStr) {
106 Module = nullptr;
107
108 // Look for the file entry. This only fails if the expected size or
109 // modification time differ.
110 OptionalFileEntryRef Entry;
111 bool IgnoreModTime = Type == MK_ExplicitModule || Type == MK_PrebuiltModule;
112 if (ImportedBy)
113 IgnoreModTime &= ImportedBy->Kind == MK_ExplicitModule ||
114 ImportedBy->Kind == MK_PrebuiltModule;
115 if (IgnoreModTime) {
116 // If neither this file nor the importer are in the module cache, this file
117 // might have a different mtime due to being moved across filesystems in
118 // a distributed build. The size must still match, though. (As must the
119 // contents, but we can't check that.)
120 ExpectedModTime = 0;
121 }
122 // Note: ExpectedSize and ExpectedModTime will be 0 for MK_ImplicitModule
123 // when using an ASTFileSignature.
124 if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, File&: Entry)) {
125 ErrorStr = IgnoreModTime ? "module file has a different size than expected"
126 : "module file has a different size or "
127 "modification time than expected";
128 return OutOfDate;
129 }
130
131 if (!Entry) {
132 ErrorStr = "module file not found";
133 return Missing;
134 }
135
136 // The ModuleManager's use of FileEntry nodes as the keys for its map of
137 // loaded modules is less than ideal. Uniqueness for FileEntry nodes is
138 // maintained by FileManager, which in turn uses inode numbers on hosts
139 // that support that. When coupled with the module cache's proclivity for
140 // turning over and deleting stale PCMs, this means entries for different
141 // module files can wind up reusing the same underlying inode. When this
142 // happens, subsequent accesses to the Modules map will disagree on the
143 // ModuleFile associated with a given file. In general, it is not sufficient
144 // to resolve this conundrum with a type like FileEntryRef that stores the
145 // name of the FileEntry node on first access because of path canonicalization
146 // issues. However, the paths constructed for implicit module builds are
147 // fully under Clang's control. We *can*, therefore, rely on their structure
148 // being consistent across operating systems and across subsequent accesses
149 // to the Modules map.
150 auto implicitModuleNamesMatch = [](ModuleKind Kind, const ModuleFile *MF,
151 FileEntryRef Entry) -> bool {
152 if (Kind != MK_ImplicitModule)
153 return true;
154 return Entry.getName() == MF->FileName;
155 };
156
157 // Check whether we already loaded this module, before
158 if (ModuleFile *ModuleEntry = Modules.lookup(Val: *Entry)) {
159 if (implicitModuleNamesMatch(Type, ModuleEntry, *Entry)) {
160 // Check the stored signature.
161 if (checkSignature(Signature: ModuleEntry->Signature, ExpectedSignature, ErrorStr))
162 return OutOfDate;
163
164 Module = ModuleEntry;
165 updateModuleImports(MF&: *ModuleEntry, ImportedBy, ImportLoc);
166 return AlreadyLoaded;
167 }
168 }
169
170 // Allocate a new module.
171 auto NewModule = std::make_unique<ModuleFile>(args&: Type, args&: *Entry, args&: Generation);
172 NewModule->Index = Chain.size();
173 NewModule->FileName = FileName.str();
174 NewModule->ImportLoc = ImportLoc;
175 NewModule->InputFilesValidationTimestamp = 0;
176
177 if (NewModule->Kind == MK_ImplicitModule)
178 NewModule->InputFilesValidationTimestamp =
179 ModCache->getModuleTimestamp(ModuleFilename: NewModule->FileName);
180
181 // Load the contents of the module
182 if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(Name: FileName)) {
183 // The buffer was already provided for us.
184 NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM(
185 Filename: FileName, Buffer: std::move(Buffer));
186 // Since the cached buffer is reused, it is safe to close the file
187 // descriptor that was opened while stat()ing the PCM in
188 // lookupModuleFile() above, it won't be needed any longer.
189 Entry->closeFile();
190 } else if (llvm::MemoryBuffer *Buffer =
191 getModuleCache().getInMemoryModuleCache().lookupPCM(
192 Filename: FileName)) {
193 NewModule->Buffer = Buffer;
194 // As above, the file descriptor is no longer needed.
195 Entry->closeFile();
196 } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM(
197 Filename: FileName)) {
198 // Report that the module is out of date, since we tried (and failed) to
199 // import it earlier.
200 Entry->closeFile();
201 return OutOfDate;
202 } else {
203 // Get a buffer of the file and close the file descriptor when done.
204 // The file is volatile because in a parallel build we expect multiple
205 // compiler processes to use the same module file rebuilding it if needed.
206 //
207 // RequiresNullTerminator is false because module files don't need it, and
208 // this allows the file to still be mmapped.
209 auto Buf = FileMgr.getBufferForFile(Entry: NewModule->File,
210 /*IsVolatile=*/isVolatile: true,
211 /*RequiresNullTerminator=*/false);
212
213 if (!Buf) {
214 ErrorStr = Buf.getError().message();
215 return Missing;
216 }
217
218 NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addPCM(
219 Filename: FileName, Buffer: std::move(*Buf));
220 }
221
222 // Initialize the stream.
223 NewModule->Data = PCHContainerRdr.ExtractPCH(Buffer: *NewModule->Buffer);
224
225 // Read the signature eagerly now so that we can check it. Avoid calling
226 // ReadSignature unless there's something to check though.
227 if (ExpectedSignature && checkSignature(Signature: ReadSignature(NewModule->Data),
228 ExpectedSignature, ErrorStr))
229 return OutOfDate;
230
231 // We're keeping this module. Store it everywhere.
232 Module = Modules[*Entry] = NewModule.get();
233
234 updateModuleImports(MF&: *NewModule, ImportedBy, ImportLoc);
235
236 if (!NewModule->isModule())
237 PCHChain.push_back(Elt: NewModule.get());
238 if (!ImportedBy)
239 Roots.push_back(Elt: NewModule.get());
240
241 Chain.push_back(Elt: std::move(NewModule));
242 return NewlyLoaded;
243}
244
245void ModuleManager::removeModules(ModuleIterator First) {
246 auto Last = end();
247 if (First == Last)
248 return;
249
250 // Explicitly clear VisitOrder since we might not notice it is stale.
251 VisitOrder.clear();
252
253 // Collect the set of module file pointers that we'll be removing.
254 llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
255 (llvm::pointer_iterator<ModuleIterator>(First)),
256 (llvm::pointer_iterator<ModuleIterator>(Last)));
257
258 auto IsVictim = [&](ModuleFile *MF) {
259 return victimSet.count(Ptr: MF);
260 };
261 // Remove any references to the now-destroyed modules.
262 for (auto I = begin(); I != First; ++I) {
263 I->Imports.remove_if(P: IsVictim);
264 I->ImportedBy.remove_if(P: IsVictim);
265 }
266 llvm::erase_if(C&: Roots, P: IsVictim);
267
268 // Remove the modules from the PCH chain.
269 for (auto I = First; I != Last; ++I) {
270 if (!I->isModule()) {
271 PCHChain.erase(CS: llvm::find(Range&: PCHChain, Val: &*I), CE: PCHChain.end());
272 break;
273 }
274 }
275
276 // Delete the modules.
277 for (ModuleIterator victim = First; victim != Last; ++victim)
278 Modules.erase(Val: victim->File);
279
280 Chain.erase(CS: Chain.begin() + (First - begin()), CE: Chain.end());
281}
282
283void
284ModuleManager::addInMemoryBuffer(StringRef FileName,
285 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
286 FileEntryRef Entry =
287 FileMgr.getVirtualFileRef(Filename: FileName, Size: Buffer->getBufferSize(), ModificationTime: 0);
288 InMemoryBuffers[Entry] = std::move(Buffer);
289}
290
291std::unique_ptr<ModuleManager::VisitState> ModuleManager::allocateVisitState() {
292 // Fast path: if we have a cached state, use it.
293 if (FirstVisitState) {
294 auto Result = std::move(FirstVisitState);
295 FirstVisitState = std::move(Result->NextState);
296 return Result;
297 }
298
299 // Allocate and return a new state.
300 return std::make_unique<VisitState>(args: size());
301}
302
303void ModuleManager::returnVisitState(std::unique_ptr<VisitState> State) {
304 assert(State->NextState == nullptr && "Visited state is in list?");
305 State->NextState = std::move(FirstVisitState);
306 FirstVisitState = std::move(State);
307}
308
309void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
310 GlobalIndex = Index;
311 if (!GlobalIndex) {
312 ModulesInCommonWithGlobalIndex.clear();
313 return;
314 }
315
316 // Notify the global module index about all of the modules we've already
317 // loaded.
318 for (ModuleFile &M : *this)
319 if (!GlobalIndex->loadedModuleFile(File: &M))
320 ModulesInCommonWithGlobalIndex.push_back(Elt: &M);
321}
322
323void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
324 if (!GlobalIndex || GlobalIndex->loadedModuleFile(File: MF))
325 return;
326
327 ModulesInCommonWithGlobalIndex.push_back(Elt: MF);
328}
329
330ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
331 const PCHContainerReader &PCHContainerRdr,
332 const HeaderSearch &HeaderSearchInfo)
333 : FileMgr(FileMgr), ModCache(&ModCache), PCHContainerRdr(PCHContainerRdr),
334 HeaderSearchInfo(HeaderSearchInfo) {}
335
336void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
337 llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
338 // If the visitation order vector is the wrong size, recompute the order.
339 if (VisitOrder.size() != Chain.size()) {
340 unsigned N = size();
341 VisitOrder.clear();
342 VisitOrder.reserve(N);
343
344 // Record the number of incoming edges for each module. When we
345 // encounter a module with no incoming edges, push it into the queue
346 // to seed the queue.
347 SmallVector<ModuleFile *, 4> Queue;
348 Queue.reserve(N);
349 llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
350 UnusedIncomingEdges.resize(N: size());
351 for (ModuleFile &M : llvm::reverse(C&: *this)) {
352 unsigned Size = M.ImportedBy.size();
353 UnusedIncomingEdges[M.Index] = Size;
354 if (!Size)
355 Queue.push_back(Elt: &M);
356 }
357
358 // Traverse the graph, making sure to visit a module before visiting any
359 // of its dependencies.
360 while (!Queue.empty()) {
361 ModuleFile *CurrentModule = Queue.pop_back_val();
362 VisitOrder.push_back(Elt: CurrentModule);
363
364 // For any module that this module depends on, push it on the
365 // stack (if it hasn't already been marked as visited).
366 for (ModuleFile *M : llvm::reverse(C&: CurrentModule->Imports)) {
367 // Remove our current module as an impediment to visiting the
368 // module we depend on. If we were the last unvisited module
369 // that depends on this particular module, push it into the
370 // queue to be visited.
371 unsigned &NumUnusedEdges = UnusedIncomingEdges[M->Index];
372 if (NumUnusedEdges && (--NumUnusedEdges == 0))
373 Queue.push_back(Elt: M);
374 }
375 }
376
377 assert(VisitOrder.size() == N && "Visitation order is wrong?");
378
379 FirstVisitState = nullptr;
380 }
381
382 auto State = allocateVisitState();
383 unsigned VisitNumber = State->NextVisitNumber++;
384
385 // If the caller has provided us with a hit-set that came from the global
386 // module index, mark every module file in common with the global module
387 // index that is *not* in that set as 'visited'.
388 if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
389 for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
390 {
391 ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
392 if (!ModuleFilesHit->count(Ptr: M))
393 State->VisitNumber[M->Index] = VisitNumber;
394 }
395 }
396
397 for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
398 ModuleFile *CurrentModule = VisitOrder[I];
399 // Should we skip this module file?
400 if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
401 continue;
402
403 // Visit the module.
404 assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
405 State->VisitNumber[CurrentModule->Index] = VisitNumber;
406 if (!Visitor(*CurrentModule))
407 continue;
408
409 // The visitor has requested that cut off visitation of any
410 // module that the current module depends on. To indicate this
411 // behavior, we mark all of the reachable modules as having been visited.
412 ModuleFile *NextModule = CurrentModule;
413 do {
414 // For any module that this module depends on, push it on the
415 // stack (if it hasn't already been marked as visited).
416 for (llvm::SetVector<ModuleFile *>::iterator
417 M = NextModule->Imports.begin(),
418 MEnd = NextModule->Imports.end();
419 M != MEnd; ++M) {
420 if (State->VisitNumber[(*M)->Index] != VisitNumber) {
421 State->Stack.push_back(Elt: *M);
422 State->VisitNumber[(*M)->Index] = VisitNumber;
423 }
424 }
425
426 if (State->Stack.empty())
427 break;
428
429 // Pop the next module off the stack.
430 NextModule = State->Stack.pop_back_val();
431 } while (true);
432 }
433
434 returnVisitState(State: std::move(State));
435}
436
437bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize,
438 time_t ExpectedModTime,
439 OptionalFileEntryRef &File) {
440 if (FileName == "-") {
441 File = expectedToOptional(E: FileMgr.getSTDIN());
442 return false;
443 }
444
445 // Open the file immediately to ensure there is no race between stat'ing and
446 // opening the file.
447 File = FileMgr.getOptionalFileRef(Filename: FileName, /*OpenFile=*/true,
448 /*CacheFailure=*/false);
449
450 if (File &&
451 ((ExpectedSize && ExpectedSize != File->getSize()) ||
452 (ExpectedModTime && ExpectedModTime != File->getModificationTime())))
453 // Do not destroy File, as it may be referenced. If we need to rebuild it,
454 // it will be destroyed by removeModules.
455 return true;
456
457 return false;
458}
459
460#ifndef NDEBUG
461namespace llvm {
462
463 template<>
464 struct GraphTraits<ModuleManager> {
465 using NodeRef = ModuleFile *;
466 using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
467 using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
468
469 static ChildIteratorType child_begin(NodeRef Node) {
470 return Node->Imports.begin();
471 }
472
473 static ChildIteratorType child_end(NodeRef Node) {
474 return Node->Imports.end();
475 }
476
477 static nodes_iterator nodes_begin(const ModuleManager &Manager) {
478 return nodes_iterator(Manager.begin());
479 }
480
481 static nodes_iterator nodes_end(const ModuleManager &Manager) {
482 return nodes_iterator(Manager.end());
483 }
484 };
485
486 template<>
487 struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
488 explicit DOTGraphTraits(bool IsSimple = false)
489 : DefaultDOTGraphTraits(IsSimple) {}
490
491 static bool renderGraphFromBottomUp() { return true; }
492
493 std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
494 return M->ModuleName;
495 }
496 };
497
498} // namespace llvm
499
500void ModuleManager::viewGraph() {
501 llvm::ViewGraph(*this, "Modules");
502}
503#endif
504