1//===- ModuleManager.cpp - Module Manager ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ModuleManager class, which manages a set of loaded
10// modules for the ASTReader.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Serialization/ModuleManager.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/LLVM.h"
17#include "clang/Lex/HeaderSearch.h"
18#include "clang/Lex/ModuleMap.h"
19#include "clang/Serialization/GlobalModuleIndex.h"
20#include "clang/Serialization/InMemoryModuleCache.h"
21#include "clang/Serialization/ModuleCache.h"
22#include "clang/Serialization/ModuleFile.h"
23#include "clang/Serialization/PCHContainerOperations.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/Support/DOTGraphTraits.h"
31#include "llvm/Support/Error.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/GraphWriter.h"
34#include "llvm/Support/MemoryBuffer.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include <cassert>
37#include <memory>
38#include <string>
39#include <system_error>
40
41using namespace clang;
42using namespace serialization;
43
44ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
45 if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
46 if (const ModuleFileName *FileName = Mod->getASTFileName())
47 return lookupByFileName(FileName: *FileName);
48
49 return nullptr;
50}
51
52ModuleFile *ModuleManager::lookupByFileName(ModuleFileName Name) const {
53 std::optional<ModuleFileKey> Key = Name.makeKey(FileMgr);
54 return Key ? lookup(Key: *Key) : nullptr;
55}
56
57ModuleFile *ModuleManager::lookup(ModuleFileKey Key) const {
58 return Modules.lookup(Val: Key);
59}
60
61std::unique_ptr<llvm::MemoryBuffer>
62ModuleManager::lookupBuffer(StringRef Name) {
63 auto Entry = FileMgr.getOptionalFileRef(Filename: Name, /*OpenFile=*/false,
64 /*CacheFailure=*/false);
65 if (!Entry)
66 return nullptr;
67 return std::move(InMemoryBuffers[*Entry]);
68}
69
70static bool checkModuleFile(off_t Size, time_t ModTime, off_t ExpectedSize,
71 time_t ExpectedModTime, std::string &ErrorStr) {
72 if (ExpectedSize && ExpectedSize != Size) {
73 ErrorStr = "module file has a different size than expected";
74 return true;
75 }
76
77 if (ExpectedModTime && ExpectedModTime != ModTime) {
78 ErrorStr = "module file has a different modification time than expected";
79 return true;
80 }
81
82 return false;
83}
84
85static bool checkSignature(ASTFileSignature Signature,
86 ASTFileSignature ExpectedSignature,
87 std::string &ErrorStr) {
88 if (!ExpectedSignature || Signature == ExpectedSignature)
89 return false;
90
91 ErrorStr =
92 Signature ? "signature mismatch" : "could not read module signature";
93 return true;
94}
95
96static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
97 SourceLocation ImportLoc) {
98 if (ImportedBy) {
99 MF.ImportedBy.insert(X: ImportedBy);
100 ImportedBy->Imports.insert(X: &MF);
101 } else {
102 if (!MF.DirectlyImported)
103 MF.ImportLoc = ImportLoc;
104
105 MF.DirectlyImported = true;
106 }
107}
108
109ModuleManager::AddModuleResult ModuleManager::addModule(
110 ModuleFileName FileName, ModuleKind Type, SourceLocation ImportLoc,
111 ModuleFile *ImportedBy, unsigned Generation, off_t ExpectedSize,
112 time_t ExpectedModTime, ASTFileSignature ExpectedSignature,
113 ASTFileSignatureReader ReadSignature, ModuleFile *&Module,
114 std::string &ErrorStr) {
115 Module = nullptr;
116
117 uint64_t InputFilesValidationTimestamp = 0;
118 if (Type == MK_ImplicitModule)
119 InputFilesValidationTimestamp = ModCache.getModuleTimestamp(ModuleFilename: FileName);
120
121 bool IgnoreModTime = Type == MK_ExplicitModule || Type == MK_PrebuiltModule;
122 if (ImportedBy)
123 IgnoreModTime &= ImportedBy->Kind == MK_ExplicitModule ||
124 ImportedBy->Kind == MK_PrebuiltModule;
125 if (IgnoreModTime) {
126 // If neither this file nor the importer are in the module cache, this file
127 // might have a different mtime due to being moved across filesystems in
128 // a distributed build. The size must still match, though. (As must the
129 // contents, but we can't check that.)
130 ExpectedModTime = 0;
131 }
132
133 std::optional<ModuleFileKey> FileKey = FileName.makeKey(FileMgr);
134 if (!FileKey) {
135 ErrorStr = "module file not found";
136 return Missing;
137 }
138
139 // Check whether we already loaded this module, before
140 if (ModuleFile *ModuleEntry = lookup(Key: *FileKey)) {
141 // Check file properties.
142 if (checkModuleFile(Size: ModuleEntry->Size, ModTime: ModuleEntry->ModTime, ExpectedSize,
143 ExpectedModTime, ErrorStr))
144 return OutOfDate;
145
146 // Check the stored signature.
147 if (checkSignature(Signature: ModuleEntry->Signature, ExpectedSignature, ErrorStr))
148 return OutOfDate;
149
150 Module = ModuleEntry;
151 updateModuleImports(MF&: *ModuleEntry, ImportedBy, ImportLoc);
152 return AlreadyLoaded;
153 }
154
155 // Load the contents of the module
156 off_t Size = ExpectedSize;
157 time_t ModTime = ExpectedModTime;
158 llvm::MemoryBuffer *ModuleBuffer = nullptr;
159 std::unique_ptr<llvm::MemoryBuffer> NewFileBuffer = nullptr;
160 if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(Name: FileName)) {
161 // The buffer was already provided for us.
162 ModuleBuffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM(
163 Filename: FileName, Buffer: std::move(Buffer));
164 } else if (llvm::MemoryBuffer *Buffer =
165 getModuleCache().getInMemoryModuleCache().lookupPCM(
166 Filename: FileName)) {
167 ModuleBuffer = Buffer;
168 if (!FileName.getImplicitModuleSuffixLength()) {
169 // Explicitly-built PCM files maintain consistency via mtime/size
170 // expectations on their imports. Even if we've previously successfully
171 // loaded a PCM file and stored it in the in-memory module cache, that
172 // does not mean its mtime/size matches current importer's expectations.
173 // Get that information so that it can be checked below.
174 // FIXME: Even though this FileManager access is likely already cached, we
175 // should store this directly in the in-memory module cache.
176 OptionalFileEntryRef Entry =
177 FileMgr.getOptionalFileRef(Filename: FileName, /*OpenFile=*/true,
178 /*CacheFailure=*/false);
179 if (!Entry) {
180 ErrorStr = "module file not found";
181 return Missing;
182 }
183 ModTime = Entry->getModificationTime();
184 Size = Entry->getSize();
185 }
186 } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM(
187 Filename: FileName)) {
188 // Report that the module is out of date, since we tried (and failed) to
189 // import it earlier.
190 return OutOfDate;
191 } else {
192 auto Buf = [&]() -> Expected<std::unique_ptr<llvm::MemoryBuffer>> {
193 // Implicit modules live in the module cache.
194 if (FileName.getImplicitModuleSuffixLength())
195 return ModCache.read(FileName, Size, ModTime);
196
197 // Explicit modules are treated as any other compiler input file, load
198 // them via FileManager.
199 Expected<FileEntryRef> Entry =
200 FileName == StringRef("-")
201 ? FileMgr.getSTDIN()
202 : FileMgr.getFileRef(Filename: FileName, /*OpenFile=*/true,
203 /*CacheFailure=*/false);
204 if (!Entry)
205 return Entry.takeError();
206
207 Size = Entry->getSize();
208 ModTime = Entry->getModificationTime();
209
210 // RequiresNullTerminator is false because module files don't need it, and
211 // this allows the file to still be mmapped.
212 return llvm::errorOrToExpected(
213 EO: FileMgr.getBufferForFile(Entry: *Entry, /*IsVolatile=*/isVolatile: false,
214 /*RequiresNullTerminator=*/false));
215 }();
216
217 if (!Buf) {
218 ErrorStr = llvm::toString(E: Buf.takeError());
219 return Missing;
220 }
221
222 NewFileBuffer = std::move(*Buf);
223 ModuleBuffer = NewFileBuffer.get();
224 }
225
226 // Allocate a new module.
227 auto NewModule = std::make_unique<ModuleFile>(args&: Type, args&: *FileKey, args&: Generation);
228 NewModule->Index = Chain.size();
229 NewModule->FileName = FileName;
230 NewModule->ImportLoc = ImportLoc;
231 NewModule->InputFilesValidationTimestamp = InputFilesValidationTimestamp;
232 NewModule->Size = Size;
233 NewModule->ModTime = ModTime;
234 NewModule->Buffer = ModuleBuffer;
235 // Initialize the stream.
236 NewModule->Data = PCHContainerRdr.ExtractPCH(Buffer: *NewModule->Buffer);
237
238 // Check file properties.
239 if (checkModuleFile(Size, ModTime, ExpectedSize, ExpectedModTime, ErrorStr))
240 return OutOfDate;
241
242 // Read the signature eagerly now so that we can check it. Avoid calling
243 // ReadSignature unless there's something to check though.
244 if (ExpectedSignature && checkSignature(Signature: ReadSignature(NewModule->Data),
245 ExpectedSignature, ErrorStr))
246 return OutOfDate;
247
248 if (NewFileBuffer)
249 getModuleCache().getInMemoryModuleCache().addPCM(Filename: FileName,
250 Buffer: std::move(NewFileBuffer));
251
252 // We're keeping this module. Store it in the map.
253 Module = Modules[*FileKey] = NewModule.get();
254
255 updateModuleImports(MF&: *NewModule, ImportedBy, ImportLoc);
256
257 if (!NewModule->isModule())
258 PCHChain.push_back(Elt: NewModule.get());
259 if (!ImportedBy)
260 Roots.push_back(Elt: NewModule.get());
261
262 Chain.push_back(Elt: std::move(NewModule));
263 return NewlyLoaded;
264}
265
266void ModuleManager::removeModules(ModuleIterator First) {
267 auto Last = end();
268 if (First == Last)
269 return;
270
271 // Explicitly clear VisitOrder since we might not notice it is stale.
272 VisitOrder.clear();
273
274 // Collect the set of module file pointers that we'll be removing.
275 llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
276 (llvm::pointer_iterator<ModuleIterator>(First)),
277 (llvm::pointer_iterator<ModuleIterator>(Last)));
278
279 auto IsVictim = [&](ModuleFile *MF) {
280 return victimSet.count(Ptr: MF);
281 };
282 // Remove any references to the now-destroyed modules.
283 for (auto I = begin(); I != First; ++I) {
284 I->Imports.remove_if(P: IsVictim);
285 I->ImportedBy.remove_if(P: IsVictim);
286 }
287 llvm::erase_if(C&: Roots, P: IsVictim);
288
289 // Remove the modules from the PCH chain.
290 for (auto I = First; I != Last; ++I) {
291 if (!I->isModule()) {
292 PCHChain.erase(CS: llvm::find(Range&: PCHChain, Val: &*I), CE: PCHChain.end());
293 break;
294 }
295 }
296
297 // Delete the modules.
298 for (ModuleIterator victim = First; victim != Last; ++victim)
299 Modules.erase(Val: victim->FileKey);
300
301 Chain.erase(CS: Chain.begin() + (First - begin()), CE: Chain.end());
302}
303
304void
305ModuleManager::addInMemoryBuffer(StringRef FileName,
306 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
307 FileEntryRef Entry =
308 FileMgr.getVirtualFileRef(Filename: FileName, Size: Buffer->getBufferSize(), ModificationTime: 0);
309 InMemoryBuffers[Entry] = std::move(Buffer);
310}
311
312std::unique_ptr<ModuleManager::VisitState> ModuleManager::allocateVisitState() {
313 // Fast path: if we have a cached state, use it.
314 if (FirstVisitState) {
315 auto Result = std::move(FirstVisitState);
316 FirstVisitState = std::move(Result->NextState);
317 return Result;
318 }
319
320 // Allocate and return a new state.
321 return std::make_unique<VisitState>(args: size());
322}
323
324void ModuleManager::returnVisitState(std::unique_ptr<VisitState> State) {
325 assert(State->NextState == nullptr && "Visited state is in list?");
326 State->NextState = std::move(FirstVisitState);
327 FirstVisitState = std::move(State);
328}
329
330void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
331 GlobalIndex = Index;
332 if (!GlobalIndex) {
333 ModulesInCommonWithGlobalIndex.clear();
334 return;
335 }
336
337 // Notify the global module index about all of the modules we've already
338 // loaded.
339 for (ModuleFile &M : *this)
340 if (!GlobalIndex->loadedModuleFile(File: &M))
341 ModulesInCommonWithGlobalIndex.push_back(Elt: &M);
342}
343
344void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
345 if (!GlobalIndex || GlobalIndex->loadedModuleFile(File: MF))
346 return;
347
348 ModulesInCommonWithGlobalIndex.push_back(Elt: MF);
349}
350
351ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
352 const PCHContainerReader &PCHContainerRdr,
353 const HeaderSearch &HeaderSearchInfo)
354 : FileMgr(FileMgr), ModCache(ModCache), PCHContainerRdr(PCHContainerRdr),
355 HeaderSearchInfo(HeaderSearchInfo) {}
356
357void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
358 llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
359 // If the visitation order vector is the wrong size, recompute the order.
360 if (VisitOrder.size() != Chain.size()) {
361 unsigned N = size();
362 VisitOrder.clear();
363 VisitOrder.reserve(N);
364
365 // Record the number of incoming edges for each module. When we
366 // encounter a module with no incoming edges, push it into the queue
367 // to seed the queue.
368 SmallVector<ModuleFile *, 4> Queue;
369 Queue.reserve(N);
370 llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
371 UnusedIncomingEdges.resize(N: size());
372 for (ModuleFile &M : llvm::reverse(C&: *this)) {
373 unsigned Size = M.ImportedBy.size();
374 UnusedIncomingEdges[M.Index] = Size;
375 if (!Size)
376 Queue.push_back(Elt: &M);
377 }
378
379 // Traverse the graph, making sure to visit a module before visiting any
380 // of its dependencies.
381 while (!Queue.empty()) {
382 ModuleFile *CurrentModule = Queue.pop_back_val();
383 VisitOrder.push_back(Elt: CurrentModule);
384
385 // For any module that this module depends on, push it on the
386 // stack (if it hasn't already been marked as visited).
387 for (ModuleFile *M : llvm::reverse(C&: CurrentModule->Imports)) {
388 // Remove our current module as an impediment to visiting the
389 // module we depend on. If we were the last unvisited module
390 // that depends on this particular module, push it into the
391 // queue to be visited.
392 unsigned &NumUnusedEdges = UnusedIncomingEdges[M->Index];
393 if (NumUnusedEdges && (--NumUnusedEdges == 0))
394 Queue.push_back(Elt: M);
395 }
396 }
397
398 assert(VisitOrder.size() == N && "Visitation order is wrong?");
399
400 FirstVisitState = nullptr;
401 }
402
403 auto State = allocateVisitState();
404 unsigned VisitNumber = State->NextVisitNumber++;
405
406 // If the caller has provided us with a hit-set that came from the global
407 // module index, mark every module file in common with the global module
408 // index that is *not* in that set as 'visited'.
409 if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
410 for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
411 {
412 ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
413 if (!ModuleFilesHit->count(Ptr: M))
414 State->VisitNumber[M->Index] = VisitNumber;
415 }
416 }
417
418 for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
419 ModuleFile *CurrentModule = VisitOrder[I];
420 // Should we skip this module file?
421 if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
422 continue;
423
424 // Visit the module.
425 assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
426 State->VisitNumber[CurrentModule->Index] = VisitNumber;
427 if (!Visitor(*CurrentModule))
428 continue;
429
430 // The visitor has requested that cut off visitation of any
431 // module that the current module depends on. To indicate this
432 // behavior, we mark all of the reachable modules as having been visited.
433 ModuleFile *NextModule = CurrentModule;
434 do {
435 // For any module that this module depends on, push it on the
436 // stack (if it hasn't already been marked as visited).
437 for (llvm::SetVector<ModuleFile *>::iterator
438 M = NextModule->Imports.begin(),
439 MEnd = NextModule->Imports.end();
440 M != MEnd; ++M) {
441 if (State->VisitNumber[(*M)->Index] != VisitNumber) {
442 State->Stack.push_back(Elt: *M);
443 State->VisitNumber[(*M)->Index] = VisitNumber;
444 }
445 }
446
447 if (State->Stack.empty())
448 break;
449
450 // Pop the next module off the stack.
451 NextModule = State->Stack.pop_back_val();
452 } while (true);
453 }
454
455 returnVisitState(State: std::move(State));
456}
457
458#ifndef NDEBUG
459namespace llvm {
460
461 template<>
462 struct GraphTraits<ModuleManager> {
463 using NodeRef = ModuleFile *;
464 using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
465 using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
466
467 static ChildIteratorType child_begin(NodeRef Node) {
468 return Node->Imports.begin();
469 }
470
471 static ChildIteratorType child_end(NodeRef Node) {
472 return Node->Imports.end();
473 }
474
475 static nodes_iterator nodes_begin(const ModuleManager &Manager) {
476 return nodes_iterator(Manager.begin());
477 }
478
479 static nodes_iterator nodes_end(const ModuleManager &Manager) {
480 return nodes_iterator(Manager.end());
481 }
482 };
483
484 template<>
485 struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
486 explicit DOTGraphTraits(bool IsSimple = false)
487 : DefaultDOTGraphTraits(IsSimple) {}
488
489 static bool renderGraphFromBottomUp() { return true; }
490
491 std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
492 return M->ModuleName;
493 }
494 };
495
496} // namespace llvm
497
498void ModuleManager::viewGraph() {
499 llvm::ViewGraph(*this, "Modules");
500}
501#endif
502