1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the GlobalModuleIndex class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Serialization/GlobalModuleIndex.h"
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Serialization/ASTBitCodes.h"
17#include "clang/Serialization/ModuleFile.h"
18#include "clang/Serialization/PCHContainerOperations.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/SmallString.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/Bitstream/BitstreamReader.h"
24#include "llvm/Bitstream/BitstreamWriter.h"
25#include "llvm/Support/DJB.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/IOSandbox.h"
28#include "llvm/Support/LockFileManager.h"
29#include "llvm/Support/MemoryBuffer.h"
30#include "llvm/Support/OnDiskHashTable.h"
31#include "llvm/Support/Path.h"
32#include "llvm/Support/TimeProfiler.h"
33#include "llvm/Support/raw_ostream.h"
34#include <cstdio>
35using namespace clang;
36using namespace serialization;
37
38//----------------------------------------------------------------------------//
39// Shared constants
40//----------------------------------------------------------------------------//
41namespace {
42 enum {
43 /// The block containing the index.
44 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
45 };
46
47 /// Describes the record types in the index.
48 enum IndexRecordTypes {
49 /// Contains version information and potentially other metadata,
50 /// used to determine if we can read this global index file.
51 INDEX_METADATA,
52 /// Describes a module, including its file name and dependencies.
53 MODULE,
54 /// The index for identifiers.
55 IDENTIFIER_INDEX
56 };
57}
58
59/// The name of the global index file.
60static const char * const IndexFileName = "modules.idx";
61
62/// The global index file version.
63static const unsigned CurrentVersion = 1;
64
65//----------------------------------------------------------------------------//
66// Global module index reader.
67//----------------------------------------------------------------------------//
68
69namespace {
70
71/// Trait used to read the identifier index from the on-disk hash
72/// table.
73class IdentifierIndexReaderTrait {
74public:
75 typedef StringRef external_key_type;
76 typedef StringRef internal_key_type;
77 typedef SmallVector<unsigned, 2> data_type;
78 typedef unsigned hash_value_type;
79 typedef unsigned offset_type;
80
81 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
82 return a == b;
83 }
84
85 static hash_value_type ComputeHash(const internal_key_type& a) {
86 return llvm::djbHash(Buffer: a);
87 }
88
89 static std::pair<unsigned, unsigned>
90 ReadKeyDataLength(const unsigned char*& d) {
91 using namespace llvm::support;
92 unsigned KeyLen = endian::readNext<uint16_t, llvm::endianness::little>(memory&: d);
93 unsigned DataLen = endian::readNext<uint16_t, llvm::endianness::little>(memory&: d);
94 return std::make_pair(x&: KeyLen, y&: DataLen);
95 }
96
97 static const internal_key_type&
98 GetInternalKey(const external_key_type& x) { return x; }
99
100 static const external_key_type&
101 GetExternalKey(const internal_key_type& x) { return x; }
102
103 static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
104 return StringRef((const char *)d, n);
105 }
106
107 static data_type ReadData(const internal_key_type& k,
108 const unsigned char* d,
109 unsigned DataLen) {
110 using namespace llvm::support;
111
112 data_type Result;
113 while (DataLen > 0) {
114 unsigned ID = endian::readNext<uint32_t, llvm::endianness::little>(memory&: d);
115 Result.push_back(Elt: ID);
116 DataLen -= 4;
117 }
118
119 return Result;
120 }
121};
122
123typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124 IdentifierIndexTable;
125
126}
127
128GlobalModuleIndex::GlobalModuleIndex(
129 std::unique_ptr<llvm::MemoryBuffer> IndexBuffer,
130 llvm::BitstreamCursor Cursor)
131 : Buffer(std::move(IndexBuffer)), IdentifierIndex(), NumIdentifierLookups(),
132 NumIdentifierLookupHits() {
133 auto Fail = [&](llvm::Error &&Err) {
134 report_fatal_error(reason: "Module index '" + Buffer->getBufferIdentifier() +
135 "' failed: " + toString(E: std::move(Err)));
136 };
137
138 llvm::TimeTraceScope TimeScope("Module LoadIndex");
139 // Read the global index.
140 bool InGlobalIndexBlock = false;
141 bool Done = false;
142 while (!Done) {
143 llvm::BitstreamEntry Entry;
144 if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
145 Entry = Res.get();
146 else
147 Fail(Res.takeError());
148
149 switch (Entry.Kind) {
150 case llvm::BitstreamEntry::Error:
151 return;
152
153 case llvm::BitstreamEntry::EndBlock:
154 if (InGlobalIndexBlock) {
155 InGlobalIndexBlock = false;
156 Done = true;
157 continue;
158 }
159 return;
160
161
162 case llvm::BitstreamEntry::Record:
163 // Entries in the global index block are handled below.
164 if (InGlobalIndexBlock)
165 break;
166
167 return;
168
169 case llvm::BitstreamEntry::SubBlock:
170 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
171 if (llvm::Error Err = Cursor.EnterSubBlock(BlockID: GLOBAL_INDEX_BLOCK_ID))
172 Fail(std::move(Err));
173 InGlobalIndexBlock = true;
174 } else if (llvm::Error Err = Cursor.SkipBlock())
175 Fail(std::move(Err));
176 continue;
177 }
178
179 SmallVector<uint64_t, 64> Record;
180 StringRef Blob;
181 Expected<unsigned> MaybeIndexRecord =
182 Cursor.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob);
183 if (!MaybeIndexRecord)
184 Fail(MaybeIndexRecord.takeError());
185 IndexRecordTypes IndexRecord =
186 static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
187 switch (IndexRecord) {
188 case INDEX_METADATA:
189 // Make sure that the version matches.
190 if (Record.size() < 1 || Record[0] != CurrentVersion)
191 return;
192 break;
193
194 case MODULE: {
195 unsigned Idx = 0;
196 unsigned ID = Record[Idx++];
197
198 // Make room for this module's information.
199 if (ID == Modules.size())
200 Modules.push_back(Elt: ModuleInfo());
201 else
202 Modules.resize(N: ID + 1);
203
204 // Size/modification time for this module file at the time the
205 // global index was built.
206 Modules[ID].Size = Record[Idx++];
207 Modules[ID].ModTime = Record[Idx++];
208
209 // File name.
210 unsigned NameLen = Record[Idx++];
211 Modules[ID].FileName.assign(first: Record.begin() + Idx,
212 last: Record.begin() + Idx + NameLen);
213 Idx += NameLen;
214
215 // Dependencies
216 unsigned NumDeps = Record[Idx++];
217 Modules[ID].Dependencies.insert(I: Modules[ID].Dependencies.end(),
218 From: Record.begin() + Idx,
219 To: Record.begin() + Idx + NumDeps);
220 Idx += NumDeps;
221
222 // Make sure we're at the end of the record.
223 assert(Idx == Record.size() && "More module info?");
224
225 // Record this module as an unresolved module.
226 // FIXME: this doesn't work correctly for module names containing path
227 // separators.
228 StringRef ModuleName = llvm::sys::path::stem(path: Modules[ID].FileName);
229 // Remove the -<hash of ModuleMapPath>
230 ModuleName = ModuleName.rsplit(Separator: '-').first;
231 UnresolvedModules[ModuleName] = ID;
232 break;
233 }
234
235 case IDENTIFIER_INDEX:
236 // Wire up the identifier index.
237 if (Record[0]) {
238 IdentifierIndex = IdentifierIndexTable::Create(
239 Buckets: (const unsigned char *)Blob.data() + Record[0],
240 Payload: (const unsigned char *)Blob.data() + sizeof(uint32_t),
241 Base: (const unsigned char *)Blob.data(), InfoObj: IdentifierIndexReaderTrait());
242 }
243 break;
244 }
245 }
246}
247
248GlobalModuleIndex::~GlobalModuleIndex() {
249 delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
250}
251
252std::pair<GlobalModuleIndex *, llvm::Error>
253GlobalModuleIndex::readIndex(StringRef Path) {
254 // This is a compiler-internal input/output, let's bypass the sandbox.
255 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
256
257 // Load the index file, if it's there.
258 llvm::SmallString<128> IndexPath;
259 IndexPath += Path;
260 llvm::sys::path::append(path&: IndexPath, a: IndexFileName);
261
262 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
263 llvm::MemoryBuffer::getFile(Filename: IndexPath.c_str());
264 if (!BufferOrErr)
265 return std::make_pair(x: nullptr,
266 y: llvm::errorCodeToError(EC: BufferOrErr.getError()));
267 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
268
269 /// The main bitstream cursor for the main block.
270 llvm::BitstreamCursor Cursor(*Buffer);
271
272 // Sniff for the signature.
273 for (unsigned char C : {'B', 'C', 'G', 'I'}) {
274 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(NumBits: 8)) {
275 if (Res.get() != C)
276 return std::make_pair(
277 x: nullptr, y: llvm::createStringError(EC: std::errc::illegal_byte_sequence,
278 Fmt: "expected signature BCGI"));
279 } else
280 return std::make_pair(x: nullptr, y: Res.takeError());
281 }
282
283 return std::make_pair(x: new GlobalModuleIndex(std::move(Buffer), std::move(Cursor)),
284 y: llvm::Error::success());
285}
286
287void GlobalModuleIndex::getModuleDependencies(
288 ModuleFile *File,
289 SmallVectorImpl<ModuleFile *> &Dependencies) {
290 // Look for information about this module file.
291 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
292 = ModulesByFile.find(Val: File);
293 if (Known == ModulesByFile.end())
294 return;
295
296 // Record dependencies.
297 Dependencies.clear();
298 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
299 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
300 if (ModuleFile *MF = Modules[I].File)
301 Dependencies.push_back(Elt: MF);
302 }
303}
304
305bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
306 Hits.clear();
307
308 // If there's no identifier index, there is nothing we can do.
309 if (!IdentifierIndex)
310 return false;
311
312 // Look into the identifier index.
313 ++NumIdentifierLookups;
314 IdentifierIndexTable &Table
315 = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
316 IdentifierIndexTable::iterator Known = Table.find(EKey: Name);
317 if (Known == Table.end()) {
318 return false;
319 }
320
321 for (unsigned ModuleID : *Known) {
322 if (ModuleFile *MF = Modules[ModuleID].File)
323 Hits.insert(Ptr: MF);
324 }
325
326 ++NumIdentifierLookupHits;
327 return true;
328}
329
330bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
331 // Look for the module in the global module index based on the module name.
332 StringRef Name = File->ModuleName;
333 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Key: Name);
334 if (Known == UnresolvedModules.end()) {
335 return true;
336 }
337
338 // Rectify this module with the global module index.
339 ModuleInfo &Info = Modules[Known->second];
340
341 // If the size and modification time match what we expected, record this
342 // module file.
343 bool Failed = true;
344 if (File->Size == Info.Size && File->ModTime == Info.ModTime) {
345 Info.File = File;
346 ModulesByFile[File] = Known->second;
347
348 Failed = false;
349 }
350
351 // One way or another, we have resolved this module file.
352 UnresolvedModules.erase(I: Known);
353 return Failed;
354}
355
356void GlobalModuleIndex::printStats() {
357 std::fprintf(stderr, format: "*** Global Module Index Statistics:\n");
358 if (NumIdentifierLookups) {
359 fprintf(stderr, format: " %u / %u identifier lookups succeeded (%f%%)\n",
360 NumIdentifierLookupHits, NumIdentifierLookups,
361 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
362 }
363 std::fprintf(stderr, format: "\n");
364}
365
366LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
367 llvm::errs() << "*** Global Module Index Dump:\n";
368 llvm::errs() << "Module files:\n";
369 for (auto &MI : Modules) {
370 llvm::errs() << "** " << MI.FileName << "\n";
371 if (MI.File)
372 MI.File->dump();
373 else
374 llvm::errs() << "\n";
375 }
376 llvm::errs() << "\n";
377}
378
379//----------------------------------------------------------------------------//
380// Global module index writer.
381//----------------------------------------------------------------------------//
382
383namespace {
384 /// Provides information about a specific module file.
385 struct ModuleFileInfo {
386 /// The numberic ID for this module file.
387 unsigned ID;
388
389 /// The set of modules on which this module depends. Each entry is
390 /// a module ID.
391 SmallVector<unsigned, 4> Dependencies;
392 ASTFileSignature Signature;
393 };
394
395 struct ImportedModuleFileInfo {
396 off_t StoredSize;
397 time_t StoredModTime;
398 ASTFileSignature StoredSignature;
399 ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
400 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
401 };
402
403 /// Builder that generates the global module index file.
404 class GlobalModuleIndexBuilder {
405 FileManager &FileMgr;
406 const PCHContainerReader &PCHContainerRdr;
407
408 /// Mapping from files to module file information.
409 using ModuleFilesMap = llvm::MapVector<FileEntryRef, ModuleFileInfo>;
410
411 /// Information about each of the known module files.
412 ModuleFilesMap ModuleFiles;
413
414 /// Mapping from the imported module file to the imported
415 /// information.
416 using ImportedModuleFilesMap =
417 std::multimap<FileEntryRef, ImportedModuleFileInfo>;
418
419 /// Information about each importing of a module file.
420 ImportedModuleFilesMap ImportedModuleFiles;
421
422 /// Mapping from identifiers to the list of module file IDs that
423 /// consider this identifier to be interesting.
424 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
425
426 /// A mapping from all interesting identifiers to the set of module
427 /// files in which those identifiers are considered interesting.
428 InterestingIdentifierMap InterestingIdentifiers;
429
430 /// Write the block-info block for the global module index file.
431 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
432
433 /// Retrieve the module file information for the given file.
434 ModuleFileInfo &getModuleFileInfo(FileEntryRef File) {
435 auto [It, Inserted] = ModuleFiles.try_emplace(Key: File);
436 if (Inserted) {
437 unsigned NewID = ModuleFiles.size();
438 ModuleFileInfo &Info = It->second;
439 Info.ID = NewID;
440 }
441 return It->second;
442 }
443
444 public:
445 explicit GlobalModuleIndexBuilder(
446 FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
447 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
448
449 /// Load the contents of the given module file into the builder.
450 llvm::Error loadModuleFile(FileEntryRef File);
451
452 /// Write the index to the given bitstream.
453 /// \returns true if an error occurred, false otherwise.
454 bool writeIndex(llvm::BitstreamWriter &Stream);
455 };
456}
457
458static void emitBlockID(unsigned ID, const char *Name,
459 llvm::BitstreamWriter &Stream,
460 SmallVectorImpl<uint64_t> &Record) {
461 Record.clear();
462 Record.push_back(Elt: ID);
463 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_SETBID, Vals: Record);
464
465 // Emit the block name if present.
466 if (!Name || Name[0] == 0) return;
467 Record.clear();
468 while (*Name)
469 Record.push_back(Elt: *Name++);
470 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Vals: Record);
471}
472
473static void emitRecordID(unsigned ID, const char *Name,
474 llvm::BitstreamWriter &Stream,
475 SmallVectorImpl<uint64_t> &Record) {
476 Record.clear();
477 Record.push_back(Elt: ID);
478 while (*Name)
479 Record.push_back(Elt: *Name++);
480 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Vals: Record);
481}
482
483void
484GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
485 SmallVector<uint64_t, 64> Record;
486 Stream.EnterBlockInfoBlock();
487
488#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
489#define RECORD(X) emitRecordID(X, #X, Stream, Record)
490 BLOCK(GLOBAL_INDEX_BLOCK);
491 RECORD(INDEX_METADATA);
492 RECORD(MODULE);
493 RECORD(IDENTIFIER_INDEX);
494#undef RECORD
495#undef BLOCK
496
497 Stream.ExitBlock();
498}
499
500namespace {
501 class InterestingASTIdentifierLookupTrait
502 : public serialization::reader::ASTIdentifierLookupTraitBase {
503
504 public:
505 /// The identifier and whether it is "interesting".
506 typedef std::pair<StringRef, bool> data_type;
507
508 data_type ReadData(const internal_key_type& k,
509 const unsigned char* d,
510 unsigned DataLen) {
511 // The first bit indicates whether this identifier is interesting.
512 // That's all we care about.
513 using namespace llvm::support;
514 IdentifierID RawID =
515 endian::readNext<IdentifierID, llvm::endianness::little>(memory&: d);
516 bool IsInteresting = RawID & 0x01;
517 return std::make_pair(x: k, y&: IsInteresting);
518 }
519 };
520}
521
522llvm::Error GlobalModuleIndexBuilder::loadModuleFile(FileEntryRef File) {
523 // Open the module file.
524
525 auto Buffer = FileMgr.getBufferForFile(Entry: File, /*isVolatile=*/true);
526 if (!Buffer)
527 return llvm::createStringError(EC: Buffer.getError(),
528 S: "failed getting buffer for module file");
529
530 // Initialize the input stream
531 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(Buffer: **Buffer));
532
533 // Sniff for the signature.
534 for (unsigned char C : {'C', 'P', 'C', 'H'})
535 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(NumBits: 8)) {
536 if (Res.get() != C)
537 return llvm::createStringError(EC: std::errc::illegal_byte_sequence,
538 Fmt: "expected signature CPCH");
539 } else
540 return Res.takeError();
541
542 // Record this module file and assign it a unique ID (if it doesn't have
543 // one already).
544 unsigned ID = getModuleFileInfo(File).ID;
545
546 // Search for the blocks and records we care about.
547 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
548 bool Done = false;
549 while (!Done) {
550 Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
551 if (!MaybeEntry)
552 return MaybeEntry.takeError();
553 llvm::BitstreamEntry Entry = MaybeEntry.get();
554
555 switch (Entry.Kind) {
556 case llvm::BitstreamEntry::Error:
557 Done = true;
558 continue;
559
560 case llvm::BitstreamEntry::Record:
561 // In the 'other' state, just skip the record. We don't care.
562 if (State == Other) {
563 if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(AbbrevID: Entry.ID))
564 continue;
565 else
566 return Skipped.takeError();
567 }
568
569 // Handle potentially-interesting records below.
570 break;
571
572 case llvm::BitstreamEntry::SubBlock:
573 if (Entry.ID == CONTROL_BLOCK_ID) {
574 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: CONTROL_BLOCK_ID))
575 return Err;
576
577 // Found the control block.
578 State = ControlBlock;
579 continue;
580 }
581
582 if (Entry.ID == AST_BLOCK_ID) {
583 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: AST_BLOCK_ID))
584 return Err;
585
586 // Found the AST block.
587 State = ASTBlock;
588 continue;
589 }
590
591 if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
592 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: UNHASHED_CONTROL_BLOCK_ID))
593 return Err;
594
595 // Found the Diagnostic Options block.
596 State = DiagnosticOptionsBlock;
597 continue;
598 }
599
600 if (llvm::Error Err = InStream.SkipBlock())
601 return Err;
602
603 continue;
604
605 case llvm::BitstreamEntry::EndBlock:
606 State = Other;
607 continue;
608 }
609
610 // Read the given record.
611 SmallVector<uint64_t, 64> Record;
612 StringRef Blob;
613 Expected<unsigned> MaybeCode = InStream.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob);
614 if (!MaybeCode)
615 return MaybeCode.takeError();
616 unsigned Code = MaybeCode.get();
617
618 // Handle module dependencies.
619 if (State == ControlBlock && Code == IMPORT) {
620 unsigned Idx = 0;
621 // Read information about the AST file.
622
623 // Skip the imported kind
624 ++Idx;
625
626 // Skip the import location
627 ++Idx;
628
629 // Skip the module name (currently this is only used for prebuilt
630 // modules while here we are only dealing with cached).
631 Blob = Blob.substr(Start: Record[Idx++]);
632
633 // Skip if it is standard C++ module
634 ++Idx;
635
636 // Load stored size/modification time.
637 off_t StoredSize = (off_t)Record[Idx++];
638 time_t StoredModTime = (time_t)Record[Idx++];
639 (void)Record[Idx++]; // ImplicitModuleSuffixLength
640
641 // Skip the stored signature.
642 // FIXME: we could read the signature out of the import and validate it.
643 StringRef SignatureBytes = Blob.substr(Start: 0, N: ASTFileSignature::size);
644 auto StoredSignature = ASTFileSignature::create(First: SignatureBytes.begin(),
645 Last: SignatureBytes.end());
646 Blob = Blob.substr(Start: ASTFileSignature::size);
647
648 // Retrieve the imported file name.
649 unsigned Length = Record[Idx++];
650 StringRef ImportedFile = Blob.substr(Start: 0, N: Length);
651 Blob = Blob.substr(Start: Length);
652
653 // Find the imported module file.
654 auto DependsOnFile =
655 FileMgr.getOptionalFileRef(Filename: ImportedFile, /*OpenFile=*/false,
656 /*CacheFailure=*/false);
657
658 if (!DependsOnFile)
659 return llvm::createStringError(EC: std::errc::bad_file_descriptor,
660 Fmt: "imported file \"%s\" not found",
661 Vals: std::string(ImportedFile).c_str());
662
663 // Save the information in ImportedModuleFileInfo so we can verify after
664 // loading all pcms.
665 ImportedModuleFiles.insert(x: std::make_pair(
666 x&: *DependsOnFile, y: ImportedModuleFileInfo(StoredSize, StoredModTime,
667 StoredSignature)));
668
669 // Record the dependency.
670 unsigned DependsOnID = getModuleFileInfo(File: *DependsOnFile).ID;
671 getModuleFileInfo(File).Dependencies.push_back(Elt: DependsOnID);
672
673 continue;
674 }
675
676 // Handle the identifier table
677 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
678 typedef llvm::OnDiskIterableChainedHashTable<
679 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
680 std::unique_ptr<InterestingIdentifierTable> Table(
681 InterestingIdentifierTable::Create(
682 Buckets: (const unsigned char *)Blob.data() + Record[0],
683 Payload: (const unsigned char *)Blob.data() + sizeof(uint32_t),
684 Base: (const unsigned char *)Blob.data()));
685 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
686 DEnd = Table->data_end();
687 D != DEnd; ++D) {
688 std::pair<StringRef, bool> Ident = *D;
689 if (Ident.second)
690 InterestingIdentifiers[Ident.first].push_back(Elt: ID);
691 else
692 (void)InterestingIdentifiers[Ident.first];
693 }
694 }
695
696 // Get Signature.
697 if (State == DiagnosticOptionsBlock && Code == SIGNATURE) {
698 auto Signature = ASTFileSignature::create(First: Blob.begin(), Last: Blob.end());
699 assert(Signature != ASTFileSignature::createDummy() &&
700 "Dummy AST file signature not backpatched in ASTWriter.");
701 getModuleFileInfo(File).Signature = Signature;
702 }
703
704 // We don't care about this record.
705 }
706
707 return llvm::Error::success();
708}
709
710namespace {
711
712/// Trait used to generate the identifier index as an on-disk hash
713/// table.
714class IdentifierIndexWriterTrait {
715public:
716 typedef StringRef key_type;
717 typedef StringRef key_type_ref;
718 typedef SmallVector<unsigned, 2> data_type;
719 typedef const SmallVector<unsigned, 2> &data_type_ref;
720 typedef unsigned hash_value_type;
721 typedef unsigned offset_type;
722
723 static hash_value_type ComputeHash(key_type_ref Key) {
724 return llvm::djbHash(Buffer: Key);
725 }
726
727 std::pair<unsigned,unsigned>
728 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
729 using namespace llvm::support;
730 endian::Writer LE(Out, llvm::endianness::little);
731 unsigned KeyLen = Key.size();
732 unsigned DataLen = Data.size() * 4;
733 LE.write<uint16_t>(Val: KeyLen);
734 LE.write<uint16_t>(Val: DataLen);
735 return std::make_pair(x&: KeyLen, y&: DataLen);
736 }
737
738 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
739 Out.write(Ptr: Key.data(), Size: KeyLen);
740 }
741
742 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
743 unsigned DataLen) {
744 using namespace llvm::support;
745 for (unsigned I = 0, N = Data.size(); I != N; ++I)
746 endian::write<uint32_t>(os&: Out, value: Data[I], endian: llvm::endianness::little);
747 }
748};
749
750}
751
752bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
753 for (auto MapEntry : ImportedModuleFiles) {
754 auto File = MapEntry.first;
755 ImportedModuleFileInfo &Info = MapEntry.second;
756 if (getModuleFileInfo(File).Signature) {
757 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
758 // Verify Signature.
759 return true;
760 } else if (Info.StoredSize != File.getSize() ||
761 Info.StoredModTime != File.getModificationTime())
762 // Verify Size and ModTime.
763 return true;
764 }
765
766 using namespace llvm;
767 llvm::TimeTraceScope TimeScope("Module WriteIndex");
768
769 // Emit the file header.
770 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
771 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
772 Stream.Emit(Val: (unsigned)'G', NumBits: 8);
773 Stream.Emit(Val: (unsigned)'I', NumBits: 8);
774
775 // Write the block-info block, which describes the records in this bitcode
776 // file.
777 emitBlockInfoBlock(Stream);
778
779 Stream.EnterSubblock(BlockID: GLOBAL_INDEX_BLOCK_ID, CodeLen: 3);
780
781 // Write the metadata.
782 SmallVector<uint64_t, 2> Record;
783 Record.push_back(Elt: CurrentVersion);
784 Stream.EmitRecord(Code: INDEX_METADATA, Vals: Record);
785
786 // Write the set of known module files.
787 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
788 MEnd = ModuleFiles.end();
789 M != MEnd; ++M) {
790 Record.clear();
791 Record.push_back(Elt: M->second.ID);
792 Record.push_back(Elt: M->first.getSize());
793 Record.push_back(Elt: M->first.getModificationTime());
794
795 // File name
796 StringRef Name(M->first.getName());
797 Record.push_back(Elt: Name.size());
798 Record.append(in_start: Name.begin(), in_end: Name.end());
799
800 // Dependencies
801 Record.push_back(Elt: M->second.Dependencies.size());
802 Record.append(in_start: M->second.Dependencies.begin(), in_end: M->second.Dependencies.end());
803 Stream.EmitRecord(Code: MODULE, Vals: Record);
804 }
805
806 // Write the identifier -> module file mapping.
807 {
808 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
809 IdentifierIndexWriterTrait Trait;
810
811 // Populate the hash table.
812 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
813 IEnd = InterestingIdentifiers.end();
814 I != IEnd; ++I) {
815 Generator.insert(Key: I->first(), Data: I->second, InfoObj&: Trait);
816 }
817
818 // Create the on-disk hash table in a buffer.
819 SmallString<4096> IdentifierTable;
820 uint32_t BucketOffset;
821 {
822 using namespace llvm::support;
823 llvm::raw_svector_ostream Out(IdentifierTable);
824 // Make sure that no bucket is at offset 0
825 endian::write<uint32_t>(os&: Out, value: 0, endian: llvm::endianness::little);
826 BucketOffset = Generator.Emit(Out, InfoObj&: Trait);
827 }
828
829 // Create a blob abbreviation
830 auto Abbrev = std::make_shared<BitCodeAbbrev>();
831 Abbrev->Add(OpInfo: BitCodeAbbrevOp(IDENTIFIER_INDEX));
832 Abbrev->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
833 Abbrev->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
834 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbrev));
835
836 // Write the identifier table
837 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
838 Stream.EmitRecordWithBlob(Abbrev: IDTableAbbrev, Vals: Record, Blob: IdentifierTable);
839 }
840
841 Stream.ExitBlock();
842 return false;
843}
844
845llvm::Error
846GlobalModuleIndex::writeIndex(FileManager &FileMgr,
847 const PCHContainerReader &PCHContainerRdr,
848 StringRef Path) {
849 // This is a compiler-internal input/output, let's bypass the sandbox.
850 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
851
852 llvm::SmallString<128> IndexPath;
853 IndexPath += Path;
854 llvm::sys::path::append(path&: IndexPath, a: IndexFileName);
855
856 // Coordinate building the global index file with other processes that might
857 // try to do the same.
858 llvm::LockFileManager Lock(IndexPath);
859 bool Owned;
860 if (llvm::Error Err = Lock.tryLock().moveInto(Value&: Owned)) {
861 llvm::consumeError(Err: std::move(Err));
862 return llvm::createStringError(EC: std::errc::io_error, Fmt: "LFS error");
863 }
864 if (!Owned) {
865 // Someone else is responsible for building the index. We don't care
866 // when they finish, so we're done.
867 return llvm::createStringError(EC: std::errc::device_or_resource_busy,
868 Fmt: "someone else is building the index");
869 }
870
871 // We're responsible for building the index ourselves.
872
873 // The module index builder.
874 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
875
876 // Load each of the module files.
877 std::error_code EC;
878 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
879 D != DEnd && !EC;
880 D.increment(ec&: EC)) {
881 // If this isn't a module file, we don't care.
882 if (llvm::sys::path::extension(path: D->path()) != ".pcm") {
883 // ... unless it's a .pcm.lock file, which indicates that someone is
884 // in the process of rebuilding a module. They'll rebuild the index
885 // at the end of that translation unit, so we don't have to.
886 if (llvm::sys::path::extension(path: D->path()) == ".pcm.lock")
887 return llvm::createStringError(EC: std::errc::device_or_resource_busy,
888 Fmt: "someone else is building the index");
889
890 continue;
891 }
892
893 // If we can't find the module file, skip it.
894 auto ModuleFile = FileMgr.getOptionalFileRef(Filename: D->path());
895 if (!ModuleFile)
896 continue;
897
898 // Load this module file.
899 if (llvm::Error Err = Builder.loadModuleFile(File: *ModuleFile))
900 return Err;
901 }
902
903 // The output buffer, into which the global index will be written.
904 SmallString<16> OutputBuffer;
905 {
906 llvm::BitstreamWriter OutputStream(OutputBuffer);
907 if (Builder.writeIndex(Stream&: OutputStream))
908 return llvm::createStringError(EC: std::errc::io_error,
909 Fmt: "failed writing index");
910 }
911
912 return llvm::writeToOutput(OutputFileName: IndexPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) {
913 OS << OutputBuffer;
914 return llvm::Error::success();
915 });
916}
917
918namespace {
919 class GlobalIndexIdentifierIterator : public IdentifierIterator {
920 /// The current position within the identifier lookup table.
921 IdentifierIndexTable::key_iterator Current;
922
923 /// The end position within the identifier lookup table.
924 IdentifierIndexTable::key_iterator End;
925
926 public:
927 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
928 Current = Idx.key_begin();
929 End = Idx.key_end();
930 }
931
932 StringRef Next() override {
933 if (Current == End)
934 return StringRef();
935
936 StringRef Result = *Current;
937 ++Current;
938 return Result;
939 }
940 };
941}
942
943IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
944 IdentifierIndexTable &Table =
945 *static_cast<IdentifierIndexTable *>(IdentifierIndex);
946 return new GlobalIndexIdentifierIterator(Table);
947}
948