1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the GlobalModuleIndex class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Serialization/GlobalModuleIndex.h"
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Serialization/ASTBitCodes.h"
17#include "clang/Serialization/ModuleFile.h"
18#include "clang/Serialization/PCHContainerOperations.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/SmallString.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/Bitstream/BitstreamReader.h"
24#include "llvm/Bitstream/BitstreamWriter.h"
25#include "llvm/Support/DJB.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/LockFileManager.h"
28#include "llvm/Support/MemoryBuffer.h"
29#include "llvm/Support/OnDiskHashTable.h"
30#include "llvm/Support/Path.h"
31#include "llvm/Support/TimeProfiler.h"
32#include "llvm/Support/raw_ostream.h"
33#include <cstdio>
34using namespace clang;
35using namespace serialization;
36
37//----------------------------------------------------------------------------//
38// Shared constants
39//----------------------------------------------------------------------------//
40namespace {
41 enum {
42 /// The block containing the index.
43 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
44 };
45
46 /// Describes the record types in the index.
47 enum IndexRecordTypes {
48 /// Contains version information and potentially other metadata,
49 /// used to determine if we can read this global index file.
50 INDEX_METADATA,
51 /// Describes a module, including its file name and dependencies.
52 MODULE,
53 /// The index for identifiers.
54 IDENTIFIER_INDEX
55 };
56}
57
58/// The name of the global index file.
59static const char * const IndexFileName = "modules.idx";
60
61/// The global index file version.
62static const unsigned CurrentVersion = 1;
63
64//----------------------------------------------------------------------------//
65// Global module index reader.
66//----------------------------------------------------------------------------//
67
68namespace {
69
70/// Trait used to read the identifier index from the on-disk hash
71/// table.
72class IdentifierIndexReaderTrait {
73public:
74 typedef StringRef external_key_type;
75 typedef StringRef internal_key_type;
76 typedef SmallVector<unsigned, 2> data_type;
77 typedef unsigned hash_value_type;
78 typedef unsigned offset_type;
79
80 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
81 return a == b;
82 }
83
84 static hash_value_type ComputeHash(const internal_key_type& a) {
85 return llvm::djbHash(Buffer: a);
86 }
87
88 static std::pair<unsigned, unsigned>
89 ReadKeyDataLength(const unsigned char*& d) {
90 using namespace llvm::support;
91 unsigned KeyLen = endian::readNext<uint16_t, llvm::endianness::little>(memory&: d);
92 unsigned DataLen = endian::readNext<uint16_t, llvm::endianness::little>(memory&: d);
93 return std::make_pair(x&: KeyLen, y&: DataLen);
94 }
95
96 static const internal_key_type&
97 GetInternalKey(const external_key_type& x) { return x; }
98
99 static const external_key_type&
100 GetExternalKey(const internal_key_type& x) { return x; }
101
102 static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
103 return StringRef((const char *)d, n);
104 }
105
106 static data_type ReadData(const internal_key_type& k,
107 const unsigned char* d,
108 unsigned DataLen) {
109 using namespace llvm::support;
110
111 data_type Result;
112 while (DataLen > 0) {
113 unsigned ID = endian::readNext<uint32_t, llvm::endianness::little>(memory&: d);
114 Result.push_back(Elt: ID);
115 DataLen -= 4;
116 }
117
118 return Result;
119 }
120};
121
122typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
123 IdentifierIndexTable;
124
125}
126
127GlobalModuleIndex::GlobalModuleIndex(
128 std::unique_ptr<llvm::MemoryBuffer> IndexBuffer,
129 llvm::BitstreamCursor Cursor)
130 : Buffer(std::move(IndexBuffer)), IdentifierIndex(), NumIdentifierLookups(),
131 NumIdentifierLookupHits() {
132 auto Fail = [&](llvm::Error &&Err) {
133 report_fatal_error(reason: "Module index '" + Buffer->getBufferIdentifier() +
134 "' failed: " + toString(E: std::move(Err)));
135 };
136
137 llvm::TimeTraceScope TimeScope("Module LoadIndex");
138 // Read the global index.
139 bool InGlobalIndexBlock = false;
140 bool Done = false;
141 while (!Done) {
142 llvm::BitstreamEntry Entry;
143 if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
144 Entry = Res.get();
145 else
146 Fail(Res.takeError());
147
148 switch (Entry.Kind) {
149 case llvm::BitstreamEntry::Error:
150 return;
151
152 case llvm::BitstreamEntry::EndBlock:
153 if (InGlobalIndexBlock) {
154 InGlobalIndexBlock = false;
155 Done = true;
156 continue;
157 }
158 return;
159
160
161 case llvm::BitstreamEntry::Record:
162 // Entries in the global index block are handled below.
163 if (InGlobalIndexBlock)
164 break;
165
166 return;
167
168 case llvm::BitstreamEntry::SubBlock:
169 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
170 if (llvm::Error Err = Cursor.EnterSubBlock(BlockID: GLOBAL_INDEX_BLOCK_ID))
171 Fail(std::move(Err));
172 InGlobalIndexBlock = true;
173 } else if (llvm::Error Err = Cursor.SkipBlock())
174 Fail(std::move(Err));
175 continue;
176 }
177
178 SmallVector<uint64_t, 64> Record;
179 StringRef Blob;
180 Expected<unsigned> MaybeIndexRecord =
181 Cursor.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob);
182 if (!MaybeIndexRecord)
183 Fail(MaybeIndexRecord.takeError());
184 IndexRecordTypes IndexRecord =
185 static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
186 switch (IndexRecord) {
187 case INDEX_METADATA:
188 // Make sure that the version matches.
189 if (Record.size() < 1 || Record[0] != CurrentVersion)
190 return;
191 break;
192
193 case MODULE: {
194 unsigned Idx = 0;
195 unsigned ID = Record[Idx++];
196
197 // Make room for this module's information.
198 if (ID == Modules.size())
199 Modules.push_back(Elt: ModuleInfo());
200 else
201 Modules.resize(N: ID + 1);
202
203 // Size/modification time for this module file at the time the
204 // global index was built.
205 Modules[ID].Size = Record[Idx++];
206 Modules[ID].ModTime = Record[Idx++];
207
208 // File name.
209 unsigned NameLen = Record[Idx++];
210 Modules[ID].FileName.assign(first: Record.begin() + Idx,
211 last: Record.begin() + Idx + NameLen);
212 Idx += NameLen;
213
214 // Dependencies
215 unsigned NumDeps = Record[Idx++];
216 Modules[ID].Dependencies.insert(I: Modules[ID].Dependencies.end(),
217 From: Record.begin() + Idx,
218 To: Record.begin() + Idx + NumDeps);
219 Idx += NumDeps;
220
221 // Make sure we're at the end of the record.
222 assert(Idx == Record.size() && "More module info?");
223
224 // Record this module as an unresolved module.
225 // FIXME: this doesn't work correctly for module names containing path
226 // separators.
227 StringRef ModuleName = llvm::sys::path::stem(path: Modules[ID].FileName);
228 // Remove the -<hash of ModuleMapPath>
229 ModuleName = ModuleName.rsplit(Separator: '-').first;
230 UnresolvedModules[ModuleName] = ID;
231 break;
232 }
233
234 case IDENTIFIER_INDEX:
235 // Wire up the identifier index.
236 if (Record[0]) {
237 IdentifierIndex = IdentifierIndexTable::Create(
238 Buckets: (const unsigned char *)Blob.data() + Record[0],
239 Payload: (const unsigned char *)Blob.data() + sizeof(uint32_t),
240 Base: (const unsigned char *)Blob.data(), InfoObj: IdentifierIndexReaderTrait());
241 }
242 break;
243 }
244 }
245}
246
247GlobalModuleIndex::~GlobalModuleIndex() {
248 delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
249}
250
251std::pair<GlobalModuleIndex *, llvm::Error>
252GlobalModuleIndex::readIndex(StringRef Path) {
253 // Load the index file, if it's there.
254 llvm::SmallString<128> IndexPath;
255 IndexPath += Path;
256 llvm::sys::path::append(path&: IndexPath, a: IndexFileName);
257
258 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
259 llvm::MemoryBuffer::getFile(Filename: IndexPath.c_str());
260 if (!BufferOrErr)
261 return std::make_pair(x: nullptr,
262 y: llvm::errorCodeToError(EC: BufferOrErr.getError()));
263 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
264
265 /// The main bitstream cursor for the main block.
266 llvm::BitstreamCursor Cursor(*Buffer);
267
268 // Sniff for the signature.
269 for (unsigned char C : {'B', 'C', 'G', 'I'}) {
270 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(NumBits: 8)) {
271 if (Res.get() != C)
272 return std::make_pair(
273 x: nullptr, y: llvm::createStringError(EC: std::errc::illegal_byte_sequence,
274 Fmt: "expected signature BCGI"));
275 } else
276 return std::make_pair(x: nullptr, y: Res.takeError());
277 }
278
279 return std::make_pair(x: new GlobalModuleIndex(std::move(Buffer), std::move(Cursor)),
280 y: llvm::Error::success());
281}
282
283void GlobalModuleIndex::getModuleDependencies(
284 ModuleFile *File,
285 SmallVectorImpl<ModuleFile *> &Dependencies) {
286 // Look for information about this module file.
287 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
288 = ModulesByFile.find(Val: File);
289 if (Known == ModulesByFile.end())
290 return;
291
292 // Record dependencies.
293 Dependencies.clear();
294 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
295 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
296 if (ModuleFile *MF = Modules[I].File)
297 Dependencies.push_back(Elt: MF);
298 }
299}
300
301bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
302 Hits.clear();
303
304 // If there's no identifier index, there is nothing we can do.
305 if (!IdentifierIndex)
306 return false;
307
308 // Look into the identifier index.
309 ++NumIdentifierLookups;
310 IdentifierIndexTable &Table
311 = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
312 IdentifierIndexTable::iterator Known = Table.find(EKey: Name);
313 if (Known == Table.end()) {
314 return false;
315 }
316
317 SmallVector<unsigned, 2> ModuleIDs = *Known;
318 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
319 if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
320 Hits.insert(Ptr: MF);
321 }
322
323 ++NumIdentifierLookupHits;
324 return true;
325}
326
327bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
328 // Look for the module in the global module index based on the module name.
329 StringRef Name = File->ModuleName;
330 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Key: Name);
331 if (Known == UnresolvedModules.end()) {
332 return true;
333 }
334
335 // Rectify this module with the global module index.
336 ModuleInfo &Info = Modules[Known->second];
337
338 // If the size and modification time match what we expected, record this
339 // module file.
340 bool Failed = true;
341 if (File->File.getSize() == Info.Size &&
342 File->File.getModificationTime() == Info.ModTime) {
343 Info.File = File;
344 ModulesByFile[File] = Known->second;
345
346 Failed = false;
347 }
348
349 // One way or another, we have resolved this module file.
350 UnresolvedModules.erase(I: Known);
351 return Failed;
352}
353
354void GlobalModuleIndex::printStats() {
355 std::fprintf(stderr, format: "*** Global Module Index Statistics:\n");
356 if (NumIdentifierLookups) {
357 fprintf(stderr, format: " %u / %u identifier lookups succeeded (%f%%)\n",
358 NumIdentifierLookupHits, NumIdentifierLookups,
359 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
360 }
361 std::fprintf(stderr, format: "\n");
362}
363
364LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
365 llvm::errs() << "*** Global Module Index Dump:\n";
366 llvm::errs() << "Module files:\n";
367 for (auto &MI : Modules) {
368 llvm::errs() << "** " << MI.FileName << "\n";
369 if (MI.File)
370 MI.File->dump();
371 else
372 llvm::errs() << "\n";
373 }
374 llvm::errs() << "\n";
375}
376
377//----------------------------------------------------------------------------//
378// Global module index writer.
379//----------------------------------------------------------------------------//
380
381namespace {
382 /// Provides information about a specific module file.
383 struct ModuleFileInfo {
384 /// The numberic ID for this module file.
385 unsigned ID;
386
387 /// The set of modules on which this module depends. Each entry is
388 /// a module ID.
389 SmallVector<unsigned, 4> Dependencies;
390 ASTFileSignature Signature;
391 };
392
393 struct ImportedModuleFileInfo {
394 off_t StoredSize;
395 time_t StoredModTime;
396 ASTFileSignature StoredSignature;
397 ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
398 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
399 };
400
401 /// Builder that generates the global module index file.
402 class GlobalModuleIndexBuilder {
403 FileManager &FileMgr;
404 const PCHContainerReader &PCHContainerRdr;
405
406 /// Mapping from files to module file information.
407 using ModuleFilesMap = llvm::MapVector<FileEntryRef, ModuleFileInfo>;
408
409 /// Information about each of the known module files.
410 ModuleFilesMap ModuleFiles;
411
412 /// Mapping from the imported module file to the imported
413 /// information.
414 using ImportedModuleFilesMap =
415 std::multimap<FileEntryRef, ImportedModuleFileInfo>;
416
417 /// Information about each importing of a module file.
418 ImportedModuleFilesMap ImportedModuleFiles;
419
420 /// Mapping from identifiers to the list of module file IDs that
421 /// consider this identifier to be interesting.
422 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
423
424 /// A mapping from all interesting identifiers to the set of module
425 /// files in which those identifiers are considered interesting.
426 InterestingIdentifierMap InterestingIdentifiers;
427
428 /// Write the block-info block for the global module index file.
429 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
430
431 /// Retrieve the module file information for the given file.
432 ModuleFileInfo &getModuleFileInfo(FileEntryRef File) {
433 auto Known = ModuleFiles.find(Key: File);
434 if (Known != ModuleFiles.end())
435 return Known->second;
436
437 unsigned NewID = ModuleFiles.size();
438 ModuleFileInfo &Info = ModuleFiles[File];
439 Info.ID = NewID;
440 return Info;
441 }
442
443 public:
444 explicit GlobalModuleIndexBuilder(
445 FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
446 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
447
448 /// Load the contents of the given module file into the builder.
449 llvm::Error loadModuleFile(FileEntryRef File);
450
451 /// Write the index to the given bitstream.
452 /// \returns true if an error occurred, false otherwise.
453 bool writeIndex(llvm::BitstreamWriter &Stream);
454 };
455}
456
457static void emitBlockID(unsigned ID, const char *Name,
458 llvm::BitstreamWriter &Stream,
459 SmallVectorImpl<uint64_t> &Record) {
460 Record.clear();
461 Record.push_back(Elt: ID);
462 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_SETBID, Vals: Record);
463
464 // Emit the block name if present.
465 if (!Name || Name[0] == 0) return;
466 Record.clear();
467 while (*Name)
468 Record.push_back(Elt: *Name++);
469 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Vals: Record);
470}
471
472static void emitRecordID(unsigned ID, const char *Name,
473 llvm::BitstreamWriter &Stream,
474 SmallVectorImpl<uint64_t> &Record) {
475 Record.clear();
476 Record.push_back(Elt: ID);
477 while (*Name)
478 Record.push_back(Elt: *Name++);
479 Stream.EmitRecord(Code: llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Vals: Record);
480}
481
482void
483GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
484 SmallVector<uint64_t, 64> Record;
485 Stream.EnterBlockInfoBlock();
486
487#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
488#define RECORD(X) emitRecordID(X, #X, Stream, Record)
489 BLOCK(GLOBAL_INDEX_BLOCK);
490 RECORD(INDEX_METADATA);
491 RECORD(MODULE);
492 RECORD(IDENTIFIER_INDEX);
493#undef RECORD
494#undef BLOCK
495
496 Stream.ExitBlock();
497}
498
499namespace {
500 class InterestingASTIdentifierLookupTrait
501 : public serialization::reader::ASTIdentifierLookupTraitBase {
502
503 public:
504 /// The identifier and whether it is "interesting".
505 typedef std::pair<StringRef, bool> data_type;
506
507 data_type ReadData(const internal_key_type& k,
508 const unsigned char* d,
509 unsigned DataLen) {
510 // The first bit indicates whether this identifier is interesting.
511 // That's all we care about.
512 using namespace llvm::support;
513 IdentifierID RawID =
514 endian::readNext<IdentifierID, llvm::endianness::little>(memory&: d);
515 bool IsInteresting = RawID & 0x01;
516 return std::make_pair(x: k, y&: IsInteresting);
517 }
518 };
519}
520
521llvm::Error GlobalModuleIndexBuilder::loadModuleFile(FileEntryRef File) {
522 // Open the module file.
523
524 auto Buffer = FileMgr.getBufferForFile(Entry: File, /*isVolatile=*/true);
525 if (!Buffer)
526 return llvm::createStringError(EC: Buffer.getError(),
527 S: "failed getting buffer for module file");
528
529 // Initialize the input stream
530 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(Buffer: **Buffer));
531
532 // Sniff for the signature.
533 for (unsigned char C : {'C', 'P', 'C', 'H'})
534 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(NumBits: 8)) {
535 if (Res.get() != C)
536 return llvm::createStringError(EC: std::errc::illegal_byte_sequence,
537 Fmt: "expected signature CPCH");
538 } else
539 return Res.takeError();
540
541 // Record this module file and assign it a unique ID (if it doesn't have
542 // one already).
543 unsigned ID = getModuleFileInfo(File).ID;
544
545 // Search for the blocks and records we care about.
546 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
547 bool Done = false;
548 while (!Done) {
549 Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
550 if (!MaybeEntry)
551 return MaybeEntry.takeError();
552 llvm::BitstreamEntry Entry = MaybeEntry.get();
553
554 switch (Entry.Kind) {
555 case llvm::BitstreamEntry::Error:
556 Done = true;
557 continue;
558
559 case llvm::BitstreamEntry::Record:
560 // In the 'other' state, just skip the record. We don't care.
561 if (State == Other) {
562 if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(AbbrevID: Entry.ID))
563 continue;
564 else
565 return Skipped.takeError();
566 }
567
568 // Handle potentially-interesting records below.
569 break;
570
571 case llvm::BitstreamEntry::SubBlock:
572 if (Entry.ID == CONTROL_BLOCK_ID) {
573 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: CONTROL_BLOCK_ID))
574 return Err;
575
576 // Found the control block.
577 State = ControlBlock;
578 continue;
579 }
580
581 if (Entry.ID == AST_BLOCK_ID) {
582 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: AST_BLOCK_ID))
583 return Err;
584
585 // Found the AST block.
586 State = ASTBlock;
587 continue;
588 }
589
590 if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
591 if (llvm::Error Err = InStream.EnterSubBlock(BlockID: UNHASHED_CONTROL_BLOCK_ID))
592 return Err;
593
594 // Found the Diagnostic Options block.
595 State = DiagnosticOptionsBlock;
596 continue;
597 }
598
599 if (llvm::Error Err = InStream.SkipBlock())
600 return Err;
601
602 continue;
603
604 case llvm::BitstreamEntry::EndBlock:
605 State = Other;
606 continue;
607 }
608
609 // Read the given record.
610 SmallVector<uint64_t, 64> Record;
611 StringRef Blob;
612 Expected<unsigned> MaybeCode = InStream.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob);
613 if (!MaybeCode)
614 return MaybeCode.takeError();
615 unsigned Code = MaybeCode.get();
616
617 // Handle module dependencies.
618 if (State == ControlBlock && Code == IMPORTS) {
619 // Load each of the imported PCH files.
620 unsigned Idx = 0, N = Record.size();
621 while (Idx < N) {
622 // Read information about the AST file.
623
624 // Skip the imported kind
625 ++Idx;
626
627 // Skip if it is standard C++ module
628 ++Idx;
629
630 // Skip the import location
631 ++Idx;
632
633 // Load stored size/modification time.
634 off_t StoredSize = (off_t)Record[Idx++];
635 time_t StoredModTime = (time_t)Record[Idx++];
636
637 // Skip the stored signature.
638 // FIXME: we could read the signature out of the import and validate it.
639 auto FirstSignatureByte = Record.begin() + Idx;
640 ASTFileSignature StoredSignature = ASTFileSignature::create(
641 First: FirstSignatureByte, Last: FirstSignatureByte + ASTFileSignature::size);
642 Idx += ASTFileSignature::size;
643
644 // Skip the module name (currently this is only used for prebuilt
645 // modules while here we are only dealing with cached).
646 Idx += Record[Idx] + 1;
647
648 // Retrieve the imported file name.
649 unsigned Length = Record[Idx++];
650 SmallString<128> ImportedFile(Record.begin() + Idx,
651 Record.begin() + Idx + Length);
652 Idx += Length;
653
654 // Find the imported module file.
655 auto DependsOnFile =
656 FileMgr.getOptionalFileRef(Filename: ImportedFile, /*OpenFile=*/false,
657 /*CacheFailure=*/false);
658
659 if (!DependsOnFile)
660 return llvm::createStringError(EC: std::errc::bad_file_descriptor,
661 Fmt: "imported file \"%s\" not found",
662 Vals: ImportedFile.c_str());
663
664 // Save the information in ImportedModuleFileInfo so we can verify after
665 // loading all pcms.
666 ImportedModuleFiles.insert(x: std::make_pair(
667 x&: *DependsOnFile, y: ImportedModuleFileInfo(StoredSize, StoredModTime,
668 StoredSignature)));
669
670 // Record the dependency.
671 unsigned DependsOnID = getModuleFileInfo(File: *DependsOnFile).ID;
672 getModuleFileInfo(File).Dependencies.push_back(Elt: DependsOnID);
673 }
674
675 continue;
676 }
677
678 // Handle the identifier table
679 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
680 typedef llvm::OnDiskIterableChainedHashTable<
681 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
682 std::unique_ptr<InterestingIdentifierTable> Table(
683 InterestingIdentifierTable::Create(
684 Buckets: (const unsigned char *)Blob.data() + Record[0],
685 Payload: (const unsigned char *)Blob.data() + sizeof(uint32_t),
686 Base: (const unsigned char *)Blob.data()));
687 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
688 DEnd = Table->data_end();
689 D != DEnd; ++D) {
690 std::pair<StringRef, bool> Ident = *D;
691 if (Ident.second)
692 InterestingIdentifiers[Ident.first].push_back(Elt: ID);
693 else
694 (void)InterestingIdentifiers[Ident.first];
695 }
696 }
697
698 // Get Signature.
699 if (State == DiagnosticOptionsBlock && Code == SIGNATURE) {
700 auto Signature = ASTFileSignature::create(First: Blob.begin(), Last: Blob.end());
701 assert(Signature != ASTFileSignature::createDummy() &&
702 "Dummy AST file signature not backpatched in ASTWriter.");
703 getModuleFileInfo(File).Signature = Signature;
704 }
705
706 // We don't care about this record.
707 }
708
709 return llvm::Error::success();
710}
711
712namespace {
713
714/// Trait used to generate the identifier index as an on-disk hash
715/// table.
716class IdentifierIndexWriterTrait {
717public:
718 typedef StringRef key_type;
719 typedef StringRef key_type_ref;
720 typedef SmallVector<unsigned, 2> data_type;
721 typedef const SmallVector<unsigned, 2> &data_type_ref;
722 typedef unsigned hash_value_type;
723 typedef unsigned offset_type;
724
725 static hash_value_type ComputeHash(key_type_ref Key) {
726 return llvm::djbHash(Buffer: Key);
727 }
728
729 std::pair<unsigned,unsigned>
730 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
731 using namespace llvm::support;
732 endian::Writer LE(Out, llvm::endianness::little);
733 unsigned KeyLen = Key.size();
734 unsigned DataLen = Data.size() * 4;
735 LE.write<uint16_t>(Val: KeyLen);
736 LE.write<uint16_t>(Val: DataLen);
737 return std::make_pair(x&: KeyLen, y&: DataLen);
738 }
739
740 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
741 Out.write(Ptr: Key.data(), Size: KeyLen);
742 }
743
744 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
745 unsigned DataLen) {
746 using namespace llvm::support;
747 for (unsigned I = 0, N = Data.size(); I != N; ++I)
748 endian::write<uint32_t>(os&: Out, value: Data[I], endian: llvm::endianness::little);
749 }
750};
751
752}
753
754bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
755 for (auto MapEntry : ImportedModuleFiles) {
756 auto File = MapEntry.first;
757 ImportedModuleFileInfo &Info = MapEntry.second;
758 if (getModuleFileInfo(File).Signature) {
759 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
760 // Verify Signature.
761 return true;
762 } else if (Info.StoredSize != File.getSize() ||
763 Info.StoredModTime != File.getModificationTime())
764 // Verify Size and ModTime.
765 return true;
766 }
767
768 using namespace llvm;
769 llvm::TimeTraceScope TimeScope("Module WriteIndex");
770
771 // Emit the file header.
772 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
773 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
774 Stream.Emit(Val: (unsigned)'G', NumBits: 8);
775 Stream.Emit(Val: (unsigned)'I', NumBits: 8);
776
777 // Write the block-info block, which describes the records in this bitcode
778 // file.
779 emitBlockInfoBlock(Stream);
780
781 Stream.EnterSubblock(BlockID: GLOBAL_INDEX_BLOCK_ID, CodeLen: 3);
782
783 // Write the metadata.
784 SmallVector<uint64_t, 2> Record;
785 Record.push_back(Elt: CurrentVersion);
786 Stream.EmitRecord(Code: INDEX_METADATA, Vals: Record);
787
788 // Write the set of known module files.
789 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
790 MEnd = ModuleFiles.end();
791 M != MEnd; ++M) {
792 Record.clear();
793 Record.push_back(Elt: M->second.ID);
794 Record.push_back(Elt: M->first.getSize());
795 Record.push_back(Elt: M->first.getModificationTime());
796
797 // File name
798 StringRef Name(M->first.getName());
799 Record.push_back(Elt: Name.size());
800 Record.append(in_start: Name.begin(), in_end: Name.end());
801
802 // Dependencies
803 Record.push_back(Elt: M->second.Dependencies.size());
804 Record.append(in_start: M->second.Dependencies.begin(), in_end: M->second.Dependencies.end());
805 Stream.EmitRecord(Code: MODULE, Vals: Record);
806 }
807
808 // Write the identifier -> module file mapping.
809 {
810 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
811 IdentifierIndexWriterTrait Trait;
812
813 // Populate the hash table.
814 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
815 IEnd = InterestingIdentifiers.end();
816 I != IEnd; ++I) {
817 Generator.insert(Key: I->first(), Data: I->second, InfoObj&: Trait);
818 }
819
820 // Create the on-disk hash table in a buffer.
821 SmallString<4096> IdentifierTable;
822 uint32_t BucketOffset;
823 {
824 using namespace llvm::support;
825 llvm::raw_svector_ostream Out(IdentifierTable);
826 // Make sure that no bucket is at offset 0
827 endian::write<uint32_t>(os&: Out, value: 0, endian: llvm::endianness::little);
828 BucketOffset = Generator.Emit(Out, InfoObj&: Trait);
829 }
830
831 // Create a blob abbreviation
832 auto Abbrev = std::make_shared<BitCodeAbbrev>();
833 Abbrev->Add(OpInfo: BitCodeAbbrevOp(IDENTIFIER_INDEX));
834 Abbrev->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
835 Abbrev->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
836 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbrev));
837
838 // Write the identifier table
839 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
840 Stream.EmitRecordWithBlob(Abbrev: IDTableAbbrev, Vals: Record, Blob: IdentifierTable);
841 }
842
843 Stream.ExitBlock();
844 return false;
845}
846
847llvm::Error
848GlobalModuleIndex::writeIndex(FileManager &FileMgr,
849 const PCHContainerReader &PCHContainerRdr,
850 StringRef Path) {
851 llvm::SmallString<128> IndexPath;
852 IndexPath += Path;
853 llvm::sys::path::append(path&: IndexPath, a: IndexFileName);
854
855 // Coordinate building the global index file with other processes that might
856 // try to do the same.
857 llvm::LockFileManager Locked(IndexPath);
858 switch (Locked) {
859 case llvm::LockFileManager::LFS_Error:
860 return llvm::createStringError(EC: std::errc::io_error, Fmt: "LFS error");
861
862 case llvm::LockFileManager::LFS_Owned:
863 // We're responsible for building the index ourselves. Do so below.
864 break;
865
866 case llvm::LockFileManager::LFS_Shared:
867 // Someone else is responsible for building the index. We don't care
868 // when they finish, so we're done.
869 return llvm::createStringError(EC: std::errc::device_or_resource_busy,
870 Fmt: "someone else is building the index");
871 }
872
873 // The module index builder.
874 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
875
876 // Load each of the module files.
877 std::error_code EC;
878 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
879 D != DEnd && !EC;
880 D.increment(ec&: EC)) {
881 // If this isn't a module file, we don't care.
882 if (llvm::sys::path::extension(path: D->path()) != ".pcm") {
883 // ... unless it's a .pcm.lock file, which indicates that someone is
884 // in the process of rebuilding a module. They'll rebuild the index
885 // at the end of that translation unit, so we don't have to.
886 if (llvm::sys::path::extension(path: D->path()) == ".pcm.lock")
887 return llvm::createStringError(EC: std::errc::device_or_resource_busy,
888 Fmt: "someone else is building the index");
889
890 continue;
891 }
892
893 // If we can't find the module file, skip it.
894 auto ModuleFile = FileMgr.getOptionalFileRef(Filename: D->path());
895 if (!ModuleFile)
896 continue;
897
898 // Load this module file.
899 if (llvm::Error Err = Builder.loadModuleFile(File: *ModuleFile))
900 return Err;
901 }
902
903 // The output buffer, into which the global index will be written.
904 SmallString<16> OutputBuffer;
905 {
906 llvm::BitstreamWriter OutputStream(OutputBuffer);
907 if (Builder.writeIndex(Stream&: OutputStream))
908 return llvm::createStringError(EC: std::errc::io_error,
909 Fmt: "failed writing index");
910 }
911
912 return llvm::writeToOutput(OutputFileName: IndexPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) {
913 OS << OutputBuffer;
914 return llvm::Error::success();
915 });
916}
917
918namespace {
919 class GlobalIndexIdentifierIterator : public IdentifierIterator {
920 /// The current position within the identifier lookup table.
921 IdentifierIndexTable::key_iterator Current;
922
923 /// The end position within the identifier lookup table.
924 IdentifierIndexTable::key_iterator End;
925
926 public:
927 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
928 Current = Idx.key_begin();
929 End = Idx.key_end();
930 }
931
932 StringRef Next() override {
933 if (Current == End)
934 return StringRef();
935
936 StringRef Result = *Current;
937 ++Current;
938 return Result;
939 }
940 };
941}
942
943IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
944 IdentifierIndexTable &Table =
945 *static_cast<IdentifierIndexTable *>(IdentifierIndex);
946 return new GlobalIndexIdentifierIterator(Table);
947}
948