1//===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper class to build precompiled preamble.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Frontend/PrecompiledPreamble.h"
14#include "clang/Basic/FileManager.h"
15#include "clang/Basic/LangStandard.h"
16#include "clang/Frontend/CompilerInstance.h"
17#include "clang/Frontend/CompilerInvocation.h"
18#include "clang/Frontend/FrontendActions.h"
19#include "clang/Frontend/FrontendOptions.h"
20#include "clang/Lex/HeaderSearch.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Preprocessor.h"
23#include "clang/Lex/PreprocessorOptions.h"
24#include "clang/Serialization/ASTWriter.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/Config/llvm-config.h"
29#include "llvm/Support/CrashRecoveryContext.h"
30#include "llvm/Support/FileSystem.h"
31#include "llvm/Support/ManagedStatic.h"
32#include "llvm/Support/Path.h"
33#include "llvm/Support/Process.h"
34#include "llvm/Support/VirtualFileSystem.h"
35#include <limits>
36#include <mutex>
37#include <utility>
38
39using namespace clang;
40
41namespace {
42
43StringRef getInMemoryPreamblePath() {
44#if defined(LLVM_ON_UNIX)
45 return "/__clang_tmp/___clang_inmemory_preamble___";
46#elif defined(_WIN32)
47 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
48#else
49#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
50 return "/__clang_tmp/___clang_inmemory_preamble___";
51#endif
52}
53
54IntrusiveRefCntPtr<llvm::vfs::FileSystem>
55createVFSOverlayForPreamblePCH(StringRef PCHFilename,
56 std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
57 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
58 // We want only the PCH file from the real filesystem to be available,
59 // so we create an in-memory VFS with just that and overlay it on top.
60 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS(
61 new llvm::vfs::InMemoryFileSystem());
62 PCHFS->addFile(Path: PCHFilename, ModificationTime: 0, Buffer: std::move(PCHBuffer));
63 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> Overlay(
64 new llvm::vfs::OverlayFileSystem(VFS));
65 Overlay->pushOverlay(FS: PCHFS);
66 return Overlay;
67}
68
69class PreambleDependencyCollector : public DependencyCollector {
70public:
71 // We want to collect all dependencies for correctness. Avoiding the real
72 // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
73 // but there is no way to distinguish between those and the ones that can be
74 // spuriously added by '-isystem' (e.g. to suppress warnings from those
75 // headers).
76 bool needSystemDependencies() override { return true; }
77};
78
79// Collects files whose existence would invalidate the preamble.
80// Collecting *all* of these would make validating it too slow though, so we
81// just find all the candidates for 'file not found' diagnostics.
82//
83// A caveat that may be significant for generated files: we'll omit files under
84// search path entries whose roots don't exist when the preamble is built.
85// These are pruned by InitHeaderSearch and so we don't see the search path.
86// It would be nice to include them but we don't want to duplicate all the rest
87// of the InitHeaderSearch logic to reconstruct them.
88class MissingFileCollector : public PPCallbacks {
89 llvm::StringSet<> &Out;
90 const HeaderSearch &Search;
91 const SourceManager &SM;
92
93public:
94 MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
95 const SourceManager &SM)
96 : Out(Out), Search(Search), SM(SM) {}
97
98 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
99 StringRef FileName, bool IsAngled,
100 CharSourceRange FilenameRange,
101 OptionalFileEntryRef File, StringRef SearchPath,
102 StringRef RelativePath, const Module *SuggestedModule,
103 bool ModuleImported,
104 SrcMgr::CharacteristicKind FileType) override {
105 // File is std::nullopt if it wasn't found.
106 // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
107 if (File)
108 return;
109
110 // If it's a rare absolute include, we know the full path already.
111 if (llvm::sys::path::is_absolute(path: FileName)) {
112 Out.insert(key: FileName);
113 return;
114 }
115
116 // Reconstruct the filenames that would satisfy this directive...
117 llvm::SmallString<256> Buf;
118 auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
119 Buf = DE.getName();
120 llvm::sys::path::append(path&: Buf, a: FileName);
121 llvm::sys::path::remove_dots(path&: Buf, /*remove_dot_dot=*/true);
122 Out.insert(key: Buf);
123 };
124 // ...relative to the including file.
125 if (!IsAngled) {
126 if (OptionalFileEntryRef IncludingFile =
127 SM.getFileEntryRefForID(FID: SM.getFileID(SpellingLoc: IncludeTok.getLocation())))
128 if (IncludingFile->getDir())
129 NotFoundRelativeTo(IncludingFile->getDir());
130 }
131 // ...relative to the search paths.
132 for (const auto &Dir : llvm::make_range(
133 x: IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
134 y: Search.search_dir_end())) {
135 // No support for frameworks or header maps yet.
136 if (Dir.isNormalDir())
137 NotFoundRelativeTo(*Dir.getDirRef());
138 }
139 }
140};
141
142/// Keeps a track of files to be deleted in destructor.
143class TemporaryFiles {
144public:
145 // A static instance to be used by all clients.
146 static TemporaryFiles &getInstance();
147
148private:
149 // Disallow constructing the class directly.
150 TemporaryFiles() = default;
151 // Disallow copy.
152 TemporaryFiles(const TemporaryFiles &) = delete;
153
154public:
155 ~TemporaryFiles();
156
157 /// Adds \p File to a set of tracked files.
158 void addFile(StringRef File);
159
160 /// Remove \p File from disk and from the set of tracked files.
161 void removeFile(StringRef File);
162
163private:
164 std::mutex Mutex;
165 llvm::StringSet<> Files;
166};
167
168TemporaryFiles &TemporaryFiles::getInstance() {
169 static TemporaryFiles Instance;
170 return Instance;
171}
172
173TemporaryFiles::~TemporaryFiles() {
174 std::lock_guard<std::mutex> Guard(Mutex);
175 for (const auto &File : Files)
176 llvm::sys::fs::remove(path: File.getKey());
177}
178
179void TemporaryFiles::addFile(StringRef File) {
180 std::lock_guard<std::mutex> Guard(Mutex);
181 auto IsInserted = Files.insert(key: File).second;
182 (void)IsInserted;
183 assert(IsInserted && "File has already been added");
184}
185
186void TemporaryFiles::removeFile(StringRef File) {
187 std::lock_guard<std::mutex> Guard(Mutex);
188 auto WasPresent = Files.erase(Key: File);
189 (void)WasPresent;
190 assert(WasPresent && "File was not tracked");
191 llvm::sys::fs::remove(path: File);
192}
193
194// A temp file that would be deleted on destructor call. If destructor is not
195// called for any reason, the file will be deleted at static objects'
196// destruction.
197// An assertion will fire if two TempPCHFiles are created with the same name,
198// so it's not intended to be used outside preamble-handling.
199class TempPCHFile {
200public:
201 // A main method used to construct TempPCHFile.
202 static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
203 // FIXME: This is a hack so that we can override the preamble file during
204 // crash-recovery testing, which is the only case where the preamble files
205 // are not necessarily cleaned up.
206 if (const char *TmpFile = ::getenv(name: "CINDEXTEST_PREAMBLE_FILE"))
207 return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
208
209 llvm::SmallString<128> File;
210 // Using the versions of createTemporaryFile() and
211 // createUniqueFile() with a file descriptor guarantees
212 // that we would never get a race condition in a multi-threaded setting
213 // (i.e., multiple threads getting the same temporary path).
214 int FD;
215 std::error_code EC;
216 if (StoragePath.empty())
217 EC = llvm::sys::fs::createTemporaryFile(Prefix: "preamble", Suffix: "pch", ResultFD&: FD, ResultPath&: File);
218 else {
219 llvm::SmallString<128> TempPath = StoragePath;
220 // Use the same filename model as fs::createTemporaryFile().
221 llvm::sys::path::append(path&: TempPath, a: "preamble-%%%%%%.pch");
222 namespace fs = llvm::sys::fs;
223 // Use the same owner-only file permissions as fs::createTemporaryFile().
224 EC = fs::createUniqueFile(Model: TempPath, ResultFD&: FD, ResultPath&: File, Flags: fs::OF_None,
225 Mode: fs::owner_read | fs::owner_write);
226 }
227 if (EC)
228 return nullptr;
229 // We only needed to make sure the file exists, close the file right away.
230 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
231 return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
232 }
233
234 TempPCHFile &operator=(const TempPCHFile &) = delete;
235 TempPCHFile(const TempPCHFile &) = delete;
236 ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(File: FilePath); };
237
238 /// A path where temporary file is stored.
239 llvm::StringRef getFilePath() const { return FilePath; };
240
241private:
242 TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
243 TemporaryFiles::getInstance().addFile(File: this->FilePath);
244 }
245
246 std::string FilePath;
247};
248
249class PrecompilePreambleAction : public ASTFrontendAction {
250public:
251 PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
252 PreambleCallbacks &Callbacks)
253 : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
254 Callbacks(Callbacks) {}
255
256 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
257 StringRef InFile) override;
258
259 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
260
261 void setEmittedPreamblePCH(ASTWriter &Writer) {
262 if (FileOS) {
263 *FileOS << Buffer->Data;
264 // Make sure it hits disk now.
265 FileOS.reset();
266 }
267
268 this->HasEmittedPreamblePCH = true;
269 Callbacks.AfterPCHEmitted(Writer);
270 }
271
272 bool BeginSourceFileAction(CompilerInstance &CI) override {
273 assert(CI.getLangOpts().CompilingPCH);
274 return ASTFrontendAction::BeginSourceFileAction(CI);
275 }
276
277 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
278 bool hasCodeCompletionSupport() const override { return false; }
279 bool hasASTFileSupport() const override { return false; }
280 TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
281
282private:
283 friend class PrecompilePreambleConsumer;
284
285 bool HasEmittedPreamblePCH = false;
286 std::shared_ptr<PCHBuffer> Buffer;
287 bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
288 std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
289 PreambleCallbacks &Callbacks;
290};
291
292class PrecompilePreambleConsumer : public PCHGenerator {
293public:
294 PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
295 InMemoryModuleCache &ModuleCache,
296 StringRef isysroot,
297 std::shared_ptr<PCHBuffer> Buffer)
298 : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
299 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
300 /*AllowASTWithErrors=*/true),
301 Action(Action) {}
302
303 bool HandleTopLevelDecl(DeclGroupRef DG) override {
304 Action.Callbacks.HandleTopLevelDecl(DG);
305 return true;
306 }
307
308 void HandleTranslationUnit(ASTContext &Ctx) override {
309 PCHGenerator::HandleTranslationUnit(Ctx);
310 if (!hasEmittedPCH())
311 return;
312 Action.setEmittedPreamblePCH(getWriter());
313 }
314
315 bool shouldSkipFunctionBody(Decl *D) override {
316 return Action.Callbacks.shouldSkipFunctionBody(D);
317 }
318
319private:
320 PrecompilePreambleAction &Action;
321};
322
323std::unique_ptr<ASTConsumer>
324PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
325 StringRef InFile) {
326 std::string Sysroot;
327 if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
328 return nullptr;
329
330 if (WritePCHFile) {
331 std::string OutputFile; // unused
332 FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
333 if (!FileOS)
334 return nullptr;
335 }
336
337 if (!CI.getFrontendOpts().RelocatablePCH)
338 Sysroot.clear();
339
340 return std::make_unique<PrecompilePreambleConsumer>(
341 args&: *this, args&: CI.getPreprocessor(), args&: CI.getModuleCache(), args&: Sysroot, args&: Buffer);
342}
343
344template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
345 if (!Val)
346 return false;
347 Output = std::move(*Val);
348 return true;
349}
350
351} // namespace
352
353PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
354 const llvm::MemoryBufferRef &Buffer,
355 unsigned MaxLines) {
356 return Lexer::ComputePreamble(Buffer: Buffer.getBuffer(), LangOpts, MaxLines);
357}
358
359class PrecompiledPreamble::PCHStorage {
360public:
361 static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
362 assert(File);
363 std::unique_ptr<PCHStorage> S(new PCHStorage());
364 S->File = std::move(File);
365 return S;
366 }
367 static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
368 std::unique_ptr<PCHStorage> S(new PCHStorage());
369 S->Memory = std::move(Buf);
370 return S;
371 }
372
373 enum class Kind { InMemory, TempFile };
374 Kind getKind() const {
375 if (Memory)
376 return Kind::InMemory;
377 if (File)
378 return Kind::TempFile;
379 llvm_unreachable("Neither Memory nor File?");
380 }
381 llvm::StringRef filePath() const {
382 assert(getKind() == Kind::TempFile);
383 return File->getFilePath();
384 }
385 llvm::StringRef memoryContents() const {
386 assert(getKind() == Kind::InMemory);
387 return StringRef(Memory->Data.data(), Memory->Data.size());
388 }
389
390 // Shrink in-memory buffers to fit.
391 // This incurs a copy, but preambles tend to be long-lived.
392 // Only safe to call once nothing can alias the buffer.
393 void shrink() {
394 if (!Memory)
395 return;
396 Memory->Data = decltype(Memory->Data)(Memory->Data);
397 }
398
399private:
400 PCHStorage() = default;
401 PCHStorage(const PCHStorage &) = delete;
402 PCHStorage &operator=(const PCHStorage &) = delete;
403
404 std::shared_ptr<PCHBuffer> Memory;
405 std::unique_ptr<TempPCHFile> File;
406};
407
408PrecompiledPreamble::~PrecompiledPreamble() = default;
409PrecompiledPreamble::PrecompiledPreamble(PrecompiledPreamble &&) = default;
410PrecompiledPreamble &
411PrecompiledPreamble::operator=(PrecompiledPreamble &&) = default;
412
413llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
414 const CompilerInvocation &Invocation,
415 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
416 DiagnosticsEngine &Diagnostics,
417 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
418 std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
419 StringRef StoragePath, PreambleCallbacks &Callbacks) {
420 assert(VFS && "VFS is null");
421
422 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
423 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
424 PreprocessorOptions &PreprocessorOpts =
425 PreambleInvocation->getPreprocessorOpts();
426
427 std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
428 std::unique_ptr<PCHStorage> Storage;
429 if (StoreInMemory) {
430 Storage = PCHStorage::inMemory(Buf: Buffer);
431 } else {
432 // Create a temporary file for the precompiled preamble. In rare
433 // circumstances, this can fail.
434 std::unique_ptr<TempPCHFile> PreamblePCHFile =
435 TempPCHFile::create(StoragePath);
436 if (!PreamblePCHFile)
437 return BuildPreambleError::CouldntCreateTempFile;
438 Storage = PCHStorage::file(File: std::move(PreamblePCHFile));
439 }
440
441 // Save the preamble text for later; we'll need to compare against it for
442 // subsequent reparses.
443 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
444 MainFileBuffer->getBufferStart() +
445 Bounds.Size);
446 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
447
448 // Tell the compiler invocation to generate a temporary precompiled header.
449 FrontendOpts.ProgramAction = frontend::GeneratePCH;
450 FrontendOpts.OutputFile = std::string(
451 StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
452 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
453 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
454 // Inform preprocessor to record conditional stack when building the preamble.
455 PreprocessorOpts.GeneratePreamble = true;
456
457 // Create the compiler instance to use for building the precompiled preamble.
458 std::unique_ptr<CompilerInstance> Clang(
459 new CompilerInstance(std::move(PCHContainerOps)));
460
461 // Recover resources if we crash before exiting this method.
462 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
463 Clang.get());
464
465 Clang->setInvocation(std::move(PreambleInvocation));
466 Clang->setDiagnostics(&Diagnostics);
467
468 // Create the target instance.
469 if (!Clang->createTarget())
470 return BuildPreambleError::CouldntCreateTargetInfo;
471
472 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
473 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
474 InputKind::Source ||
475 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
476 Language::LLVM_IR) {
477 return BuildPreambleError::BadInputs;
478 }
479
480 // Clear out old caches and data.
481 Diagnostics.Reset();
482 ProcessWarningOptions(Diags&: Diagnostics, Opts: Clang->getDiagnosticOpts());
483
484 VFS =
485 createVFSFromCompilerInvocation(CI: Clang->getInvocation(), Diags&: Diagnostics, BaseFS: VFS);
486
487 // Create a file manager object to provide access to and cache the filesystem.
488 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
489
490 // Create the source manager.
491 Clang->setSourceManager(
492 new SourceManager(Diagnostics, Clang->getFileManager()));
493
494 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
495 Clang->addDependencyCollector(Listener: PreambleDepCollector);
496
497 Clang->getLangOpts().CompilingPCH = true;
498
499 // Remap the main source file to the preamble buffer.
500 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
501 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
502 InputData: MainFileBuffer->getBuffer().slice(Start: 0, End: Bounds.Size), BufferName: MainFilePath);
503 if (PreprocessorOpts.RetainRemappedFileBuffers) {
504 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
505 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: PreambleInputBuffer.get());
506 } else {
507 // In that case, remapped buffer will be deleted by CompilerInstance on
508 // BeginSourceFile, so we call release() to avoid double deletion.
509 PreprocessorOpts.addRemappedFile(From: MainFilePath,
510 To: PreambleInputBuffer.release());
511 }
512
513 auto Act = std::make_unique<PrecompilePreambleAction>(
514 args: std::move(Buffer),
515 /*WritePCHFile=*/args: Storage->getKind() == PCHStorage::Kind::TempFile,
516 args&: Callbacks);
517 if (!Act->BeginSourceFile(CI&: *Clang.get(), Input: Clang->getFrontendOpts().Inputs[0]))
518 return BuildPreambleError::BeginSourceFileFailed;
519
520 // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
521 // referenced in the callback.
522 Callbacks.BeforeExecute(CI&: *Clang);
523
524 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
525 Callbacks.createPPCallbacks();
526 if (DelegatedPPCallbacks)
527 Clang->getPreprocessor().addPPCallbacks(C: std::move(DelegatedPPCallbacks));
528 if (auto CommentHandler = Callbacks.getCommentHandler())
529 Clang->getPreprocessor().addCommentHandler(Handler: CommentHandler);
530 llvm::StringSet<> MissingFiles;
531 Clang->getPreprocessor().addPPCallbacks(
532 C: std::make_unique<MissingFileCollector>(
533 args&: MissingFiles, args&: Clang->getPreprocessor().getHeaderSearchInfo(),
534 args&: Clang->getSourceManager()));
535
536 if (llvm::Error Err = Act->Execute())
537 return errorToErrorCode(Err: std::move(Err));
538
539 // Run the callbacks.
540 Callbacks.AfterExecute(CI&: *Clang);
541
542 Act->EndSourceFile();
543
544 if (!Act->hasEmittedPreamblePCH())
545 return BuildPreambleError::CouldntEmitPCH;
546 Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
547
548 // Keep track of all of the files that the source manager knows about,
549 // so we can verify whether they have changed or not.
550 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
551
552 SourceManager &SourceMgr = Clang->getSourceManager();
553 for (auto &Filename : PreambleDepCollector->getDependencies()) {
554 auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
555 if (!MaybeFile ||
556 MaybeFile == SourceMgr.getFileEntryRefForID(FID: SourceMgr.getMainFileID()))
557 continue;
558 auto File = *MaybeFile;
559 if (time_t ModTime = File.getModificationTime()) {
560 FilesInPreamble[File.getName()] =
561 PrecompiledPreamble::PreambleFileHash::createForFile(Size: File.getSize(),
562 ModTime);
563 } else {
564 llvm::MemoryBufferRef Buffer =
565 SourceMgr.getMemoryBufferForFileOrFake(File);
566 FilesInPreamble[File.getName()] =
567 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
568 }
569 }
570
571 // Shrinking the storage requires extra temporary memory.
572 // Destroying clang first reduces peak memory usage.
573 CICleanup.unregister();
574 Clang.reset();
575 Storage->shrink();
576 return PrecompiledPreamble(
577 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
578 std::move(FilesInPreamble), std::move(MissingFiles));
579}
580
581PreambleBounds PrecompiledPreamble::getBounds() const {
582 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
583}
584
585std::size_t PrecompiledPreamble::getSize() const {
586 switch (Storage->getKind()) {
587 case PCHStorage::Kind::InMemory:
588 return Storage->memoryContents().size();
589 case PCHStorage::Kind::TempFile: {
590 uint64_t Result;
591 if (llvm::sys::fs::file_size(Path: Storage->filePath(), Result))
592 return 0;
593
594 assert(Result <= std::numeric_limits<std::size_t>::max() &&
595 "file size did not fit into size_t");
596 return Result;
597 }
598 }
599 llvm_unreachable("Unhandled storage kind");
600}
601
602bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
603 const llvm::MemoryBufferRef &MainFileBuffer,
604 PreambleBounds Bounds,
605 llvm::vfs::FileSystem &VFS) const {
606
607 assert(
608 Bounds.Size <= MainFileBuffer.getBufferSize() &&
609 "Buffer is too large. Bounds were calculated from a different buffer?");
610
611 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
612 PreprocessorOptions &PreprocessorOpts =
613 PreambleInvocation->getPreprocessorOpts();
614
615 // We've previously computed a preamble. Check whether we have the same
616 // preamble now that we did before, and that there's enough space in
617 // the main-file buffer within the precompiled preamble to fit the
618 // new main file.
619 if (PreambleBytes.size() != Bounds.Size ||
620 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
621 !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
622 MainFileBuffer.getBuffer().begin()))
623 return false;
624 // The preamble has not changed. We may be able to re-use the precompiled
625 // preamble.
626
627 // Check that none of the files used by the preamble have changed.
628 // First, make a record of those files that have been overridden via
629 // remapping or unsaved_files.
630 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
631 llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
632 for (const auto &R : PreprocessorOpts.RemappedFiles) {
633 llvm::vfs::Status Status;
634 if (!moveOnNoError(Val: VFS.status(Path: R.second), Output&: Status)) {
635 // If we can't stat the file we're remapping to, assume that something
636 // horrible happened.
637 return false;
638 }
639 // If a mapped file was previously missing, then it has changed.
640 llvm::SmallString<128> MappedPath(R.first);
641 if (!VFS.makeAbsolute(Path&: MappedPath))
642 OverriddenAbsPaths.insert(key: MappedPath);
643
644 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
645 Size: Status.getSize(), ModTime: llvm::sys::toTimeT(TP: Status.getLastModificationTime()));
646 }
647
648 // OverridenFileBuffers tracks only the files not found in VFS.
649 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
650 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
651 const PrecompiledPreamble::PreambleFileHash PreambleHash =
652 PreambleFileHash::createForMemoryBuffer(Buffer: RB.second->getMemBufferRef());
653 llvm::vfs::Status Status;
654 if (moveOnNoError(Val: VFS.status(Path: RB.first), Output&: Status))
655 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
656 else
657 OverridenFileBuffers[RB.first] = PreambleHash;
658
659 llvm::SmallString<128> MappedPath(RB.first);
660 if (!VFS.makeAbsolute(Path&: MappedPath))
661 OverriddenAbsPaths.insert(key: MappedPath);
662 }
663
664 // Check whether anything has changed.
665 for (const auto &F : FilesInPreamble) {
666 auto OverridenFileBuffer = OverridenFileBuffers.find(Key: F.first());
667 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
668 // The file's buffer was remapped and the file was not found in VFS.
669 // Check whether it matches up with the previous mapping.
670 if (OverridenFileBuffer->second != F.second)
671 return false;
672 continue;
673 }
674
675 llvm::vfs::Status Status;
676 if (!moveOnNoError(Val: VFS.status(Path: F.first()), Output&: Status)) {
677 // If the file's buffer is not remapped and we can't stat it,
678 // assume that something horrible happened.
679 return false;
680 }
681
682 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
683 OverriddenFiles.find(x: Status.getUniqueID());
684 if (Overridden != OverriddenFiles.end()) {
685 // This file was remapped; check whether the newly-mapped file
686 // matches up with the previous mapping.
687 if (Overridden->second != F.second)
688 return false;
689 continue;
690 }
691
692 // Neither the file's buffer nor the file itself was remapped;
693 // check whether it has changed on disk.
694 if (Status.getSize() != uint64_t(F.second.Size) ||
695 llvm::sys::toTimeT(TP: Status.getLastModificationTime()) !=
696 F.second.ModTime)
697 return false;
698 }
699 for (const auto &F : MissingFiles) {
700 // A missing file may be "provided" by an override buffer or file.
701 if (OverriddenAbsPaths.count(Key: F.getKey()))
702 return false;
703 // If a file previously recorded as missing exists as a regular file, then
704 // consider the preamble out-of-date.
705 if (auto Status = VFS.status(Path: F.getKey())) {
706 if (Status->isRegularFile())
707 return false;
708 }
709 }
710 return true;
711}
712
713void PrecompiledPreamble::AddImplicitPreamble(
714 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
715 llvm::MemoryBuffer *MainFileBuffer) const {
716 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
717 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
718}
719
720void PrecompiledPreamble::OverridePreamble(
721 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
722 llvm::MemoryBuffer *MainFileBuffer) const {
723 auto Bounds = ComputePreambleBounds(LangOpts: CI.getLangOpts(), Buffer: *MainFileBuffer, MaxLines: 0);
724 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
725}
726
727PrecompiledPreamble::PrecompiledPreamble(
728 std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
729 bool PreambleEndsAtStartOfLine,
730 llvm::StringMap<PreambleFileHash> FilesInPreamble,
731 llvm::StringSet<> MissingFiles)
732 : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
733 MissingFiles(std::move(MissingFiles)),
734 PreambleBytes(std::move(PreambleBytes)),
735 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
736 assert(this->Storage != nullptr);
737}
738
739PrecompiledPreamble::PreambleFileHash
740PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
741 time_t ModTime) {
742 PreambleFileHash Result;
743 Result.Size = Size;
744 Result.ModTime = ModTime;
745 Result.MD5 = {};
746 return Result;
747}
748
749PrecompiledPreamble::PreambleFileHash
750PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
751 const llvm::MemoryBufferRef &Buffer) {
752 PreambleFileHash Result;
753 Result.Size = Buffer.getBufferSize();
754 Result.ModTime = 0;
755
756 llvm::MD5 MD5Ctx;
757 MD5Ctx.update(Str: Buffer.getBuffer().data());
758 MD5Ctx.final(Result&: Result.MD5);
759
760 return Result;
761}
762
763void PrecompiledPreamble::configurePreamble(
764 PreambleBounds Bounds, CompilerInvocation &CI,
765 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
766 llvm::MemoryBuffer *MainFileBuffer) const {
767 assert(VFS);
768
769 auto &PreprocessorOpts = CI.getPreprocessorOpts();
770
771 // Remap main file to point to MainFileBuffer.
772 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
773 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: MainFileBuffer);
774
775 // Configure ImpicitPCHInclude.
776 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
777 PreprocessorOpts.PrecompiledPreambleBytes.second =
778 Bounds.PreambleEndsAtStartOfLine;
779 PreprocessorOpts.DisablePCHOrModuleValidation =
780 DisableValidationForModuleKind::PCH;
781
782 // Don't bother generating the long version of the predefines buffer.
783 // The preamble is going to overwrite it anyway.
784 PreprocessorOpts.UsePredefines = false;
785
786 setupPreambleStorage(Storage: *Storage, PreprocessorOpts, VFS);
787}
788
789void PrecompiledPreamble::setupPreambleStorage(
790 const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
791 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS) {
792 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
793 llvm::StringRef PCHPath = Storage.filePath();
794 PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
795
796 // Make sure we can access the PCH file even if we're using a VFS
797 IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS =
798 llvm::vfs::getRealFileSystem();
799 if (VFS == RealFS || VFS->exists(Path: PCHPath))
800 return;
801 auto Buf = RealFS->getBufferForFile(Name: PCHPath);
802 if (!Buf) {
803 // We can't read the file even from RealFS, this is clearly an error,
804 // but we'll just leave the current VFS as is and let clang's code
805 // figure out what to do with missing PCH.
806 return;
807 }
808
809 // We have a slight inconsistency here -- we're using the VFS to
810 // read files, but the PCH was generated in the real file system.
811 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(*Buf), VFS);
812 } else {
813 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
814 // For in-memory preamble, we have to provide a VFS overlay that makes it
815 // accessible.
816 StringRef PCHPath = getInMemoryPreamblePath();
817 PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
818
819 auto Buf = llvm::MemoryBuffer::getMemBuffer(
820 InputData: Storage.memoryContents(), BufferName: PCHPath, /*RequiresNullTerminator=*/false);
821 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(Buf), VFS);
822 }
823}
824
825void PreambleCallbacks::BeforeExecute(CompilerInstance &CI) {}
826void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
827void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
828void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
829std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
830 return nullptr;
831}
832CommentHandler *PreambleCallbacks::getCommentHandler() { return nullptr; }
833
834static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
835
836std::error_code clang::make_error_code(BuildPreambleError Error) {
837 return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
838}
839
840const char *BuildPreambleErrorCategory::name() const noexcept {
841 return "build-preamble.error";
842}
843
844std::string BuildPreambleErrorCategory::message(int condition) const {
845 switch (static_cast<BuildPreambleError>(condition)) {
846 case BuildPreambleError::CouldntCreateTempFile:
847 return "Could not create temporary file for PCH";
848 case BuildPreambleError::CouldntCreateTargetInfo:
849 return "CreateTargetInfo() return null";
850 case BuildPreambleError::BeginSourceFileFailed:
851 return "BeginSourceFile() return an error";
852 case BuildPreambleError::CouldntEmitPCH:
853 return "Could not emit PCH";
854 case BuildPreambleError::BadInputs:
855 return "Command line arguments must contain exactly one source file";
856 }
857 llvm_unreachable("unexpected BuildPreambleError");
858}
859