1//===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper class to build precompiled preamble.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Frontend/PrecompiledPreamble.h"
14#include "clang/Basic/FileManager.h"
15#include "clang/Basic/LangStandard.h"
16#include "clang/Frontend/CompilerInstance.h"
17#include "clang/Frontend/CompilerInvocation.h"
18#include "clang/Frontend/FrontendActions.h"
19#include "clang/Frontend/FrontendOptions.h"
20#include "clang/Lex/HeaderSearch.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Preprocessor.h"
23#include "clang/Lex/PreprocessorOptions.h"
24#include "clang/Serialization/ASTWriter.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/Config/llvm-config.h"
29#include "llvm/Support/CrashRecoveryContext.h"
30#include "llvm/Support/FileSystem.h"
31#include "llvm/Support/ManagedStatic.h"
32#include "llvm/Support/Path.h"
33#include "llvm/Support/Process.h"
34#include "llvm/Support/VirtualFileSystem.h"
35#include <limits>
36#include <mutex>
37#include <utility>
38
39using namespace clang;
40
41namespace {
42
43StringRef getInMemoryPreamblePath() {
44#if defined(LLVM_ON_UNIX)
45 return "/__clang_tmp/___clang_inmemory_preamble___";
46#elif defined(_WIN32)
47 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
48#else
49#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
50 return "/__clang_tmp/___clang_inmemory_preamble___";
51#endif
52}
53
54IntrusiveRefCntPtr<llvm::vfs::FileSystem>
55createVFSOverlayForPreamblePCH(StringRef PCHFilename,
56 std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
57 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
58 // We want only the PCH file from the real filesystem to be available,
59 // so we create an in-memory VFS with just that and overlay it on top.
60 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS(
61 new llvm::vfs::InMemoryFileSystem());
62 PCHFS->addFile(Path: PCHFilename, ModificationTime: 0, Buffer: std::move(PCHBuffer));
63 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> Overlay(
64 new llvm::vfs::OverlayFileSystem(VFS));
65 Overlay->pushOverlay(FS: PCHFS);
66 return Overlay;
67}
68
69class PreambleDependencyCollector : public DependencyCollector {
70public:
71 // We want to collect all dependencies for correctness. Avoiding the real
72 // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
73 // but there is no way to distinguish between those and the ones that can be
74 // spuriously added by '-isystem' (e.g. to suppress warnings from those
75 // headers).
76 bool needSystemDependencies() override { return true; }
77};
78
79// Collects files whose existence would invalidate the preamble.
80// Collecting *all* of these would make validating it too slow though, so we
81// just find all the candidates for 'file not found' diagnostics.
82//
83// A caveat that may be significant for generated files: we'll omit files under
84// search path entries whose roots don't exist when the preamble is built.
85// These are pruned by InitHeaderSearch and so we don't see the search path.
86// It would be nice to include them but we don't want to duplicate all the rest
87// of the InitHeaderSearch logic to reconstruct them.
88class MissingFileCollector : public PPCallbacks {
89 llvm::StringSet<> &Out;
90 const HeaderSearch &Search;
91 const SourceManager &SM;
92
93public:
94 MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
95 const SourceManager &SM)
96 : Out(Out), Search(Search), SM(SM) {}
97
98 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
99 StringRef FileName, bool IsAngled,
100 CharSourceRange FilenameRange,
101 OptionalFileEntryRef File, StringRef SearchPath,
102 StringRef RelativePath, const Module *SuggestedModule,
103 bool ModuleImported,
104 SrcMgr::CharacteristicKind FileType) override {
105 // File is std::nullopt if it wasn't found.
106 // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
107 if (File)
108 return;
109
110 // If it's a rare absolute include, we know the full path already.
111 if (llvm::sys::path::is_absolute(path: FileName)) {
112 Out.insert(key: FileName);
113 return;
114 }
115
116 // Reconstruct the filenames that would satisfy this directive...
117 llvm::SmallString<256> Buf;
118 auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
119 Buf = DE.getName();
120 llvm::sys::path::append(path&: Buf, a: FileName);
121 llvm::sys::path::remove_dots(path&: Buf, /*remove_dot_dot=*/true);
122 Out.insert(key: Buf);
123 };
124 // ...relative to the including file.
125 if (!IsAngled) {
126 if (OptionalFileEntryRef IncludingFile =
127 SM.getFileEntryRefForID(FID: SM.getFileID(SpellingLoc: IncludeTok.getLocation())))
128 if (IncludingFile->getDir())
129 NotFoundRelativeTo(IncludingFile->getDir());
130 }
131 // ...relative to the search paths.
132 for (const auto &Dir : llvm::make_range(
133 x: IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
134 y: Search.search_dir_end())) {
135 // No support for frameworks or header maps yet.
136 if (Dir.isNormalDir())
137 NotFoundRelativeTo(*Dir.getDirRef());
138 }
139 }
140};
141
142/// Keeps a track of files to be deleted in destructor.
143class TemporaryFiles {
144public:
145 // A static instance to be used by all clients.
146 static TemporaryFiles &getInstance();
147
148private:
149 // Disallow constructing the class directly.
150 TemporaryFiles() = default;
151 // Disallow copy.
152 TemporaryFiles(const TemporaryFiles &) = delete;
153
154public:
155 ~TemporaryFiles();
156
157 /// Adds \p File to a set of tracked files.
158 void addFile(StringRef File);
159
160 /// Remove \p File from disk and from the set of tracked files.
161 void removeFile(StringRef File);
162
163private:
164 std::mutex Mutex;
165 llvm::StringSet<> Files;
166};
167
168TemporaryFiles &TemporaryFiles::getInstance() {
169 static TemporaryFiles Instance;
170 return Instance;
171}
172
173TemporaryFiles::~TemporaryFiles() {
174 std::lock_guard<std::mutex> Guard(Mutex);
175 for (const auto &File : Files)
176 llvm::sys::fs::remove(path: File.getKey());
177}
178
179void TemporaryFiles::addFile(StringRef File) {
180 std::lock_guard<std::mutex> Guard(Mutex);
181 auto IsInserted = Files.insert(key: File).second;
182 (void)IsInserted;
183 assert(IsInserted && "File has already been added");
184}
185
186void TemporaryFiles::removeFile(StringRef File) {
187 std::lock_guard<std::mutex> Guard(Mutex);
188 auto WasPresent = Files.erase(Key: File);
189 (void)WasPresent;
190 assert(WasPresent && "File was not tracked");
191 llvm::sys::fs::remove(path: File);
192}
193
194// A temp file that would be deleted on destructor call. If destructor is not
195// called for any reason, the file will be deleted at static objects'
196// destruction.
197// An assertion will fire if two TempPCHFiles are created with the same name,
198// so it's not intended to be used outside preamble-handling.
199class TempPCHFile {
200public:
201 // A main method used to construct TempPCHFile.
202 static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
203 // FIXME: This is a hack so that we can override the preamble file during
204 // crash-recovery testing, which is the only case where the preamble files
205 // are not necessarily cleaned up.
206 if (const char *TmpFile = ::getenv(name: "CINDEXTEST_PREAMBLE_FILE"))
207 return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
208
209 llvm::SmallString<128> File;
210 // Using the versions of createTemporaryFile() and
211 // createUniqueFile() with a file descriptor guarantees
212 // that we would never get a race condition in a multi-threaded setting
213 // (i.e., multiple threads getting the same temporary path).
214 int FD;
215 std::error_code EC;
216 if (StoragePath.empty())
217 EC = llvm::sys::fs::createTemporaryFile(Prefix: "preamble", Suffix: "pch", ResultFD&: FD, ResultPath&: File);
218 else {
219 llvm::SmallString<128> TempPath = StoragePath;
220 // Use the same filename model as fs::createTemporaryFile().
221 llvm::sys::path::append(path&: TempPath, a: "preamble-%%%%%%.pch");
222 namespace fs = llvm::sys::fs;
223 // Use the same owner-only file permissions as fs::createTemporaryFile().
224 EC = fs::createUniqueFile(Model: TempPath, ResultFD&: FD, ResultPath&: File, Flags: fs::OF_None,
225 Mode: fs::owner_read | fs::owner_write);
226 }
227 if (EC)
228 return nullptr;
229 // We only needed to make sure the file exists, close the file right away.
230 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
231 return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
232 }
233
234 TempPCHFile &operator=(const TempPCHFile &) = delete;
235 TempPCHFile(const TempPCHFile &) = delete;
236 ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(File: FilePath); };
237
238 /// A path where temporary file is stored.
239 llvm::StringRef getFilePath() const { return FilePath; };
240
241private:
242 TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
243 TemporaryFiles::getInstance().addFile(File: this->FilePath);
244 }
245
246 std::string FilePath;
247};
248
249class PrecompilePreambleAction : public ASTFrontendAction {
250public:
251 PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
252 PreambleCallbacks &Callbacks)
253 : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
254 Callbacks(Callbacks) {}
255
256 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
257 StringRef InFile) override;
258
259 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
260
261 void setEmittedPreamblePCH(ASTWriter &Writer) {
262 if (FileOS) {
263 *FileOS << Buffer->Data;
264 // Make sure it hits disk now.
265 FileOS.reset();
266 }
267
268 this->HasEmittedPreamblePCH = true;
269 Callbacks.AfterPCHEmitted(Writer);
270 }
271
272 bool BeginSourceFileAction(CompilerInstance &CI) override {
273 assert(CI.getLangOpts().CompilingPCH);
274 return ASTFrontendAction::BeginSourceFileAction(CI);
275 }
276
277 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
278 bool hasCodeCompletionSupport() const override { return false; }
279 bool hasASTFileSupport() const override { return false; }
280 TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
281
282private:
283 friend class PrecompilePreambleConsumer;
284
285 bool HasEmittedPreamblePCH = false;
286 std::shared_ptr<PCHBuffer> Buffer;
287 bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
288 std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
289 PreambleCallbacks &Callbacks;
290};
291
292class PrecompilePreambleConsumer : public PCHGenerator {
293public:
294 PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
295 ModuleCache &ModCache, StringRef isysroot,
296 std::shared_ptr<PCHBuffer> Buffer)
297 : PCHGenerator(PP, ModCache, "", isysroot, std::move(Buffer),
298 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
299 /*AllowASTWithErrors=*/true),
300 Action(Action) {}
301
302 bool HandleTopLevelDecl(DeclGroupRef DG) override {
303 Action.Callbacks.HandleTopLevelDecl(DG);
304 return true;
305 }
306
307 void HandleTranslationUnit(ASTContext &Ctx) override {
308 PCHGenerator::HandleTranslationUnit(Ctx);
309 if (!hasEmittedPCH())
310 return;
311 Action.setEmittedPreamblePCH(getWriter());
312 }
313
314 bool shouldSkipFunctionBody(Decl *D) override {
315 return Action.Callbacks.shouldSkipFunctionBody(D);
316 }
317
318private:
319 PrecompilePreambleAction &Action;
320};
321
322std::unique_ptr<ASTConsumer>
323PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
324 StringRef InFile) {
325 std::string Sysroot;
326 if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
327 return nullptr;
328
329 if (WritePCHFile) {
330 std::string OutputFile; // unused
331 FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
332 if (!FileOS)
333 return nullptr;
334 }
335
336 if (!CI.getFrontendOpts().RelocatablePCH)
337 Sysroot.clear();
338
339 return std::make_unique<PrecompilePreambleConsumer>(
340 args&: *this, args&: CI.getPreprocessor(), args&: CI.getModuleCache(), args&: Sysroot, args&: Buffer);
341}
342
343template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
344 if (!Val)
345 return false;
346 Output = std::move(*Val);
347 return true;
348}
349
350} // namespace
351
352PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
353 const llvm::MemoryBufferRef &Buffer,
354 unsigned MaxLines) {
355 return Lexer::ComputePreamble(Buffer: Buffer.getBuffer(), LangOpts, MaxLines);
356}
357
358class PrecompiledPreamble::PCHStorage {
359public:
360 static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
361 assert(File);
362 std::unique_ptr<PCHStorage> S(new PCHStorage());
363 S->File = std::move(File);
364 return S;
365 }
366 static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
367 std::unique_ptr<PCHStorage> S(new PCHStorage());
368 S->Memory = std::move(Buf);
369 return S;
370 }
371
372 enum class Kind { InMemory, TempFile };
373 Kind getKind() const {
374 if (Memory)
375 return Kind::InMemory;
376 if (File)
377 return Kind::TempFile;
378 llvm_unreachable("Neither Memory nor File?");
379 }
380 llvm::StringRef filePath() const {
381 assert(getKind() == Kind::TempFile);
382 return File->getFilePath();
383 }
384 llvm::StringRef memoryContents() const {
385 assert(getKind() == Kind::InMemory);
386 return StringRef(Memory->Data.data(), Memory->Data.size());
387 }
388
389 // Shrink in-memory buffers to fit.
390 // This incurs a copy, but preambles tend to be long-lived.
391 // Only safe to call once nothing can alias the buffer.
392 void shrink() {
393 if (!Memory)
394 return;
395 Memory->Data = decltype(Memory->Data)(Memory->Data);
396 }
397
398private:
399 PCHStorage() = default;
400 PCHStorage(const PCHStorage &) = delete;
401 PCHStorage &operator=(const PCHStorage &) = delete;
402
403 std::shared_ptr<PCHBuffer> Memory;
404 std::unique_ptr<TempPCHFile> File;
405};
406
407PrecompiledPreamble::~PrecompiledPreamble() = default;
408PrecompiledPreamble::PrecompiledPreamble(PrecompiledPreamble &&) = default;
409PrecompiledPreamble &
410PrecompiledPreamble::operator=(PrecompiledPreamble &&) = default;
411
412llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
413 const CompilerInvocation &Invocation,
414 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
415 DiagnosticsEngine &Diagnostics,
416 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
417 std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
418 StringRef StoragePath, PreambleCallbacks &Callbacks) {
419 assert(VFS && "VFS is null");
420
421 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
422 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
423 PreprocessorOptions &PreprocessorOpts =
424 PreambleInvocation->getPreprocessorOpts();
425
426 std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
427 std::unique_ptr<PCHStorage> Storage;
428 if (StoreInMemory) {
429 Storage = PCHStorage::inMemory(Buf: Buffer);
430 } else {
431 // Create a temporary file for the precompiled preamble. In rare
432 // circumstances, this can fail.
433 std::unique_ptr<TempPCHFile> PreamblePCHFile =
434 TempPCHFile::create(StoragePath);
435 if (!PreamblePCHFile)
436 return BuildPreambleError::CouldntCreateTempFile;
437 Storage = PCHStorage::file(File: std::move(PreamblePCHFile));
438 }
439
440 // Save the preamble text for later; we'll need to compare against it for
441 // subsequent reparses.
442 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
443 MainFileBuffer->getBufferStart() +
444 Bounds.Size);
445 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
446
447 // Tell the compiler invocation to generate a temporary precompiled header.
448 FrontendOpts.ProgramAction = frontend::GeneratePCH;
449 FrontendOpts.OutputFile = std::string(
450 StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
451 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
452 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
453 // Inform preprocessor to record conditional stack when building the preamble.
454 PreprocessorOpts.GeneratePreamble = true;
455
456 // Create the compiler instance to use for building the precompiled preamble.
457 auto Clang = std::make_unique<CompilerInstance>(args: std::move(PreambleInvocation),
458 args: std::move(PCHContainerOps));
459
460 // Recover resources if we crash before exiting this method.
461 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
462 Clang.get());
463
464 Clang->setDiagnostics(&Diagnostics);
465
466 // Create the target instance.
467 if (!Clang->createTarget())
468 return BuildPreambleError::CouldntCreateTargetInfo;
469
470 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
471 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
472 InputKind::Source ||
473 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
474 Language::LLVM_IR) {
475 return BuildPreambleError::BadInputs;
476 }
477
478 // Clear out old caches and data.
479 Diagnostics.Reset();
480 ProcessWarningOptions(Diags&: Diagnostics, Opts: Clang->getDiagnosticOpts(), VFS&: *VFS);
481
482 VFS =
483 createVFSFromCompilerInvocation(CI: Clang->getInvocation(), Diags&: Diagnostics, BaseFS: VFS);
484
485 // Create a file manager object to provide access to and cache the filesystem.
486 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
487
488 // Create the source manager.
489 Clang->setSourceManager(
490 new SourceManager(Diagnostics, Clang->getFileManager()));
491
492 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
493 Clang->addDependencyCollector(Listener: PreambleDepCollector);
494
495 Clang->getLangOpts().CompilingPCH = true;
496
497 // Remap the main source file to the preamble buffer.
498 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
499 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
500 InputData: MainFileBuffer->getBuffer().slice(Start: 0, End: Bounds.Size), BufferName: MainFilePath);
501 if (PreprocessorOpts.RetainRemappedFileBuffers) {
502 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
503 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: PreambleInputBuffer.get());
504 } else {
505 // In that case, remapped buffer will be deleted by CompilerInstance on
506 // BeginSourceFile, so we call release() to avoid double deletion.
507 PreprocessorOpts.addRemappedFile(From: MainFilePath,
508 To: PreambleInputBuffer.release());
509 }
510
511 auto Act = std::make_unique<PrecompilePreambleAction>(
512 args: std::move(Buffer),
513 /*WritePCHFile=*/args: Storage->getKind() == PCHStorage::Kind::TempFile,
514 args&: Callbacks);
515 if (!Act->BeginSourceFile(CI&: *Clang, Input: Clang->getFrontendOpts().Inputs[0]))
516 return BuildPreambleError::BeginSourceFileFailed;
517
518 // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
519 // referenced in the callback.
520 Callbacks.BeforeExecute(CI&: *Clang);
521
522 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
523 Callbacks.createPPCallbacks();
524 if (DelegatedPPCallbacks)
525 Clang->getPreprocessor().addPPCallbacks(C: std::move(DelegatedPPCallbacks));
526 if (auto CommentHandler = Callbacks.getCommentHandler())
527 Clang->getPreprocessor().addCommentHandler(Handler: CommentHandler);
528 llvm::StringSet<> MissingFiles;
529 Clang->getPreprocessor().addPPCallbacks(
530 C: std::make_unique<MissingFileCollector>(
531 args&: MissingFiles, args&: Clang->getPreprocessor().getHeaderSearchInfo(),
532 args&: Clang->getSourceManager()));
533
534 if (llvm::Error Err = Act->Execute())
535 return errorToErrorCode(Err: std::move(Err));
536
537 // Run the callbacks.
538 Callbacks.AfterExecute(CI&: *Clang);
539
540 Act->EndSourceFile();
541
542 if (!Act->hasEmittedPreamblePCH())
543 return BuildPreambleError::CouldntEmitPCH;
544 Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
545
546 // Keep track of all of the files that the source manager knows about,
547 // so we can verify whether they have changed or not.
548 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
549
550 SourceManager &SourceMgr = Clang->getSourceManager();
551 for (auto &Filename : PreambleDepCollector->getDependencies()) {
552 auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
553 if (!MaybeFile ||
554 MaybeFile == SourceMgr.getFileEntryRefForID(FID: SourceMgr.getMainFileID()))
555 continue;
556 auto File = *MaybeFile;
557 if (time_t ModTime = File.getModificationTime()) {
558 FilesInPreamble[File.getName()] =
559 PrecompiledPreamble::PreambleFileHash::createForFile(Size: File.getSize(),
560 ModTime);
561 } else {
562 llvm::MemoryBufferRef Buffer =
563 SourceMgr.getMemoryBufferForFileOrFake(File);
564 FilesInPreamble[File.getName()] =
565 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
566 }
567 }
568
569 // Shrinking the storage requires extra temporary memory.
570 // Destroying clang first reduces peak memory usage.
571 CICleanup.unregister();
572 Clang.reset();
573 Storage->shrink();
574 return PrecompiledPreamble(
575 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
576 std::move(FilesInPreamble), std::move(MissingFiles));
577}
578
579PreambleBounds PrecompiledPreamble::getBounds() const {
580 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
581}
582
583std::size_t PrecompiledPreamble::getSize() const {
584 switch (Storage->getKind()) {
585 case PCHStorage::Kind::InMemory:
586 return Storage->memoryContents().size();
587 case PCHStorage::Kind::TempFile: {
588 uint64_t Result;
589 if (llvm::sys::fs::file_size(Path: Storage->filePath(), Result))
590 return 0;
591
592 assert(Result <= std::numeric_limits<std::size_t>::max() &&
593 "file size did not fit into size_t");
594 return Result;
595 }
596 }
597 llvm_unreachable("Unhandled storage kind");
598}
599
600bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
601 const llvm::MemoryBufferRef &MainFileBuffer,
602 PreambleBounds Bounds,
603 llvm::vfs::FileSystem &VFS) const {
604
605 assert(
606 Bounds.Size <= MainFileBuffer.getBufferSize() &&
607 "Buffer is too large. Bounds were calculated from a different buffer?");
608
609 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
610 PreprocessorOptions &PreprocessorOpts =
611 PreambleInvocation->getPreprocessorOpts();
612
613 // We've previously computed a preamble. Check whether we have the same
614 // preamble now that we did before, and that there's enough space in
615 // the main-file buffer within the precompiled preamble to fit the
616 // new main file.
617 if (PreambleBytes.size() != Bounds.Size ||
618 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
619 !std::equal(first1: PreambleBytes.begin(), last1: PreambleBytes.end(),
620 first2: MainFileBuffer.getBuffer().begin()))
621 return false;
622 // The preamble has not changed. We may be able to re-use the precompiled
623 // preamble.
624
625 // Check that none of the files used by the preamble have changed.
626 // First, make a record of those files that have been overridden via
627 // remapping or unsaved_files.
628 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
629 llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
630 for (const auto &R : PreprocessorOpts.RemappedFiles) {
631 llvm::vfs::Status Status;
632 if (!moveOnNoError(Val: VFS.status(Path: R.second), Output&: Status)) {
633 // If we can't stat the file we're remapping to, assume that something
634 // horrible happened.
635 return false;
636 }
637 // If a mapped file was previously missing, then it has changed.
638 llvm::SmallString<128> MappedPath(R.first);
639 if (!VFS.makeAbsolute(Path&: MappedPath))
640 OverriddenAbsPaths.insert(key: MappedPath);
641
642 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
643 Size: Status.getSize(), ModTime: llvm::sys::toTimeT(TP: Status.getLastModificationTime()));
644 }
645
646 // OverridenFileBuffers tracks only the files not found in VFS.
647 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
648 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
649 const PrecompiledPreamble::PreambleFileHash PreambleHash =
650 PreambleFileHash::createForMemoryBuffer(Buffer: RB.second->getMemBufferRef());
651 llvm::vfs::Status Status;
652 if (moveOnNoError(Val: VFS.status(Path: RB.first), Output&: Status))
653 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
654 else
655 OverridenFileBuffers[RB.first] = PreambleHash;
656
657 llvm::SmallString<128> MappedPath(RB.first);
658 if (!VFS.makeAbsolute(Path&: MappedPath))
659 OverriddenAbsPaths.insert(key: MappedPath);
660 }
661
662 // Check whether anything has changed.
663 for (const auto &F : FilesInPreamble) {
664 auto OverridenFileBuffer = OverridenFileBuffers.find(Key: F.first());
665 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
666 // The file's buffer was remapped and the file was not found in VFS.
667 // Check whether it matches up with the previous mapping.
668 if (OverridenFileBuffer->second != F.second)
669 return false;
670 continue;
671 }
672
673 llvm::vfs::Status Status;
674 if (!moveOnNoError(Val: VFS.status(Path: F.first()), Output&: Status)) {
675 // If the file's buffer is not remapped and we can't stat it,
676 // assume that something horrible happened.
677 return false;
678 }
679
680 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
681 OverriddenFiles.find(x: Status.getUniqueID());
682 if (Overridden != OverriddenFiles.end()) {
683 // This file was remapped; check whether the newly-mapped file
684 // matches up with the previous mapping.
685 if (Overridden->second != F.second)
686 return false;
687 continue;
688 }
689
690 // Neither the file's buffer nor the file itself was remapped;
691 // check whether it has changed on disk.
692 if (Status.getSize() != uint64_t(F.second.Size) ||
693 llvm::sys::toTimeT(TP: Status.getLastModificationTime()) !=
694 F.second.ModTime)
695 return false;
696 }
697 for (const auto &F : MissingFiles) {
698 // A missing file may be "provided" by an override buffer or file.
699 if (OverriddenAbsPaths.count(Key: F.getKey()))
700 return false;
701 // If a file previously recorded as missing exists as a regular file, then
702 // consider the preamble out-of-date.
703 if (auto Status = VFS.status(Path: F.getKey())) {
704 if (Status->isRegularFile())
705 return false;
706 }
707 }
708 return true;
709}
710
711void PrecompiledPreamble::AddImplicitPreamble(
712 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
713 llvm::MemoryBuffer *MainFileBuffer) const {
714 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
715 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
716}
717
718void PrecompiledPreamble::OverridePreamble(
719 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
720 llvm::MemoryBuffer *MainFileBuffer) const {
721 auto Bounds = ComputePreambleBounds(LangOpts: CI.getLangOpts(), Buffer: *MainFileBuffer, MaxLines: 0);
722 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
723}
724
725PrecompiledPreamble::PrecompiledPreamble(
726 std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
727 bool PreambleEndsAtStartOfLine,
728 llvm::StringMap<PreambleFileHash> FilesInPreamble,
729 llvm::StringSet<> MissingFiles)
730 : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
731 MissingFiles(std::move(MissingFiles)),
732 PreambleBytes(std::move(PreambleBytes)),
733 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
734 assert(this->Storage != nullptr);
735}
736
737PrecompiledPreamble::PreambleFileHash
738PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
739 time_t ModTime) {
740 PreambleFileHash Result;
741 Result.Size = Size;
742 Result.ModTime = ModTime;
743 Result.MD5 = {};
744 return Result;
745}
746
747PrecompiledPreamble::PreambleFileHash
748PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
749 const llvm::MemoryBufferRef &Buffer) {
750 PreambleFileHash Result;
751 Result.Size = Buffer.getBufferSize();
752 Result.ModTime = 0;
753
754 llvm::MD5 MD5Ctx;
755 MD5Ctx.update(Str: Buffer.getBuffer().data());
756 MD5Ctx.final(Result&: Result.MD5);
757
758 return Result;
759}
760
761void PrecompiledPreamble::configurePreamble(
762 PreambleBounds Bounds, CompilerInvocation &CI,
763 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
764 llvm::MemoryBuffer *MainFileBuffer) const {
765 assert(VFS);
766
767 auto &PreprocessorOpts = CI.getPreprocessorOpts();
768
769 // Remap main file to point to MainFileBuffer.
770 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
771 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: MainFileBuffer);
772
773 // Configure ImpicitPCHInclude.
774 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
775 PreprocessorOpts.PrecompiledPreambleBytes.second =
776 Bounds.PreambleEndsAtStartOfLine;
777 PreprocessorOpts.DisablePCHOrModuleValidation =
778 DisableValidationForModuleKind::PCH;
779
780 // Don't bother generating the long version of the predefines buffer.
781 // The preamble is going to overwrite it anyway.
782 PreprocessorOpts.UsePredefines = false;
783
784 setupPreambleStorage(Storage: *Storage, PreprocessorOpts, VFS);
785}
786
787void PrecompiledPreamble::setupPreambleStorage(
788 const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
789 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS) {
790 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
791 llvm::StringRef PCHPath = Storage.filePath();
792 PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
793
794 // Make sure we can access the PCH file even if we're using a VFS
795 IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS =
796 llvm::vfs::getRealFileSystem();
797 if (VFS == RealFS || VFS->exists(Path: PCHPath))
798 return;
799 auto Buf = RealFS->getBufferForFile(Name: PCHPath);
800 if (!Buf) {
801 // We can't read the file even from RealFS, this is clearly an error,
802 // but we'll just leave the current VFS as is and let clang's code
803 // figure out what to do with missing PCH.
804 return;
805 }
806
807 // We have a slight inconsistency here -- we're using the VFS to
808 // read files, but the PCH was generated in the real file system.
809 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(*Buf), VFS);
810 } else {
811 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
812 // For in-memory preamble, we have to provide a VFS overlay that makes it
813 // accessible.
814 StringRef PCHPath = getInMemoryPreamblePath();
815 PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
816
817 auto Buf = llvm::MemoryBuffer::getMemBuffer(
818 InputData: Storage.memoryContents(), BufferName: PCHPath, /*RequiresNullTerminator=*/false);
819 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(Buf), VFS);
820 }
821}
822
823void PreambleCallbacks::BeforeExecute(CompilerInstance &CI) {}
824void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
825void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
826void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
827std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
828 return nullptr;
829}
830CommentHandler *PreambleCallbacks::getCommentHandler() { return nullptr; }
831
832static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
833
834std::error_code clang::make_error_code(BuildPreambleError Error) {
835 return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
836}
837
838const char *BuildPreambleErrorCategory::name() const noexcept {
839 return "build-preamble.error";
840}
841
842std::string BuildPreambleErrorCategory::message(int condition) const {
843 switch (static_cast<BuildPreambleError>(condition)) {
844 case BuildPreambleError::CouldntCreateTempFile:
845 return "Could not create temporary file for PCH";
846 case BuildPreambleError::CouldntCreateTargetInfo:
847 return "CreateTargetInfo() return null";
848 case BuildPreambleError::BeginSourceFileFailed:
849 return "BeginSourceFile() return an error";
850 case BuildPreambleError::CouldntEmitPCH:
851 return "Could not emit PCH";
852 case BuildPreambleError::BadInputs:
853 return "Command line arguments must contain exactly one source file";
854 }
855 llvm_unreachable("unexpected BuildPreambleError");
856}
857