1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Serialization/ModuleCache.h"
10
11#include "clang/Serialization/InMemoryModuleCache.h"
12#include "clang/Serialization/ModuleFile.h"
13#include "llvm/ADT/ScopeExit.h"
14#include "llvm/Support/Error.h"
15#include "llvm/Support/FileSystem.h"
16#include "llvm/Support/IOSandbox.h"
17#include "llvm/Support/LockFileManager.h"
18#include "llvm/Support/Path.h"
19
20using namespace clang;
21
22const ModuleCacheDirectory *ModuleCache::getDirectoryPtr(StringRef Path) {
23 auto [ByNameIt, ByNameInserted] = ByPath.insert(KV: {Path, nullptr});
24 if (!ByNameIt->second) {
25 // This is a compiler-internal input/output, let's bypass the sandbox.
26 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
27
28 // If we cannot get status of the module cache directory, try if trying to
29 // create it helps.
30 llvm::sys::fs::file_status Status;
31 if (std::error_code EC = llvm::sys::fs::status(path: Path, result&: Status)) {
32 // Unless the status failed because the directory does not exist yet.
33 if (EC != std::errc::no_such_file_or_directory)
34 return nullptr;
35 // If we're unable to create the directory.
36 if (llvm::sys::fs::create_directories(path: Path))
37 return nullptr;
38 // If we're unable to stat the newly created directory.
39 if (llvm::sys::fs::status(path: Path, result&: Status))
40 return nullptr;
41 }
42
43 llvm::sys::fs::UniqueID UID = Status.getUniqueID();
44 auto [ByUIDIt, ByUIDInserted] = ByUID.insert(KV: {UID, nullptr});
45 if (!ByUIDIt->second)
46 ByUIDIt->second = std::make_unique<ModuleCacheDirectory>();
47 ByNameIt->second = ByUIDIt->second.get();
48 }
49 return ByNameIt->second;
50}
51
52/// Write a new timestamp file with the given path.
53static void writeTimestampFile(StringRef TimestampFile) {
54 std::error_code EC;
55 llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
56}
57
58void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
59 time_t PruneAfter, bool PruneTopLevel,
60 llvm::function_ref<void(StringRef)> OnPrune) {
61 if (PruneInterval <= 0 || PruneAfter <= 0)
62 return;
63
64 // This is a compiler-internal input/output, let's bypass the sandbox.
65 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
66
67 llvm::SmallString<256> RootPath(Path);
68 (void)llvm::sys::fs::make_absolute(path&: RootPath);
69
70 llvm::SmallString<128> TimestampFile(RootPath);
71 llvm::sys::path::append(path&: TimestampFile, a: "modules.timestamp");
72
73 // Try to stat() the timestamp file.
74 llvm::sys::fs::file_status StatBuf;
75 if (std::error_code EC = llvm::sys::fs::status(path: TimestampFile, result&: StatBuf)) {
76 // If the timestamp file wasn't there, create one now.
77 if (EC == std::errc::no_such_file_or_directory)
78 writeTimestampFile(TimestampFile);
79 return;
80 }
81
82 // Check whether the time stamp is older than our pruning interval.
83 // If not, do nothing.
84 time_t TimestampModTime =
85 llvm::sys::toTimeT(TP: StatBuf.getLastModificationTime());
86 time_t CurrentTime = time(timer: nullptr);
87 if (CurrentTime - TimestampModTime <= PruneInterval)
88 return;
89
90 // Write a new timestamp file so that nobody else attempts to prune.
91 // There is a benign race condition here, if two Clang instances happen to
92 // notice at the same time that the timestamp is out-of-date.
93 writeTimestampFile(TimestampFile);
94
95 auto NotifyPruned = [&](StringRef RemovedPath) {
96 if (OnPrune)
97 OnPrune(RemovedPath);
98 };
99
100 // Walk the entire module cache, looking for unused module files and module
101 // indices.
102 std::error_code EC;
103 auto TryPruneFile = [&](StringRef FilePath) {
104 // We only care about module and global module index files.
105 StringRef Filename = llvm::sys::path::filename(path: FilePath);
106 StringRef Extension = llvm::sys::path::extension(path: FilePath);
107 if (Extension != ".pcm" && Extension != ".timestamp" &&
108 Filename != "modules.idx")
109 return;
110
111 // Don't prune the pruning timestamp file.
112 if (Filename == "modules.timestamp")
113 return;
114
115 // Look at this file. If we can't stat it, there's nothing interesting
116 // there.
117 if (llvm::sys::fs::status(path: FilePath, result&: StatBuf))
118 return;
119
120 // If the file has been used recently enough, leave it there.
121 time_t FileAccessTime = llvm::sys::toTimeT(TP: StatBuf.getLastAccessedTime());
122 if (CurrentTime - FileAccessTime <= PruneAfter)
123 return;
124
125 // Remove the file.
126 if (!llvm::sys::fs::remove(path: FilePath))
127 NotifyPruned(FilePath);
128
129 // Remove the timestamp file created by implicit module builds.
130 std::string TimestampFilename = FilePath.str() + ".timestamp";
131 if (!llvm::sys::fs::remove(path: TimestampFilename))
132 NotifyPruned(TimestampFilename);
133 };
134
135 for (llvm::sys::fs::directory_iterator Dir(RootPath, EC), DirEnd;
136 Dir != DirEnd && !EC; Dir.increment(ec&: EC)) {
137 // If we don't have a directory, try to prune it as a file in the root.
138 if (!llvm::sys::fs::is_directory(Path: Dir->path())) {
139 if (PruneTopLevel)
140 TryPruneFile(Dir->path());
141 continue;
142 }
143
144 // Walk all the files within this directory.
145 for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
146 File != FileEnd && !EC; File.increment(ec&: EC))
147 TryPruneFile(File->path());
148
149 // If we removed all the files in the directory, remove the directory
150 // itself.
151 if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
152 llvm::sys::fs::directory_iterator() &&
153 !EC) {
154 if (!llvm::sys::fs::remove(path: Dir->path()))
155 NotifyPruned(Dir->path());
156 }
157 }
158}
159
160std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
161 off_t &Size, time_t &ModTime) {
162 StringRef Extension = llvm::sys::path::extension(path: Path);
163 SmallString<128> ModelPath = StringRef(Path).drop_back(N: Extension.size());
164 ModelPath += "-%%%%%%%%";
165 ModelPath += Extension;
166 ModelPath += ".tmp";
167
168 std::error_code EC;
169 int FD;
170 SmallString<128> TmpPath;
171 if ((EC = llvm::sys::fs::createUniqueFile(Model: ModelPath, ResultFD&: FD, ResultPath&: TmpPath))) {
172 if (EC != std::errc::no_such_file_or_directory)
173 return EC;
174
175 StringRef Dir = llvm::sys::path::parent_path(path: Path);
176 if (std::error_code InnerEC = llvm::sys::fs::create_directories(path: Dir))
177 return InnerEC;
178
179 if ((EC = llvm::sys::fs::createUniqueFile(Model: ModelPath, ResultFD&: FD, ResultPath&: TmpPath)))
180 return EC;
181 }
182
183 llvm::sys::fs::file_status Status;
184 {
185 llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
186 OS << Buffer.getBuffer();
187 // Using the status from an open file descriptor ensures this is not racy.
188 if ((EC = llvm::sys::fs::status(FD, Result&: Status)))
189 return EC;
190 }
191
192 Size = Status.getSize();
193 ModTime = llvm::sys::toTimeT(TP: Status.getLastModificationTime());
194
195 // This preserves both size and modification time.
196 if ((EC = llvm::sys::fs::rename(from: TmpPath, to: Path)))
197 return EC;
198
199 return {};
200}
201
202Expected<std::unique_ptr<llvm::MemoryBuffer>>
203clang::readImpl(StringRef FileName, off_t &Size, time_t &ModTime) {
204 Expected<llvm::sys::fs::file_t> FD =
205 llvm::sys::fs::openNativeFileForRead(Name: FileName);
206 if (!FD)
207 return FD.takeError();
208 llvm::scope_exit CloseFD([&FD]() { llvm::sys::fs::closeFile(F&: *FD); });
209 llvm::sys::fs::file_status Status;
210 if (std::error_code EC = llvm::sys::fs::status(FD: *FD, Result&: Status))
211 return llvm::errorCodeToError(EC);
212 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
213 llvm::MemoryBuffer::getOpenFile(FD: *FD, Filename: FileName, FileSize: Status.getSize(),
214 /*RequiresNullTerminator=*/false);
215 if (!Buf)
216 return llvm::errorCodeToError(EC: Buf.getError());
217 Size = Status.getSize();
218 ModTime = llvm::sys::toTimeT(TP: Status.getLastModificationTime());
219 return std::move(*Buf);
220}
221
222namespace {
223class CrossProcessModuleCache : public ModuleCache {
224 InMemoryModuleCache InMemory;
225
226public:
227 std::unique_ptr<llvm::AdvisoryLock>
228 getLock(StringRef ModuleFilename) override {
229 return std::make_unique<llvm::LockFileManager>(args&: ModuleFilename);
230 }
231
232 std::time_t getModuleTimestamp(StringRef ModuleFilename) override {
233 // This is a compiler-internal input/output, let's bypass the sandbox.
234 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
235
236 std::string TimestampFilename =
237 serialization::ModuleFile::getTimestampFilename(FileName: ModuleFilename);
238 llvm::sys::fs::file_status Status;
239 if (llvm::sys::fs::status(path: TimestampFilename, result&: Status) != std::error_code{})
240 return 0;
241 return llvm::sys::toTimeT(TP: Status.getLastModificationTime());
242 }
243
244 void updateModuleTimestamp(StringRef ModuleFilename) override {
245 // This is a compiler-internal input/output, let's bypass the sandbox.
246 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
247
248 // Overwrite the timestamp file contents so that file's mtime changes.
249 std::error_code EC;
250 llvm::raw_fd_ostream OS(
251 serialization::ModuleFile::getTimestampFilename(FileName: ModuleFilename), EC,
252 llvm::sys::fs::OF_TextWithCRLF);
253 if (EC)
254 return;
255 OS << "Timestamp file\n";
256 OS.close();
257 OS.clear_error(); // Avoid triggering a fatal error.
258 }
259
260 void maybePrune(StringRef Path, time_t PruneInterval,
261 time_t PruneAfter) override {
262 // This is a compiler-internal input/output, let's bypass the sandbox.
263 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
264
265 maybePruneImpl(Path, PruneInterval, PruneAfter);
266 }
267
268 InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
269 const InMemoryModuleCache &getInMemoryModuleCache() const override {
270 return InMemory;
271 }
272
273 std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
274 off_t &Size, time_t &ModTime) override {
275 // This is a compiler-internal input/output, let's bypass the sandbox.
276 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
277
278 return writeImpl(Path, Buffer, Size, ModTime);
279 }
280
281 Expected<std::unique_ptr<llvm::MemoryBuffer>>
282 read(StringRef FileName, off_t &Size, time_t &ModTime) override {
283 // This is a compiler-internal input/output, let's bypass the sandbox.
284 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
285
286 return readImpl(FileName, Size, ModTime);
287 }
288};
289} // namespace
290
291std::shared_ptr<ModuleCache> clang::createCrossProcessModuleCache() {
292 return std::make_shared<CrossProcessModuleCache>();
293}
294