1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Serialization/ModuleCache.h"
10
11#include "clang/Basic/AtomicLineLogger.h"
12#include "clang/Serialization/InMemoryModuleCache.h"
13#include "clang/Serialization/ModuleFile.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/Support/Error.h"
16#include "llvm/Support/FileSystem.h"
17#include "llvm/Support/IOSandbox.h"
18#include "llvm/Support/LockFileManager.h"
19#include "llvm/Support/MemoryBuffer.h"
20#include "llvm/Support/Path.h"
21
22#include <chrono>
23
24using namespace clang;
25
26const ModuleCacheDirectory *ModuleCache::getDirectoryPtr(StringRef Path) {
27 auto [ByNameIt, ByNameInserted] = ByPath.insert(KV: {Path, nullptr});
28 if (!ByNameIt->second) {
29 // This is a compiler-internal input/output, let's bypass the sandbox.
30 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
31
32 // If we cannot get status of the module cache directory, try if trying to
33 // create it helps.
34 llvm::sys::fs::file_status Status;
35 if (std::error_code EC = llvm::sys::fs::status(path: Path, result&: Status)) {
36 // Unless the status failed because the directory does not exist yet.
37 if (EC != std::errc::no_such_file_or_directory)
38 return nullptr;
39 // If we're unable to create the directory.
40 if (llvm::sys::fs::create_directories(path: Path))
41 return nullptr;
42 // If we're unable to stat the newly created directory.
43 if (llvm::sys::fs::status(path: Path, result&: Status))
44 return nullptr;
45 }
46
47 llvm::sys::fs::UniqueID UID = Status.getUniqueID();
48 auto [ByUIDIt, ByUIDInserted] = ByUID.insert(KV: {UID, nullptr});
49 if (!ByUIDIt->second)
50 ByUIDIt->second = std::make_unique<ModuleCacheDirectory>();
51 ByNameIt->second = ByUIDIt->second.get();
52 }
53 return ByNameIt->second;
54}
55
56/// Write a new timestamp file with the given path.
57static void writeTimestampFile(StringRef TimestampFile) {
58 std::error_code EC;
59 llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
60}
61
62void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
63 time_t PruneAfter, bool PruneTopLevel,
64 llvm::function_ref<void(StringRef)> OnPrune) {
65 if (PruneInterval <= 0 || PruneAfter <= 0)
66 return;
67
68 // This is a compiler-internal input/output, let's bypass the sandbox.
69 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
70
71 llvm::SmallString<256> RootPath(Path);
72 (void)llvm::sys::fs::make_absolute(path&: RootPath);
73
74 llvm::SmallString<128> TimestampFile(RootPath);
75 llvm::sys::path::append(path&: TimestampFile, a: "modules.timestamp");
76
77 // Try to stat() the timestamp file.
78 llvm::sys::fs::file_status StatBuf;
79 if (std::error_code EC = llvm::sys::fs::status(path: TimestampFile, result&: StatBuf)) {
80 // If the timestamp file wasn't there, create one now.
81 if (EC == std::errc::no_such_file_or_directory)
82 writeTimestampFile(TimestampFile);
83 return;
84 }
85
86 // Check whether the time stamp is older than our pruning interval.
87 // If not, do nothing.
88 time_t TimestampModTime =
89 llvm::sys::toTimeT(TP: StatBuf.getLastModificationTime());
90 time_t CurrentTime = time(timer: nullptr);
91 if (CurrentTime - TimestampModTime <= PruneInterval)
92 return;
93
94 // Write a new timestamp file so that nobody else attempts to prune.
95 // There is a benign race condition here, if two Clang instances happen to
96 // notice at the same time that the timestamp is out-of-date.
97 writeTimestampFile(TimestampFile);
98
99 auto NotifyPruned = [&](StringRef RemovedPath) {
100 if (OnPrune)
101 OnPrune(RemovedPath);
102 };
103
104 // Walk the entire module cache, looking for unused module files and module
105 // indices.
106 std::error_code EC;
107 auto TryPruneFile = [&](StringRef FilePath) {
108 // We only care about module and global module index files.
109 StringRef Filename = llvm::sys::path::filename(path: FilePath);
110 StringRef Extension = llvm::sys::path::extension(path: FilePath);
111 if (Extension != ".pcm" && Extension != ".timestamp" &&
112 Filename != "modules.idx")
113 return;
114
115 // Don't prune the pruning timestamp file.
116 if (Filename == "modules.timestamp")
117 return;
118
119 // Look at this file. If we can't stat it, there's nothing interesting
120 // there.
121 if (llvm::sys::fs::status(path: FilePath, result&: StatBuf))
122 return;
123
124 // If the file has been used recently enough, leave it there.
125 time_t FileAccessTime = llvm::sys::toTimeT(TP: StatBuf.getLastAccessedTime());
126 if (CurrentTime - FileAccessTime <= PruneAfter)
127 return;
128
129 // Remove the file.
130 if (!llvm::sys::fs::remove(path: FilePath))
131 NotifyPruned(FilePath);
132
133 // Remove the timestamp file created by implicit module builds.
134 std::string TimestampFilename = FilePath.str() + ".timestamp";
135 if (!llvm::sys::fs::remove(path: TimestampFilename))
136 NotifyPruned(TimestampFilename);
137 };
138
139 for (llvm::sys::fs::directory_iterator Dir(RootPath, EC), DirEnd;
140 Dir != DirEnd && !EC; Dir.increment(ec&: EC)) {
141 // If we don't have a directory, try to prune it as a file in the root.
142 if (!llvm::sys::fs::is_directory(Path: Dir->path())) {
143 if (PruneTopLevel)
144 TryPruneFile(Dir->path());
145 continue;
146 }
147
148 // Walk all the files within this directory.
149 for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
150 File != FileEnd && !EC; File.increment(ec&: EC))
151 TryPruneFile(File->path());
152
153 // If we removed all the files in the directory, remove the directory
154 // itself.
155 if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
156 llvm::sys::fs::directory_iterator() &&
157 !EC) {
158 if (!llvm::sys::fs::remove(path: Dir->path()))
159 NotifyPruned(Dir->path());
160 }
161 }
162}
163
164llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
165clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
166 time_t &ModTime, bool ReadBack) {
167 StringRef Extension = llvm::sys::path::extension(path: Path);
168 SmallString<128> ModelPath = StringRef(Path).drop_back(N: Extension.size());
169 ModelPath += "-%%%%%%%%";
170 ModelPath += Extension;
171 ModelPath += ".tmp";
172
173 // The temporary file is removed on discard() and also when crashing, so that
174 // failing to produce the PCM doesn't leave it behind.
175 llvm::ErrorOr<llvm::sys::fs::TempFile> Temp =
176 llvm::sys::fs::TempFile::createWithErrorCode(Model: ModelPath);
177 if (!Temp) {
178 if (Temp.getError() != std::errc::no_such_file_or_directory)
179 return Temp.getError();
180
181 StringRef Dir = llvm::sys::path::parent_path(path: Path);
182 if (std::error_code InnerEC = llvm::sys::fs::create_directories(path: Dir))
183 return InnerEC;
184
185 Temp = llvm::sys::fs::TempFile::createWithErrorCode(Model: ModelPath);
186 if (!Temp)
187 return Temp.getError();
188 }
189
190 llvm::sys::fs::file_status Status;
191 std::unique_ptr<llvm::MemoryBuffer> Written;
192 auto WriteContentsAndReadBack = [&]() -> std::error_code {
193 // The file descriptor is owned (and closed) by the temporary file.
194 llvm::raw_fd_ostream OS(Temp->FD, /*shouldClose=*/false);
195 OS << Buffer.getBuffer();
196 // Make sure all contents reach the file before querying its status.
197 OS.flush();
198 if (std::error_code EC = OS.error()) {
199 // Prevent the destructor from reporting a fatal error.
200 OS.clear_error();
201 return EC;
202 }
203 // Using the status from an open file descriptor ensures this is not racy.
204 if (std::error_code EC = llvm::sys::fs::status(FD: Temp->FD, Result&: Status))
205 return EC;
206 if (ReadBack) {
207 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
208 llvm::MemoryBuffer::getOpenFile(
209 FD: llvm::sys::fs::convertFDToNativeFile(FD: Temp->FD), Filename: Path,
210 FileSize: Status.getSize(), /*RequiresNullTerminator=*/false);
211 if (!Buf)
212 return Buf.getError();
213 Written = std::move(*Buf);
214 }
215 return {};
216 };
217
218 if (std::error_code EC = WriteContentsAndReadBack()) {
219 llvm::consumeError(Err: Temp->discard());
220 return EC;
221 }
222
223 // Rename to the final path and close. This preserves both size and
224 // modification time. On failure, the temporary file is removed.
225 if (std::error_code EC = Temp->keepWithErrorCode(Name: Path))
226 return EC;
227
228 Size = Status.getSize();
229 ModTime = llvm::sys::toTimeT(TP: Status.getLastModificationTime());
230
231 return std::move(Written);
232}
233
234#ifndef _WIN32
235/// The access time of a module file is only refreshed explicitly when it is
236/// older than this interval so we avoid frequent yet unnecessary updates.
237static constexpr std::chrono::seconds AccessTimeRefreshInterval(60);
238#endif
239
240/// Record that the module file open as \p FD has just been used by setting its
241/// access time to now, if it is older than AccessTimeRefreshInterval. Module
242/// cache pruning goes by access time, and file systems don't reliably update it
243/// when a file is read (e.g., Linux relatime).
244static void
245refreshAccessTime([[maybe_unused]] llvm::sys::fs::file_t FD,
246 [[maybe_unused]] const llvm::sys::fs::file_status &Status) {
247// FIXME: Support Windows. The handle (FD) is opened for reading only, and
248// SetFileTime needs one with FILE_WRITE_ATTRIBUTES access.
249#ifndef _WIN32
250 llvm::sys::TimePoint<> Now = std::chrono::system_clock::now();
251 if (Now - Status.getLastAccessedTime() <= AccessTimeRefreshInterval)
252 return;
253
254 // Keep the modification time: modules importing this one record it.
255 // Go through the descriptor so a module file replaced meanwhile is left
256 // alone.
257 (void)llvm::sys::fs::setLastAccessAndModificationTime(
258 FD: FD.get(), AccessTime: Now, ModificationTime: Status.getLastModificationTime());
259#endif
260}
261
262Expected<std::unique_ptr<llvm::MemoryBuffer>>
263clang::readImpl(StringRef FileName, off_t &Size, time_t &ModTime) {
264 Expected<llvm::sys::fs::file_t> FD =
265 llvm::sys::fs::openNativeFileForRead(Name: FileName);
266 if (!FD)
267 return FD.takeError();
268 llvm::scope_exit CloseFD([&FD]() { llvm::sys::fs::closeFile(F&: *FD); });
269 llvm::sys::fs::file_status Status;
270 if (std::error_code EC = llvm::sys::fs::status(F: *FD, Result&: Status))
271 return llvm::errorCodeToError(EC);
272 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
273 llvm::MemoryBuffer::getOpenFile(FD: *FD, Filename: FileName, FileSize: Status.getSize(),
274 /*RequiresNullTerminator=*/false);
275 if (!Buf)
276 return llvm::errorCodeToError(EC: Buf.getError());
277 Size = Status.getSize();
278 ModTime = llvm::sys::toTimeT(TP: Status.getLastModificationTime());
279 refreshAccessTime(FD: *FD, Status);
280 return std::move(*Buf);
281}
282
283namespace {
284
285static AtomicLineLogger NoOpLogger;
286class CrossProcessModuleCache : public ModuleCache {
287 InMemoryModuleCache InMemory;
288
289public:
290 explicit CrossProcessModuleCache()
291 : ModuleCache(NoOpLogger), InMemory(NoOpLogger) {}
292
293 std::unique_ptr<llvm::AdvisoryLock>
294 getLock(StringRef ModuleFilename) override {
295 return std::make_unique<llvm::LockFileManager>(args&: ModuleFilename);
296 }
297
298 std::time_t getModuleTimestamp(StringRef ModuleFilename) override {
299 // This is a compiler-internal input/output, let's bypass the sandbox.
300 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
301
302 std::string TimestampFilename =
303 serialization::ModuleFile::getTimestampFilename(FileName: ModuleFilename);
304 llvm::sys::fs::file_status Status;
305 if (llvm::sys::fs::status(path: TimestampFilename, result&: Status) != std::error_code{})
306 return 0;
307 return llvm::sys::toTimeT(TP: Status.getLastModificationTime());
308 }
309
310 void updateModuleTimestamp(StringRef ModuleFilename) override {
311 // This is a compiler-internal input/output, let's bypass the sandbox.
312 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
313
314 // Overwrite the timestamp file contents so that file's mtime changes.
315 std::error_code EC;
316 llvm::raw_fd_ostream OS(
317 serialization::ModuleFile::getTimestampFilename(FileName: ModuleFilename), EC,
318 llvm::sys::fs::OF_TextWithCRLF);
319 if (EC)
320 return;
321 OS << "Timestamp file\n";
322 OS.close();
323 OS.clear_error(); // Avoid triggering a fatal error.
324 }
325
326 void maybePrune(StringRef Path, time_t PruneInterval,
327 time_t PruneAfter) override {
328 // This is a compiler-internal input/output, let's bypass the sandbox.
329 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
330
331 maybePruneImpl(Path, PruneInterval, PruneAfter);
332 }
333
334 InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
335 const InMemoryModuleCache &getInMemoryModuleCache() const override {
336 return InMemory;
337 }
338
339 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
340 write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
341 time_t &ModTime) override {
342 // This is a compiler-internal input/output, let's bypass the sandbox.
343 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
344
345 return writeImpl(Path, Buffer: *Buffer, Size, ModTime, /*ReadBack=*/true);
346 }
347
348 Expected<std::unique_ptr<llvm::MemoryBuffer>>
349 read(StringRef FileName, off_t &Size, time_t &ModTime) override {
350 // This is a compiler-internal input/output, let's bypass the sandbox.
351 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
352
353 return readImpl(FileName, Size, ModTime);
354 }
355};
356} // namespace
357
358std::shared_ptr<ModuleCache> clang::createCrossProcessModuleCache() {
359 return std::make_shared<CrossProcessModuleCache>();
360}
361