| 1 | //===- InProcessModuleCache.cpp - Implicit Module Cache ---------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "clang/DependencyScanning/InProcessModuleCache.h" |
| 10 | |
| 11 | #include "clang/Basic/AtomicLineLogger.h" |
| 12 | #include "clang/Serialization/InMemoryModuleCache.h" |
| 13 | #include "clang/Serialization/ModuleCache.h" |
| 14 | #include "llvm/Support/AdvisoryLock.h" |
| 15 | #include "llvm/Support/Chrono.h" |
| 16 | #include "llvm/Support/Error.h" |
| 17 | #include "llvm/Support/FileSystem.h" |
| 18 | #include "llvm/Support/IOSandbox.h" |
| 19 | #include "llvm/Support/MemoryBuffer.h" |
| 20 | #include "llvm/Support/Path.h" |
| 21 | |
| 22 | using namespace clang; |
| 23 | using namespace dependencies; |
| 24 | |
| 25 | void ModuleCacheEntries::addInvalidatedPath(StringRef Path) { |
| 26 | #ifndef NDEBUG |
| 27 | SmallString<256> Canonical(Path); |
| 28 | assert(llvm::sys::path::is_absolute(Path) && |
| 29 | !llvm::sys::path::remove_dots(Canonical, /*remove_dot_dot=*/true) && |
| 30 | "invalidated paths must be spelled like ModuleDeps::DirectoryDeps" ); |
| 31 | #endif |
| 32 | { |
| 33 | std::lock_guard<std::mutex> Lock(InvalidatedPathsMutex); |
| 34 | InvalidatedPaths.insert(key: Path); |
| 35 | } |
| 36 | AnyInvalidatedPaths.store(i: true, m: std::memory_order_release); |
| 37 | } |
| 38 | |
| 39 | std::optional<bool> |
| 40 | ModuleCacheEntries::isDirectoryInvalidated(StringRef Directory) const { |
| 41 | if (!ValidateAgainstInvalidatedPaths) |
| 42 | return std::nullopt; |
| 43 | if (!AnyInvalidatedPaths.load(m: std::memory_order_acquire)) |
| 44 | return false; |
| 45 | std::lock_guard<std::mutex> Lock(InvalidatedPathsMutex); |
| 46 | return InvalidatedPaths.contains(key: Directory); |
| 47 | } |
| 48 | |
| 49 | ModuleCacheEntries::~ModuleCacheEntries() { |
| 50 | assert(!Flusher.joinable() && "Concurrent flushing was not stopped" ); |
| 51 | } |
| 52 | |
| 53 | void ModuleCacheEntries::scheduleFlush(ModuleCacheMapEntry &MapEntry) { |
| 54 | // We're not flushing at all, nothing to schedule. |
| 55 | if (!Flusher.joinable()) |
| 56 | return; |
| 57 | |
| 58 | { |
| 59 | std::lock_guard<std::mutex> Lock(FlushMutex); |
| 60 | FlushEntries.push_back(x: &MapEntry); |
| 61 | } |
| 62 | FlushCondVar.notify_one(); |
| 63 | } |
| 64 | |
| 65 | void ModuleCacheEntries::startBackgroundFlushing(AtomicLineLogger &Logger) { |
| 66 | assert(!Flusher.joinable() && "Already started concurrent flushing" ); |
| 67 | Flusher = std::thread([this, &Logger] { |
| 68 | std::unique_lock<std::mutex> PendingLock(FlushMutex); |
| 69 | while (true) { |
| 70 | FlushCondVar.wait(lock&: PendingLock, |
| 71 | p: [&] { return StopFlushing || !FlushEntries.empty(); }); |
| 72 | // Leave the remaining entries to flushLeftovers(). |
| 73 | if (StopFlushing) |
| 74 | return; |
| 75 | ModuleCacheMapEntry &MapEntry = *FlushEntries.front(); |
| 76 | FlushEntries.pop_front(); |
| 77 | PendingLock.unlock(); |
| 78 | flushEntry(Path: MapEntry.getKey(), Entry&: *MapEntry.second, Logger, |
| 79 | /*ReadBack=*/true); |
| 80 | PendingLock.lock(); |
| 81 | } |
| 82 | }); |
| 83 | } |
| 84 | |
| 85 | void ModuleCacheEntries::stopBackgroundFlushing() { |
| 86 | assert(Flusher.joinable() && "Never started concurrent flushing" ); |
| 87 | { |
| 88 | std::lock_guard<std::mutex> Lock(FlushMutex); |
| 89 | StopFlushing = true; |
| 90 | } |
| 91 | FlushCondVar.notify_one(); |
| 92 | Flusher.join(); |
| 93 | } |
| 94 | |
| 95 | void ModuleCacheEntries::flushLeftovers(AtomicLineLogger &Logger) { |
| 96 | assert(!Flusher.joinable() && "Flushing leftovers concurrently" ); |
| 97 | for (auto &[Name, Entry] : llvm::make_pointee_range(Range&: FlushEntries)) |
| 98 | flushEntry(Path: Name, Entry&: *Entry, Logger, /*ReadBack=*/false); |
| 99 | } |
| 100 | |
| 101 | void ModuleCacheEntries::flushEntry(StringRef Path, ModuleCacheEntry &Entry, |
| 102 | AtomicLineLogger &Logger, bool ReadBack) { |
| 103 | // This is a compiler-internal output, let's bypass the sandbox. |
| 104 | auto BypassSandbox = llvm::sys::sandbox::scopedDisable(); |
| 105 | |
| 106 | std::shared_ptr<const llvm::MemoryBuffer> InMemoryBuffer; |
| 107 | { |
| 108 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 109 | if (Entry.State != ModuleCacheEntry::S_Written) |
| 110 | return; |
| 111 | InMemoryBuffer = Entry.Buffer; |
| 112 | } |
| 113 | |
| 114 | off_t Size; |
| 115 | time_t ModTime; |
| 116 | auto Written = writeImpl(Path, Buffer: *InMemoryBuffer, Size, ModTime, ReadBack); |
| 117 | if (!Written) { |
| 118 | // Best-effort: keep the buffer in memory and let flush() try again. |
| 119 | Logger.log() << "pcm_flush_failed: " << Path; |
| 120 | return; |
| 121 | } |
| 122 | Logger.log() << "pcm_flush: " << Path; |
| 123 | |
| 124 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 125 | // Note: We keep Entry.ModTime, since that's what readers have already seen. |
| 126 | Entry.Buffer = std::move(*Written); |
| 127 | Entry.State = ModuleCacheEntry::S_Flushed; |
| 128 | } |
| 129 | |
| 130 | namespace { |
| 131 | /// A buffer that keeps the contents of a \c ModuleCacheEntry alive. |
| 132 | class SharedMemoryBuffer : public llvm::MemoryBuffer { |
| 133 | std::shared_ptr<const llvm::MemoryBuffer> Underlying; |
| 134 | |
| 135 | public: |
| 136 | SharedMemoryBuffer(std::shared_ptr<const llvm::MemoryBuffer> Underlying) |
| 137 | : Underlying(std::move(Underlying)) { |
| 138 | init(BufStart: this->Underlying->getBufferStart(), BufEnd: this->Underlying->getBufferEnd(), |
| 139 | /*RequiresNullTerminator=*/false); |
| 140 | } |
| 141 | |
| 142 | StringRef getBufferIdentifier() const override { |
| 143 | return Underlying->getBufferIdentifier(); |
| 144 | } |
| 145 | |
| 146 | BufferKind getBufferKind() const override { |
| 147 | return Underlying->getBufferKind(); |
| 148 | } |
| 149 | }; |
| 150 | |
| 151 | class ReaderWriterLock : public llvm::AdvisoryLock { |
| 152 | ModuleCacheEntry &Entry; |
| 153 | std::optional<unsigned> OwnedGeneration; |
| 154 | |
| 155 | public: |
| 156 | ReaderWriterLock(ModuleCacheEntry &Entry) : Entry(Entry) {} |
| 157 | |
| 158 | Expected<bool> tryLock() override { |
| 159 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 160 | if (Entry.Locked) |
| 161 | return false; |
| 162 | Entry.Locked = true; |
| 163 | OwnedGeneration = Entry.Generation; |
| 164 | return true; |
| 165 | } |
| 166 | |
| 167 | llvm::WaitForUnlockResult |
| 168 | waitForUnlockFor(std::chrono::seconds MaxSeconds) override { |
| 169 | assert(!OwnedGeneration); |
| 170 | std::unique_lock<std::mutex> Lock(Entry.Mutex); |
| 171 | unsigned CurrentGeneration = Entry.Generation; |
| 172 | bool Success = Entry.CondVar.wait_for(lock&: Lock, rtime: MaxSeconds, p: [&] { |
| 173 | // We check not only Locked, but also Generation to break the wait in case |
| 174 | // of unsafeUnlock() and successful tryLock(). |
| 175 | return !Entry.Locked || Entry.Generation != CurrentGeneration; |
| 176 | }); |
| 177 | return Success ? llvm::WaitForUnlockResult::Success |
| 178 | : llvm::WaitForUnlockResult::Timeout; |
| 179 | } |
| 180 | |
| 181 | std::error_code unsafeUnlock() override { |
| 182 | { |
| 183 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 184 | Entry.Generation += 1; |
| 185 | Entry.Locked = false; |
| 186 | } |
| 187 | Entry.CondVar.notify_all(); |
| 188 | return {}; |
| 189 | } |
| 190 | |
| 191 | ~ReaderWriterLock() override { |
| 192 | if (OwnedGeneration) { |
| 193 | { |
| 194 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 195 | // Avoid stomping over the state managed by someone else after |
| 196 | // unsafeUnlock() and successful tryLock(). |
| 197 | if (*OwnedGeneration == Entry.Generation) |
| 198 | Entry.Locked = false; |
| 199 | } |
| 200 | Entry.CondVar.notify_all(); |
| 201 | } |
| 202 | } |
| 203 | }; |
| 204 | |
| 205 | class InProcessModuleCache : public ModuleCache { |
| 206 | ModuleCacheEntries &Entries; |
| 207 | |
| 208 | // TODO: If we changed the InMemoryModuleCache API and relied on strict |
| 209 | // context hash, we could probably create more efficient thread-safe |
| 210 | // implementation of the InMemoryModuleCache such that it doesn't need to be |
| 211 | // recreated for each translation unit. |
| 212 | InMemoryModuleCache InMemory; |
| 213 | |
| 214 | ModuleCacheMapEntry &getOrCreateMapEntry(StringRef Filename) { |
| 215 | std::lock_guard<std::mutex> Lock(Entries.Mutex); |
| 216 | auto &MapEntry = *Entries.Map.try_emplace(Key: Filename).first; |
| 217 | if (!MapEntry.second) |
| 218 | MapEntry.second = std::make_unique<ModuleCacheEntry>(); |
| 219 | return MapEntry; |
| 220 | } |
| 221 | |
| 222 | ModuleCacheEntry &getOrCreateEntry(StringRef Filename) { |
| 223 | return *getOrCreateMapEntry(Filename).second; |
| 224 | } |
| 225 | |
| 226 | public: |
| 227 | InProcessModuleCache(ModuleCacheEntries &Entries, AtomicLineLogger &Logger) |
| 228 | : ModuleCache(Logger), Entries(Entries), InMemory(Logger) {} |
| 229 | |
| 230 | std::unique_ptr<llvm::AdvisoryLock> getLock(StringRef Filename) override { |
| 231 | auto &Entry = getOrCreateEntry(Filename); |
| 232 | return std::make_unique<ReaderWriterLock>(args&: Entry); |
| 233 | } |
| 234 | |
| 235 | std::time_t getModuleTimestamp(StringRef Filename) override { |
| 236 | auto &Timestamp = getOrCreateEntry(Filename).Timestamp; |
| 237 | |
| 238 | Logger.log() << "timestamp_read: " << Filename; |
| 239 | return Timestamp.load(); |
| 240 | } |
| 241 | |
| 242 | void updateModuleTimestamp(StringRef Filename) override { |
| 243 | // Note: This essentially replaces FS contention with mutex contention. |
| 244 | auto &Timestamp = getOrCreateEntry(Filename).Timestamp; |
| 245 | |
| 246 | Logger.log() << "timestamp_write: " << Filename; |
| 247 | Timestamp.store(i: llvm::sys::toTimeT(TP: std::chrono::system_clock::now())); |
| 248 | } |
| 249 | |
| 250 | std::optional<bool> isDirectoryInvalidated(StringRef Directory) override { |
| 251 | return Entries.isDirectoryInvalidated(Directory); |
| 252 | } |
| 253 | |
| 254 | void maybePrune(StringRef Path, time_t PruneInterval, |
| 255 | time_t PruneAfter) override { |
| 256 | // FIXME: This only needs to be ran once per build, not in every |
| 257 | // compilation. Call it once per service. |
| 258 | maybePruneImpl(Path, PruneInterval, PruneAfter); |
| 259 | } |
| 260 | |
| 261 | InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } |
| 262 | const InMemoryModuleCache &getInMemoryModuleCache() const override { |
| 263 | return InMemory; |
| 264 | } |
| 265 | |
| 266 | llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> |
| 267 | write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size, |
| 268 | time_t &ModTime) override { |
| 269 | ModuleCacheMapEntry &MapEntry = getOrCreateMapEntry(Filename: Path); |
| 270 | ModuleCacheEntry &Entry = *MapEntry.second; |
| 271 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 272 | Logger.log() << "pcm_write: " << Path; |
| 273 | if (Entry.State == ModuleCacheEntry::S_Written || |
| 274 | Entry.State == ModuleCacheEntry::S_Flushed) { |
| 275 | assert(Entry.Buffer && "Wrote PCM with no contents" ); |
| 276 | assert(Entry.Buffer->getBuffer() == Buffer->getBuffer() && |
| 277 | "Wrote the same PCM with different contents" ); |
| 278 | Size = Entry.Buffer->getBufferSize(); |
| 279 | ModTime = Entry.ModTime; |
| 280 | return std::make_unique<SharedMemoryBuffer>(args&: Entry.Buffer); |
| 281 | } |
| 282 | // Readers of the previous buffer (if any) keep it alive on their own. |
| 283 | Entry.Buffer = std::move(Buffer); |
| 284 | Entry.ModTime = llvm::sys::toTimeT(TP: std::chrono::system_clock::now()); |
| 285 | Entry.State = ModuleCacheEntry::S_Written; |
| 286 | Entries.scheduleFlush(MapEntry); |
| 287 | Size = Entry.Buffer->getBufferSize(); |
| 288 | ModTime = Entry.ModTime; |
| 289 | return std::make_unique<SharedMemoryBuffer>(args&: Entry.Buffer); |
| 290 | } |
| 291 | |
| 292 | Expected<std::unique_ptr<llvm::MemoryBuffer>> |
| 293 | read(StringRef FileName, off_t &Size, time_t &ModTime) override { |
| 294 | Logger.log() << "pcm_read_disk: " << FileName; |
| 295 | ModuleCacheEntry &Entry = getOrCreateEntry(Filename: FileName); |
| 296 | std::lock_guard<std::mutex> Lock(Entry.Mutex); |
| 297 | if (Entry.State == ModuleCacheEntry::S_Unknown) { |
| 298 | // This is a compiler-internal input/output, let's bypass the sandbox. |
| 299 | auto BypassSandbox = llvm::sys::sandbox::scopedDisable(); |
| 300 | off_t ReadSize; |
| 301 | time_t ReadModTime; |
| 302 | auto ReadBuffer = readImpl(FileName, Size&: ReadSize, ModTime&: ReadModTime); |
| 303 | if (!ReadBuffer) |
| 304 | return ReadBuffer.takeError(); |
| 305 | Entry.Buffer = std::move(*ReadBuffer); |
| 306 | Entry.ModTime = ReadModTime; |
| 307 | Entry.State = ModuleCacheEntry::S_Read; |
| 308 | } |
| 309 | Size = Entry.Buffer->getBufferSize(); |
| 310 | ModTime = Entry.ModTime; |
| 311 | return std::make_unique<SharedMemoryBuffer>(args&: Entry.Buffer); |
| 312 | } |
| 313 | }; |
| 314 | } // namespace |
| 315 | |
| 316 | std::shared_ptr<ModuleCache> |
| 317 | dependencies::makeInProcessModuleCache(ModuleCacheEntries &Entries, |
| 318 | AtomicLineLogger &Logger) { |
| 319 | return std::make_shared<InProcessModuleCache>(args&: Entries, args&: Logger); |
| 320 | } |
| 321 | |