1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "BuiltinCAS.h"
10#include "OnDiskCommon.h"
11#include "llvm/ADT/ScopeExit.h"
12#include "llvm/CAS/BuiltinCASContext.h"
13#include "llvm/CAS/BuiltinObjectHasher.h"
14#include "llvm/CAS/OnDiskCASLogger.h"
15#include "llvm/CAS/OnDiskGraphDB.h"
16#include "llvm/CAS/UnifiedOnDiskCache.h"
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/Error.h"
19#include "llvm/Support/IOSandbox.h"
20#include "llvm/Support/MemoryBuffer.h"
21#include "llvm/Support/Path.h"
22
23using namespace llvm;
24using namespace llvm::cas;
25using namespace llvm::cas::builtin;
26
27namespace {
28
29class OnDiskCAS : public BuiltinCAS {
30public:
31 Expected<ObjectRef> storeImpl(ArrayRef<uint8_t> ComputedHash,
32 ArrayRef<ObjectRef> Refs,
33 ArrayRef<char> Data) final;
34
35 Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) final;
36
37 CASID getID(ObjectRef Ref) const final;
38
39 std::optional<ObjectRef> getReference(const CASID &ID) const final;
40
41 Expected<bool> isMaterialized(ObjectRef Ref) const final;
42
43 ArrayRef<char> getDataConst(ObjectHandle Node) const final;
44
45 Expected<ObjectRef> storeFromFile(StringRef Path) final;
46
47 Error exportDataToFile(ObjectHandle Node, StringRef Path) const final;
48
49 std::unique_ptr<MemoryBuffer>
50 getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
51 bool RequiresNullTerminator) final;
52
53 void print(raw_ostream &OS) const final;
54 Error validate(bool CheckHash) const final;
55
56 static Expected<std::unique_ptr<OnDiskCAS>> open(StringRef Path);
57
58 OnDiskCAS(std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB)
59 : UnifiedDB(std::move(UniDB)), DB(&UnifiedDB->getGraphDB()) {}
60
61private:
62 ObjectHandle convertHandle(ondisk::ObjectHandle Node) const {
63 return makeObjectHandle(InternalRef: Node.getOpaqueData());
64 }
65
66 ondisk::ObjectHandle convertHandle(ObjectHandle Node) const {
67 return ondisk::ObjectHandle(Node.getInternalRef(ExpectedCAS: *this));
68 }
69
70 ObjectRef convertRef(ondisk::ObjectID Ref) const {
71 return makeObjectRef(InternalRef: Ref.getOpaqueData());
72 }
73
74 ondisk::ObjectID convertRef(ObjectRef Ref) const {
75 return ondisk::ObjectID::fromOpaqueData(Opaque: Ref.getInternalRef(ExpectedCAS: *this));
76 }
77
78 size_t getNumRefs(ObjectHandle Node) const final {
79 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
80 return llvm::size(Range&: RefsRange);
81 }
82
83 ObjectRef readRef(ObjectHandle Node, size_t I) const final {
84 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
85 return convertRef(Ref: RefsRange.begin()[I]);
86 }
87
88 Error forEachRef(ObjectHandle Node,
89 function_ref<Error(ObjectRef)> Callback) const final;
90
91 Error setSizeLimit(std::optional<uint64_t> SizeLimit) final;
92 Expected<std::optional<uint64_t>> getStorageSize() const final;
93 Error pruneStorageData() final;
94
95 OnDiskCAS(std::unique_ptr<ondisk::OnDiskGraphDB> GraphDB)
96 : OwnedDB(std::move(GraphDB)), DB(OwnedDB.get()) {}
97
98 std::unique_ptr<ondisk::OnDiskGraphDB> OwnedDB;
99 std::shared_ptr<ondisk::UnifiedOnDiskCache> UnifiedDB;
100 ondisk::OnDiskGraphDB *DB;
101};
102
103} // end anonymous namespace
104
105void OnDiskCAS::print(raw_ostream &OS) const { DB->print(OS); }
106Error OnDiskCAS::validate(bool CheckHash) const {
107 if (auto E = DB->validate(Deep: CheckHash, Hasher: builtin::hashingFunc))
108 return E;
109
110 return Error::success();
111}
112
113CASID OnDiskCAS::getID(ObjectRef Ref) const {
114 ArrayRef<uint8_t> Hash = DB->getDigest(Ref: convertRef(Ref));
115 return CASID::create(Context: &getContext(), Hash: toStringRef(Input: Hash));
116}
117
118std::optional<ObjectRef> OnDiskCAS::getReference(const CASID &ID) const {
119 std::optional<ondisk::ObjectID> ObjID =
120 DB->getExistingReference(Digest: ID.getHash());
121 if (!ObjID)
122 return std::nullopt;
123 return convertRef(Ref: *ObjID);
124}
125
126Expected<bool> OnDiskCAS::isMaterialized(ObjectRef ExternalRef) const {
127 return DB->isMaterialized(Ref: convertRef(Ref: ExternalRef));
128}
129
130ArrayRef<char> OnDiskCAS::getDataConst(ObjectHandle Node) const {
131 return DB->getObjectData(Node: convertHandle(Node));
132}
133
134Expected<std::optional<ObjectHandle>>
135OnDiskCAS::loadIfExists(ObjectRef ExternalRef) {
136 Expected<std::optional<ondisk::ObjectHandle>> ObjHnd =
137 DB->load(Ref: convertRef(Ref: ExternalRef));
138 if (!ObjHnd)
139 return ObjHnd.takeError();
140 if (!*ObjHnd)
141 return std::nullopt;
142 return convertHandle(Node: **ObjHnd);
143}
144
145Expected<ObjectRef> OnDiskCAS::storeImpl(ArrayRef<uint8_t> ComputedHash,
146 ArrayRef<ObjectRef> Refs,
147 ArrayRef<char> Data) {
148 SmallVector<ondisk::ObjectID, 64> IDs;
149 IDs.reserve(N: Refs.size());
150 for (ObjectRef Ref : Refs) {
151 IDs.push_back(Elt: convertRef(Ref));
152 }
153
154 auto StoredID = DB->getReference(Hash: ComputedHash);
155 if (LLVM_UNLIKELY(!StoredID))
156 return StoredID.takeError();
157 if (Error E = DB->store(ID: *StoredID, Refs: IDs, Data))
158 return std::move(E);
159 return convertRef(Ref: *StoredID);
160}
161
162Expected<ObjectRef> OnDiskCAS::storeFromFile(StringRef Path) {
163 auto Hash = BuiltinObjectHasher<HasherT>::hashFile(FilePath: Path);
164 if (LLVM_UNLIKELY(!Hash))
165 return Hash.takeError();
166 auto StoredID = DB->getReference(Hash: *Hash);
167 if (LLVM_UNLIKELY(!StoredID))
168 return StoredID.takeError();
169 if (Error E = DB->storeFile(ID: *StoredID, FilePath: Path))
170 return E;
171 return convertRef(Ref: *StoredID);
172}
173
174std::unique_ptr<MemoryBuffer>
175OnDiskCAS::getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
176 bool RequiresNullTerminator) {
177 return DB->getStandaloneMemoryBuffer(Node: convertHandle(Node), Name,
178 RequiresNullTerminator);
179}
180
181Error OnDiskCAS::exportDataToFile(ObjectHandle Node, StringRef Path) const {
182 auto FBData = DB->getInternalFileBackedObjectData(Node: convertHandle(Node));
183 if (!FBData.FileInfo.has_value())
184 return BuiltinCAS::exportDataToFile(Node, Path);
185
186 // Optimized version using the underlying database file.
187 assert(FBData.FileInfo.has_value());
188
189 auto BypassSandbox = sys::sandbox::scopedDisable();
190
191 ondisk::UniqueTempFile UniqueTmp;
192 auto ExpectedPath = UniqueTmp.createAndCopyFrom(ParentPath: sys::path::parent_path(path: Path),
193 CopyFromPath: FBData.FileInfo->FilePath);
194 if (!ExpectedPath)
195 return ExpectedPath.takeError();
196 StringRef TmpPath = *ExpectedPath;
197
198 if (FBData.FileInfo->IsFileNulTerminated) {
199 // Remove the nul terminator.
200 int FD;
201 if (std::error_code EC =
202 sys::fs::openFileForWrite(Name: TmpPath, ResultFD&: FD, Disp: sys::fs::CD_OpenExisting))
203 return createFileError(F: TmpPath, EC);
204 auto CloseFile = scope_exit([&FD] {
205 sys::fs::file_t File = sys::fs::convertFDToNativeFile(FD);
206 sys::fs::closeFile(F&: File);
207 });
208 if (std::error_code EC = sys::fs::resize_file(FD, Size: FBData.Data.size()))
209 return createFileError(F: TmpPath, EC);
210 }
211
212 if (Error E = UniqueTmp.renameTo(RenameToPath: Path))
213 return E;
214
215 return Error::success();
216}
217
218Error OnDiskCAS::forEachRef(ObjectHandle Node,
219 function_ref<Error(ObjectRef)> Callback) const {
220 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
221 for (ondisk::ObjectID Ref : RefsRange) {
222 if (Error E = Callback(convertRef(Ref)))
223 return E;
224 }
225 return Error::success();
226}
227
228Error OnDiskCAS::setSizeLimit(std::optional<uint64_t> SizeLimit) {
229 UnifiedDB->setSizeLimit(SizeLimit);
230 return Error::success();
231}
232
233Expected<std::optional<uint64_t>> OnDiskCAS::getStorageSize() const {
234 return UnifiedDB->getStorageSize();
235}
236
237Error OnDiskCAS::pruneStorageData() { return UnifiedDB->collectGarbage(); }
238
239Expected<std::unique_ptr<OnDiskCAS>> OnDiskCAS::open(StringRef AbsPath) {
240 std::shared_ptr<ondisk::OnDiskCASLogger> Logger;
241#ifndef _WIN32
242 if (Error E =
243 ondisk::OnDiskCASLogger::openIfEnabled(Path: AbsPath).moveInto(Value&: Logger))
244 return std::move(E);
245#endif
246
247 Expected<std::unique_ptr<ondisk::OnDiskGraphDB>> DB =
248 ondisk::OnDiskGraphDB::open(Path: AbsPath, HashName: BuiltinCASContext::getHashName(),
249 HashByteSize: sizeof(HashType), /*UpstreamDB=*/nullptr,
250 Logger: std::move(Logger));
251 if (!DB)
252 return DB.takeError();
253 return std::unique_ptr<OnDiskCAS>(new OnDiskCAS(std::move(*DB)));
254}
255
256bool cas::isOnDiskCASEnabled() {
257#if LLVM_ENABLE_ONDISK_CAS
258 return true;
259#else
260 return false;
261#endif
262}
263
264Expected<std::unique_ptr<ObjectStore>> cas::createOnDiskCAS(const Twine &Path) {
265#if LLVM_ENABLE_ONDISK_CAS
266 // FIXME: An absolute path isn't really good enough. Should open a directory
267 // and use openat() for files underneath.
268 SmallString<256> AbsPath;
269 Path.toVector(Out&: AbsPath);
270 sys::fs::make_absolute(path&: AbsPath);
271
272 return OnDiskCAS::open(AbsPath);
273#else
274 return createStringError(inconvertibleErrorCode(), "OnDiskCAS is disabled");
275#endif /* LLVM_ENABLE_ONDISK_CAS */
276}
277
278std::unique_ptr<ObjectStore>
279cas::builtin::createObjectStoreFromUnifiedOnDiskCache(
280 std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB) {
281 return std::make_unique<OnDiskCAS>(args: std::move(UniDB));
282}
283