1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "BuiltinCAS.h"
10#include "llvm/CAS/BuiltinCASContext.h"
11#include "llvm/CAS/OnDiskCASLogger.h"
12#include "llvm/CAS/OnDiskGraphDB.h"
13#include "llvm/CAS/UnifiedOnDiskCache.h"
14#include "llvm/Support/Compiler.h"
15#include "llvm/Support/Error.h"
16
17using namespace llvm;
18using namespace llvm::cas;
19using namespace llvm::cas::builtin;
20
21namespace {
22
23class OnDiskCAS : public BuiltinCAS {
24public:
25 Expected<ObjectRef> storeImpl(ArrayRef<uint8_t> ComputedHash,
26 ArrayRef<ObjectRef> Refs,
27 ArrayRef<char> Data) final;
28
29 Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) final;
30
31 CASID getID(ObjectRef Ref) const final;
32
33 std::optional<ObjectRef> getReference(const CASID &ID) const final;
34
35 Expected<bool> isMaterialized(ObjectRef Ref) const final;
36
37 ArrayRef<char> getDataConst(ObjectHandle Node) const final;
38
39 void print(raw_ostream &OS) const final;
40 Error validate(bool CheckHash) const final;
41
42 static Expected<std::unique_ptr<OnDiskCAS>> open(StringRef Path);
43
44 OnDiskCAS(std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB)
45 : UnifiedDB(std::move(UniDB)), DB(&UnifiedDB->getGraphDB()) {}
46
47private:
48 ObjectHandle convertHandle(ondisk::ObjectHandle Node) const {
49 return makeObjectHandle(InternalRef: Node.getOpaqueData());
50 }
51
52 ondisk::ObjectHandle convertHandle(ObjectHandle Node) const {
53 return ondisk::ObjectHandle(Node.getInternalRef(ExpectedCAS: *this));
54 }
55
56 ObjectRef convertRef(ondisk::ObjectID Ref) const {
57 return makeObjectRef(InternalRef: Ref.getOpaqueData());
58 }
59
60 ondisk::ObjectID convertRef(ObjectRef Ref) const {
61 return ondisk::ObjectID::fromOpaqueData(Opaque: Ref.getInternalRef(ExpectedCAS: *this));
62 }
63
64 size_t getNumRefs(ObjectHandle Node) const final {
65 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
66 return llvm::size(Range&: RefsRange);
67 }
68
69 ObjectRef readRef(ObjectHandle Node, size_t I) const final {
70 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
71 return convertRef(Ref: RefsRange.begin()[I]);
72 }
73
74 Error forEachRef(ObjectHandle Node,
75 function_ref<Error(ObjectRef)> Callback) const final;
76
77 Error setSizeLimit(std::optional<uint64_t> SizeLimit) final;
78 Expected<std::optional<uint64_t>> getStorageSize() const final;
79 Error pruneStorageData() final;
80
81 OnDiskCAS(std::unique_ptr<ondisk::OnDiskGraphDB> GraphDB)
82 : OwnedDB(std::move(GraphDB)), DB(OwnedDB.get()) {}
83
84 std::unique_ptr<ondisk::OnDiskGraphDB> OwnedDB;
85 std::shared_ptr<ondisk::UnifiedOnDiskCache> UnifiedDB;
86 ondisk::OnDiskGraphDB *DB;
87};
88
89} // end anonymous namespace
90
91void OnDiskCAS::print(raw_ostream &OS) const { DB->print(OS); }
92Error OnDiskCAS::validate(bool CheckHash) const {
93 if (auto E = DB->validate(Deep: CheckHash, Hasher: builtin::hashingFunc))
94 return E;
95
96 return Error::success();
97}
98
99CASID OnDiskCAS::getID(ObjectRef Ref) const {
100 ArrayRef<uint8_t> Hash = DB->getDigest(Ref: convertRef(Ref));
101 return CASID::create(Context: &getContext(), Hash: toStringRef(Input: Hash));
102}
103
104std::optional<ObjectRef> OnDiskCAS::getReference(const CASID &ID) const {
105 std::optional<ondisk::ObjectID> ObjID =
106 DB->getExistingReference(Digest: ID.getHash());
107 if (!ObjID)
108 return std::nullopt;
109 return convertRef(Ref: *ObjID);
110}
111
112Expected<bool> OnDiskCAS::isMaterialized(ObjectRef ExternalRef) const {
113 return DB->isMaterialized(Ref: convertRef(Ref: ExternalRef));
114}
115
116ArrayRef<char> OnDiskCAS::getDataConst(ObjectHandle Node) const {
117 return DB->getObjectData(Node: convertHandle(Node));
118}
119
120Expected<std::optional<ObjectHandle>>
121OnDiskCAS::loadIfExists(ObjectRef ExternalRef) {
122 Expected<std::optional<ondisk::ObjectHandle>> ObjHnd =
123 DB->load(Ref: convertRef(Ref: ExternalRef));
124 if (!ObjHnd)
125 return ObjHnd.takeError();
126 if (!*ObjHnd)
127 return std::nullopt;
128 return convertHandle(Node: **ObjHnd);
129}
130
131Expected<ObjectRef> OnDiskCAS::storeImpl(ArrayRef<uint8_t> ComputedHash,
132 ArrayRef<ObjectRef> Refs,
133 ArrayRef<char> Data) {
134 SmallVector<ondisk::ObjectID, 64> IDs;
135 IDs.reserve(N: Refs.size());
136 for (ObjectRef Ref : Refs) {
137 IDs.push_back(Elt: convertRef(Ref));
138 }
139
140 auto StoredID = DB->getReference(Hash: ComputedHash);
141 if (LLVM_UNLIKELY(!StoredID))
142 return StoredID.takeError();
143 if (Error E = DB->store(ID: *StoredID, Refs: IDs, Data))
144 return std::move(E);
145 return convertRef(Ref: *StoredID);
146}
147
148Error OnDiskCAS::forEachRef(ObjectHandle Node,
149 function_ref<Error(ObjectRef)> Callback) const {
150 auto RefsRange = DB->getObjectRefs(Node: convertHandle(Node));
151 for (ondisk::ObjectID Ref : RefsRange) {
152 if (Error E = Callback(convertRef(Ref)))
153 return E;
154 }
155 return Error::success();
156}
157
158Error OnDiskCAS::setSizeLimit(std::optional<uint64_t> SizeLimit) {
159 UnifiedDB->setSizeLimit(SizeLimit);
160 return Error::success();
161}
162
163Expected<std::optional<uint64_t>> OnDiskCAS::getStorageSize() const {
164 return UnifiedDB->getStorageSize();
165}
166
167Error OnDiskCAS::pruneStorageData() { return UnifiedDB->collectGarbage(); }
168
169Expected<std::unique_ptr<OnDiskCAS>> OnDiskCAS::open(StringRef AbsPath) {
170 std::shared_ptr<ondisk::OnDiskCASLogger> Logger;
171#ifndef _WIN32
172 if (Error E =
173 ondisk::OnDiskCASLogger::openIfEnabled(Path: AbsPath).moveInto(Value&: Logger))
174 return std::move(E);
175#endif
176
177 Expected<std::unique_ptr<ondisk::OnDiskGraphDB>> DB =
178 ondisk::OnDiskGraphDB::open(Path: AbsPath, HashName: BuiltinCASContext::getHashName(),
179 HashByteSize: sizeof(HashType), /*UpstreamDB=*/nullptr,
180 Logger: std::move(Logger));
181 if (!DB)
182 return DB.takeError();
183 return std::unique_ptr<OnDiskCAS>(new OnDiskCAS(std::move(*DB)));
184}
185
186bool cas::isOnDiskCASEnabled() {
187#if LLVM_ENABLE_ONDISK_CAS
188 return true;
189#else
190 return false;
191#endif
192}
193
194Expected<std::unique_ptr<ObjectStore>> cas::createOnDiskCAS(const Twine &Path) {
195#if LLVM_ENABLE_ONDISK_CAS
196 // FIXME: An absolute path isn't really good enough. Should open a directory
197 // and use openat() for files underneath.
198 SmallString<256> AbsPath;
199 Path.toVector(Out&: AbsPath);
200 sys::fs::make_absolute(path&: AbsPath);
201
202 return OnDiskCAS::open(AbsPath);
203#else
204 return createStringError(inconvertibleErrorCode(), "OnDiskCAS is disabled");
205#endif /* LLVM_ENABLE_ONDISK_CAS */
206}
207
208std::unique_ptr<ObjectStore>
209cas::builtin::createObjectStoreFromUnifiedOnDiskCache(
210 std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB) {
211 return std::make_unique<OnDiskCAS>(args: std::move(UniDB));
212}
213