1//===- llvm/CAS/ObjectStore.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the declaration of the ObjectStore class.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CAS_OBJECTSTORE_H
15#define LLVM_CAS_OBJECTSTORE_H
16
17#include "llvm/ADT/StringRef.h"
18#include "llvm/CAS/CASID.h"
19#include "llvm/CAS/CASReference.h"
20#include "llvm/Support/Error.h"
21#include "llvm/Support/FileSystem.h"
22#include <cstddef>
23
24namespace llvm {
25
26class MemoryBuffer;
27template <typename T> class unique_function;
28
29namespace cas {
30
31class ObjectStore;
32class ObjectProxy;
33class ActionCache;
34
35/// Content-addressable storage for objects.
36///
37/// Conceptually, objects are stored in a "unique set".
38///
39/// - Objects are immutable ("value objects") that are defined by their
40/// content. They are implicitly deduplicated by content.
41/// - Each object has a unique identifier (UID) that's derived from its content,
42/// called a \a CASID.
43/// - This UID is a fixed-size (strong) hash of the transitive content of a
44/// CAS object.
45/// - It's comparable between any two CAS instances that have the same \a
46/// CASIDContext::getHashSchemaIdentifier().
47/// - The UID can be printed (e.g., \a CASID::toString()) and it can parsed
48/// by the same or a different CAS instance with \a
49/// ObjectStore::parseID().
50/// - An object can be looked up by content or by UID.
51/// - \a store() is "get-or-create" methods, writing an object if it
52/// doesn't exist yet, and return a ref to it in any case.
53/// - \a loadObject(const CASID&) looks up an object by its UID.
54/// - Objects can reference other objects, forming an arbitrary DAG.
55///
56/// The \a ObjectStore interface has a few ways of referencing objects:
57///
58/// - \a ObjectRef encapsulates a reference to something in the CAS. It is an
59/// opaque type that references an object inside a specific CAS. It is
60/// implementation defined if the underlying object exists or not for an
61/// ObjectRef, and it can used to speed up CAS lookup as an implementation
62/// detail. However, you don't know anything about the underlying objects.
63/// "Loading" the object is a separate step that may not have happened
64/// yet, and which can fail (e.g. due to filesystem corruption) or introduce
65/// latency (if downloading from a remote store).
66/// - \a ObjectHandle encapulates a *loaded* object in the CAS. You need one of
67/// these to inspect the content of an object: to look at its stored
68/// data and references. This is internal to CAS implementation and not
69/// availble from CAS public APIs.
70/// - \a CASID: the UID for an object in the CAS, obtained through \a
71/// ObjectStore::getID() or \a ObjectStore::parseID(). This is a valid CAS
72/// identifier, but may reference an object that is unknown to this CAS
73/// instance.
74/// - \a ObjectProxy pairs an ObjectHandle (subclass) with a ObjectStore, and
75/// wraps access APIs to avoid having to pass extra parameters. It is the
76/// object used for accessing underlying data and refs by CAS users.
77///
78/// Both ObjectRef and ObjectHandle are lightweight, wrapping a `uint64_t` and
79/// are only valid with the associated ObjectStore instance.
80///
81/// There are a few options for accessing content of objects, with different
82/// lifetime tradeoffs:
83///
84/// - \a getData() accesses data without exposing lifetime at all.
85/// - \a getMemoryBuffer() returns a \a MemoryBuffer that may alias storage
86/// owned by the CAS, so it must not outlive the \a ObjectStore.
87/// - \a getStandaloneMemoryBuffer() returns a \a MemoryBuffer whose lifetime
88/// is independent of the CAS (it can live longer).
89/// - \a getDataString() return StringRef with lifetime is guaranteed to last as
90/// long as \a ObjectStore.
91/// - \a readRef() and \a forEachRef() iterate through the references in an
92/// object. There is no lifetime assumption.
93class LLVM_ABI ObjectStore {
94 friend class ObjectProxy;
95 void anchor();
96
97public:
98 /// Get a \p CASID from a \p ID, which should have been generated by \a
99 /// CASID::print(). This succeeds as long as \a validateID() would pass. The
100 /// object may be unknown to this CAS instance.
101 ///
102 /// TODO: Remove, and update callers to use \a validateID() or \a
103 /// extractHashFromID().
104 virtual Expected<CASID> parseID(StringRef ID) = 0;
105
106 /// Store object into ObjectStore.
107 virtual Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
108 ArrayRef<char> Data) = 0;
109 /// Get an ID for \p Ref.
110 virtual CASID getID(ObjectRef Ref) const = 0;
111
112 /// Stores the data of a file into ObjectStore.
113 ///
114 /// An underlying implementation could perform optimizations that reduce I/O
115 /// and disk space consumption.
116 ///
117 /// If there are any concurrent modifications to the file, the contents in the
118 /// CAS may be corrupt.
119 ///
120 /// \param FilePath the path of the file data.
121 virtual Expected<ObjectRef> storeFromFile(StringRef Path);
122
123 /// Exports the data of an object to a file path. It does not include any
124 /// references of the object.
125 ///
126 /// An underlying implementation could perform optimizations that reduce I/O
127 /// and disk space consumption.
128 ///
129 /// \param Node the object to read data from.
130 /// \param FilePath the path of the file data.
131 virtual Error exportDataToFile(ObjectHandle Node, StringRef Path) const;
132
133 /// Get an existing reference to the object called \p ID.
134 ///
135 /// Returns \c None if the object is not stored in this CAS.
136 virtual std::optional<ObjectRef> getReference(const CASID &ID) const = 0;
137
138 /// \returns true if the object is directly available from the local CAS, for
139 /// implementations that have this kind of distinction.
140 virtual Expected<bool> isMaterialized(ObjectRef Ref) const = 0;
141
142 /// Validate the underlying object referred by CASID.
143 virtual Error validateObject(const CASID &ID) = 0;
144
145 /// Validate the entire ObjectStore.
146 virtual Error validate(bool CheckHash) const = 0;
147
148protected:
149 /// Load the object referenced by \p Ref.
150 ///
151 /// Errors if the object cannot be loaded.
152 /// \returns \c std::nullopt if the object is missing from the CAS.
153 virtual Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) = 0;
154
155 /// Like \c loadIfExists but returns an error if the object is missing.
156 Expected<ObjectHandle> load(ObjectRef Ref);
157
158 /// Get the size of some data.
159 virtual uint64_t getDataSize(ObjectHandle Node) const = 0;
160
161 /// Methods for handling objects. CAS implementations need to override to
162 /// provide functions to access stored CAS objects and references.
163 virtual Error forEachRef(ObjectHandle Node,
164 function_ref<Error(ObjectRef)> Callback) const = 0;
165 virtual ObjectRef readRef(ObjectHandle Node, size_t I) const = 0;
166 virtual size_t getNumRefs(ObjectHandle Node) const = 0;
167 virtual ArrayRef<char> getData(ObjectHandle Node,
168 bool RequiresNullTerminator = false) const = 0;
169
170 /// Get ObjectRef from open file.
171 virtual Expected<ObjectRef>
172 storeFromOpenFileImpl(sys::fs::file_t FD,
173 std::optional<sys::fs::file_status> Status);
174
175 /// Customization point for \a getStandaloneMemoryBuffer(). The default
176 /// implementation copies the data, which always satisfies the lifetime
177 /// requirement; implementations that can hand out storage outliving
178 /// themselves, e.g. a mapping of a file they do not keep open, should
179 /// override this to avoid the copy. Must not return \c nullptr: fall back
180 /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
181 /// path does not apply.
182 virtual std::unique_ptr<MemoryBuffer>
183 getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
184 bool RequiresNullTerminator);
185
186 /// Get a lifetime-extended StringRef pointing at \p Data.
187 ///
188 /// Depending on the CAS implementation, this may involve in-memory storage
189 /// overhead.
190 StringRef getDataString(ObjectHandle Node) {
191 return toStringRef(Input: getData(Node));
192 }
193
194 /// Get a MemoryBuffer pointing at \p Data.
195 ///
196 /// The buffer may alias storage owned by this ObjectStore, in which case it
197 /// is only valid for as long as the store is.
198 std::unique_ptr<MemoryBuffer>
199 getMemoryBuffer(ObjectHandle Node, StringRef Name = "",
200 bool RequiresNullTerminator = true);
201
202 /// Get a MemoryBuffer for \p Node that stays valid after this ObjectStore is
203 /// destroyed.
204 ///
205 /// May be more expensive than \a getMemoryBuffer(), which is free to alias
206 /// storage the store already has mapped; prefer that one whenever the buffer
207 /// cannot outlive the store. Never returns \c nullptr: copying the data
208 /// always satisfies the lifetime requirement.
209 std::unique_ptr<MemoryBuffer>
210 getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name = "",
211 bool RequiresNullTerminator = true);
212
213 /// Read all the refs from object in a SmallVector.
214 virtual void readRefs(ObjectHandle Node,
215 SmallVectorImpl<ObjectRef> &Refs) const;
216
217 /// Allow ObjectStore implementations to create internal handles.
218#define MAKE_CAS_HANDLE_CONSTRUCTOR(HandleKind) \
219 HandleKind make##HandleKind(uint64_t InternalRef) const { \
220 return HandleKind(*this, InternalRef); \
221 }
222 MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectHandle)
223 MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectRef)
224#undef MAKE_CAS_HANDLE_CONSTRUCTOR
225
226public:
227 /// Helper functions to store object and returns a ObjectProxy.
228 Expected<ObjectProxy> createProxy(ArrayRef<ObjectRef> Refs, StringRef Data);
229
230 /// Store object from StringRef.
231 Expected<ObjectRef> storeFromString(ArrayRef<ObjectRef> Refs,
232 StringRef String) {
233 return store(Refs, Data: arrayRefFromStringRef<char>(Input: String));
234 }
235
236 /// Default implementation reads \p FD and calls \a storeNode(). Does not
237 /// take ownership of \p FD; the caller is responsible for closing it.
238 ///
239 /// If \p Status is sent in it is to be treated as a hint. Implementations
240 /// must protect against the file size potentially growing after the status
241 /// was taken (i.e., they cannot assume that an mmap will be null-terminated
242 /// where \p Status implies).
243 ///
244 /// Returns the \a CASID and the size of the file.
245 Expected<ObjectRef>
246 storeFromOpenFile(sys::fs::file_t FD,
247 std::optional<sys::fs::file_status> Status = std::nullopt) {
248 return storeFromOpenFileImpl(FD, Status);
249 }
250
251 static Error createUnknownObjectError(const CASID &ID);
252
253 /// Create ObjectProxy from CASID. If the object doesn't exist, get an error.
254 Expected<ObjectProxy> getProxy(const CASID &ID);
255 /// Create ObjectProxy from ObjectRef. If the object can't be loaded, get an
256 /// error.
257 Expected<ObjectProxy> getProxy(ObjectRef Ref);
258
259 /// \returns \c std::nullopt if the object is missing from the CAS.
260 Expected<std::optional<ObjectProxy>> getProxyIfExists(ObjectRef Ref);
261
262 /// Read the data from \p Data into \p OS.
263 uint64_t readData(ObjectHandle Node, raw_ostream &OS, uint64_t Offset = 0,
264 uint64_t MaxBytes = -1ULL) const {
265 ArrayRef<char> Data = getData(Node);
266 assert(Offset < Data.size() && "Expected valid offset");
267 Data = Data.drop_front(N: Offset).take_front(N: MaxBytes);
268 OS << toStringRef(Input: Data);
269 return Data.size();
270 }
271
272 /// Set the size for limiting growth of on-disk storage. This has an effect
273 /// for when the instance is closed.
274 ///
275 /// Implementations may leave this unimplemented.
276 virtual Error setSizeLimit(std::optional<uint64_t> SizeLimit) {
277 return Error::success();
278 }
279
280 /// \returns the storage size of the on-disk CAS data.
281 ///
282 /// Implementations that don't have an implementation for this should return
283 /// \p std::nullopt.
284 virtual Expected<std::optional<uint64_t>> getStorageSize() const {
285 return std::nullopt;
286 }
287
288 /// Prune local storage to reduce its size according to the desired size
289 /// limit. Pruning can happen concurrently with other operations.
290 ///
291 /// Implementations may leave this unimplemented.
292 virtual Error pruneStorageData() { return Error::success(); }
293
294 /// Validate the whole node tree.
295 Error validateTree(ObjectRef Ref);
296
297 /// Import object from another CAS. This will import the full tree from the
298 /// other CAS.
299 Expected<ObjectRef> importObject(ObjectStore &Upstream, ObjectRef Other);
300
301 /// Print the ObjectStore internals for debugging purpose.
302 virtual void print(raw_ostream &) const {}
303 void dump() const;
304
305 /// Get CASContext
306 const CASContext &getContext() const { return Context; }
307
308 virtual ~ObjectStore() = default;
309
310protected:
311 ObjectStore(const CASContext &Context) : Context(Context) {}
312
313private:
314 const CASContext &Context;
315};
316
317/// Reference to an abstract hierarchical node, with data and references.
318/// Reference is passed by value and is expected to be valid as long as the \a
319/// ObjectStore is.
320class ObjectProxy {
321public:
322 ObjectStore &getCAS() const { return *CAS; }
323 CASID getID() const { return CAS->getID(Ref); }
324 ObjectRef getRef() const { return Ref; }
325 size_t getNumReferences() const { return CAS->getNumRefs(Node: H); }
326 ObjectRef getReference(size_t I) const { return CAS->readRef(Node: H, I); }
327
328 operator CASID() const { return getID(); }
329 CASID getReferenceID(size_t I) const {
330 std::optional<CASID> ID = getCAS().getID(Ref: getReference(I));
331 assert(ID && "Expected reference to be first-class object");
332 return *ID;
333 }
334
335 /// Visit each reference in order, returning an error from \p Callback to
336 /// stop early.
337 Error forEachReference(function_ref<Error(ObjectRef)> Callback) const {
338 return CAS->forEachRef(Node: H, Callback);
339 }
340
341 LLVM_ABI std::unique_ptr<MemoryBuffer>
342 getMemoryBuffer(StringRef Name = "",
343 bool RequiresNullTerminator = true) const;
344
345 /// Get a MemoryBuffer that stays valid after the CAS is destroyed.
346 LLVM_ABI std::unique_ptr<MemoryBuffer>
347 getStandaloneMemoryBuffer(StringRef Name = "",
348 bool RequiresNullTerminator = true) const;
349
350 /// Get the content of the node. Valid as long as the CAS is valid.
351 StringRef getData() const { return CAS->getDataString(Node: H); }
352
353 /// Exports the data of an object to a file path.
354 Error exportDataToFile(StringRef Path) const {
355 return CAS->exportDataToFile(Node: H, Path);
356 }
357
358 friend bool operator==(const ObjectProxy &Proxy, ObjectRef Ref) {
359 return Proxy.getRef() == Ref;
360 }
361 friend bool operator==(ObjectRef Ref, const ObjectProxy &Proxy) {
362 return Proxy.getRef() == Ref;
363 }
364 friend bool operator!=(const ObjectProxy &Proxy, ObjectRef Ref) {
365 return !(Proxy.getRef() == Ref);
366 }
367 friend bool operator!=(ObjectRef Ref, const ObjectProxy &Proxy) {
368 return !(Proxy.getRef() == Ref);
369 }
370
371public:
372 ObjectProxy() = delete;
373
374 static ObjectProxy load(ObjectStore &CAS, ObjectRef Ref, ObjectHandle Node) {
375 return ObjectProxy(CAS, Ref, Node);
376 }
377
378private:
379 ObjectProxy(ObjectStore &CAS, ObjectRef Ref, ObjectHandle H)
380 : CAS(&CAS), Ref(Ref), H(H) {}
381
382 ObjectStore *CAS;
383 ObjectRef Ref;
384 ObjectHandle H;
385};
386
387/// Create an in memory CAS.
388LLVM_ABI std::unique_ptr<ObjectStore> createInMemoryCAS();
389
390/// \returns true if \c LLVM_ENABLE_ONDISK_CAS configuration was enabled.
391LLVM_ABI bool isOnDiskCASEnabled();
392
393/// Create a persistent on-disk path at \p Path.
394LLVM_ABI Expected<std::unique_ptr<ObjectStore>>
395createOnDiskCAS(const Twine &Path);
396
397/// Create \c ObjectStore and \c ActionCache instances backed by a plugin that
398/// implements the C API in \c "llvm-c/CAS/PluginAPI_functions.h".
399///
400/// \param PluginPath path of the dynamic library to load.
401/// \param OnDiskPath local path that the plugin should use for any on-disk
402/// resources/caches.
403/// \param PluginArgs name/value pairs passed to the plugin as custom options;
404/// they are opaque to the client.
405LLVM_ABI Expected<
406 std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
407createPluginCASDatabases(
408 StringRef PluginPath, StringRef OnDiskPath,
409 ArrayRef<std::pair<std::string, std::string>> PluginArgs);
410
411} // namespace cas
412} // namespace llvm
413
414#endif // LLVM_CAS_OBJECTSTORE_H
415