1//===- llvm/CAS/ObjectStore.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the declaration of the ObjectStore class.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CAS_OBJECTSTORE_H
15#define LLVM_CAS_OBJECTSTORE_H
16
17#include "llvm/ADT/StringRef.h"
18#include "llvm/CAS/CASID.h"
19#include "llvm/CAS/CASReference.h"
20#include "llvm/CAS/ValidationResult.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/FileSystem.h"
23#include <cstddef>
24
25namespace llvm {
26
27class MemoryBuffer;
28template <typename T> class unique_function;
29
30namespace cas {
31
32class ObjectStore;
33class ObjectProxy;
34class ActionCache;
35
36/// Content-addressable storage for objects.
37///
38/// Conceptually, objects are stored in a "unique set".
39///
40/// - Objects are immutable ("value objects") that are defined by their
41/// content. They are implicitly deduplicated by content.
42/// - Each object has a unique identifier (UID) that's derived from its content,
43/// called a \a CASID.
44/// - This UID is a fixed-size (strong) hash of the transitive content of a
45/// CAS object.
46/// - It's comparable between any two CAS instances that have the same \a
47/// CASIDContext::getHashSchemaIdentifier().
48/// - The UID can be printed (e.g., \a CASID::toString()) and it can parsed
49/// by the same or a different CAS instance with \a
50/// ObjectStore::parseID().
51/// - An object can be looked up by content or by UID.
52/// - \a store() is "get-or-create" methods, writing an object if it
53/// doesn't exist yet, and return a ref to it in any case.
54/// - \a loadObject(const CASID&) looks up an object by its UID.
55/// - Objects can reference other objects, forming an arbitrary DAG.
56///
57/// The \a ObjectStore interface has a few ways of referencing objects:
58///
59/// - \a ObjectRef encapsulates a reference to something in the CAS. It is an
60/// opaque type that references an object inside a specific CAS. It is
61/// implementation defined if the underlying object exists or not for an
62/// ObjectRef, and it can used to speed up CAS lookup as an implementation
63/// detail. However, you don't know anything about the underlying objects.
64/// "Loading" the object is a separate step that may not have happened
65/// yet, and which can fail (e.g. due to filesystem corruption) or introduce
66/// latency (if downloading from a remote store).
67/// - \a ObjectHandle encapulates a *loaded* object in the CAS. You need one of
68/// these to inspect the content of an object: to look at its stored
69/// data and references. This is internal to CAS implementation and not
70/// availble from CAS public APIs.
71/// - \a CASID: the UID for an object in the CAS, obtained through \a
72/// ObjectStore::getID() or \a ObjectStore::parseID(). This is a valid CAS
73/// identifier, but may reference an object that is unknown to this CAS
74/// instance.
75/// - \a ObjectProxy pairs an ObjectHandle (subclass) with a ObjectStore, and
76/// wraps access APIs to avoid having to pass extra parameters. It is the
77/// object used for accessing underlying data and refs by CAS users.
78///
79/// Both ObjectRef and ObjectHandle are lightweight, wrapping a `uint64_t` and
80/// are only valid with the associated ObjectStore instance.
81///
82/// There are a few options for accessing content of objects, with different
83/// lifetime tradeoffs:
84///
85/// - \a getData() accesses data without exposing lifetime at all.
86/// - \a getMemoryBuffer() returns a \a MemoryBuffer that may alias storage
87/// owned by the CAS, so it must not outlive the \a ObjectStore.
88/// - \a getStandaloneMemoryBuffer() returns a \a MemoryBuffer whose lifetime
89/// is independent of the CAS (it can live longer).
90/// - \a getDataString() return StringRef with lifetime is guaranteed to last as
91/// long as \a ObjectStore.
92/// - \a readRef() and \a forEachRef() iterate through the references in an
93/// object. There is no lifetime assumption.
94class LLVM_ABI ObjectStore {
95 friend class ObjectProxy;
96 void anchor();
97
98public:
99 /// Get a \p CASID from a \p ID, which should have been generated by \a
100 /// CASID::print(). This succeeds as long as \a validateID() would pass. The
101 /// object may be unknown to this CAS instance.
102 ///
103 /// TODO: Remove, and update callers to use \a validateID() or \a
104 /// extractHashFromID().
105 virtual Expected<CASID> parseID(StringRef ID) = 0;
106
107 /// Store object into ObjectStore.
108 virtual Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
109 ArrayRef<char> Data) = 0;
110 /// Get an ID for \p Ref.
111 virtual CASID getID(ObjectRef Ref) const = 0;
112
113 /// Stores the data of a file into ObjectStore.
114 ///
115 /// An underlying implementation could perform optimizations that reduce I/O
116 /// and disk space consumption.
117 ///
118 /// If there are any concurrent modifications to the file, the contents in the
119 /// CAS may be corrupt.
120 ///
121 /// \param FilePath the path of the file data.
122 virtual Expected<ObjectRef> storeFromFile(StringRef Path);
123
124 /// Exports the data of an object to a file path. It does not include any
125 /// references of the object.
126 ///
127 /// An underlying implementation could perform optimizations that reduce I/O
128 /// and disk space consumption.
129 ///
130 /// \param Node the object to read data from.
131 /// \param FilePath the path of the file data.
132 virtual Error exportDataToFile(ObjectHandle Node, StringRef Path) const;
133
134 /// Get an existing reference to the object called \p ID.
135 ///
136 /// Returns \c None if the object is not stored in this CAS.
137 virtual std::optional<ObjectRef> getReference(const CASID &ID) const = 0;
138
139 /// \returns true if the object is directly available from the local CAS, for
140 /// implementations that have this kind of distinction.
141 virtual Expected<bool> isMaterialized(ObjectRef Ref) const = 0;
142
143 /// Validate the underlying object referred by CASID.
144 virtual Error validateObject(const CASID &ID) = 0;
145
146 /// Validate the entire ObjectStore.
147 virtual Error validate(bool CheckHash) const = 0;
148
149protected:
150 /// Load the object referenced by \p Ref.
151 ///
152 /// Errors if the object cannot be loaded.
153 /// \returns \c std::nullopt if the object is missing from the CAS.
154 virtual Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) = 0;
155
156 /// Like \c loadIfExists but returns an error if the object is missing.
157 Expected<ObjectHandle> load(ObjectRef Ref);
158
159 /// Get the size of some data.
160 virtual uint64_t getDataSize(ObjectHandle Node) const = 0;
161
162 /// Methods for handling objects. CAS implementations need to override to
163 /// provide functions to access stored CAS objects and references.
164 virtual Error forEachRef(ObjectHandle Node,
165 function_ref<Error(ObjectRef)> Callback) const = 0;
166 virtual ObjectRef readRef(ObjectHandle Node, size_t I) const = 0;
167 virtual size_t getNumRefs(ObjectHandle Node) const = 0;
168 virtual ArrayRef<char> getData(ObjectHandle Node,
169 bool RequiresNullTerminator = false) const = 0;
170
171 /// Get ObjectRef from open file.
172 virtual Expected<ObjectRef>
173 storeFromOpenFileImpl(sys::fs::file_t FD,
174 std::optional<sys::fs::file_status> Status);
175
176 /// Customization point for \a getStandaloneMemoryBuffer(). The default
177 /// implementation copies the data, which always satisfies the lifetime
178 /// requirement; implementations that can hand out storage outliving
179 /// themselves, e.g. a mapping of a file they do not keep open, should
180 /// override this to avoid the copy. Must not return \c nullptr: fall back
181 /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
182 /// path does not apply.
183 virtual std::unique_ptr<MemoryBuffer>
184 getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
185 bool RequiresNullTerminator);
186
187 /// Get a lifetime-extended StringRef pointing at \p Data.
188 ///
189 /// Depending on the CAS implementation, this may involve in-memory storage
190 /// overhead.
191 StringRef getDataString(ObjectHandle Node) {
192 return toStringRef(Input: getData(Node));
193 }
194
195 /// Get a MemoryBuffer pointing at \p Data.
196 ///
197 /// The buffer may alias storage owned by this ObjectStore, in which case it
198 /// is only valid for as long as the store is.
199 std::unique_ptr<MemoryBuffer>
200 getMemoryBuffer(ObjectHandle Node, StringRef Name = "",
201 bool RequiresNullTerminator = true);
202
203 /// Get a MemoryBuffer for \p Node that stays valid after this ObjectStore is
204 /// destroyed.
205 ///
206 /// May be more expensive than \a getMemoryBuffer(), which is free to alias
207 /// storage the store already has mapped; prefer that one whenever the buffer
208 /// cannot outlive the store. Never returns \c nullptr: copying the data
209 /// always satisfies the lifetime requirement.
210 std::unique_ptr<MemoryBuffer>
211 getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name = "",
212 bool RequiresNullTerminator = true);
213
214 /// Read all the refs from object in a SmallVector.
215 virtual void readRefs(ObjectHandle Node,
216 SmallVectorImpl<ObjectRef> &Refs) const;
217
218 /// Allow ObjectStore implementations to create internal handles.
219#define MAKE_CAS_HANDLE_CONSTRUCTOR(HandleKind) \
220 HandleKind make##HandleKind(uint64_t InternalRef) const { \
221 return HandleKind(*this, InternalRef); \
222 }
223 MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectHandle)
224 MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectRef)
225#undef MAKE_CAS_HANDLE_CONSTRUCTOR
226
227public:
228 /// Helper functions to store object and returns a ObjectProxy.
229 Expected<ObjectProxy> createProxy(ArrayRef<ObjectRef> Refs, StringRef Data);
230
231 /// Store object from StringRef.
232 Expected<ObjectRef> storeFromString(ArrayRef<ObjectRef> Refs,
233 StringRef String) {
234 return store(Refs, Data: arrayRefFromStringRef<char>(Input: String));
235 }
236
237 /// Default implementation reads \p FD and calls \a storeNode(). Does not
238 /// take ownership of \p FD; the caller is responsible for closing it.
239 ///
240 /// If \p Status is sent in it is to be treated as a hint. Implementations
241 /// must protect against the file size potentially growing after the status
242 /// was taken (i.e., they cannot assume that an mmap will be null-terminated
243 /// where \p Status implies).
244 ///
245 /// Returns the \a CASID and the size of the file.
246 Expected<ObjectRef>
247 storeFromOpenFile(sys::fs::file_t FD,
248 std::optional<sys::fs::file_status> Status = std::nullopt) {
249 return storeFromOpenFileImpl(FD, Status);
250 }
251
252 static Error createUnknownObjectError(const CASID &ID);
253
254 /// Create ObjectProxy from CASID. If the object doesn't exist, get an error.
255 Expected<ObjectProxy> getProxy(const CASID &ID);
256 /// Create ObjectProxy from ObjectRef. If the object can't be loaded, get an
257 /// error.
258 Expected<ObjectProxy> getProxy(ObjectRef Ref);
259
260 /// \returns \c std::nullopt if the object is missing from the CAS.
261 Expected<std::optional<ObjectProxy>> getProxyIfExists(ObjectRef Ref);
262
263 /// Read the data from \p Data into \p OS.
264 uint64_t readData(ObjectHandle Node, raw_ostream &OS, uint64_t Offset = 0,
265 uint64_t MaxBytes = -1ULL) const {
266 ArrayRef<char> Data = getData(Node);
267 assert(Offset < Data.size() && "Expected valid offset");
268 Data = Data.drop_front(N: Offset).take_front(N: MaxBytes);
269 OS << toStringRef(Input: Data);
270 return Data.size();
271 }
272
273 /// Set the size for limiting growth of on-disk storage. This has an effect
274 /// for when the instance is closed.
275 ///
276 /// Implementations may leave this unimplemented.
277 virtual Error setSizeLimit(std::optional<uint64_t> SizeLimit) {
278 return Error::success();
279 }
280
281 /// \returns the storage size of the on-disk CAS data.
282 ///
283 /// Implementations that don't have an implementation for this should return
284 /// \p std::nullopt.
285 virtual Expected<std::optional<uint64_t>> getStorageSize() const {
286 return std::nullopt;
287 }
288
289 /// Prune local storage to reduce its size according to the desired size
290 /// limit. Pruning can happen concurrently with other operations.
291 ///
292 /// Implementations may leave this unimplemented.
293 virtual Error pruneStorageData() { return Error::success(); }
294
295 /// Validate the whole node tree.
296 Error validateTree(ObjectRef Ref);
297
298 /// Import object from another CAS. This will import the full tree from the
299 /// other CAS.
300 Expected<ObjectRef> importObject(ObjectStore &Upstream, ObjectRef Other);
301
302 /// Print the ObjectStore internals for debugging purpose.
303 virtual void print(raw_ostream &) const {}
304 void dump() const;
305
306 /// Get CASContext
307 const CASContext &getContext() const { return Context; }
308
309 virtual ~ObjectStore() = default;
310
311protected:
312 ObjectStore(const CASContext &Context) : Context(Context) {}
313
314private:
315 const CASContext &Context;
316};
317
318/// Reference to an abstract hierarchical node, with data and references.
319/// Reference is passed by value and is expected to be valid as long as the \a
320/// ObjectStore is.
321class ObjectProxy {
322public:
323 ObjectStore &getCAS() const { return *CAS; }
324 CASID getID() const { return CAS->getID(Ref); }
325 ObjectRef getRef() const { return Ref; }
326 size_t getNumReferences() const { return CAS->getNumRefs(Node: H); }
327 ObjectRef getReference(size_t I) const { return CAS->readRef(Node: H, I); }
328
329 operator CASID() const { return getID(); }
330 CASID getReferenceID(size_t I) const {
331 std::optional<CASID> ID = getCAS().getID(Ref: getReference(I));
332 assert(ID && "Expected reference to be first-class object");
333 return *ID;
334 }
335
336 /// Visit each reference in order, returning an error from \p Callback to
337 /// stop early.
338 Error forEachReference(function_ref<Error(ObjectRef)> Callback) const {
339 return CAS->forEachRef(Node: H, Callback);
340 }
341
342 LLVM_ABI std::unique_ptr<MemoryBuffer>
343 getMemoryBuffer(StringRef Name = "",
344 bool RequiresNullTerminator = true) const;
345
346 /// Get a MemoryBuffer that stays valid after the CAS is destroyed.
347 LLVM_ABI std::unique_ptr<MemoryBuffer>
348 getStandaloneMemoryBuffer(StringRef Name = "",
349 bool RequiresNullTerminator = true) const;
350
351 /// Get the content of the node. Valid as long as the CAS is valid.
352 StringRef getData() const { return CAS->getDataString(Node: H); }
353
354 /// Exports the data of an object to a file path.
355 Error exportDataToFile(StringRef Path) const {
356 return CAS->exportDataToFile(Node: H, Path);
357 }
358
359 friend bool operator==(const ObjectProxy &Proxy, ObjectRef Ref) {
360 return Proxy.getRef() == Ref;
361 }
362 friend bool operator==(ObjectRef Ref, const ObjectProxy &Proxy) {
363 return Proxy.getRef() == Ref;
364 }
365 friend bool operator!=(const ObjectProxy &Proxy, ObjectRef Ref) {
366 return !(Proxy.getRef() == Ref);
367 }
368 friend bool operator!=(ObjectRef Ref, const ObjectProxy &Proxy) {
369 return !(Proxy.getRef() == Ref);
370 }
371
372public:
373 ObjectProxy() = delete;
374
375 static ObjectProxy load(ObjectStore &CAS, ObjectRef Ref, ObjectHandle Node) {
376 return ObjectProxy(CAS, Ref, Node);
377 }
378
379private:
380 ObjectProxy(ObjectStore &CAS, ObjectRef Ref, ObjectHandle H)
381 : CAS(&CAS), Ref(Ref), H(H) {}
382
383 ObjectStore *CAS;
384 ObjectRef Ref;
385 ObjectHandle H;
386};
387
388/// Create an in memory CAS.
389LLVM_ABI std::unique_ptr<ObjectStore> createInMemoryCAS();
390
391/// \returns true if \c LLVM_ENABLE_ONDISK_CAS configuration was enabled.
392LLVM_ABI bool isOnDiskCASEnabled();
393
394/// Create a persistent on-disk path at \p Path.
395LLVM_ABI Expected<std::unique_ptr<ObjectStore>>
396createOnDiskCAS(const Twine &Path);
397
398/// Create \c ObjectStore and \c ActionCache instances backed by a plugin that
399/// implements the C API in \c "llvm-c/CAS/PluginAPI_functions.h".
400///
401/// \param PluginPath path of the dynamic library to load.
402/// \param OnDiskPath local path that the plugin should use for any on-disk
403/// resources/caches.
404/// \param PluginArgs name/value pairs passed to the plugin as custom options;
405/// they are opaque to the client.
406LLVM_ABI Expected<
407 std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
408createPluginCASDatabases(
409 StringRef PluginPath, StringRef OnDiskPath,
410 ArrayRef<std::pair<std::string, std::string>> PluginArgs);
411
412/// Validate the on-disk data of a plugin-backed CAS in-process if needed, by
413/// calling the plugin's \c llcas_cas_validate_if_needed. The plugin decides
414/// whether validation is needed.
415///
416/// Clients that want to be resilient to unexpected crashes during validation
417/// may call this from a separate process (e.g. via
418/// \c llvm-cas -validate-if-needed) and call \c recoverPluginCASDatabases if
419/// it fails.
420///
421/// \param PluginPath path of the dynamic library to load.
422/// \param OnDiskPath local path that the plugin uses for any on-disk
423/// resources/caches.
424/// \param PluginArgs name/value pairs passed to the plugin as custom options;
425/// they are opaque to the client.
426/// \param CheckHash Whether to validate hashes match the data.
427/// \param ForceValidation Whether to force validation to occur even if it
428/// should not be necessary.
429///
430/// \returns \c Valid if the data is valid, \c Skipped if validation is not
431/// needed, or an \c Error if validation cannot be performed (including if the
432/// plugin does not support it) or the data is invalid.
433LLVM_ABI Expected<ValidationResult> validatePluginCASDatabasesIfNeeded(
434 StringRef PluginPath, StringRef OnDiskPath,
435 ArrayRef<std::pair<std::string, std::string>> PluginArgs, bool CheckHash,
436 bool ForceValidation);
437
438/// Recover the on-disk data of a plugin-backed CAS after a failed
439/// \c validatePluginCASDatabasesIfNeeded, by calling the plugin's
440/// \c llcas_cas_recover_ondisk_data.
441///
442/// \returns \c Recovered if the data has been recovered, \c Skipped if
443/// recovery is not needed (e.g. a concurrent process already recovered), or an
444/// \c Error if recovery cannot be performed (including if the plugin does not
445/// support it).
446LLVM_ABI Expected<ValidationResult> recoverPluginCASDatabases(
447 StringRef PluginPath, StringRef OnDiskPath,
448 ArrayRef<std::pair<std::string, std::string>> PluginArgs);
449
450} // namespace cas
451} // namespace llvm
452
453#endif // LLVM_CAS_OBJECTSTORE_H
454