1//===- OnDiskKeyValueDB.cpp -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file implements OnDiskKeyValueDB, an ondisk key value database.
11///
12/// The KeyValue database file is named `actions.<version>` inside the CAS
13/// directory. The database stores a mapping between a fixed-sized key and a
14/// fixed-sized value, where the size of key and value can be configured when
15/// opening the database.
16///
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/CAS/OnDiskKeyValueDB.h"
21#include "OnDiskCommon.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CAS/OnDiskTrieRawHashMap.h"
24#include "llvm/CAS/UnifiedOnDiskCache.h"
25#include "llvm/Support/Alignment.h"
26#include "llvm/Support/Compiler.h"
27#include "llvm/Support/Errc.h"
28#include "llvm/Support/Path.h"
29
30using namespace llvm;
31using namespace llvm::cas;
32using namespace llvm::cas::ondisk;
33
34static constexpr StringLiteral ActionCacheFile = "actions.";
35
36Expected<ArrayRef<char>> OnDiskKeyValueDB::put(ArrayRef<uint8_t> Key,
37 ArrayRef<char> Value) {
38 if (LLVM_UNLIKELY(Value.size() != ValueSize))
39 return createStringError(EC: errc::invalid_argument,
40 S: "expected value size of " + itostr(X: ValueSize) +
41 ", got: " + itostr(X: Value.size()));
42 assert(Value.size() == ValueSize);
43 auto ActionP = Cache.insertLazy(
44 Hash: Key, OnConstruct: [&](FileOffset TentativeOffset,
45 OnDiskTrieRawHashMap::ValueProxy TentativeValue) {
46 assert(TentativeValue.Data.size() == ValueSize);
47 llvm::copy(Range&: Value, Out: TentativeValue.Data.data());
48 });
49 if (LLVM_UNLIKELY(!ActionP))
50 return ActionP.takeError();
51 return (*ActionP)->Data;
52}
53
54Expected<std::optional<ArrayRef<char>>>
55OnDiskKeyValueDB::get(ArrayRef<uint8_t> Key) {
56 // Check the result cache.
57 OnDiskTrieRawHashMap::ConstOnDiskPtr ActionP = Cache.find(Hash: Key);
58 if (ActionP) {
59 assert(isAddrAligned(Align(8), ActionP->Data.data()));
60 return ActionP->Data;
61 }
62 if (!UnifiedCache || !UnifiedCache->UpstreamKVDB)
63 return std::nullopt;
64
65 // Try to fault in from upstream.
66 return UnifiedCache->faultInFromUpstreamKV(Key);
67}
68
69Expected<std::unique_ptr<OnDiskKeyValueDB>>
70OnDiskKeyValueDB::open(StringRef Path, StringRef HashName, unsigned KeySize,
71 StringRef ValueName, size_t ValueSize,
72 UnifiedOnDiskCache *Cache,
73 std::shared_ptr<OnDiskCASLogger> Logger) {
74 if (std::error_code EC = sys::fs::create_directories(path: Path))
75 return createFileError(F: Path, EC);
76
77 SmallString<256> CachePath(Path);
78 sys::path::append(path&: CachePath, a: ActionCacheFile + CASFormatVersion);
79 constexpr uint64_t MB = 1024ull * 1024ull;
80 constexpr uint64_t GB = 1024ull * 1024ull * 1024ull;
81
82 uint64_t MaxFileSize = GB;
83 auto CustomSize = getOverriddenMaxMappingSize();
84 if (!CustomSize)
85 return CustomSize.takeError();
86 if (*CustomSize)
87 MaxFileSize = **CustomSize;
88
89 std::optional<OnDiskTrieRawHashMap> ActionCache;
90 if (Error E = OnDiskTrieRawHashMap::create(
91 Path: CachePath,
92 TrieName: "llvm.actioncache[" + HashName + "->" + ValueName + "]",
93 NumHashBits: KeySize * 8,
94 /*DataSize=*/ValueSize, MaxFileSize, /*MinFileSize=*/NewFileInitialSize: MB,
95 Logger: std::move(Logger))
96 .moveInto(Value&: ActionCache))
97 return std::move(E);
98
99 return std::unique_ptr<OnDiskKeyValueDB>(
100 new OnDiskKeyValueDB(ValueSize, std::move(*ActionCache), Cache));
101}
102
103static Error validateOnDiskKeyValueDB(const OnDiskTrieRawHashMap &Cache,
104 size_t ValueSize, OnDiskGraphDB *CAS) {
105 return Cache.validate(
106 RecordVerifier: [&](FileOffset Offset,
107 OnDiskTrieRawHashMap::ConstValueProxy Record) -> Error {
108 auto formatError = [&](Twine Msg) {
109 return createStringError(
110 EC: llvm::errc::illegal_byte_sequence,
111 S: "bad cache value at 0x" +
112 utohexstr(X: (unsigned)Offset.get(), /*LowerCase=*/true) + ": " +
113 Msg.str());
114 };
115
116 if (Record.Data.size() != ValueSize)
117 return formatError("wrong cache value size");
118 if (!isAddrAligned(Lhs: Align(8), Addr: Record.Data.data()))
119 return formatError("wrong cache value alignment");
120 if (CAS) {
121 auto ID =
122 ondisk::UnifiedOnDiskCache::getObjectIDFromValue(Value: Record.Data);
123 if (Error E = CAS->validateObjectID(ID))
124 return formatError(llvm::toString(E: std::move(E)));
125 }
126 return Error::success();
127 });
128}
129
130Error OnDiskKeyValueDB::validate() const {
131 if (UnifiedCache && UnifiedCache->UpstreamKVDB) {
132 assert(UnifiedCache->UpstreamGraphDB &&
133 "upstream cache and cas must be paired");
134 if (auto E = validateOnDiskKeyValueDB(Cache: UnifiedCache->UpstreamKVDB->Cache,
135 ValueSize: UnifiedCache->UpstreamKVDB->ValueSize,
136 CAS: UnifiedCache->UpstreamGraphDB.get()))
137 return E;
138 }
139 return validateOnDiskKeyValueDB(
140 Cache, ValueSize,
141 CAS: UnifiedCache ? UnifiedCache->PrimaryGraphDB.get() : nullptr);
142}
143