1//===- OnDiskKeyValueDB.cpp -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file implements OnDiskKeyValueDB, an ondisk key value database.
11///
12/// The KeyValue database file is named `actions.<version>` inside the CAS
13/// directory. The database stores a mapping between a fixed-sized key and a
14/// fixed-sized value, where the size of key and value can be configured when
15/// opening the database.
16///
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/CAS/OnDiskKeyValueDB.h"
21#include "OnDiskCommon.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CAS/UnifiedOnDiskCache.h"
24#include "llvm/Support/Alignment.h"
25#include "llvm/Support/Compiler.h"
26#include "llvm/Support/Errc.h"
27#include "llvm/Support/Path.h"
28
29using namespace llvm;
30using namespace llvm::cas;
31using namespace llvm::cas::ondisk;
32
33static constexpr StringLiteral ActionCacheFile = "actions.";
34
35Expected<ArrayRef<char>> OnDiskKeyValueDB::put(ArrayRef<uint8_t> Key,
36 ArrayRef<char> Value) {
37 if (LLVM_UNLIKELY(Value.size() != ValueSize))
38 return createStringError(EC: errc::invalid_argument,
39 S: "expected value size of " + itostr(X: ValueSize) +
40 ", got: " + itostr(X: Value.size()));
41 assert(Value.size() == ValueSize);
42 auto ActionP = Cache.insertLazy(
43 Hash: Key, OnConstruct: [&](FileOffset TentativeOffset,
44 OnDiskTrieRawHashMap::ValueProxy TentativeValue) {
45 assert(TentativeValue.Data.size() == ValueSize);
46 llvm::copy(Range&: Value, Out: TentativeValue.Data.data());
47 });
48 if (LLVM_UNLIKELY(!ActionP))
49 return ActionP.takeError();
50 return (*ActionP)->Data;
51}
52
53Expected<std::optional<ArrayRef<char>>>
54OnDiskKeyValueDB::get(ArrayRef<uint8_t> Key) {
55 // Check the result cache.
56 OnDiskTrieRawHashMap::ConstOnDiskPtr ActionP = Cache.find(Hash: Key);
57 if (ActionP) {
58 assert(isAddrAligned(Align(8), ActionP->Data.data()));
59 return ActionP->Data;
60 }
61 if (!UnifiedCache || !UnifiedCache->UpstreamKVDB)
62 return std::nullopt;
63
64 // Try to fault in from upstream.
65 return UnifiedCache->faultInFromUpstreamKV(Key);
66}
67
68Expected<std::unique_ptr<OnDiskKeyValueDB>>
69OnDiskKeyValueDB::open(StringRef Path, StringRef HashName, unsigned KeySize,
70 StringRef ValueName, size_t ValueSize,
71 UnifiedOnDiskCache *Cache,
72 std::shared_ptr<OnDiskCASLogger> Logger) {
73 if (std::error_code EC = sys::fs::create_directories(path: Path))
74 return createFileError(F: Path, EC);
75
76 SmallString<256> CachePath(Path);
77 sys::path::append(path&: CachePath, a: ActionCacheFile + CASFormatVersion);
78 constexpr uint64_t MB = 1024ull * 1024ull;
79 constexpr uint64_t GB = 1024ull * 1024ull * 1024ull;
80
81 uint64_t MaxFileSize = GB;
82 auto CustomSize = getOverriddenMaxMappingSize();
83 if (!CustomSize)
84 return CustomSize.takeError();
85 if (*CustomSize)
86 MaxFileSize = **CustomSize;
87
88 std::optional<OnDiskTrieRawHashMap> ActionCache;
89 if (Error E = OnDiskTrieRawHashMap::create(
90 Path: CachePath,
91 TrieName: "llvm.actioncache[" + HashName + "->" + ValueName + "]",
92 NumHashBits: KeySize * 8,
93 /*DataSize=*/ValueSize, MaxFileSize, /*MinFileSize=*/NewFileInitialSize: MB,
94 Logger: std::move(Logger))
95 .moveInto(Value&: ActionCache))
96 return std::move(E);
97
98 return std::unique_ptr<OnDiskKeyValueDB>(
99 new OnDiskKeyValueDB(ValueSize, std::move(*ActionCache), Cache));
100}
101
102Error OnDiskKeyValueDB::validate(CheckValueT CheckValue) const {
103 if (UnifiedCache && UnifiedCache->UpstreamKVDB) {
104 if (auto E = UnifiedCache->UpstreamKVDB->validate(CheckValue))
105 return E;
106 }
107 return Cache.validate(
108 RecordVerifier: [&](FileOffset Offset,
109 OnDiskTrieRawHashMap::ConstValueProxy Record) -> Error {
110 auto formatError = [&](Twine Msg) {
111 return createStringError(
112 EC: llvm::errc::illegal_byte_sequence,
113 S: "bad cache value at 0x" +
114 utohexstr(X: (unsigned)Offset.get(), /*LowerCase=*/true) + ": " +
115 Msg.str());
116 };
117
118 if (Record.Data.size() != ValueSize)
119 return formatError("wrong cache value size");
120 if (!isAddrAligned(Lhs: Align(8), Addr: Record.Data.data()))
121 return formatError("wrong cache value alignment");
122 if (CheckValue)
123 return CheckValue(Offset, Record.Data);
124 return Error::success();
125 });
126}
127