1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file A utility for operating on LLVM CAS.
10///
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CAS/ActionCache.h"
14#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
15#include "llvm/CAS/ObjectStore.h"
16#include "llvm/Option/Arg.h"
17#include "llvm/Option/ArgList.h"
18#include "llvm/Option/Option.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/Error.h"
21#include "llvm/Support/InitLLVM.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include "llvm/Support/raw_ostream.h"
24
25using namespace llvm;
26using namespace llvm::cas;
27
28namespace {
29enum ID {
30 OPT_INVALID = 0, // This is not an option ID.
31#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
32#include "Options.inc"
33#undef OPTION
34};
35
36#define OPTTABLE_STR_TABLE_CODE
37#include "Options.inc"
38#undef OPTTABLE_STR_TABLE_CODE
39
40#define OPTTABLE_PREFIXES_TABLE_CODE
41#include "Options.inc"
42#undef OPTTABLE_PREFIXES_TABLE_CODE
43
44using namespace llvm::opt;
45static constexpr opt::OptTable::Info InfoTable[] = {
46#define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),
47#include "Options.inc"
48#undef OPTION
49};
50
51class LLVMCASOptTable : public opt::GenericOptTable {
52public:
53 LLVMCASOptTable()
54 : opt::GenericOptTable(OptionStrTable, OptionPrefixesTable, InfoTable) {}
55};
56
57enum class CommandKind {
58 Invalid,
59 Dump,
60 CatNodeData,
61 MakeBlob,
62 MakeNode,
63 ListObjectReferences,
64 Import,
65 PutCacheKey,
66 GetCacheResult,
67 Validate,
68 ValidateObject,
69 ValidateIfNeeded,
70 Prune,
71};
72
73struct CommandOptions {
74 CommandKind Command = CommandKind::Invalid;
75 std::vector<std::string> Inputs;
76 std::string CASPath;
77 std::string UpstreamCASPath;
78 std::string DataPath;
79 bool CheckHash;
80 bool AllowRecovery;
81 bool Force;
82 bool InProcess;
83
84 static CommandKind getCommandKind(opt::Arg &A) {
85 switch (A.getOption().getID()) {
86 case OPT_cas_dump:
87 return CommandKind::Dump;
88 case OPT_cat_node_data:
89 return CommandKind::CatNodeData;
90 case OPT_make_blob:
91 return CommandKind::MakeBlob;
92 case OPT_make_node:
93 return CommandKind::MakeNode;
94 case OPT_ls_node_refs:
95 return CommandKind::ListObjectReferences;
96 case OPT_import:
97 return CommandKind::Import;
98 case OPT_put_cache_key:
99 return CommandKind::PutCacheKey;
100 case OPT_get_cache_result:
101 return CommandKind::GetCacheResult;
102 case OPT_validate:
103 return CommandKind::Validate;
104 case OPT_validate_object:
105 return CommandKind::ValidateObject;
106 case OPT_validate_if_needed:
107 return CommandKind::ValidateIfNeeded;
108 case OPT_prune:
109 return CommandKind::Prune;
110 }
111 return CommandKind::Invalid;
112 }
113
114 // Command requires input.
115 static bool requiresInput(CommandKind Kind) {
116 return Kind != CommandKind::ValidateIfNeeded &&
117 Kind != CommandKind::Validate && Kind != CommandKind::MakeBlob &&
118 Kind != CommandKind::MakeNode && Kind != CommandKind::Dump &&
119 Kind != CommandKind::Prune;
120 }
121};
122} // namespace
123
124static int dump(ObjectStore &CAS);
125static int listObjectReferences(ObjectStore &CAS, const CASID &ID);
126static int catNodeData(ObjectStore &CAS, const CASID &ID);
127static int makeBlob(ObjectStore &CAS, StringRef DataPath);
128static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,
129 StringRef DataPath);
130static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
131 ArrayRef<std::string> Objects);
132static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
133 ArrayRef<std::string> Objects);
134static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
135static int validateObject(ObjectStore &CAS, const CASID &ID);
136static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);
137static int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,
138 bool AllowRecovery, bool InProcess,
139 const char *Argv0);
140static int prune(cas::ObjectStore &CAS);
141
142static Expected<CommandOptions> parseOptions(int Argc, char **Argv) {
143 BumpPtrAllocator Alloc;
144 StringSaver Saver(Alloc);
145 SmallVector<const char *> ExpanedArgs;
146 if (!cl::expandResponseFiles(Argc, Argv, EnvVar: nullptr, Saver, NewArgv&: ExpanedArgs))
147 return createStringError(Fmt: "cannot expand response file");
148
149 LLVMCASOptTable T;
150 unsigned MI, MC;
151 opt::InputArgList Args = T.ParseArgs(Args: ExpanedArgs, MissingArgIndex&: MI, MissingArgCount&: MC);
152
153 for (auto *Arg : Args.filtered(Ids: OPT_UNKNOWN)) {
154 llvm::errs() << "ignoring unknown option: " << Arg->getSpelling() << '\n';
155 }
156
157 if (Args.hasArg(Ids: OPT_help)) {
158 T.printHelp(
159 OS&: outs(),
160 Usage: (std::string(Argv[0]) + " [action] [options] <input files>").c_str(),
161 Title: "llvm-cas tool that performs CAS actions.", ShowHidden: false);
162 exit(status: 0);
163 }
164
165 CommandOptions Opts;
166 for (auto *A : Args.filtered(Ids: OPT_grp_action))
167 Opts.Command = CommandOptions::getCommandKind(A&: *A);
168
169 if (Opts.Command == CommandKind::Invalid)
170 return createStringError(Fmt: "no command action is specified");
171
172 for (auto *File : Args.filtered(Ids: OPT_INPUT))
173 Opts.Inputs.push_back(x: File->getValue());
174 Opts.CASPath = Args.getLastArgValue(Id: OPT_cas_path);
175 Opts.UpstreamCASPath = Args.getLastArgValue(Id: OPT_upstream_cas);
176 Opts.DataPath = Args.getLastArgValue(Id: OPT_data);
177 Opts.CheckHash = Args.hasArg(Ids: OPT_check_hash);
178 Opts.AllowRecovery = Args.hasArg(Ids: OPT_allow_recovery);
179 Opts.Force = Args.hasArg(Ids: OPT_force);
180 Opts.InProcess = Args.hasArg(Ids: OPT_in_process);
181
182 // Validate options.
183 if (Opts.CASPath.empty())
184 return createStringError(Fmt: "missing --cas <path>");
185
186 if (Opts.Inputs.empty() && CommandOptions::requiresInput(Kind: Opts.Command))
187 return createStringError(Fmt: "missing <input> to operate on");
188
189 return Opts;
190}
191
192int main(int Argc, char **Argv) {
193 InitLLVM X(Argc, Argv);
194
195 ExitOnError ExitOnErr;
196 auto Opts = ExitOnErr(parseOptions(Argc, Argv));
197
198 if (Opts.Command == CommandKind::ValidateIfNeeded)
199 return validateIfNeeded(Path: Opts.CASPath, CheckHash: Opts.CheckHash, Force: Opts.Force,
200 AllowRecovery: Opts.AllowRecovery, InProcess: Opts.InProcess, Argv0: Argv[0]);
201
202 auto [CAS, AC] = ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.CASPath));
203 assert(CAS);
204
205 if (Opts.Command == CommandKind::Dump)
206 return dump(CAS&: *CAS);
207
208 if (Opts.Command == CommandKind::Validate)
209 return validate(CAS&: *CAS, AC&: *AC, CheckHash: Opts.CheckHash);
210
211 if (Opts.Command == CommandKind::MakeBlob)
212 return makeBlob(CAS&: *CAS, DataPath: Opts.DataPath);
213
214 if (Opts.Command == CommandKind::MakeNode)
215 return makeNode(CAS&: *CAS, References: Opts.Inputs, DataPath: Opts.DataPath);
216
217 if (Opts.Command == CommandKind::Prune)
218 return prune(CAS&: *CAS);
219
220 if (Opts.Command == CommandKind::Import) {
221 if (Opts.UpstreamCASPath.empty())
222 ExitOnErr(createStringError(Fmt: "missing '-upstream-cas'"));
223
224 auto [UpstreamCAS, _] =
225 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.UpstreamCASPath));
226 return import(FromCAS&: *UpstreamCAS, ToCAS&: *CAS, Objects: Opts.Inputs);
227 }
228
229 if (Opts.Command == CommandKind::PutCacheKey ||
230 Opts.Command == CommandKind::GetCacheResult) {
231 if (!AC)
232 ExitOnErr(createStringError(Fmt: "no action-cache available"));
233 }
234
235 if (Opts.Command == CommandKind::PutCacheKey)
236 return putCacheKey(CAS&: *CAS, AC&: *AC, Objects: Opts.Inputs);
237
238 // Remaining commands need exactly one CAS object.
239 if (Opts.Inputs.size() > 1)
240 ExitOnErr(createStringError(Fmt: "too many <object>s, expected 1"));
241 CASID ID = ExitOnErr(CAS->parseID(ID: Opts.Inputs.front()));
242
243 if (Opts.Command == CommandKind::GetCacheResult)
244 return getCacheResult(CAS&: *CAS, AC&: *AC, ID);
245
246 if (Opts.Command == CommandKind::ListObjectReferences)
247 return listObjectReferences(CAS&: *CAS, ID);
248
249 if (Opts.Command == CommandKind::CatNodeData)
250 return catNodeData(CAS&: *CAS, ID);
251
252 assert(Opts.Command == CommandKind::ValidateObject);
253 return validateObject(CAS&: *CAS, ID);
254}
255
256static Expected<std::unique_ptr<MemoryBuffer>> openBuffer(StringRef DataPath) {
257 if (DataPath.empty())
258 return createStringError(Fmt: "--data missing");
259 return errorOrToExpected(EO: DataPath == "-"
260 ? llvm::MemoryBuffer::getSTDIN()
261 : llvm::MemoryBuffer::getFile(Filename: DataPath));
262}
263
264int dump(ObjectStore &CAS) {
265 ExitOnError ExitOnErr("llvm-cas: dump: ");
266 CAS.print(llvm::outs());
267 return 0;
268}
269
270int makeBlob(ObjectStore &CAS, StringRef DataPath) {
271 ExitOnError ExitOnErr("llvm-cas: make-blob: ");
272 std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));
273
274 ObjectProxy Blob = ExitOnErr(CAS.createProxy(Refs: {}, Data: Buffer->getBuffer()));
275 llvm::outs() << Blob.getID() << "\n";
276 return 0;
277}
278
279int catNodeData(ObjectStore &CAS, const CASID &ID) {
280 ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");
281 llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();
282 return 0;
283}
284
285int listObjectReferences(ObjectStore &CAS, const CASID &ID) {
286 ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");
287
288 ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));
289 ExitOnErr(Object.forEachReference(Callback: [&](ObjectRef Ref) -> Error {
290 llvm::outs() << CAS.getID(Ref) << "\n";
291 return Error::success();
292 }));
293
294 return 0;
295}
296
297static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,
298 StringRef DataPath) {
299 std::unique_ptr<MemoryBuffer> Data =
300 ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));
301
302 SmallVector<ObjectRef> IDs;
303 for (StringRef Object : Objects) {
304 ExitOnError ObjectErr("llvm-cas: make-node: ref: ");
305 std::optional<ObjectRef> ID =
306 CAS.getReference(ID: ObjectErr(CAS.parseID(ID: Object)));
307 if (!ID)
308 ObjectErr(createStringError(S: "unknown object '" + Object + "'"));
309 IDs.push_back(Elt: *ID);
310 }
311
312 ExitOnError ExitOnErr("llvm-cas: make-node: ");
313 ObjectProxy Object = ExitOnErr(CAS.createProxy(Refs: IDs, Data: Data->getBuffer()));
314 llvm::outs() << Object.getID() << "\n";
315 return 0;
316}
317
318static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
319 ArrayRef<std::string> Objects) {
320 ExitOnError ExitOnErr("llvm-cas: import: ");
321
322 for (StringRef Object : Objects) {
323 CASID ID = ExitOnErr(FromCAS.parseID(ID: Object));
324 auto Ref = FromCAS.getReference(ID);
325 if (!Ref)
326 ExitOnErr(createStringError(S: "input not found: " + ID.toString()));
327
328 auto Imported = ExitOnErr(ToCAS.importObject(Upstream&: FromCAS, Other: *Ref));
329 llvm::outs() << ToCAS.getID(Ref: Imported).toString() << "\n";
330 }
331 return 0;
332}
333
334static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
335 ArrayRef<std::string> Objects) {
336 ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");
337
338 if (Objects.size() % 2 != 0)
339 ExitOnErr(createStringError(Fmt: "expected pairs of inputs"));
340 while (!Objects.empty()) {
341 CASID Key = ExitOnErr(CAS.parseID(ID: Objects[0]));
342 CASID Result = ExitOnErr(CAS.parseID(ID: Objects[1]));
343 Objects = Objects.drop_front(N: 2);
344 ExitOnErr(AC.put(ActionKey: Key, Result));
345 }
346 return 0;
347}
348
349static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {
350 ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");
351
352 auto Result = ExitOnErr(AC.get(ActionKey: ID));
353 if (!Result) {
354 outs() << "result not found\n";
355 return 1;
356 }
357 outs() << *Result << "\n";
358 return 0;
359}
360
361int validateObject(ObjectStore &CAS, const CASID &ID) {
362 ExitOnError ExitOnErr("llvm-cas: validate-object: ");
363 ExitOnErr(CAS.validateObject(ID));
364 outs() << ID << ": validated successfully\n";
365 return 0;
366}
367
368int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {
369 ExitOnError ExitOnErr("llvm-cas: validate: ");
370 ExitOnErr(CAS.validate(CheckHash));
371 ExitOnErr(AC.validate());
372 outs() << "validated successfully\n";
373 return 0;
374}
375
376int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,
377 bool AllowRecovery, bool InProcess, const char *Argv0) {
378 ExitOnError ExitOnErr("llvm-cas: validate-if-needed: ");
379 std::string ExecStorage;
380 std::optional<StringRef> Exec;
381 if (!InProcess) {
382 ExecStorage = sys::fs::getMainExecutable(argv0: Argv0, MainExecAddr: (void *)validateIfNeeded);
383 Exec = ExecStorage;
384 }
385 ValidationResult Result = ExitOnErr(validateOnDiskUnifiedCASDatabasesIfNeeded(
386 Path, CheckHash, AllowRecovery, ForceValidation: Force, LLVMCasBinaryPath: Exec));
387 switch (Result) {
388 case ValidationResult::Valid:
389 outs() << "validated successfully\n";
390 break;
391 case ValidationResult::Recovered:
392 outs() << "recovered from invalid data\n";
393 break;
394 case ValidationResult::Skipped:
395 outs() << "validation skipped\n";
396 break;
397 }
398 return 0;
399}
400
401static int prune(cas::ObjectStore &CAS) {
402 ExitOnError ExitOnErr("llvm-cas: prune: ");
403 ExitOnErr(CAS.pruneStorageData());
404 return 0;
405}
406