1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file A utility for operating on LLVM CAS.
10///
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/StringExtras.h"
14#include "llvm/CAS/ActionCache.h"
15#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
16#include "llvm/CAS/ObjectStore.h"
17#include "llvm/Option/Arg.h"
18#include "llvm/Option/ArgList.h"
19#include "llvm/Option/Option.h"
20#include "llvm/Support/CommandLine.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/InitLLVM.h"
23#include "llvm/Support/MemoryBuffer.h"
24#include "llvm/Support/Program.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28using namespace llvm::cas;
29
30namespace {
31enum ID {
32 OPT_INVALID = 0, // This is not an option ID.
33#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
34#include "Options.inc"
35#undef OPTION
36};
37
38using namespace llvm::opt;
39#define OPTTABLE_CODE
40#include "Options.inc"
41
42class LLVMCASOptTable : public opt::OptTable {
43public:
44 LLVMCASOptTable() : opt::OptTable(optionTables()) {}
45};
46
47enum class CommandKind {
48 Invalid,
49 Dump,
50 CatNodeData,
51 MakeBlob,
52 MakeNode,
53 ListObjectReferences,
54 Import,
55 PutCacheKey,
56 GetCacheResult,
57 Validate,
58 ValidateObject,
59 ValidateIfNeeded,
60 Prune,
61};
62
63struct CommandOptions {
64 CommandKind Command = CommandKind::Invalid;
65 std::vector<std::string> Inputs;
66 std::string CASPath;
67 std::string CASPluginPath;
68 SmallVector<std::pair<std::string, std::string>> CASPluginOpts;
69 std::string UpstreamCASPath;
70 std::string DataPath;
71 bool CheckHash;
72 bool AllowRecovery;
73 bool Force;
74 bool InProcess;
75 bool Verbose;
76
77 static CommandKind getCommandKind(opt::Arg &A) {
78 switch (A.getOption().getID()) {
79 case OPT_cas_dump:
80 return CommandKind::Dump;
81 case OPT_cat_node_data:
82 return CommandKind::CatNodeData;
83 case OPT_make_blob:
84 return CommandKind::MakeBlob;
85 case OPT_make_node:
86 return CommandKind::MakeNode;
87 case OPT_ls_node_refs:
88 return CommandKind::ListObjectReferences;
89 case OPT_import:
90 return CommandKind::Import;
91 case OPT_put_cache_key:
92 return CommandKind::PutCacheKey;
93 case OPT_get_cache_result:
94 return CommandKind::GetCacheResult;
95 case OPT_validate:
96 return CommandKind::Validate;
97 case OPT_validate_object:
98 return CommandKind::ValidateObject;
99 case OPT_validate_if_needed:
100 return CommandKind::ValidateIfNeeded;
101 case OPT_prune:
102 return CommandKind::Prune;
103 }
104 return CommandKind::Invalid;
105 }
106
107 // Command requires input.
108 static bool requiresInput(CommandKind Kind) {
109 return Kind != CommandKind::ValidateIfNeeded &&
110 Kind != CommandKind::Validate && Kind != CommandKind::MakeBlob &&
111 Kind != CommandKind::MakeNode && Kind != CommandKind::Dump &&
112 Kind != CommandKind::Prune;
113 }
114};
115} // namespace
116
117static int dump(ObjectStore &CAS);
118static int listObjectReferences(ObjectStore &CAS, const CASID &ID);
119static int catNodeData(ObjectStore &CAS, const CASID &ID);
120static int makeBlob(ObjectStore &CAS, StringRef DataPath);
121static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,
122 StringRef DataPath);
123static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
124 ArrayRef<std::string> Objects);
125static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
126 ArrayRef<std::string> Objects);
127static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
128static int validateObject(ObjectStore &CAS, const CASID &ID);
129static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);
130static int validateIfNeeded(const CommandOptions &Opts, const char *Argv0);
131static int prune(cas::ObjectStore &CAS);
132
133static Expected<CommandOptions> parseOptions(int Argc, char **Argv) {
134 BumpPtrAllocator Alloc;
135 StringSaver Saver(Alloc);
136 SmallVector<const char *> ExpanedArgs;
137 if (!cl::expandResponseFiles(Argc, Argv, EnvVar: nullptr, Saver, NewArgv&: ExpanedArgs))
138 return createStringError(Fmt: "cannot expand response file");
139
140 LLVMCASOptTable T;
141 unsigned MI, MC;
142 opt::InputArgList Args = T.ParseArgs(Args: ExpanedArgs, MissingArgIndex&: MI, MissingArgCount&: MC);
143
144 for (auto *Arg : Args.filtered(Ids: OPT_UNKNOWN)) {
145 llvm::errs() << "ignoring unknown option: " << Arg->getSpelling() << '\n';
146 }
147
148 if (Args.hasArg(Ids: OPT_help)) {
149 T.printHelp(
150 OS&: outs(),
151 Usage: (std::string(Argv[0]) + " [action] [options] <input files>").c_str(),
152 Title: "llvm-cas tool that performs CAS actions.", ShowHidden: false);
153 exit(status: 0);
154 }
155
156 CommandOptions Opts;
157 for (auto *A : Args.filtered(Ids: OPT_grp_action))
158 Opts.Command = CommandOptions::getCommandKind(A&: *A);
159
160 if (Opts.Command == CommandKind::Invalid)
161 return createStringError(Fmt: "no command action is specified");
162
163 for (auto *File : Args.filtered(Ids: OPT_INPUT))
164 Opts.Inputs.push_back(x: File->getValue());
165 Opts.CASPath = Args.getLastArgValue(Id: OPT_cas_path);
166 Opts.CASPluginPath = Args.getLastArgValue(Id: OPT_cas_plugin_path);
167 for (StringRef PluginOpt : Args.getAllArgValues(Id: OPT_cas_plugin_option)) {
168 auto [Name, Value] = PluginOpt.split(Separator: '=');
169 Opts.CASPluginOpts.emplace_back(Args&: Name, Args&: Value);
170 }
171 Opts.UpstreamCASPath = Args.getLastArgValue(Id: OPT_upstream_cas);
172 Opts.DataPath = Args.getLastArgValue(Id: OPT_data);
173 Opts.CheckHash = Args.hasArg(Ids: OPT_check_hash);
174 Opts.AllowRecovery = Args.hasArg(Ids: OPT_allow_recovery);
175 Opts.Force = Args.hasArg(Ids: OPT_force);
176 Opts.InProcess = Args.hasArg(Ids: OPT_in_process);
177 Opts.Verbose = Args.hasArg(Ids: OPT_verbose);
178
179 // Validate options.
180 if (Opts.CASPath.empty())
181 return createStringError(Fmt: "missing --cas <path>");
182
183 if (Opts.Inputs.empty() && CommandOptions::requiresInput(Kind: Opts.Command))
184 return createStringError(Fmt: "missing <input> to operate on");
185
186 return Opts;
187}
188
189int main(int Argc, char **Argv) {
190 InitLLVM X(Argc, Argv);
191
192 ExitOnError ExitOnErr;
193 auto Opts = ExitOnErr(parseOptions(Argc, Argv));
194
195 if (Opts.Command == CommandKind::ValidateIfNeeded)
196 return validateIfNeeded(Opts, Argv0: Argv[0]);
197
198 std::shared_ptr<ObjectStore> CAS;
199 std::shared_ptr<ActionCache> AC;
200 if (!Opts.CASPluginPath.empty())
201 std::tie(args&: CAS, args&: AC) = ExitOnErr(createPluginCASDatabases(
202 PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath, PluginArgs: Opts.CASPluginOpts));
203 else
204 std::tie(args&: CAS, args&: AC) =
205 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.CASPath));
206 assert(CAS);
207
208 if (Opts.Command == CommandKind::Dump)
209 return dump(CAS&: *CAS);
210
211 if (Opts.Command == CommandKind::Validate)
212 return validate(CAS&: *CAS, AC&: *AC, CheckHash: Opts.CheckHash);
213
214 if (Opts.Command == CommandKind::MakeBlob)
215 return makeBlob(CAS&: *CAS, DataPath: Opts.DataPath);
216
217 if (Opts.Command == CommandKind::MakeNode)
218 return makeNode(CAS&: *CAS, References: Opts.Inputs, DataPath: Opts.DataPath);
219
220 if (Opts.Command == CommandKind::Prune)
221 return prune(CAS&: *CAS);
222
223 if (Opts.Command == CommandKind::Import) {
224 if (Opts.UpstreamCASPath.empty())
225 ExitOnErr(createStringError(Fmt: "missing '-upstream-cas'"));
226
227 auto [UpstreamCAS, _] =
228 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.UpstreamCASPath));
229 return import(FromCAS&: *UpstreamCAS, ToCAS&: *CAS, Objects: Opts.Inputs);
230 }
231
232 if (Opts.Command == CommandKind::PutCacheKey ||
233 Opts.Command == CommandKind::GetCacheResult) {
234 if (!AC)
235 ExitOnErr(createStringError(Fmt: "no action-cache available"));
236 }
237
238 if (Opts.Command == CommandKind::PutCacheKey)
239 return putCacheKey(CAS&: *CAS, AC&: *AC, Objects: Opts.Inputs);
240
241 // Remaining commands need exactly one CAS object.
242 if (Opts.Inputs.size() > 1)
243 ExitOnErr(createStringError(Fmt: "too many <object>s, expected 1"));
244 CASID ID = ExitOnErr(CAS->parseID(ID: Opts.Inputs.front()));
245
246 if (Opts.Command == CommandKind::GetCacheResult)
247 return getCacheResult(CAS&: *CAS, AC&: *AC, ID);
248
249 if (Opts.Command == CommandKind::ListObjectReferences)
250 return listObjectReferences(CAS&: *CAS, ID);
251
252 if (Opts.Command == CommandKind::CatNodeData)
253 return catNodeData(CAS&: *CAS, ID);
254
255 assert(Opts.Command == CommandKind::ValidateObject);
256 return validateObject(CAS&: *CAS, ID);
257}
258
259static Expected<std::unique_ptr<MemoryBuffer>> openBuffer(StringRef DataPath) {
260 if (DataPath.empty())
261 return createStringError(Fmt: "--data missing");
262 return errorOrToExpected(EO: DataPath == "-"
263 ? llvm::MemoryBuffer::getSTDIN()
264 : llvm::MemoryBuffer::getFile(Filename: DataPath));
265}
266
267int dump(ObjectStore &CAS) {
268 ExitOnError ExitOnErr("llvm-cas: dump: ");
269 CAS.print(llvm::outs());
270 return 0;
271}
272
273int makeBlob(ObjectStore &CAS, StringRef DataPath) {
274 ExitOnError ExitOnErr("llvm-cas: make-blob: ");
275 std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));
276
277 ObjectProxy Blob = ExitOnErr(CAS.createProxy(Refs: {}, Data: Buffer->getBuffer()));
278 llvm::outs() << Blob.getID() << "\n";
279 return 0;
280}
281
282int catNodeData(ObjectStore &CAS, const CASID &ID) {
283 ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");
284 llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();
285 return 0;
286}
287
288int listObjectReferences(ObjectStore &CAS, const CASID &ID) {
289 ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");
290
291 ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));
292 ExitOnErr(Object.forEachReference(Callback: [&](ObjectRef Ref) -> Error {
293 llvm::outs() << CAS.getID(Ref) << "\n";
294 return Error::success();
295 }));
296
297 return 0;
298}
299
300static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,
301 StringRef DataPath) {
302 std::unique_ptr<MemoryBuffer> Data =
303 ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));
304
305 SmallVector<ObjectRef> IDs;
306 for (StringRef Object : Objects) {
307 ExitOnError ObjectErr("llvm-cas: make-node: ref: ");
308 std::optional<ObjectRef> ID =
309 CAS.getReference(ID: ObjectErr(CAS.parseID(ID: Object)));
310 if (!ID)
311 ObjectErr(createStringError(S: "unknown object '" + Object + "'"));
312 IDs.push_back(Elt: *ID);
313 }
314
315 ExitOnError ExitOnErr("llvm-cas: make-node: ");
316 ObjectProxy Object = ExitOnErr(CAS.createProxy(Refs: IDs, Data: Data->getBuffer()));
317 llvm::outs() << Object.getID() << "\n";
318 return 0;
319}
320
321static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
322 ArrayRef<std::string> Objects) {
323 ExitOnError ExitOnErr("llvm-cas: import: ");
324
325 for (StringRef Object : Objects) {
326 CASID ID = ExitOnErr(FromCAS.parseID(ID: Object));
327 auto Ref = FromCAS.getReference(ID);
328 if (!Ref)
329 ExitOnErr(createStringError(S: "input not found: " + ID.toString()));
330
331 auto Imported = ExitOnErr(ToCAS.importObject(Upstream&: FromCAS, Other: *Ref));
332 llvm::outs() << ToCAS.getID(Ref: Imported).toString() << "\n";
333 }
334 return 0;
335}
336
337static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
338 ArrayRef<std::string> Objects) {
339 ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");
340
341 if (Objects.size() % 2 != 0)
342 ExitOnErr(createStringError(Fmt: "expected pairs of inputs"));
343 while (!Objects.empty()) {
344 CASID Key = ExitOnErr(CAS.parseID(ID: Objects[0]));
345 CASID Result = ExitOnErr(CAS.parseID(ID: Objects[1]));
346 Objects = Objects.drop_front(N: 2);
347 ExitOnErr(AC.put(ActionKey: Key, Result));
348 }
349 return 0;
350}
351
352static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {
353 ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");
354
355 auto Result = ExitOnErr(AC.get(ActionKey: ID));
356 if (!Result) {
357 outs() << "result not found\n";
358 return 1;
359 }
360 outs() << *Result << "\n";
361 return 0;
362}
363
364int validateObject(ObjectStore &CAS, const CASID &ID) {
365 ExitOnError ExitOnErr("llvm-cas: validate-object: ");
366 ExitOnErr(CAS.validateObject(ID));
367 outs() << ID << ": validated successfully\n";
368 return 0;
369}
370
371int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {
372 ExitOnError ExitOnErr("llvm-cas: validate: ");
373 ExitOnErr(CAS.validate(CheckHash));
374 ExitOnErr(AC.validate());
375 outs() << "validated successfully\n";
376 return 0;
377}
378
379/// Validates the CAS in this process and prints the result.
380static Error validateInProcess(const CommandOptions &Opts) {
381 ValidationResult Result;
382 if (Error E = (Opts.CASPluginPath.empty()
383 ? validateOnDiskUnifiedCASDatabasesIfNeeded(
384 Path: Opts.CASPath, CheckHash: Opts.CheckHash, ForceValidation: Opts.Force)
385 : validatePluginCASDatabasesIfNeeded(
386 PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath, PluginArgs: Opts.CASPluginOpts,
387 CheckHash: Opts.CheckHash, ForceValidation: Opts.Force))
388 .moveInto(Value&: Result))
389 return E;
390 outs() << (Result == ValidationResult::Skipped ? "validation skipped\n"
391 : "validated successfully\n");
392 return Error::success();
393}
394
395/// Validates the CAS by re-executing llvm-cas with --in-process, which
396/// protects against crashes during validation. The output of the child process
397/// is forwarded, except for its errors when they would be followed by recovery
398/// and are not requested with --verbose.
399///
400/// \returns false if validation failed or crashed.
401static Expected<bool> validateOutOfProcess(const CommandOptions &Opts,
402 const char *Argv0) {
403 std::string Exec =
404 sys::fs::getMainExecutable(argv0: Argv0, MainExecAddr: (void *)validateOutOfProcess);
405 SmallVector<std::string> PluginOpts;
406 for (const auto &[Name, Value] : Opts.CASPluginOpts)
407 PluginOpts.push_back(Elt: Name + "=" + Value);
408
409 SmallVector<StringRef> Args{Exec, "--cas", Opts.CASPath};
410 if (!Opts.CASPluginPath.empty()) {
411 Args.append(IL: {"--fcas-plugin-path", Opts.CASPluginPath});
412 for (StringRef PluginOpt : PluginOpts)
413 Args.append(IL: {"--fcas-plugin-option", PluginOpt});
414 }
415 Args.append(IL: {"--validate-if-needed", "--in-process"});
416 if (Opts.CheckHash)
417 Args.push_back(Elt: "--check-hash");
418 if (Opts.Force)
419 Args.push_back(Elt: "--force");
420
421 // Discard stderr if quiet, and otherwise inherit all of stdin, stdout and
422 // stderr.
423 bool QuietErrors = Opts.AllowRecovery && !Opts.Verbose;
424 SmallVector<std::optional<StringRef>, 3> Redirects;
425 if (QuietErrors)
426 Redirects = {std::nullopt, std::nullopt, StringRef("")};
427
428 outs().flush();
429 std::string ErrMsg;
430 int Result = sys::ExecuteAndWait(Program: Exec, Args, /*Env=*/std::nullopt, Redirects,
431 /*SecondsToWait=*/120,
432 /*MemoryLimit=*/0, ErrMsg: &ErrMsg);
433 if (Result == -1)
434 return createStringError(S: "failed to exec " + join(R&: Args, Separator: " ") + ": " +
435 ErrMsg);
436 if (Result == -2 && !QuietErrors)
437 errs() << "llvm-cas: validate-if-needed: validation crashed: " << ErrMsg
438 << "\n";
439 return Result == 0;
440}
441
442int validateIfNeeded(const CommandOptions &Opts, const char *Argv0) {
443 ExitOnError ExitOnErr("llvm-cas: validate-if-needed: ");
444 if (Opts.InProcess) {
445 Error E = validateInProcess(Opts);
446 if (!E)
447 return 0;
448 if (!Opts.AllowRecovery)
449 ExitOnErr(std::move(E));
450 // The error is expected when recovering, so only print it on request. It
451 // is also recorded in the CAS log.
452 if (Opts.Verbose)
453 errs() << "llvm-cas: validate-if-needed: " << toString(E: std::move(E))
454 << "\n";
455 else
456 consumeError(Err: std::move(E));
457 } else {
458 if (ExitOnErr(validateOutOfProcess(Opts, Argv0)))
459 return 0;
460 if (!Opts.AllowRecovery)
461 ExitOnErr(createStringError(Fmt: "cas contents invalid"));
462 }
463
464 ValidationResult Result = ExitOnErr(
465 Opts.CASPluginPath.empty()
466 ? recoverOnDiskUnifiedCASDatabases(Path: Opts.CASPath)
467 : recoverPluginCASDatabases(PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath,
468 PluginArgs: Opts.CASPluginOpts));
469 outs() << (Result == ValidationResult::Skipped
470 ? "recovery skipped\n"
471 : "recovered from invalid data\n");
472 return 0;
473}
474
475static int prune(cas::ObjectStore &CAS) {
476 ExitOnError ExitOnErr("llvm-cas: prune: ");
477 ExitOnErr(CAS.pruneStorageData());
478 return 0;
479}
480