1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file A utility for operating on LLVM CAS.
10///
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/StringExtras.h"
14#include "llvm/ADT/StringSwitch.h"
15#include "llvm/CAS/ActionCache.h"
16#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
17#include "llvm/CAS/ObjectStore.h"
18#include "llvm/Option/Arg.h"
19#include "llvm/Option/ArgList.h"
20#include "llvm/Option/Option.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/Error.h"
23#include "llvm/Support/InitLLVM.h"
24#include "llvm/Support/MemoryBuffer.h"
25#include "llvm/Support/Program.h"
26#include "llvm/Support/raw_ostream.h"
27
28using namespace llvm;
29using namespace llvm::cas;
30
31namespace {
32enum ID {
33 OPT_INVALID = 0, // This is not an option ID.
34#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
35#include "Options.inc"
36#undef OPTION
37};
38
39using namespace llvm::opt;
40#define OPTTABLE_CODE
41#include "Options.inc"
42
43class LLVMCASOptTable : public opt::OptTable {
44public:
45 LLVMCASOptTable() : opt::OptTable(optionTables()) {}
46};
47
48enum class CommandKind {
49 Invalid,
50 Dump,
51 CatNodeData,
52 MakeBlob,
53 MakeNode,
54 ListObjectReferences,
55 Import,
56 PutCacheKey,
57 GetCacheResult,
58 Validate,
59 ValidateObject,
60 ValidateIfNeeded,
61 Prune,
62};
63
64struct CommandOptions {
65 CommandKind Command = CommandKind::Invalid;
66 std::vector<std::string> Inputs;
67 std::string CASPath;
68 std::string CASPluginPath;
69 SmallVector<std::pair<std::string, std::string>> CASPluginOpts;
70 std::string UpstreamCASPath;
71 std::string DataPath;
72 bool CheckHash;
73 bool AllowRecovery;
74 bool Force;
75 bool InProcess;
76 bool Verbose;
77
78 static CommandKind getCommandKind(StringRef SubCommand) {
79 return StringSwitch<CommandKind>(SubCommand)
80 .Case(S: "dump", Value: CommandKind::Dump)
81 .Case(S: "cat-node-data", Value: CommandKind::CatNodeData)
82 .Case(S: "make-blob", Value: CommandKind::MakeBlob)
83 .Case(S: "make-node", Value: CommandKind::MakeNode)
84 .Case(S: "ls-node-refs", Value: CommandKind::ListObjectReferences)
85 .Case(S: "import", Value: CommandKind::Import)
86 .Case(S: "put-cache-key", Value: CommandKind::PutCacheKey)
87 .Case(S: "get-cache-result", Value: CommandKind::GetCacheResult)
88 .Case(S: "validate", Value: CommandKind::Validate)
89 .Case(S: "validate-object", Value: CommandKind::ValidateObject)
90 .Case(S: "validate-if-needed", Value: CommandKind::ValidateIfNeeded)
91 .Case(S: "prune", Value: CommandKind::Prune)
92 .Default(Value: CommandKind::Invalid);
93 }
94
95 // Command requires input.
96 static bool requiresInput(CommandKind Kind) {
97 return Kind != CommandKind::ValidateIfNeeded &&
98 Kind != CommandKind::Validate && Kind != CommandKind::MakeBlob &&
99 Kind != CommandKind::MakeNode && Kind != CommandKind::Dump &&
100 Kind != CommandKind::Prune;
101 }
102};
103} // namespace
104
105static int dump(ObjectStore &CAS);
106static int listObjectReferences(ObjectStore &CAS, const CASID &ID);
107static int catNodeData(ObjectStore &CAS, const CASID &ID);
108static int makeBlob(ObjectStore &CAS, StringRef DataPath);
109static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,
110 StringRef DataPath);
111static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
112 ArrayRef<std::string> Objects);
113static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
114 ArrayRef<std::string> Objects);
115static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
116static int validateObject(ObjectStore &CAS, const CASID &ID);
117static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);
118static int validateIfNeeded(const CommandOptions &Opts, const char *Argv0);
119static int prune(cas::ObjectStore &CAS);
120
121/// Rewrites the legacy option spellings of subcommands, e.g. `--make-blob`
122/// or `-make-blob`, into the subcommand, e.g. `make-blob`, so that the old
123/// style of invoking llvm-cas keeps working.
124static void rewriteLegacySubCommands(const opt::OptTable &T,
125 MutableArrayRef<const char *> Args) {
126 for (const char *&Arg : Args) {
127 StringRef Name = Arg;
128 if (!Name.consume_front(Prefix: "-"))
129 continue;
130 Name.consume_front(Prefix: "-");
131 if (any_of(Range: T.getSubCommands(), P: [&](const opt::OptTable::SubCommand &SC) {
132 return Name == SC.Name;
133 }))
134 Arg = Name.data();
135 }
136}
137
138static Expected<CommandOptions> parseOptions(int Argc, char **Argv) {
139 BumpPtrAllocator Alloc;
140 StringSaver Saver(Alloc);
141 SmallVector<const char *> ExpanedArgs;
142 if (!cl::expandResponseFiles(Argc, Argv, EnvVar: nullptr, Saver, NewArgv&: ExpanedArgs))
143 return createStringError(Fmt: "cannot expand response file");
144
145 LLVMCASOptTable T;
146 rewriteLegacySubCommands(T, Args: ExpanedArgs);
147 unsigned MI, MC;
148 opt::InputArgList Args = T.ParseArgs(Args: ExpanedArgs, MissingArgIndex&: MI, MissingArgCount&: MC);
149
150 // The first positional argument that is a subcommand name is the
151 // subcommand, the rest are inputs, e.g. `cat-node-data make-blob` has input
152 // `make-blob`.
153 CommandOptions Opts;
154 StringRef SubCommand = Args.getSubCommand(
155 AllSubCommands: T.getSubCommands(),
156 HandleMultipleSubcommands: [](ArrayRef<StringRef>) {
157 llvm_unreachable("later subcommand names are inputs");
158 },
159 HandleOtherPositionals: [&](ArrayRef<StringRef> Inputs) {
160 Opts.Inputs.assign(first: Inputs.begin(), last: Inputs.end());
161 },
162 /*AllowSubCommandNamesAsPositionals=*/true);
163
164 for (auto *Arg : Args.filtered(Ids: OPT_UNKNOWN)) {
165 llvm::errs() << "ignoring unknown option: " << Arg->getSpelling() << '\n';
166 }
167
168 if (Args.hasArg(Ids: OPT_help)) {
169 T.printHelp(OS&: outs(),
170 Usage: (std::string(Argv[0]) + " <subcommand> [options] <input files>")
171 .c_str(),
172 Title: "llvm-cas tool that performs CAS actions.",
173 /*ShowHidden=*/false,
174 /*ShowAllAliases=*/false, VisibilityMask: opt::Visibility(), SubCommand);
175 exit(status: 0);
176 }
177
178 Opts.Command = CommandOptions::getCommandKind(SubCommand);
179 if (Opts.Command == CommandKind::Invalid)
180 return createStringError(Fmt: "no subcommand is specified");
181 Opts.CASPath = Args.getLastArgValue(Id: OPT_cas_path);
182 Opts.CASPluginPath = Args.getLastArgValue(Id: OPT_cas_plugin_path);
183 for (StringRef PluginOpt : Args.getAllArgValues(Id: OPT_cas_plugin_option)) {
184 auto [Name, Value] = PluginOpt.split(Separator: '=');
185 Opts.CASPluginOpts.emplace_back(Args&: Name, Args&: Value);
186 }
187 Opts.UpstreamCASPath = Args.getLastArgValue(Id: OPT_upstream_cas);
188 Opts.DataPath = Args.getLastArgValue(Id: OPT_data);
189 Opts.CheckHash = Args.hasArg(Ids: OPT_check_hash);
190 Opts.AllowRecovery = Args.hasArg(Ids: OPT_allow_recovery);
191 Opts.Force = Args.hasArg(Ids: OPT_force);
192 Opts.InProcess = Args.hasArg(Ids: OPT_in_process);
193 Opts.Verbose = Args.hasArg(Ids: OPT_verbose);
194
195 // Validate options.
196 if (Opts.CASPath.empty())
197 return createStringError(Fmt: "missing --cas <path>");
198
199 if (Opts.Inputs.empty() && CommandOptions::requiresInput(Kind: Opts.Command))
200 return createStringError(Fmt: "missing <input> to operate on");
201
202 return Opts;
203}
204
205int main(int Argc, char **Argv) {
206 InitLLVM X(Argc, Argv);
207
208 ExitOnError ExitOnErr;
209 auto Opts = ExitOnErr(parseOptions(Argc, Argv));
210
211 if (Opts.Command == CommandKind::ValidateIfNeeded)
212 return validateIfNeeded(Opts, Argv0: Argv[0]);
213
214 std::shared_ptr<ObjectStore> CAS;
215 std::shared_ptr<ActionCache> AC;
216 if (!Opts.CASPluginPath.empty())
217 std::tie(args&: CAS, args&: AC) = ExitOnErr(createPluginCASDatabases(
218 PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath, PluginArgs: Opts.CASPluginOpts));
219 else
220 std::tie(args&: CAS, args&: AC) =
221 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.CASPath));
222 assert(CAS);
223
224 if (Opts.Command == CommandKind::Dump)
225 return dump(CAS&: *CAS);
226
227 if (Opts.Command == CommandKind::Validate)
228 return validate(CAS&: *CAS, AC&: *AC, CheckHash: Opts.CheckHash);
229
230 if (Opts.Command == CommandKind::MakeBlob)
231 return makeBlob(CAS&: *CAS, DataPath: Opts.DataPath);
232
233 if (Opts.Command == CommandKind::MakeNode)
234 return makeNode(CAS&: *CAS, References: Opts.Inputs, DataPath: Opts.DataPath);
235
236 if (Opts.Command == CommandKind::Prune)
237 return prune(CAS&: *CAS);
238
239 if (Opts.Command == CommandKind::Import) {
240 if (Opts.UpstreamCASPath.empty())
241 ExitOnErr(createStringError(Fmt: "missing '-upstream-cas'"));
242
243 auto [UpstreamCAS, _] =
244 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.UpstreamCASPath));
245 return import(FromCAS&: *UpstreamCAS, ToCAS&: *CAS, Objects: Opts.Inputs);
246 }
247
248 if (Opts.Command == CommandKind::PutCacheKey ||
249 Opts.Command == CommandKind::GetCacheResult) {
250 if (!AC)
251 ExitOnErr(createStringError(Fmt: "no action-cache available"));
252 }
253
254 if (Opts.Command == CommandKind::PutCacheKey)
255 return putCacheKey(CAS&: *CAS, AC&: *AC, Objects: Opts.Inputs);
256
257 // Remaining commands need exactly one CAS object.
258 if (Opts.Inputs.size() > 1)
259 ExitOnErr(createStringError(Fmt: "too many <object>s, expected 1"));
260 CASID ID = ExitOnErr(CAS->parseID(ID: Opts.Inputs.front()));
261
262 if (Opts.Command == CommandKind::GetCacheResult)
263 return getCacheResult(CAS&: *CAS, AC&: *AC, ID);
264
265 if (Opts.Command == CommandKind::ListObjectReferences)
266 return listObjectReferences(CAS&: *CAS, ID);
267
268 if (Opts.Command == CommandKind::CatNodeData)
269 return catNodeData(CAS&: *CAS, ID);
270
271 assert(Opts.Command == CommandKind::ValidateObject);
272 return validateObject(CAS&: *CAS, ID);
273}
274
275static Expected<std::unique_ptr<MemoryBuffer>> openBuffer(StringRef DataPath) {
276 if (DataPath.empty())
277 return createStringError(Fmt: "--data missing");
278 return errorOrToExpected(EO: DataPath == "-"
279 ? llvm::MemoryBuffer::getSTDIN()
280 : llvm::MemoryBuffer::getFile(Filename: DataPath));
281}
282
283int dump(ObjectStore &CAS) {
284 ExitOnError ExitOnErr("llvm-cas: dump: ");
285 CAS.print(llvm::outs());
286 return 0;
287}
288
289int makeBlob(ObjectStore &CAS, StringRef DataPath) {
290 ExitOnError ExitOnErr("llvm-cas: make-blob: ");
291 std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));
292
293 ObjectProxy Blob = ExitOnErr(CAS.createProxy(Refs: {}, Data: Buffer->getBuffer()));
294 llvm::outs() << Blob.getID() << "\n";
295 return 0;
296}
297
298int catNodeData(ObjectStore &CAS, const CASID &ID) {
299 ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");
300 llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();
301 return 0;
302}
303
304int listObjectReferences(ObjectStore &CAS, const CASID &ID) {
305 ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");
306
307 ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));
308 ExitOnErr(Object.forEachReference(Callback: [&](ObjectRef Ref) -> Error {
309 llvm::outs() << CAS.getID(Ref) << "\n";
310 return Error::success();
311 }));
312
313 return 0;
314}
315
316static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,
317 StringRef DataPath) {
318 std::unique_ptr<MemoryBuffer> Data =
319 ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));
320
321 SmallVector<ObjectRef> IDs;
322 for (StringRef Object : Objects) {
323 ExitOnError ObjectErr("llvm-cas: make-node: ref: ");
324 std::optional<ObjectRef> ID =
325 CAS.getReference(ID: ObjectErr(CAS.parseID(ID: Object)));
326 if (!ID)
327 ObjectErr(createStringError(S: "unknown object '" + Object + "'"));
328 IDs.push_back(Elt: *ID);
329 }
330
331 ExitOnError ExitOnErr("llvm-cas: make-node: ");
332 ObjectProxy Object = ExitOnErr(CAS.createProxy(Refs: IDs, Data: Data->getBuffer()));
333 llvm::outs() << Object.getID() << "\n";
334 return 0;
335}
336
337static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
338 ArrayRef<std::string> Objects) {
339 ExitOnError ExitOnErr("llvm-cas: import: ");
340
341 for (StringRef Object : Objects) {
342 CASID ID = ExitOnErr(FromCAS.parseID(ID: Object));
343 auto Ref = FromCAS.getReference(ID);
344 if (!Ref)
345 ExitOnErr(createStringError(S: "input not found: " + ID.toString()));
346
347 auto Imported = ExitOnErr(ToCAS.importObject(Upstream&: FromCAS, Other: *Ref));
348 llvm::outs() << ToCAS.getID(Ref: Imported).toString() << "\n";
349 }
350 return 0;
351}
352
353static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
354 ArrayRef<std::string> Objects) {
355 ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");
356
357 if (Objects.size() % 2 != 0)
358 ExitOnErr(createStringError(Fmt: "expected pairs of inputs"));
359 while (!Objects.empty()) {
360 CASID Key = ExitOnErr(CAS.parseID(ID: Objects[0]));
361 CASID Result = ExitOnErr(CAS.parseID(ID: Objects[1]));
362 Objects = Objects.drop_front(N: 2);
363 ExitOnErr(AC.put(ActionKey: Key, Result));
364 }
365 return 0;
366}
367
368static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {
369 ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");
370
371 auto Result = ExitOnErr(AC.get(ActionKey: ID));
372 if (!Result) {
373 outs() << "result not found\n";
374 return 1;
375 }
376 outs() << *Result << "\n";
377 return 0;
378}
379
380int validateObject(ObjectStore &CAS, const CASID &ID) {
381 ExitOnError ExitOnErr("llvm-cas: validate-object: ");
382 ExitOnErr(CAS.validateObject(ID));
383 outs() << ID << ": validated successfully\n";
384 return 0;
385}
386
387int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {
388 ExitOnError ExitOnErr("llvm-cas: validate: ");
389 ExitOnErr(CAS.validate(CheckHash));
390 ExitOnErr(AC.validate());
391 outs() << "validated successfully\n";
392 return 0;
393}
394
395/// Validates the CAS in this process and prints the result.
396static Error validateInProcess(const CommandOptions &Opts) {
397 ValidationResult Result;
398 if (Error E = (Opts.CASPluginPath.empty()
399 ? validateOnDiskUnifiedCASDatabasesIfNeeded(
400 Path: Opts.CASPath, CheckHash: Opts.CheckHash, ForceValidation: Opts.Force)
401 : validatePluginCASDatabasesIfNeeded(
402 PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath, PluginArgs: Opts.CASPluginOpts,
403 CheckHash: Opts.CheckHash, ForceValidation: Opts.Force))
404 .moveInto(Value&: Result))
405 return E;
406 outs() << (Result == ValidationResult::Skipped ? "validation skipped\n"
407 : "validated successfully\n");
408 return Error::success();
409}
410
411/// Validates the CAS by re-executing llvm-cas with --in-process, which
412/// protects against crashes during validation. The output of the child process
413/// is forwarded, except for its errors when they would be followed by recovery
414/// and are not requested with --verbose.
415///
416/// \returns false if validation failed or crashed.
417static Expected<bool> validateOutOfProcess(const CommandOptions &Opts,
418 const char *Argv0) {
419 std::string Exec =
420 sys::fs::getMainExecutable(argv0: Argv0, MainExecAddr: (void *)validateOutOfProcess);
421 SmallVector<std::string> PluginOpts;
422 for (const auto &[Name, Value] : Opts.CASPluginOpts)
423 PluginOpts.push_back(Elt: Name + "=" + Value);
424
425 SmallVector<StringRef> Args{Exec, "validate-if-needed", "--in-process",
426 "--cas", Opts.CASPath};
427 if (!Opts.CASPluginPath.empty()) {
428 Args.append(IL: {"--fcas-plugin-path", Opts.CASPluginPath});
429 for (StringRef PluginOpt : PluginOpts)
430 Args.append(IL: {"--fcas-plugin-option", PluginOpt});
431 }
432 if (Opts.CheckHash)
433 Args.push_back(Elt: "--check-hash");
434 if (Opts.Force)
435 Args.push_back(Elt: "--force");
436
437 // Discard stderr if quiet, and otherwise inherit all of stdin, stdout and
438 // stderr.
439 bool QuietErrors = Opts.AllowRecovery && !Opts.Verbose;
440 SmallVector<std::optional<StringRef>, 3> Redirects;
441 if (QuietErrors)
442 Redirects = {std::nullopt, std::nullopt, StringRef("")};
443
444 outs().flush();
445 std::string ErrMsg;
446 int Result = sys::ExecuteAndWait(Program: Exec, Args, /*Env=*/std::nullopt, Redirects,
447 /*SecondsToWait=*/120,
448 /*MemoryLimit=*/0, ErrMsg: &ErrMsg);
449 if (Result == -1)
450 return createStringError(S: "failed to exec " + join(R&: Args, Separator: " ") + ": " +
451 ErrMsg);
452 if (Result == -2 && !QuietErrors)
453 errs() << "llvm-cas: validate-if-needed: validation crashed: " << ErrMsg
454 << "\n";
455 return Result == 0;
456}
457
458int validateIfNeeded(const CommandOptions &Opts, const char *Argv0) {
459 ExitOnError ExitOnErr("llvm-cas: validate-if-needed: ");
460 if (Opts.InProcess) {
461 Error E = validateInProcess(Opts);
462 if (!E)
463 return 0;
464 if (!Opts.AllowRecovery)
465 ExitOnErr(std::move(E));
466 // The error is expected when recovering, so only print it on request. It
467 // is also recorded in the CAS log.
468 if (Opts.Verbose)
469 errs() << "llvm-cas: validate-if-needed: " << toString(E: std::move(E))
470 << "\n";
471 else
472 consumeError(Err: std::move(E));
473 } else {
474 if (ExitOnErr(validateOutOfProcess(Opts, Argv0)))
475 return 0;
476 if (!Opts.AllowRecovery)
477 ExitOnErr(createStringError(Fmt: "cas contents invalid"));
478 }
479
480 ValidationResult Result = ExitOnErr(
481 Opts.CASPluginPath.empty()
482 ? recoverOnDiskUnifiedCASDatabases(Path: Opts.CASPath)
483 : recoverPluginCASDatabases(PluginPath: Opts.CASPluginPath, OnDiskPath: Opts.CASPath,
484 PluginArgs: Opts.CASPluginOpts));
485 outs() << (Result == ValidationResult::Skipped
486 ? "recovery skipped\n"
487 : "recovered from invalid data\n");
488 return 0;
489}
490
491static int prune(cas::ObjectStore &CAS) {
492 ExitOnError ExitOnErr("llvm-cas: prune: ");
493 ExitOnErr(CAS.pruneStorageData());
494 return 0;
495}
496