1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file A utility for operating on LLVM CAS.
10///
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CAS/ActionCache.h"
14#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
15#include "llvm/CAS/ObjectStore.h"
16#include "llvm/Option/Arg.h"
17#include "llvm/Option/ArgList.h"
18#include "llvm/Option/Option.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/Error.h"
21#include "llvm/Support/InitLLVM.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include "llvm/Support/raw_ostream.h"
24
25using namespace llvm;
26using namespace llvm::cas;
27
28namespace {
29enum ID {
30 OPT_INVALID = 0, // This is not an option ID.
31#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
32#include "Options.inc"
33#undef OPTION
34};
35
36using namespace llvm::opt;
37#define OPTTABLE_CODE
38#include "Options.inc"
39
40class LLVMCASOptTable : public opt::OptTable {
41public:
42 LLVMCASOptTable() : opt::OptTable(optionTables()) {}
43};
44
45enum class CommandKind {
46 Invalid,
47 Dump,
48 CatNodeData,
49 MakeBlob,
50 MakeNode,
51 ListObjectReferences,
52 Import,
53 PutCacheKey,
54 GetCacheResult,
55 Validate,
56 ValidateObject,
57 ValidateIfNeeded,
58 Prune,
59};
60
61struct CommandOptions {
62 CommandKind Command = CommandKind::Invalid;
63 std::vector<std::string> Inputs;
64 std::string CASPath;
65 std::string UpstreamCASPath;
66 std::string DataPath;
67 bool CheckHash;
68 bool AllowRecovery;
69 bool Force;
70 bool InProcess;
71
72 static CommandKind getCommandKind(opt::Arg &A) {
73 switch (A.getOption().getID()) {
74 case OPT_cas_dump:
75 return CommandKind::Dump;
76 case OPT_cat_node_data:
77 return CommandKind::CatNodeData;
78 case OPT_make_blob:
79 return CommandKind::MakeBlob;
80 case OPT_make_node:
81 return CommandKind::MakeNode;
82 case OPT_ls_node_refs:
83 return CommandKind::ListObjectReferences;
84 case OPT_import:
85 return CommandKind::Import;
86 case OPT_put_cache_key:
87 return CommandKind::PutCacheKey;
88 case OPT_get_cache_result:
89 return CommandKind::GetCacheResult;
90 case OPT_validate:
91 return CommandKind::Validate;
92 case OPT_validate_object:
93 return CommandKind::ValidateObject;
94 case OPT_validate_if_needed:
95 return CommandKind::ValidateIfNeeded;
96 case OPT_prune:
97 return CommandKind::Prune;
98 }
99 return CommandKind::Invalid;
100 }
101
102 // Command requires input.
103 static bool requiresInput(CommandKind Kind) {
104 return Kind != CommandKind::ValidateIfNeeded &&
105 Kind != CommandKind::Validate && Kind != CommandKind::MakeBlob &&
106 Kind != CommandKind::MakeNode && Kind != CommandKind::Dump &&
107 Kind != CommandKind::Prune;
108 }
109};
110} // namespace
111
112static int dump(ObjectStore &CAS);
113static int listObjectReferences(ObjectStore &CAS, const CASID &ID);
114static int catNodeData(ObjectStore &CAS, const CASID &ID);
115static int makeBlob(ObjectStore &CAS, StringRef DataPath);
116static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,
117 StringRef DataPath);
118static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
119 ArrayRef<std::string> Objects);
120static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
121 ArrayRef<std::string> Objects);
122static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
123static int validateObject(ObjectStore &CAS, const CASID &ID);
124static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);
125static int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,
126 bool AllowRecovery, bool InProcess,
127 const char *Argv0);
128static int prune(cas::ObjectStore &CAS);
129
130static Expected<CommandOptions> parseOptions(int Argc, char **Argv) {
131 BumpPtrAllocator Alloc;
132 StringSaver Saver(Alloc);
133 SmallVector<const char *> ExpanedArgs;
134 if (!cl::expandResponseFiles(Argc, Argv, EnvVar: nullptr, Saver, NewArgv&: ExpanedArgs))
135 return createStringError(Fmt: "cannot expand response file");
136
137 LLVMCASOptTable T;
138 unsigned MI, MC;
139 opt::InputArgList Args = T.ParseArgs(Args: ExpanedArgs, MissingArgIndex&: MI, MissingArgCount&: MC);
140
141 for (auto *Arg : Args.filtered(Ids: OPT_UNKNOWN)) {
142 llvm::errs() << "ignoring unknown option: " << Arg->getSpelling() << '\n';
143 }
144
145 if (Args.hasArg(Ids: OPT_help)) {
146 T.printHelp(
147 OS&: outs(),
148 Usage: (std::string(Argv[0]) + " [action] [options] <input files>").c_str(),
149 Title: "llvm-cas tool that performs CAS actions.", ShowHidden: false);
150 exit(status: 0);
151 }
152
153 CommandOptions Opts;
154 for (auto *A : Args.filtered(Ids: OPT_grp_action))
155 Opts.Command = CommandOptions::getCommandKind(A&: *A);
156
157 if (Opts.Command == CommandKind::Invalid)
158 return createStringError(Fmt: "no command action is specified");
159
160 for (auto *File : Args.filtered(Ids: OPT_INPUT))
161 Opts.Inputs.push_back(x: File->getValue());
162 Opts.CASPath = Args.getLastArgValue(Id: OPT_cas_path);
163 Opts.UpstreamCASPath = Args.getLastArgValue(Id: OPT_upstream_cas);
164 Opts.DataPath = Args.getLastArgValue(Id: OPT_data);
165 Opts.CheckHash = Args.hasArg(Ids: OPT_check_hash);
166 Opts.AllowRecovery = Args.hasArg(Ids: OPT_allow_recovery);
167 Opts.Force = Args.hasArg(Ids: OPT_force);
168 Opts.InProcess = Args.hasArg(Ids: OPT_in_process);
169
170 // Validate options.
171 if (Opts.CASPath.empty())
172 return createStringError(Fmt: "missing --cas <path>");
173
174 if (Opts.Inputs.empty() && CommandOptions::requiresInput(Kind: Opts.Command))
175 return createStringError(Fmt: "missing <input> to operate on");
176
177 return Opts;
178}
179
180int main(int Argc, char **Argv) {
181 InitLLVM X(Argc, Argv);
182
183 ExitOnError ExitOnErr;
184 auto Opts = ExitOnErr(parseOptions(Argc, Argv));
185
186 if (Opts.Command == CommandKind::ValidateIfNeeded)
187 return validateIfNeeded(Path: Opts.CASPath, CheckHash: Opts.CheckHash, Force: Opts.Force,
188 AllowRecovery: Opts.AllowRecovery, InProcess: Opts.InProcess, Argv0: Argv[0]);
189
190 auto [CAS, AC] = ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.CASPath));
191 assert(CAS);
192
193 if (Opts.Command == CommandKind::Dump)
194 return dump(CAS&: *CAS);
195
196 if (Opts.Command == CommandKind::Validate)
197 return validate(CAS&: *CAS, AC&: *AC, CheckHash: Opts.CheckHash);
198
199 if (Opts.Command == CommandKind::MakeBlob)
200 return makeBlob(CAS&: *CAS, DataPath: Opts.DataPath);
201
202 if (Opts.Command == CommandKind::MakeNode)
203 return makeNode(CAS&: *CAS, References: Opts.Inputs, DataPath: Opts.DataPath);
204
205 if (Opts.Command == CommandKind::Prune)
206 return prune(CAS&: *CAS);
207
208 if (Opts.Command == CommandKind::Import) {
209 if (Opts.UpstreamCASPath.empty())
210 ExitOnErr(createStringError(Fmt: "missing '-upstream-cas'"));
211
212 auto [UpstreamCAS, _] =
213 ExitOnErr(createOnDiskUnifiedCASDatabases(Path: Opts.UpstreamCASPath));
214 return import(FromCAS&: *UpstreamCAS, ToCAS&: *CAS, Objects: Opts.Inputs);
215 }
216
217 if (Opts.Command == CommandKind::PutCacheKey ||
218 Opts.Command == CommandKind::GetCacheResult) {
219 if (!AC)
220 ExitOnErr(createStringError(Fmt: "no action-cache available"));
221 }
222
223 if (Opts.Command == CommandKind::PutCacheKey)
224 return putCacheKey(CAS&: *CAS, AC&: *AC, Objects: Opts.Inputs);
225
226 // Remaining commands need exactly one CAS object.
227 if (Opts.Inputs.size() > 1)
228 ExitOnErr(createStringError(Fmt: "too many <object>s, expected 1"));
229 CASID ID = ExitOnErr(CAS->parseID(ID: Opts.Inputs.front()));
230
231 if (Opts.Command == CommandKind::GetCacheResult)
232 return getCacheResult(CAS&: *CAS, AC&: *AC, ID);
233
234 if (Opts.Command == CommandKind::ListObjectReferences)
235 return listObjectReferences(CAS&: *CAS, ID);
236
237 if (Opts.Command == CommandKind::CatNodeData)
238 return catNodeData(CAS&: *CAS, ID);
239
240 assert(Opts.Command == CommandKind::ValidateObject);
241 return validateObject(CAS&: *CAS, ID);
242}
243
244static Expected<std::unique_ptr<MemoryBuffer>> openBuffer(StringRef DataPath) {
245 if (DataPath.empty())
246 return createStringError(Fmt: "--data missing");
247 return errorOrToExpected(EO: DataPath == "-"
248 ? llvm::MemoryBuffer::getSTDIN()
249 : llvm::MemoryBuffer::getFile(Filename: DataPath));
250}
251
252int dump(ObjectStore &CAS) {
253 ExitOnError ExitOnErr("llvm-cas: dump: ");
254 CAS.print(llvm::outs());
255 return 0;
256}
257
258int makeBlob(ObjectStore &CAS, StringRef DataPath) {
259 ExitOnError ExitOnErr("llvm-cas: make-blob: ");
260 std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));
261
262 ObjectProxy Blob = ExitOnErr(CAS.createProxy(Refs: {}, Data: Buffer->getBuffer()));
263 llvm::outs() << Blob.getID() << "\n";
264 return 0;
265}
266
267int catNodeData(ObjectStore &CAS, const CASID &ID) {
268 ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");
269 llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();
270 return 0;
271}
272
273int listObjectReferences(ObjectStore &CAS, const CASID &ID) {
274 ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");
275
276 ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));
277 ExitOnErr(Object.forEachReference(Callback: [&](ObjectRef Ref) -> Error {
278 llvm::outs() << CAS.getID(Ref) << "\n";
279 return Error::success();
280 }));
281
282 return 0;
283}
284
285static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,
286 StringRef DataPath) {
287 std::unique_ptr<MemoryBuffer> Data =
288 ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));
289
290 SmallVector<ObjectRef> IDs;
291 for (StringRef Object : Objects) {
292 ExitOnError ObjectErr("llvm-cas: make-node: ref: ");
293 std::optional<ObjectRef> ID =
294 CAS.getReference(ID: ObjectErr(CAS.parseID(ID: Object)));
295 if (!ID)
296 ObjectErr(createStringError(S: "unknown object '" + Object + "'"));
297 IDs.push_back(Elt: *ID);
298 }
299
300 ExitOnError ExitOnErr("llvm-cas: make-node: ");
301 ObjectProxy Object = ExitOnErr(CAS.createProxy(Refs: IDs, Data: Data->getBuffer()));
302 llvm::outs() << Object.getID() << "\n";
303 return 0;
304}
305
306static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,
307 ArrayRef<std::string> Objects) {
308 ExitOnError ExitOnErr("llvm-cas: import: ");
309
310 for (StringRef Object : Objects) {
311 CASID ID = ExitOnErr(FromCAS.parseID(ID: Object));
312 auto Ref = FromCAS.getReference(ID);
313 if (!Ref)
314 ExitOnErr(createStringError(S: "input not found: " + ID.toString()));
315
316 auto Imported = ExitOnErr(ToCAS.importObject(Upstream&: FromCAS, Other: *Ref));
317 llvm::outs() << ToCAS.getID(Ref: Imported).toString() << "\n";
318 }
319 return 0;
320}
321
322static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
323 ArrayRef<std::string> Objects) {
324 ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");
325
326 if (Objects.size() % 2 != 0)
327 ExitOnErr(createStringError(Fmt: "expected pairs of inputs"));
328 while (!Objects.empty()) {
329 CASID Key = ExitOnErr(CAS.parseID(ID: Objects[0]));
330 CASID Result = ExitOnErr(CAS.parseID(ID: Objects[1]));
331 Objects = Objects.drop_front(N: 2);
332 ExitOnErr(AC.put(ActionKey: Key, Result));
333 }
334 return 0;
335}
336
337static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {
338 ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");
339
340 auto Result = ExitOnErr(AC.get(ActionKey: ID));
341 if (!Result) {
342 outs() << "result not found\n";
343 return 1;
344 }
345 outs() << *Result << "\n";
346 return 0;
347}
348
349int validateObject(ObjectStore &CAS, const CASID &ID) {
350 ExitOnError ExitOnErr("llvm-cas: validate-object: ");
351 ExitOnErr(CAS.validateObject(ID));
352 outs() << ID << ": validated successfully\n";
353 return 0;
354}
355
356int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {
357 ExitOnError ExitOnErr("llvm-cas: validate: ");
358 ExitOnErr(CAS.validate(CheckHash));
359 ExitOnErr(AC.validate());
360 outs() << "validated successfully\n";
361 return 0;
362}
363
364int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,
365 bool AllowRecovery, bool InProcess, const char *Argv0) {
366 ExitOnError ExitOnErr("llvm-cas: validate-if-needed: ");
367 std::string ExecStorage;
368 std::optional<StringRef> Exec;
369 if (!InProcess) {
370 ExecStorage = sys::fs::getMainExecutable(argv0: Argv0, MainExecAddr: (void *)validateIfNeeded);
371 Exec = ExecStorage;
372 }
373 ValidationResult Result = ExitOnErr(validateOnDiskUnifiedCASDatabasesIfNeeded(
374 Path, CheckHash, AllowRecovery, ForceValidation: Force, LLVMCasBinaryPath: Exec));
375 switch (Result) {
376 case ValidationResult::Valid:
377 outs() << "validated successfully\n";
378 break;
379 case ValidationResult::Recovered:
380 outs() << "recovered from invalid data\n";
381 break;
382 case ValidationResult::Skipped:
383 outs() << "validation skipped\n";
384 break;
385 }
386 return 0;
387}
388
389static int prune(cas::ObjectStore &CAS) {
390 ExitOnError ExitOnErr("llvm-cas: prune: ");
391 ExitOnErr(CAS.pruneStorageData());
392 return 0;
393}
394