1//===- llvm-ir2vec.cpp - IR2Vec/MIR2Vec Embedding Generation Tool --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the IR2Vec and MIR2Vec embedding generation tool.
11///
12/// This tool supports two modes:
13/// - LLVM IR mode (-mode=llvm): Process LLVM IR
14/// - Machine IR mode (-mode=mir): Process Machine IR
15///
16/// Available subcommands:
17///
18/// 1. Triplet Generation (triplets):
19/// Generates numeric triplets (head, tail, relation) for vocabulary
20/// training. Output format: MAX_RELATION=N header followed by
21/// head\ttail\trelation lines. Relations: 0=Type, 1=Next, 2+=Arg0,Arg1,...
22///
23/// For LLVM IR:
24/// llvm-ir2vec triplets input.bc -o train2id.txt
25///
26/// For Machine IR:
27/// llvm-ir2vec triplets -mode=mir input.mir -o train2id.txt
28///
29/// 2. Entity Mappings (entities):
30/// Generates entity mappings for vocabulary training.
31/// Output format: <total_entities> header followed by entity\tid lines.
32///
33/// For LLVM IR:
34/// llvm-ir2vec entities input.bc -o entity2id.txt
35///
36/// For Machine IR:
37/// llvm-ir2vec entities -mode=mir input.mir -o entity2id.txt
38///
39/// 3. Embedding Generation (embeddings):
40/// Generates IR2Vec/MIR2Vec embeddings using a trained vocabulary.
41///
42/// For LLVM IR:
43/// llvm-ir2vec embeddings --ir2vec-vocab-path=vocab.json
44/// --ir2vec-kind=<kind> --level=<level> input.bc -o embeddings.txt
45/// Kind: --ir2vec-kind=symbolic (default), --ir2vec-kind=flow-aware
46///
47/// For Machine IR:
48/// llvm-ir2vec embeddings -mode=mir --mir2vec-vocab-path=vocab.json
49/// --level=<level> input.mir -o embeddings.txt
50///
51/// Levels: --level=inst (instructions), --level=bb (basic blocks),
52/// --level=func (functions) (See IR2Vec.cpp/MIR2Vec.cpp for more embedding
53/// generation options)
54///
55//===----------------------------------------------------------------------===//
56
57#include "IRUtils/IRUtils.h"
58#include "MIRUtils/MIRUtils.h"
59#include "llvm/ADT/ArrayRef.h"
60#include "llvm/Analysis/IR2Vec.h"
61#include "llvm/CodeGen/CommandFlags.h"
62#include "llvm/CodeGen/MIR2Vec.h"
63#include "llvm/CodeGen/MIRParser/MIRParser.h"
64#include "llvm/CodeGen/MachineFunction.h"
65#include "llvm/CodeGen/MachineModuleInfo.h"
66#include "llvm/CodeGen/TargetInstrInfo.h"
67#include "llvm/CodeGen/TargetRegisterInfo.h"
68#include "llvm/IR/BasicBlock.h"
69#include "llvm/IR/Function.h"
70#include "llvm/IR/Instructions.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/Module.h"
73#include "llvm/IR/PassInstrumentation.h"
74#include "llvm/IR/PassManager.h"
75#include "llvm/IR/Type.h"
76#include "llvm/IRReader/IRReader.h"
77#include "llvm/MC/TargetRegistry.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Debug.h"
80#include "llvm/Support/Errc.h"
81#include "llvm/Support/InitLLVM.h"
82#include "llvm/Support/LibraryOptionsContext.h"
83#include "llvm/Support/SourceMgr.h"
84#include "llvm/Support/TargetSelect.h"
85#include "llvm/Support/WithColor.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Target/TargetMachine.h"
88#include "llvm/TargetParser/Host.h"
89
90#define DEBUG_TYPE "ir2vec"
91
92namespace llvm {
93
94// Common option category for options shared between IR2Vec and MIR2Vec
95static cl::OptionCategory CommonCategory("Common Options",
96 "Options applicable to both IR2Vec "
97 "and MIR2Vec modes");
98
99enum IRKind {
100 LLVMIR = 0, ///< LLVM IR
101 MIR ///< Machine IR
102};
103
104static cl::opt<IRKind>
105 IRMode("mode", cl::desc("Tool operation mode:"),
106 cl::values(clEnumValN(LLVMIR, "llvm", "Process LLVM IR"),
107 clEnumValN(MIR, "mir", "Process Machine IR")),
108 cl::init(Val: LLVMIR), cl::cat(CommonCategory));
109
110// Subcommands
111static cl::SubCommand
112 TripletsSubCmd("triplets", "Generate triplets for vocabulary training");
113static cl::SubCommand
114 EntitiesSubCmd("entities",
115 "Generate entity mappings for vocabulary training");
116static cl::SubCommand
117 EmbeddingsSubCmd("embeddings",
118 "Generate embeddings using trained vocabulary");
119
120// Common options
121static cl::opt<std::string> InputFilename(
122 cl::Positional, cl::desc("<input bitcode/MIR file or '-' for stdin>"),
123 cl::init(Val: "-"), cl::sub(TripletsSubCmd), cl::sub(EntitiesSubCmd),
124 cl::sub(EmbeddingsSubCmd), cl::cat(CommonCategory));
125
126static cl::opt<std::string> OutputFilename("o", cl::desc("Output filename"),
127 cl::value_desc("filename"),
128 cl::init(Val: "-"),
129 cl::cat(CommonCategory));
130
131// Embedding-specific options
132static cl::opt<std::string>
133 FunctionName("function", cl::desc("Process specific function only"),
134 cl::value_desc("name"), cl::init(Val: ""),
135 cl::sub(EmbeddingsSubCmd), cl::cat(CommonCategory));
136
137static cl::opt<EmbeddingLevel>
138 Level("level", cl::desc("Embedding generation level:"),
139 cl::values(clEnumValN(InstructionLevel, "inst",
140 "Generate instruction-level embeddings"),
141 clEnumValN(BasicBlockLevel, "bb",
142 "Generate basic block-level embeddings"),
143 clEnumValN(FunctionLevel, "func",
144 "Generate function-level embeddings")),
145 cl::init(Val: FunctionLevel), cl::sub(EmbeddingsSubCmd),
146 cl::cat(CommonCategory));
147
148namespace ir2vec {
149
150/// Process the module and generate output based on selected subcommand
151static Error processModule(Module &M, raw_ostream &OS) {
152 IR2VecTool Tool(M);
153
154 if (EmbeddingsSubCmd) {
155 // Initialize vocabulary for embedding generation
156 // Note: Requires --ir2vec-vocab-path option to be set
157 StringRef VocabFile = ir2vec::getVocabFile(C: M.getContext());
158 if (VocabFile.empty()) {
159 return createStringError(
160 EC: errc::invalid_argument,
161 S: "IR2Vec vocabulary file path not specified; "
162 "You may need to set it using --ir2vec-vocab-path");
163 }
164
165 std::shared_ptr<Vocabulary> Vocab;
166 if (auto Err = ir2vec::loadVocabulary(VocabPath: VocabFile).moveInto(Value&: Vocab))
167 return Err;
168 if (auto Err = Tool.setVocabulary(std::move(Vocab)))
169 return Err;
170
171 if (!FunctionName.empty()) {
172 // Process single function
173 if (const Function *F = M.getFunction(Name: FunctionName))
174 Tool.writeEmbeddingsToStream(F: *F, OS, Level);
175 else
176 return createStringError(EC: errc::invalid_argument,
177 Fmt: "Function '%s' not found",
178 Vals: FunctionName.c_str());
179 } else {
180 // Process all functions
181 Tool.writeEmbeddingsToStream(OS, Level);
182 }
183 } else {
184 // Both triplets and entities use triplet generation
185 Tool.writeTripletsToStream(OS);
186 }
187 return Error::success();
188}
189} // namespace ir2vec
190
191namespace mir2vec {
192
193static cl::opt<std::string>
194 TargetTriple("mtriple", cl::desc("Override target triple for module"),
195 cl::cat(MIR2VecCategory));
196
197/// Setup MIR context from input file
198static Error setupMIRContext(const std::string &InputFile, MIRContext &Ctx) {
199 SMDiagnostic Err;
200
201 auto MIR = createMIRParserFromFile(Filename: InputFile, Error&: Err, Context&: Ctx.Context);
202 if (!MIR) {
203 Err.print(ProgName: ToolName, S&: errs());
204 return createStringError(EC: errc::invalid_argument,
205 S: "Failed to parse MIR file");
206 }
207
208 auto SetDataLayout = [&](StringRef DataLayoutTargetTriple,
209 StringRef OldDLStr) -> std::optional<std::string> {
210 std::string IRTargetTriple = DataLayoutTargetTriple.str();
211 if (!TargetTriple.empty())
212 IRTargetTriple = Triple::normalize(Str: TargetTriple);
213 Triple TheTriple = Triple(IRTargetTriple);
214 if (TheTriple.getTriple().empty())
215 TheTriple.setTriple(sys::getDefaultTargetTriple());
216
217 auto TMOrErr = codegen::createTargetMachineForTriple(
218 LibraryOptions: Ctx.Context.getLibraryOptions(), TargetTriple: TheTriple);
219 if (!TMOrErr) {
220 WithColor::error(OS&: errs(), Prefix: ToolName)
221 << toString(E: TMOrErr.takeError()) << "\n";
222 exit(status: 1); // Match original behavior
223 }
224 Ctx.TM = std::move(*TMOrErr);
225 return TheTriple.computeDataLayout();
226 };
227
228 Ctx.M = MIR->parseIRModule(DataLayoutCallback: SetDataLayout);
229 if (!Ctx.M) {
230 Err.print(ProgName: ToolName, S&: errs());
231 return createStringError(EC: errc::invalid_argument,
232 S: "Failed to parse IR module");
233 }
234
235 Ctx.MMI = std::make_unique<MachineModuleInfo>(args: Ctx.TM.get());
236 if (!Ctx.MMI || MIR->parseMachineFunctions(M&: *Ctx.M, MMI&: *Ctx.MMI)) {
237 Err.print(ProgName: ToolName, S&: errs());
238 return createStringError(EC: errc::invalid_argument,
239 S: "Failed to parse machine functions");
240 }
241
242 return Error::success();
243}
244
245/// Generic vocabulary initialization and processing
246template <typename ProcessFunc>
247static Error processWithVocabulary(MIRContext &Ctx, raw_ostream &OS,
248 bool useLayoutVocab, ProcessFunc processFn) {
249 MIR2VecTool Tool(*Ctx.MMI);
250
251 // Initialize appropriate vocabulary type
252 bool success = useLayoutVocab ? Tool.initializeVocabularyForLayout(M: *Ctx.M)
253 : Tool.initializeVocabulary(M: *Ctx.M);
254
255 if (!success) {
256 WithColor::error(OS&: errs(), Prefix: ToolName)
257 << "Failed to initialize MIR2Vec vocabulary"
258 << (useLayoutVocab ? " for layout" : "") << ".\n";
259 return createStringError(EC: errc::invalid_argument,
260 S: "Vocabulary initialization failed");
261 }
262
263 assert(Tool.getVocabulary() &&
264 "MIR2Vec vocabulary should be initialized at this point");
265
266 LLVM_DEBUG(dbgs() << "MIR2Vec vocabulary loaded successfully.\n"
267 << "Vocabulary dimension: "
268 << Tool.getVocabulary()->getDimension() << "\n"
269 << "Vocabulary size: "
270 << Tool.getVocabulary()->getCanonicalSize() << "\n");
271
272 // Execute the specific processing logic
273 return processFn(Tool);
274}
275
276/// Process module for triplet generation
277static Error processModuleForTriplets(MIRContext &Ctx, raw_ostream &OS) {
278 return processWithVocabulary(Ctx, OS, /*useLayoutVocab=*/true,
279 processFn: [&](MIR2VecTool &Tool) -> Error {
280 Tool.writeTripletsToStream(M: *Ctx.M, OS);
281 return Error::success();
282 });
283}
284
285/// Process module for entity generation
286static Error processModuleForEntities(MIRContext &Ctx, raw_ostream &OS) {
287 return processWithVocabulary(Ctx, OS, /*useLayoutVocab=*/true,
288 processFn: [&](MIR2VecTool &Tool) -> Error {
289 Tool.writeEntitiesToStream(OS);
290 return Error::success();
291 });
292}
293
294/// Process module for embedding generation
295static Error processModuleForEmbeddings(MIRContext &Ctx, raw_ostream &OS) {
296 return processWithVocabulary(
297 Ctx, OS, /*useLayoutVocab=*/false, processFn: [&](MIR2VecTool &Tool) -> Error {
298 if (!FunctionName.empty()) {
299 // Process single function
300 Function *F = Ctx.M->getFunction(Name: FunctionName);
301 if (!F) {
302 WithColor::error(OS&: errs(), Prefix: ToolName)
303 << "Function '" << FunctionName << "' not found\n";
304 return createStringError(EC: errc::invalid_argument,
305 S: "Function not found");
306 }
307
308 MachineFunction *MF = Ctx.MMI->getMachineFunction(F: *F);
309 if (!MF) {
310 WithColor::error(OS&: errs(), Prefix: ToolName)
311 << "No MachineFunction for " << FunctionName << "\n";
312 return createStringError(EC: errc::invalid_argument,
313 S: "No MachineFunction");
314 }
315
316 Tool.writeEmbeddingsToStream(MF&: *MF, OS, Level);
317 } else {
318 // Process all functions
319 Tool.writeEmbeddingsToStream(M: *Ctx.M, OS, Level);
320 }
321 return Error::success();
322 });
323}
324
325/// Main entry point for MIR processing
326static Error processModule(const cl::LibraryOptionsContext &LibraryOptions,
327 const std::string &InputFile, raw_ostream &OS) {
328 MIRContext Ctx;
329 Ctx.Context.getLibraryOptions().assign(Other: LibraryOptions);
330
331 // Setup MIR context (parse file, setup target machine, etc.)
332 if (auto Err = setupMIRContext(InputFile, Ctx))
333 return Err;
334
335 // Process based on subcommand
336 if (TripletsSubCmd)
337 return processModuleForTriplets(Ctx, OS);
338 else if (EntitiesSubCmd)
339 return processModuleForEntities(Ctx, OS);
340 else if (EmbeddingsSubCmd)
341 return processModuleForEmbeddings(Ctx, OS);
342 else {
343 WithColor::error(OS&: errs(), Prefix: ToolName)
344 << "Please specify a subcommand: triplets, entities, or embeddings\n";
345 return createStringError(EC: errc::invalid_argument, S: "No subcommand specified");
346 }
347}
348
349} // namespace mir2vec
350
351} // namespace llvm
352
353int main(int argc, char **argv) {
354 using namespace llvm;
355 using namespace llvm::ir2vec;
356 using namespace llvm::mir2vec;
357
358 InitLLVM X(argc, argv);
359 // Show Common and MIR2Vec option categories
360 cl::HideUnrelatedOptions(Categories: ArrayRef<const cl::OptionCategory *>{
361 &CommonCategory, &mir2vec::MIR2VecCategory});
362 cl::LibraryOptionsContext LibraryOptions;
363 cl::parseCommandLineOptions(
364 Context&: LibraryOptions, ToolOptions: codegen::getToolOptions(), Args: ArrayRef(argv, argc),
365 Overview: "IR2Vec/MIR2Vec - Embedding Generation Tool\n"
366 "Generates embeddings for a given LLVM IR or MIR and "
367 "supports triplet generation for vocabulary "
368 "training and embedding generation.\n\n"
369 "See https://llvm.org/docs/CommandGuide/llvm-ir2vec.html for more "
370 "information.\n");
371
372 std::error_code EC;
373 raw_fd_ostream OS(OutputFilename, EC);
374 if (EC) {
375 WithColor::error(OS&: errs(), Prefix: ToolName)
376 << "opening output file: " << EC.message() << "\n";
377 return 1;
378 }
379
380 if (IRMode == IRKind::LLVMIR) {
381 if (EntitiesSubCmd) {
382 // Just dump entity mappings without processing any IR
383 IR2VecTool::writeEntitiesToStream(OS);
384 return 0;
385 }
386
387 // Parse the input LLVM IR file or stdin
388 SMDiagnostic Err;
389 LLVMContext Context;
390 Context.getLibraryOptions().assign(Other: LibraryOptions);
391 std::unique_ptr<Module> M = parseIRFile(Filename: InputFilename, Err, Context);
392 if (!M) {
393 Err.print(ProgName: ToolName, S&: errs());
394 return 1;
395 }
396
397 if (Error Err = processModule(M&: *M, OS)) {
398 handleAllErrors(E: std::move(Err), Handlers: [&](const ErrorInfoBase &EIB) {
399 WithColor::error(OS&: errs(), Prefix: ToolName) << EIB.message() << "\n";
400 });
401 return 1;
402 }
403 return 0;
404 }
405 if (IRMode == IRKind::MIR) {
406 // Initialize targets for Machine IR processing
407 InitializeAllTargets();
408 InitializeAllTargetMCs();
409 InitializeAllAsmParsers();
410 InitializeAllAsmPrinters();
411
412 if (Error Err = mir2vec::processModule(LibraryOptions, InputFile: InputFilename, OS)) {
413 handleAllErrors(E: std::move(Err), Handlers: [&](const ErrorInfoBase &EIB) {
414 WithColor::error(OS&: errs(), Prefix: ToolName) << EIB.message() << "\n";
415 });
416 return 1;
417 }
418
419 return 0;
420 }
421
422 return 0;
423}
424