1//===- MlirOptMain.cpp - MLIR Optimizer Driver ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is a utility that runs an optimization pass and prints the result back
10// out. It is designed to support unit testing.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Tools/mlir-opt/MlirOptMain.h"
15#include "mlir/Bytecode/BytecodeImplementation.h"
16#include "mlir/Bytecode/BytecodeWriter.h"
17#include "mlir/Debug/CLOptionsSetup.h"
18#include "mlir/Debug/Counter.h"
19#include "mlir/Dialect/IRDL/IR/IRDL.h"
20#include "mlir/Dialect/IRDL/IRDLLoading.h"
21#include "mlir/IR/AsmState.h"
22#include "mlir/IR/Attributes.h"
23#include "mlir/IR/BuiltinOps.h"
24#include "mlir/IR/Diagnostics.h"
25#include "mlir/IR/Location.h"
26#include "mlir/IR/MLIRContext.h"
27#include "mlir/IR/Remarks.h"
28#include "mlir/Parser/Parser.h"
29#include "mlir/Pass/PassManager.h"
30#include "mlir/Pass/PassRegistry.h"
31#include "mlir/Remark/RemarkStreamer.h"
32#include "mlir/Support/FileUtilities.h"
33#include "mlir/Support/Timing.h"
34#include "mlir/Support/ToolUtilities.h"
35#include "mlir/Tools/ParseUtilities.h"
36#include "mlir/Tools/Plugins/DialectPlugin.h"
37#include "mlir/Tools/Plugins/PassPlugin.h"
38#include "llvm/ADT/ScopeExit.h"
39#include "llvm/ADT/StringRef.h"
40#include "llvm/ADT/StringSet.h"
41#include "llvm/Remarks/RemarkFormat.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/InitLLVM.h"
45#include "llvm/Support/LogicalResult.h"
46#include "llvm/Support/ManagedStatic.h"
47#include "llvm/Support/Process.h"
48#include "llvm/Support/Regex.h"
49#include "llvm/Support/SourceMgr.h"
50#include "llvm/Support/ThreadPool.h"
51#include "llvm/Support/ToolOutputFile.h"
52
53using namespace mlir;
54using namespace llvm;
55
56namespace {
57class BytecodeVersionParser : public cl::parser<std::optional<int64_t>> {
58public:
59 BytecodeVersionParser(cl::Option &o)
60 : cl::parser<std::optional<int64_t>>(o) {}
61
62 bool parse(cl::Option &o, StringRef /*argName*/, StringRef arg,
63 std::optional<int64_t> &v) {
64 long long w;
65 if (getAsSignedInteger(Str: arg, Radix: 10, Result&: w))
66 return o.error(Message: "Invalid argument '" + arg +
67 "', only integer is supported.");
68 v = w;
69 return false;
70 }
71};
72
73/// This class is intended to manage the handling of command line options for
74/// creating a *-opt config. This is a singleton.
75struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
76 MlirOptMainConfigCLOptions() {
77 // These options are static but all uses ExternalStorage to initialize the
78 // members of the parent class. This is unusual but since this class is a
79 // singleton it basically attaches command line option to the singleton
80 // members.
81
82 static cl::opt<bool, /*ExternalStorage=*/true> allowUnregisteredDialects(
83 "allow-unregistered-dialect",
84 cl::desc("Allow operation with no registered dialects"),
85 cl::location(L&: allowUnregisteredDialectsFlag), cl::init(Val: false));
86
87 static cl::opt<bool, /*ExternalStorage=*/true> dumpPassPipeline(
88 "dump-pass-pipeline", cl::desc("Print the pipeline that will be run"),
89 cl::location(L&: dumpPassPipelineFlag), cl::init(Val: false));
90
91 static cl::opt<bool, /*ExternalStorage=*/true> emitBytecode(
92 "emit-bytecode", cl::desc("Emit bytecode when generating output"),
93 cl::location(L&: emitBytecodeFlag), cl::init(Val: false));
94
95 static cl::opt<bool, /*ExternalStorage=*/true> elideResourcesFromBytecode(
96 "elide-resource-data-from-bytecode",
97 cl::desc("Elide resources when generating bytecode"),
98 cl::location(L&: elideResourceDataFromBytecodeFlag), cl::init(Val: false));
99
100 static cl::opt<std::string, /*ExternalStorage=*/true> emitBytecodeProducer(
101 "emit-bytecode-producer",
102 cl::desc("Use specified producer when generating bytecode output"),
103 cl::location(L&: emitBytecodeProducerFlag), cl::init(Val: ""));
104
105 static cl::opt<std::optional<int64_t>, /*ExternalStorage=*/true,
106 BytecodeVersionParser>
107 bytecodeVersion(
108 "emit-bytecode-version",
109 cl::desc("Use specified bytecode when generating output"),
110 cl::location(L&: emitBytecodeVersion), cl::init(Val: std::nullopt));
111
112 static cl::list<std::string, std::vector<std::string>>
113 bytecodeDialectVersions(
114 "emit-bytecode-dialect-version",
115 cl::desc(
116 "Use the specified dialect version when generating bytecode "
117 "(may be repeated for different dialects)"),
118 cl::value_desc("dialect=version"),
119 cl::location(L&: emitBytecodeDialectVersions));
120
121 static cl::opt<std::string, /*ExternalStorage=*/true> irdlFile(
122 "irdl-file",
123 cl::desc("IRDL file to register before processing the input"),
124 cl::location(L&: irdlFileFlag), cl::init(Val: ""), cl::value_desc("filename"));
125
126 static cl::opt<VerbosityLevel, /*ExternalStorage=*/true>
127 diagnosticVerbosityLevel(
128 "mlir-diagnostic-verbosity-level",
129 cl::desc("Choose level of diagnostic information"),
130 cl::location(L&: diagnosticVerbosityLevelFlag),
131 cl::init(Val: VerbosityLevel::ErrorsWarningsAndRemarks),
132 cl::values(
133 clEnumValN(VerbosityLevel::ErrorsOnly, "errors", "Errors only"),
134 clEnumValN(VerbosityLevel::ErrorsAndWarnings, "warnings",
135 "Errors and warnings"),
136 clEnumValN(VerbosityLevel::ErrorsWarningsAndRemarks, "remarks",
137 "Errors, warnings and remarks")));
138
139 static cl::opt<bool, /*ExternalStorage=*/true> disableDiagnosticNotes(
140 "mlir-disable-diagnostic-notes", cl::desc("Disable diagnostic notes."),
141 cl::location(L&: disableDiagnosticNotesFlag), cl::init(Val: false));
142
143 static cl::opt<bool, /*ExternalStorage=*/true> explicitModule(
144 "no-implicit-module",
145 cl::desc("Disable implicit addition of a top-level module op during "
146 "parsing"),
147 cl::location(L&: useExplicitModuleFlag), cl::init(Val: false));
148
149 static cl::opt<bool, /*ExternalStorage=*/true> listPasses(
150 "list-passes", cl::desc("Print the list of registered passes and exit"),
151 cl::location(L&: listPassesFlag), cl::init(Val: false));
152
153 static cl::opt<bool, /*ExternalStorage=*/true> runReproducer(
154 "run-reproducer", cl::desc("Run the pipeline stored in the reproducer"),
155 cl::location(L&: runReproducerFlag), cl::init(Val: false));
156
157 static cl::opt<bool, /*ExternalStorage=*/true> showDialects(
158 "show-dialects",
159 cl::desc("Print the list of registered dialects and exit"),
160 cl::location(L&: showDialectsFlag), cl::init(Val: false));
161
162 static cl::opt<std::string, /*ExternalStorage=*/true> splitInputFile{
163 "split-input-file",
164 llvm::cl::ValueOptional,
165 cl::callback(CB: [&](const std::string &str) {
166 // Implicit value: use default marker if flag was used without value.
167 if (str.empty())
168 splitInputFile.setValue(V: kDefaultSplitMarker);
169 }),
170 cl::desc("Split the input file into chunks using the given or "
171 "default marker and process each chunk independently"),
172 cl::location(L&: splitInputFileFlag),
173 cl::init(Val: "")};
174
175 static cl::opt<std::string, /*ExternalStorage=*/true> outputSplitMarker(
176 "output-split-marker",
177 cl::desc("Split marker to use for merging the ouput"),
178 cl::location(L&: outputSplitMarkerFlag), cl::init(Val: kDefaultSplitMarker));
179
180 static cl::opt<SourceMgrDiagnosticVerifierHandler::Level,
181 /*ExternalStorage=*/true>
182 verifyDiagnostics{
183 "verify-diagnostics", llvm::cl::ValueOptional,
184 cl::desc("Check that emitted diagnostics match expected-* lines on "
185 "the corresponding line"),
186 cl::location(L&: verifyDiagnosticsFlag),
187 cl::values(
188 clEnumValN(SourceMgrDiagnosticVerifierHandler::Level::All,
189 "all",
190 "Check all diagnostics (expected, unexpected, "
191 "near-misses)"),
192 // Implicit value: when passed with no arguments, e.g.
193 // `--verify-diagnostics` or `--verify-diagnostics=`.
194 clEnumValN(SourceMgrDiagnosticVerifierHandler::Level::All, "",
195 "Check all diagnostics (expected, unexpected, "
196 "near-misses)"),
197 clEnumValN(
198 SourceMgrDiagnosticVerifierHandler::Level::OnlyExpected,
199 "only-expected", "Check only expected diagnostics"))};
200
201 static cl::opt<bool, /*ExternalStorage=*/true> verifyPasses(
202 "verify-each",
203 cl::desc("Run the verifier after each transformation pass"),
204 cl::location(L&: verifyPassesFlag), cl::init(Val: true));
205
206 static cl::opt<bool, /*ExternalStorage=*/true> disableVerifyOnParsing(
207 "mlir-very-unsafe-disable-verifier-on-parsing",
208 cl::desc("Disable the verifier on parsing (very unsafe)"),
209 cl::location(L&: disableVerifierOnParsingFlag), cl::init(Val: false));
210
211 static cl::opt<bool, /*ExternalStorage=*/true> verifyRoundtrip(
212 "verify-roundtrip",
213 cl::desc("Round-trip the IR after parsing and ensure it succeeds"),
214 cl::location(L&: verifyRoundtripFlag), cl::init(Val: false));
215
216 static cl::list<std::string> passPlugins(
217 "load-pass-plugin", cl::desc("Load passes from plugin library"));
218
219 static cl::opt<std::string, /*ExternalStorage=*/true>
220 generateReproducerFile(
221 "mlir-generate-reproducer",
222 llvm::cl::desc(
223 "Generate an mlir reproducer at the provided filename"
224 " (no crash required)"),
225 cl::location(L&: generateReproducerFileFlag), cl::init(Val: ""),
226 cl::value_desc("filename"));
227
228 static cl::OptionCategory remarkCategory(
229 "Remark Options",
230 "Filter remarks by regular expression (llvm::Regex syntax).");
231
232 static llvm::cl::opt<RemarkFormat, /*ExternalStorage=*/true> remarkFormat{
233 "remark-format",
234 llvm::cl::desc("Specify the format for remark output."),
235 cl::location(L&: remarkFormatFlag),
236 llvm::cl::value_desc("format"),
237 llvm::cl::init(Val: RemarkFormat::REMARK_FORMAT_STDOUT),
238 llvm::cl::values(clEnumValN(RemarkFormat::REMARK_FORMAT_STDOUT,
239 "emitRemark",
240 "Print as emitRemark to command-line"),
241 clEnumValN(RemarkFormat::REMARK_FORMAT_YAML, "yaml",
242 "Print yaml file"),
243 clEnumValN(RemarkFormat::REMARK_FORMAT_BITSTREAM,
244 "bitstream", "Print bitstream file")),
245 llvm::cl::cat(remarkCategory)};
246
247 static llvm::cl::opt<RemarkPolicy, /*ExternalStorage=*/true> remarkPolicy{
248 "remark-policy",
249 llvm::cl::desc("Specify the policy for remark output."),
250 cl::location(L&: remarkPolicyFlag),
251 llvm::cl::value_desc("format"),
252 llvm::cl::init(Val: RemarkPolicy::REMARK_POLICY_ALL),
253 llvm::cl::values(clEnumValN(RemarkPolicy::REMARK_POLICY_ALL, "all",
254 "Print all remarks"),
255 clEnumValN(RemarkPolicy::REMARK_POLICY_FINAL, "final",
256 "Print final remarks")),
257 llvm::cl::cat(remarkCategory)};
258
259 static cl::opt<std::string, /*ExternalStorage=*/true> remarksAll(
260 "remarks-filter",
261 cl::desc("Show all remarks: passed, missed, failed, analysis"),
262 cl::location(L&: remarksAllFilterFlag), cl::init(Val: ""),
263 cl::cat(remarkCategory));
264
265 static cl::opt<std::string, /*ExternalStorage=*/true> remarksFile(
266 "remarks-output-file",
267 cl::desc(
268 "Output file for yaml and bitstream remark formats. Default is "
269 "mlir-remarks.yaml or mlir-remarks.bitstream"),
270 cl::location(L&: remarksOutputFileFlag), cl::init(Val: ""),
271 cl::cat(remarkCategory));
272
273 static cl::opt<std::string, /*ExternalStorage=*/true> remarksPassed(
274 "remarks-filter-passed", cl::desc("Show passed remarks"),
275 cl::location(L&: remarksPassedFilterFlag), cl::init(Val: ""),
276 cl::cat(remarkCategory));
277
278 static cl::opt<std::string, /*ExternalStorage=*/true> remarksFailed(
279 "remarks-filter-failed", cl::desc("Show failed remarks"),
280 cl::location(L&: remarksFailedFilterFlag), cl::init(Val: ""),
281 cl::cat(remarkCategory));
282
283 static cl::opt<std::string, /*ExternalStorage=*/true> remarksMissed(
284 "remarks-filter-missed", cl::desc("Show missed remarks"),
285 cl::location(L&: remarksMissedFilterFlag), cl::init(Val: ""),
286 cl::cat(remarkCategory));
287
288 static cl::opt<std::string, /*ExternalStorage=*/true> remarksAnalyse(
289 "remarks-filter-analyse", cl::desc("Show analysis remarks"),
290 cl::location(L&: remarksAnalyseFilterFlag), cl::init(Val: ""),
291 cl::cat(remarkCategory));
292
293 /// Set the callback to load a pass plugin.
294 passPlugins.setCallback([&](const std::string &pluginPath) {
295 auto plugin = PassPlugin::load(filename: pluginPath);
296 if (!plugin) {
297 errs() << "Failed to load passes from '" << pluginPath
298 << "'. Request ignored.\n";
299 return;
300 }
301 plugin.get().registerPassRegistryCallbacks();
302 });
303
304 static cl::list<std::string> dialectPlugins(
305 "load-dialect-plugin", cl::desc("Load dialects from plugin library"));
306 this->dialectPlugins = std::addressof(r&: dialectPlugins);
307
308 static PassPipelineCLParser passPipeline("", "Compiler passes to run", "p");
309 setPassPipelineParser(passPipeline);
310 }
311
312 /// Set the callback to load a dialect plugin.
313 void setDialectPluginsCallback(DialectRegistry &registry);
314
315 /// Pointer to static dialectPlugins variable in constructor, needed by
316 /// setDialectPluginsCallback(DialectRegistry&).
317 cl::list<std::string> *dialectPlugins = nullptr;
318};
319
320/// A scoped diagnostic handler that suppresses certain diagnostics based on
321/// the verbosity level and whether the diagnostic is a note.
322class DiagnosticFilter : public ScopedDiagnosticHandler {
323public:
324 DiagnosticFilter(MLIRContext *ctx, VerbosityLevel verbosityLevel,
325 bool showNotes = true)
326 : ScopedDiagnosticHandler(ctx) {
327 setHandler([verbosityLevel, showNotes](Diagnostic &diag) {
328 auto severity = diag.getSeverity();
329 switch (severity) {
330 case mlir::DiagnosticSeverity::Error:
331 // failure indicates that the error is not handled by the filter and
332 // goes through to the default handler. Therefore, the error can be
333 // successfully printed.
334 return failure();
335 case mlir::DiagnosticSeverity::Warning:
336 if (verbosityLevel == VerbosityLevel::ErrorsOnly)
337 return success();
338 else
339 return failure();
340 case mlir::DiagnosticSeverity::Remark:
341 if (verbosityLevel == VerbosityLevel::ErrorsOnly ||
342 verbosityLevel == VerbosityLevel::ErrorsAndWarnings)
343 return success();
344 else
345 return failure();
346 case mlir::DiagnosticSeverity::Note:
347 if (showNotes)
348 return failure();
349 else
350 return success();
351 }
352 llvm_unreachable("Unknown diagnostic severity");
353 });
354 }
355};
356} // namespace
357
358ManagedStatic<MlirOptMainConfigCLOptions> clOptionsConfig;
359
360void MlirOptMainConfig::registerCLOptions(DialectRegistry &registry) {
361 clOptionsConfig->setDialectPluginsCallback(registry);
362 tracing::DebugConfig::registerCLOptions();
363}
364
365MlirOptMainConfig MlirOptMainConfig::createFromCLOptions() {
366 clOptionsConfig->setDebugConfig(tracing::DebugConfig::createFromCLOptions());
367 return *clOptionsConfig;
368}
369
370MlirOptMainConfig &MlirOptMainConfig::setPassPipelineParser(
371 const PassPipelineCLParser &passPipeline) {
372 passPipelineCallback = [&](PassManager &pm) {
373 auto errorHandler = [&](const Twine &msg) {
374 emitError(loc: UnknownLoc::get(context: pm.getContext())) << msg;
375 return failure();
376 };
377 if (failed(Result: passPipeline.addToPipeline(pm, errorHandler)))
378 return failure();
379 if (this->shouldDumpPassPipeline()) {
380 llvm::errs() << "Pass Manager with " << pm.size() << " passes:\n";
381 pm.printAsTextualPipeline(os&: llvm::errs(), /*pretty=*/true);
382 llvm::errs() << "\n";
383 }
384 return success();
385 };
386 return *this;
387}
388
389void MlirOptMainConfigCLOptions::setDialectPluginsCallback(
390 DialectRegistry &registry) {
391 dialectPlugins->setCallback([&](const std::string &pluginPath) {
392 auto plugin = DialectPlugin::load(filename: pluginPath);
393 if (!plugin) {
394 errs() << "Failed to load dialect plugin from '" << pluginPath
395 << "'. Request ignored.\n";
396 return;
397 };
398 plugin.get().registerDialectRegistryCallbacks(dialectRegistry&: registry);
399 });
400}
401
402LogicalResult loadIRDLDialects(StringRef irdlFile, MLIRContext &ctx) {
403 DialectRegistry registry;
404 registry.insert<irdl::IRDLDialect>();
405 ctx.appendDialectRegistry(registry);
406
407 // Set up the input file.
408 std::string errorMessage;
409 std::unique_ptr<MemoryBuffer> file = openInputFile(inputFilename: irdlFile, errorMessage: &errorMessage);
410 if (!file) {
411 emitError(loc: UnknownLoc::get(context: &ctx)) << errorMessage;
412 return failure();
413 }
414
415 // Give the buffer to the source manager.
416 // This will be picked up by the parser.
417 SourceMgr sourceMgr;
418 sourceMgr.AddNewSourceBuffer(F: std::move(file), IncludeLoc: SMLoc());
419
420 SourceMgrDiagnosticHandler sourceMgrHandler(sourceMgr, &ctx);
421
422 // Parse the input file.
423 OwningOpRef<ModuleOp> module(parseSourceFile<ModuleOp>(sourceMgr, config: &ctx));
424 if (!module)
425 return failure();
426
427 // Load IRDL dialects.
428 return irdl::loadDialects(op: module.get());
429}
430
431// Return success if the module can correctly round-trip. This intended to test
432// that the custom printers/parsers are complete.
433static LogicalResult doVerifyRoundTrip(Operation *op,
434 const MlirOptMainConfig &config,
435 bool useBytecode) {
436 // We use a new context to avoid resource handle renaming issue in the diff.
437 MLIRContext roundtripContext;
438 OwningOpRef<Operation *> roundtripModule;
439 roundtripContext.appendDialectRegistry(
440 registry: op->getContext()->getDialectRegistry());
441 if (op->getContext()->allowsUnregisteredDialects())
442 roundtripContext.allowUnregisteredDialects();
443 StringRef irdlFile = config.getIrdlFile();
444 if (!irdlFile.empty() && failed(Result: loadIRDLDialects(irdlFile, ctx&: roundtripContext)))
445 return failure();
446
447 std::string testType = (useBytecode) ? "bytecode" : "textual";
448 // Print a first time with custom format (or bytecode) and parse it back to
449 // the roundtripModule.
450 {
451 std::string buffer;
452 llvm::raw_string_ostream ostream(buffer);
453 if (useBytecode) {
454 if (failed(Result: writeBytecodeToFile(op, os&: ostream))) {
455 op->emitOpError()
456 << "failed to write bytecode, cannot verify round-trip.\n";
457 return failure();
458 }
459 } else {
460 op->print(os&: ostream,
461 flags: OpPrintingFlags().printGenericOpForm().enableDebugInfo());
462 }
463 FallbackAsmResourceMap fallbackResourceMap;
464 ParserConfig parseConfig(&roundtripContext, config.shouldVerifyOnParsing(),
465 &fallbackResourceMap);
466 roundtripModule = parseSourceString<Operation *>(sourceStr: buffer, config: parseConfig);
467 if (!roundtripModule) {
468 op->emitOpError() << "failed to parse " << testType
469 << " content back, cannot verify round-trip.\n";
470 return failure();
471 }
472 }
473
474 // Print in the generic form for the reference module and the round-tripped
475 // one and compare the outputs.
476 std::string reference, roundtrip;
477 {
478 llvm::raw_string_ostream ostreamref(reference);
479 op->print(os&: ostreamref,
480 flags: OpPrintingFlags().printGenericOpForm().enableDebugInfo());
481 llvm::raw_string_ostream ostreamrndtrip(roundtrip);
482 roundtripModule.get()->print(
483 os&: ostreamrndtrip,
484 flags: OpPrintingFlags().printGenericOpForm().enableDebugInfo());
485 }
486 if (reference != roundtrip) {
487 // TODO implement a diff.
488 return op->emitOpError()
489 << testType
490 << " roundTrip testing roundtripped module differs "
491 "from reference:\n<<<<<<Reference\n"
492 << reference << "\n=====\n"
493 << roundtrip << "\n>>>>>roundtripped\n";
494 }
495
496 return success();
497}
498
499static LogicalResult doVerifyRoundTrip(Operation *op,
500 const MlirOptMainConfig &config) {
501 auto txtStatus = doVerifyRoundTrip(op, config, /*useBytecode=*/false);
502 auto bcStatus = doVerifyRoundTrip(op, config, /*useBytecode=*/true);
503 return success(IsSuccess: succeeded(Result: txtStatus) && succeeded(Result: bcStatus));
504}
505
506/// Perform the actions on the input file indicated by the command line flags
507/// within the specified context.
508///
509/// This typically parses the main source file, runs zero or more optimization
510/// passes, then prints the output.
511///
512static LogicalResult
513performActions(raw_ostream &os,
514 const std::shared_ptr<llvm::SourceMgr> &sourceMgr,
515 MLIRContext *context, const MlirOptMainConfig &config) {
516 DefaultTimingManager tm;
517 applyDefaultTimingManagerCLOptions(tm);
518 TimingScope timing = tm.getRootScope();
519
520 // Disable multi-threading when parsing the input file. This removes the
521 // unnecessary/costly context synchronization when parsing.
522 bool wasThreadingEnabled = context->isMultithreadingEnabled();
523 context->disableMultithreading();
524
525 // Prepare the parser config, and attach any useful/necessary resource
526 // handlers. Unhandled external resources are treated as passthrough, i.e.
527 // they are not processed and will be emitted directly to the output
528 // untouched.
529 PassReproducerOptions reproOptions;
530 FallbackAsmResourceMap fallbackResourceMap;
531 ParserConfig parseConfig(context, config.shouldVerifyOnParsing(),
532 &fallbackResourceMap);
533 if (config.shouldRunReproducer())
534 reproOptions.attachResourceParser(config&: parseConfig);
535
536 // Parse the input file and reset the context threading state.
537 TimingScope parserTiming = timing.nest(args: "Parser");
538 OwningOpRef<Operation *> op = parseSourceFileForTool(
539 sourceMgr, config: parseConfig, insertImplicitModule: !config.shouldUseExplicitModule());
540 parserTiming.stop();
541 if (!op)
542 return failure();
543
544 // Perform round-trip verification if requested
545 if (config.shouldVerifyRoundtrip() &&
546 failed(Result: doVerifyRoundTrip(op: op.get(), config)))
547 return failure();
548
549 context->enableMultithreading(enable: wasThreadingEnabled);
550 // Set the remark categories and policy.
551 remark::RemarkCategories cats{
552 .all: config.getRemarksAllFilter(), .passed: config.getRemarksPassedFilter(),
553 .missed: config.getRemarksMissedFilter(), .analysis: config.getRemarksAnalyseFilter(),
554 .failed: config.getRemarksFailedFilter()};
555
556 mlir::MLIRContext &ctx = *context;
557 // Helper to create the appropriate policy based on configuration
558 auto createPolicy = [&config]()
559 -> std::unique_ptr<mlir::remark::detail::RemarkEmittingPolicyBase> {
560 if (config.getRemarkPolicy() == RemarkPolicy::REMARK_POLICY_ALL)
561 return std::make_unique<mlir::remark::RemarkEmittingPolicyAll>();
562 if (config.getRemarkPolicy() == RemarkPolicy::REMARK_POLICY_FINAL)
563 return std::make_unique<mlir::remark::RemarkEmittingPolicyFinal>();
564
565 llvm_unreachable("Invalid remark policy");
566 };
567
568 switch (config.getRemarkFormat()) {
569 case RemarkFormat::REMARK_FORMAT_STDOUT:
570 if (failed(Result: mlir::remark::enableOptimizationRemarks(
571 ctx, streamer: nullptr, remarkEmittingPolicy: createPolicy(), cats, printAsEmitRemarks: true /*printAsEmitRemarks*/)))
572 return failure();
573 break;
574
575 case RemarkFormat::REMARK_FORMAT_YAML: {
576 std::string file = config.getRemarksOutputFile().empty()
577 ? "mlir-remarks.yaml"
578 : config.getRemarksOutputFile();
579 if (failed(Result: mlir::remark::enableOptimizationRemarksWithLLVMStreamer(
580 ctx, filePath: file, fmt: llvm::remarks::Format::YAML, remarkEmittingPolicy: createPolicy(), cat: cats)))
581 return failure();
582 break;
583 }
584
585 case RemarkFormat::REMARK_FORMAT_BITSTREAM: {
586 std::string file = config.getRemarksOutputFile().empty()
587 ? "mlir-remarks.bitstream"
588 : config.getRemarksOutputFile();
589 if (failed(Result: mlir::remark::enableOptimizationRemarksWithLLVMStreamer(
590 ctx, filePath: file, fmt: llvm::remarks::Format::Bitstream, remarkEmittingPolicy: createPolicy(), cat: cats)))
591 return failure();
592 break;
593 }
594 }
595 // Emit the remarks the policy deferred on every exit, while the caller's
596 // diagnostic handlers are still registered; ~MLIRContext would be too late.
597 llvm::scope_exit finalizeRemarks([&ctx] {
598 if (remark::detail::RemarkEngine *engine = ctx.getRemarkEngine())
599 engine->getRemarkEmittingPolicy()->finalize();
600 });
601
602 // Prepare the pass manager, applying command-line and reproducer options.
603 PassManager pm(op.get()->getName(), PassManager::Nesting::Implicit);
604 pm.enableVerifier(enabled: config.shouldVerifyPasses());
605 if (failed(Result: applyPassManagerCLOptions(pm)))
606 return failure();
607 pm.enableTiming(timingScope&: timing);
608 if (config.shouldRunReproducer() && failed(Result: reproOptions.apply(pm)))
609 return failure();
610 if (failed(Result: config.setupPassPipeline(pm)))
611 return failure();
612
613 // Run the pipeline.
614 if (failed(Result: pm.run(op: *op)))
615 return failure();
616
617 // Generate reproducers if requested
618 if (!config.getReproducerFilename().empty()) {
619 StringRef anchorName = pm.getOpAnchorName();
620 const auto &passes = pm.getPasses();
621 makeReproducer(anchorName, passes, op: op.get(),
622 outputFile: config.getReproducerFilename());
623 }
624
625 // Print the output.
626 TimingScope outputTiming = timing.nest(args: "Output");
627 if (config.shouldEmitBytecode()) {
628 std::optional<StringRef> producer = config.bytecodeProducerToEmit();
629 BytecodeWriterConfig writerConfig =
630 producer ? BytecodeWriterConfig(fallbackResourceMap, producer.value())
631 : BytecodeWriterConfig(fallbackResourceMap);
632 if (auto v = config.bytecodeVersionToEmit())
633 writerConfig.setDesiredBytecodeVersion(*v);
634 llvm::StringSet<> versionedDialects;
635 for (StringRef entry : config.bytecodeDialectVersionsToEmit()) {
636 auto [name, version] = entry.split(Separator: '=');
637 if (name.empty() || version.empty())
638 return op.get()->emitError() << "expected 'dialect=version' for "
639 "--emit-bytecode-dialect-version, got '"
640 << entry << "'";
641 if (!versionedDialects.insert(key: name).second)
642 return op.get()->emitError()
643 << "duplicate bytecode version for dialect '" << name << "'";
644 Dialect *dialect = context->getOrLoadDialect(name);
645 if (!dialect)
646 return op.get()->emitError()
647 << "unknown dialect '" << name << "' for bytecode emission";
648 auto *interface =
649 dialect->getRegisteredInterface<BytecodeDialectInterface>();
650 std::unique_ptr<DialectVersion> parsedVersion =
651 interface ? interface->parseVersion(version) : nullptr;
652 if (!parsedVersion)
653 return op.get()->emitError()
654 << "dialect '" << name << "' does not support emitting bytecode "
655 << "version '" << version << "'";
656 writerConfig.setDialectVersion(dialectName: name, dialectVersion: std::move(parsedVersion));
657 }
658 if (config.shouldElideResourceDataFromBytecode())
659 writerConfig.setElideResourceDataFlag();
660 return writeBytecodeToFile(op: op.get(), os, config: writerConfig);
661 }
662
663 if (config.bytecodeVersionToEmit().has_value())
664 return emitError(loc: UnknownLoc::get(context: pm.getContext()))
665 << "bytecode version while not emitting bytecode";
666
667 if (!config.bytecodeDialectVersionsToEmit().empty())
668 return emitError(loc: UnknownLoc::get(context: pm.getContext()))
669 << "dialect bytecode version while not emitting bytecode";
670
671 // Don't re-run the verifier if we already ran the verifier at the end of the
672 // pass pipeline.
673 AsmState asmState(op.get(),
674 OpPrintingFlags().assumeVerified(
675 enable: config.shouldVerifyPasses() && !pm.empty()),
676 /*locationMap=*/nullptr, &fallbackResourceMap);
677 os << OpWithState(op.get(), asmState) << '\n';
678 return success();
679}
680
681/// Parses the memory buffer. If successfully, run a series of passes against
682/// it and print the result.
683static LogicalResult
684processBuffer(raw_ostream &os, std::unique_ptr<MemoryBuffer> ownedBuffer,
685 llvm::MemoryBufferRef sourceBuffer,
686 const MlirOptMainConfig &config, DialectRegistry &registry,
687 SourceMgrDiagnosticVerifierHandler *verifyHandler,
688 llvm::ThreadPoolInterface *threadPool) {
689 // Tell sourceMgr about this buffer, which is what the parser will pick up.
690 auto sourceMgr = std::make_shared<SourceMgr>();
691 // Add the original buffer to the source manager to use for determining
692 // locations.
693 sourceMgr->AddNewSourceBuffer(
694 F: llvm::MemoryBuffer::getMemBuffer(Ref: sourceBuffer,
695 /*RequiresNullTerminator=*/false),
696 IncludeLoc: SMLoc());
697 sourceMgr->AddNewSourceBuffer(F: std::move(ownedBuffer), IncludeLoc: SMLoc());
698
699 // Create a context just for the current buffer. Disable threading on
700 // creation since we'll inject the thread-pool separately.
701 MLIRContext context(registry, MLIRContext::Threading::DISABLED);
702 if (threadPool)
703 context.setThreadPool(*threadPool);
704 // Keep the registration within the lifetimes of the context and verifier.
705 std::unique_ptr<ScopedDiagnosticHandler> verifierRegistration;
706 if (verifyHandler)
707 verifierRegistration = verifyHandler->registerInContext(ctx: &context);
708
709 StringRef irdlFile = config.getIrdlFile();
710 if (!irdlFile.empty() && failed(Result: loadIRDLDialects(irdlFile, ctx&: context)))
711 return failure();
712
713 // Parse the input file.
714 context.allowUnregisteredDialects(allow: config.shouldAllowUnregisteredDialects());
715 if (config.shouldVerifyDiagnostics())
716 context.printOpOnDiagnostic(enable: false);
717
718 tracing::InstallDebugHandler installDebugHandler(context,
719 config.getDebugConfig());
720
721 // If we are in verify diagnostics mode then we have a lot of work to do,
722 // otherwise just perform the actions without worrying about it.
723 if (!config.shouldVerifyDiagnostics()) {
724 SourceMgrDiagnosticHandler sourceMgrHandler(*sourceMgr, &context);
725 DiagnosticFilter diagnosticFilter(&context,
726 config.getDiagnosticVerbosityLevel(),
727 config.shouldShowNotes());
728 return performActions(os, sourceMgr, context: &context, config);
729 }
730
731 // Do any processing requested by command line flags. We don't care whether
732 // these actions succeed or fail, we only care what diagnostics they produce
733 // and whether they match our expectations.
734 (void)performActions(os, sourceMgr, context: &context, config);
735
736 return success();
737}
738
739std::string mlir::registerCLIOptions(llvm::StringRef toolName,
740 DialectRegistry &registry) {
741 MlirOptMainConfig::registerCLOptions(registry);
742 registerAsmPrinterCLOptions();
743 registerMLIRContextCLOptions();
744 registerPassManagerCLOptions();
745 registerDefaultTimingManagerCLOptions();
746 tracing::DebugCounter::registerCLOptions();
747
748 // Build the list of dialects as a header for the --help message.
749 std::string helpHeader = (toolName + "\nAvailable Dialects: ").str();
750 {
751 llvm::raw_string_ostream os(helpHeader);
752 interleaveComma(c: registry.getRegisteredDialectNames(), os,
753 each_fn: [&](auto name) { os << name; });
754 }
755 return helpHeader;
756}
757
758std::pair<std::string, std::string>
759mlir::parseCLIOptions(int argc, char **argv, llvm::StringRef helpHeader) {
760 static cl::opt<std::string> inputFilename(
761 cl::Positional, cl::desc("<input file>"), cl::init(Val: "-"));
762
763 static cl::opt<std::string> outputFilename("o", cl::desc("Output filename"),
764 cl::value_desc("filename"),
765 cl::init(Val: "-"));
766 cl::ParseCommandLineOptions(argc, argv, Overview: helpHeader);
767 return std::make_pair(x&: inputFilename.getValue(), y&: outputFilename.getValue());
768}
769
770std::pair<std::string, std::string>
771mlir::registerAndParseCLIOptions(int argc, char **argv,
772 llvm::StringRef toolName,
773 DialectRegistry &registry) {
774 auto helpHeader = registerCLIOptions(toolName, registry);
775 return parseCLIOptions(argc, argv, helpHeader);
776}
777
778static LogicalResult printRegisteredDialects(DialectRegistry &registry) {
779 llvm::outs() << "Available Dialects: ";
780 interleave(c: registry.getRegisteredDialectNames(), os&: llvm::outs(), separator: ",");
781 llvm::outs() << "\n";
782 return success();
783}
784
785static LogicalResult printRegisteredPassesAndReturn() {
786 mlir::printRegisteredPasses();
787 return success();
788}
789
790LogicalResult mlir::MlirOptMain(llvm::raw_ostream &outputStream,
791 std::unique_ptr<llvm::MemoryBuffer> buffer,
792 DialectRegistry &registry,
793 const MlirOptMainConfig &config) {
794 if (config.shouldShowDialects())
795 return printRegisteredDialects(registry);
796
797 if (config.shouldListPasses())
798 return printRegisteredPassesAndReturn();
799
800 // The split-input-file mode is a very specific mode that slices the file
801 // up into small pieces and checks each independently.
802 // We use an explicit threadpool to avoid creating and joining/destroying
803 // threads for each of the split.
804 ThreadPoolInterface *threadPool = nullptr;
805
806 // Create a temporary context for the sake of checking if
807 // --mlir-disable-threading was passed on the command line.
808 // We use the thread-pool this context is creating, and avoid
809 // creating any thread when disabled.
810 MLIRContext threadPoolCtx;
811 if (threadPoolCtx.isMultithreadingEnabled())
812 threadPool = &threadPoolCtx.getThreadPool();
813
814 SourceMgr sourceMgr;
815 sourceMgr.AddNewSourceBuffer(
816 F: llvm::MemoryBuffer::getMemBuffer(Ref: buffer->getMemBufferRef(),
817 /*RequiresNullTerminator=*/false),
818 IncludeLoc: SMLoc());
819 // Note: this creates a verifier handler independent of the the flag set, as
820 // internally if the flag is not set, a new scoped diagnostic handler is
821 // created which would intercept the diagnostics and verify them.
822 SourceMgrDiagnosticVerifierHandler sourceMgrHandler(
823 sourceMgr, &threadPoolCtx, config.verifyDiagnosticsLevel());
824 auto chunkFn = [&](std::unique_ptr<MemoryBuffer> chunkBuffer,
825 llvm::MemoryBufferRef sourceBuffer, raw_ostream &os) {
826 return processBuffer(
827 os, ownedBuffer: std::move(chunkBuffer), sourceBuffer, config, registry,
828 verifyHandler: config.shouldVerifyDiagnostics() ? &sourceMgrHandler : nullptr,
829 threadPool);
830 };
831 LogicalResult status = splitAndProcessBuffer(
832 originalBuffer: llvm::MemoryBuffer::getMemBuffer(Ref: buffer->getMemBufferRef(),
833 /*RequiresNullTerminator=*/false),
834 processChunkBuffer: chunkFn, os&: outputStream, inputSplitMarker: config.inputSplitMarker(),
835 outputSplitMarker: config.outputSplitMarker());
836 if (config.shouldVerifyDiagnostics() && failed(Result: sourceMgrHandler.verify()))
837 status = failure();
838 return status;
839}
840
841LogicalResult mlir::MlirOptMain(int argc, char **argv,
842 llvm::StringRef inputFilename,
843 llvm::StringRef outputFilename,
844 DialectRegistry &registry) {
845
846 InitLLVM y(argc, argv);
847
848 MlirOptMainConfig config = MlirOptMainConfig::createFromCLOptions();
849
850 if (config.shouldShowDialects())
851 return printRegisteredDialects(registry);
852
853 if (config.shouldListPasses())
854 return printRegisteredPassesAndReturn();
855
856 // When reading from stdin and the input is a tty, it is often a user
857 // mistake and the process "appears to be stuck". Print a message to let the
858 // user know about it!
859 if (inputFilename == "-" &&
860 sys::Process::FileDescriptorIsDisplayed(fd: fileno(stdin)))
861 llvm::errs() << "(processing input from stdin now, hit ctrl-c/ctrl-d to "
862 "interrupt)\n";
863
864 // Set up the input file.
865 std::string errorMessage;
866 auto file = openInputFile(inputFilename, errorMessage: &errorMessage);
867 if (!file) {
868 llvm::errs() << errorMessage << "\n";
869 return failure();
870 }
871
872 auto output = openOutputFile(outputFilename, errorMessage: &errorMessage);
873 if (!output) {
874 llvm::errs() << errorMessage << "\n";
875 return failure();
876 }
877 if (failed(Result: MlirOptMain(outputStream&: output->os(), buffer: std::move(file), registry, config)))
878 return failure();
879
880 // Keep the output file if the invocation of MlirOptMain was successful.
881 output->keep();
882 return success();
883}
884
885LogicalResult mlir::MlirOptMain(int argc, char **argv, llvm::StringRef toolName,
886 DialectRegistry &registry) {
887
888 // Register and parse command line options.
889 std::string inputFilename, outputFilename;
890 std::tie(args&: inputFilename, args&: outputFilename) =
891 registerAndParseCLIOptions(argc, argv, toolName, registry);
892
893 return MlirOptMain(argc, argv, inputFilename, outputFilename, registry);
894}
895