| 1 | //===- Main.cpp - Top-Level TableGen implementation -----------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // TableGen is a tool which can be used to build up a description of something, |
| 10 | // then invoke one or more "tablegen backends" to emit information about the |
| 11 | // description in some predefined format. In practice, this is used by the LLVM |
| 12 | // code generators to automate generation of a code generator through a |
| 13 | // high-level description of the target. |
| 14 | // |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | |
| 17 | #include "llvm/TableGen/Main.h" |
| 18 | #include "TGLexer.h" |
| 19 | #include "TGParser.h" |
| 20 | #include "llvm/ADT/SmallString.h" |
| 21 | #include "llvm/ADT/StringRef.h" |
| 22 | #include "llvm/ADT/Twine.h" |
| 23 | #include "llvm/Support/CommandLine.h" |
| 24 | #include "llvm/Support/ErrorOr.h" |
| 25 | #include "llvm/Support/FileSystem.h" |
| 26 | #include "llvm/Support/MemoryBuffer.h" |
| 27 | #include "llvm/Support/Path.h" |
| 28 | #include "llvm/Support/SMLoc.h" |
| 29 | #include "llvm/Support/SourceMgr.h" |
| 30 | #include "llvm/Support/ToolOutputFile.h" |
| 31 | #include "llvm/Support/VirtualFileSystem.h" |
| 32 | #include "llvm/Support/raw_ostream.h" |
| 33 | #include "llvm/TableGen/Error.h" |
| 34 | #include "llvm/TableGen/Record.h" |
| 35 | #include "llvm/TableGen/TGTimer.h" |
| 36 | #include "llvm/TableGen/TableGenBackend.h" |
| 37 | #include <memory> |
| 38 | #include <set> |
| 39 | #include <string> |
| 40 | #include <system_error> |
| 41 | #include <utility> |
| 42 | using namespace llvm; |
| 43 | |
| 44 | static cl::opt<std::string> |
| 45 | OutputFilename("o" , cl::desc("Output filename" ), cl::value_desc("filename" ), |
| 46 | cl::init(Val: "-" )); |
| 47 | |
| 48 | static cl::opt<std::string> |
| 49 | DependFilename("d" , |
| 50 | cl::desc("Dependency filename" ), |
| 51 | cl::value_desc("filename" ), |
| 52 | cl::init(Val: "" )); |
| 53 | |
| 54 | static cl::opt<std::string> |
| 55 | InputFilename(cl::Positional, cl::desc("<input file>" ), cl::init(Val: "-" )); |
| 56 | |
| 57 | static cl::list<std::string> |
| 58 | IncludeDirs("I" , cl::desc("Directory of include files" ), |
| 59 | cl::value_desc("directory" ), cl::Prefix); |
| 60 | |
| 61 | static cl::list<std::string> |
| 62 | MacroNames("D" , cl::desc("Name of the macro to be defined" ), |
| 63 | cl::value_desc("macro name" ), cl::Prefix); |
| 64 | |
| 65 | static cl::opt<bool> |
| 66 | WriteIfChanged("write-if-changed" , cl::desc("Only write output if it changed" )); |
| 67 | |
| 68 | static cl::opt<bool> |
| 69 | TimePhases("time-phases" , cl::desc("Time phases of parser and backend" )); |
| 70 | |
| 71 | cl::opt<bool> llvm::EmitLongStrLiterals( |
| 72 | "long-string-literals" , |
| 73 | cl::desc("when emitting large string tables, prefer string literals over " |
| 74 | "comma-separated char literals. This can be a readability and " |
| 75 | "compile-time performance win, but upsets some compilers" ), |
| 76 | cl::Hidden, cl::init(Val: true)); |
| 77 | |
| 78 | static cl::opt<bool> NoWarnOnUnusedTemplateArgs( |
| 79 | "no-warn-on-unused-template-args" , |
| 80 | cl::desc("Disable unused template argument warnings." )); |
| 81 | |
| 82 | static cl::opt<bool> Preprocess("E" , cl::desc("Write preprocessed output" )); |
| 83 | |
| 84 | static int reportError(const char *ProgName, Twine Msg) { |
| 85 | errs() << ProgName << ": " << Msg; |
| 86 | errs().flush(); |
| 87 | return 1; |
| 88 | } |
| 89 | |
| 90 | namespace { |
| 91 | /// Encapsulate file, line and column numbers from SourceMgr. |
| 92 | struct SMCoords { |
| 93 | unsigned Buf = 0; |
| 94 | unsigned Line = 0; |
| 95 | unsigned Col = 0; |
| 96 | SMCoords() = default; |
| 97 | SMCoords(const SourceMgr &Mgr, SMLoc Loc) { |
| 98 | Buf = Mgr.FindBufferContainingLoc(Loc); |
| 99 | // TODO: Find a faster way to do this. For example we don't need the column |
| 100 | // number for every token, only the first token on each output line. |
| 101 | std::tie(args&: Line, args&: Col) = Mgr.getLineAndColumn(Loc, BufferID: Buf); |
| 102 | } |
| 103 | }; |
| 104 | } // end anonymous namespace |
| 105 | |
| 106 | /// Create preprocessed output for `-E` option. |
| 107 | static int preprocessInput(TGLexer &Lex, raw_ostream &OS) { |
| 108 | SMCoords Last; |
| 109 | while (true) { |
| 110 | Lex.Lex(); |
| 111 | if (Lex.getCode() == tgtok::Eof || Lex.getCode() == tgtok::Error) |
| 112 | break; |
| 113 | SMCoords This(SrcMgr, Lex.getLoc()); |
| 114 | if (This.Buf == Last.Buf && This.Line == Last.Line) { |
| 115 | // Add a single space between tokens on the same line. This is overkill in |
| 116 | // many cases but at least it will parse correctly. |
| 117 | OS << ' '; |
| 118 | } else { |
| 119 | // Always start a new line when including a new file or popping back out |
| 120 | // to the previous file. This is just a heuristic to make the output look |
| 121 | // reasonably pretty. |
| 122 | if (Last.Buf) |
| 123 | OS << '\n'; |
| 124 | // Indent the first token on a line to its original indentation, to make |
| 125 | // the output look pretty. |
| 126 | OS.indent(NumSpaces: This.Col - 1); |
| 127 | } |
| 128 | |
| 129 | const char *Start = Lex.getLoc().getPointer(); |
| 130 | const char *End = Lex.getLocRange().End.getPointer(); |
| 131 | StringRef Text(Start, End - Start); |
| 132 | OS << Text; |
| 133 | |
| 134 | Last = This; |
| 135 | // A code block is the only kind of token that can span multiple lines. |
| 136 | // Record the line it ends on, so that a following token on that line stays |
| 137 | // on the same output line. Count '\n' characters to match how SourceMgr |
| 138 | // counts lines. |
| 139 | if (Lex.getCode() == tgtok::CodeFragment) |
| 140 | Last.Line += Text.count(C: '\n'); |
| 141 | } |
| 142 | if (Last.Buf != 0) |
| 143 | OS << '\n'; |
| 144 | return Lex.getCode() == tgtok::Error; |
| 145 | } |
| 146 | |
| 147 | /// Escape a filename in the dependency file so that it is correctly |
| 148 | /// interpreted by `make`. This is consistent with Clang, GCC, and lld. |
| 149 | static std::string escapeDependencyFilename(StringRef Filename) { |
| 150 | std::string Res; |
| 151 | raw_string_ostream OS(Res); |
| 152 | |
| 153 | // Only transform the path to native on Windows, where backslashes are valid |
| 154 | // path separators. On non-Windows platforms, we don't want backslashes in |
| 155 | // filenames to be incorrectly treated as path separators. |
| 156 | #ifdef _WIN32 |
| 157 | SmallString<256> NativePath; |
| 158 | sys::path::native(Filename, NativePath); |
| 159 | #else |
| 160 | StringRef NativePath = Filename; |
| 161 | #endif |
| 162 | |
| 163 | for (unsigned I = 0, E = NativePath.size(); I != E; ++I) { |
| 164 | if (NativePath[I] == '#') |
| 165 | OS << '\\'; |
| 166 | else if (NativePath[I] == ' ') { |
| 167 | OS << '\\'; |
| 168 | unsigned J = I; |
| 169 | while (J > 0 && NativePath[--J] == '\\') |
| 170 | OS << '\\'; |
| 171 | } else if (NativePath[I] == '$') |
| 172 | OS << '$'; |
| 173 | OS << NativePath[I]; |
| 174 | } |
| 175 | |
| 176 | return Res; |
| 177 | } |
| 178 | |
| 179 | /// Create a dependency file for `-d` option. |
| 180 | /// |
| 181 | /// This functionality is really only for the benefit of the build system. |
| 182 | /// It is similar to GCC's `-M*` family of options. |
| 183 | static int createDependencyFile(const char *argv0) { |
| 184 | if (OutputFilename == "-" ) |
| 185 | return reportError(ProgName: argv0, Msg: "the option -d must be used together with -o\n" ); |
| 186 | |
| 187 | std::error_code EC; |
| 188 | ToolOutputFile DepOut(DependFilename, EC, sys::fs::OF_Text); |
| 189 | if (EC) |
| 190 | return reportError(ProgName: argv0, Msg: "error opening " + DependFilename + ":" + |
| 191 | EC.message() + "\n" ); |
| 192 | DepOut.os() << escapeDependencyFilename(Filename: OutputFilename) << ":" ; |
| 193 | |
| 194 | // Emit the primary input file as a dependency. This matches C compilers like |
| 195 | // Clang and GCC. Without it, a .td file with no `include` directives would |
| 196 | // produce a depfile listing zero dependencies. CMake's |
| 197 | // `cmake_transform_depfile` then collapses that to a 0-byte file, which Ninja |
| 198 | // treats as a missing depfile and re-runs the rule on every incremental |
| 199 | // build. |
| 200 | if (InputFilename != "-" ) |
| 201 | DepOut.os() << ' ' << escapeDependencyFilename(Filename: InputFilename); |
| 202 | |
| 203 | // Every other buffer in SrcMgr was added by an include directive, and its |
| 204 | // identifier is the path that the include resolved to. Buffer 1 is the main |
| 205 | // file. Use a set to sort the list and remove duplicates. |
| 206 | std::set<std::string> Dependencies; |
| 207 | for (unsigned I = 2, E = SrcMgr.getNumBuffers(); I <= E; ++I) |
| 208 | Dependencies.insert(x: SrcMgr.getMemoryBuffer(i: I)->getBufferIdentifier().str()); |
| 209 | for (const std::string &Dep : Dependencies) |
| 210 | DepOut.os() << ' ' << escapeDependencyFilename(Filename: Dep); |
| 211 | DepOut.os() << "\n" ; |
| 212 | DepOut.keep(); |
| 213 | return 0; |
| 214 | } |
| 215 | |
| 216 | static int WriteOutput(const char *argv0, StringRef Filename, |
| 217 | StringRef Content) { |
| 218 | if (WriteIfChanged) { |
| 219 | // Only updates the real output file if there are any differences. |
| 220 | // This prevents recompilation of all the files depending on it if there |
| 221 | // aren't any. |
| 222 | if (auto ExistingOrErr = MemoryBuffer::getFile(Filename, /*IsText=*/true)) |
| 223 | if (std::move(ExistingOrErr.get())->getBuffer() == Content) |
| 224 | return 0; |
| 225 | } |
| 226 | std::error_code EC; |
| 227 | // On z/OS, text files default to IBM-1047, but generated files are UTF-8. |
| 228 | // OF_ForceUTF8 has no effect on other platforms. |
| 229 | ToolOutputFile OutFile(Filename, EC, |
| 230 | sys::fs::OF_Text | sys::fs::OF_ForceUTF8); |
| 231 | if (EC) |
| 232 | return reportError(ProgName: argv0, Msg: "error opening " + Filename + ": " + |
| 233 | EC.message() + "\n" ); |
| 234 | OutFile.os() << Content; |
| 235 | if (ErrorsPrinted == 0) |
| 236 | OutFile.keep(); |
| 237 | |
| 238 | return 0; |
| 239 | } |
| 240 | |
| 241 | int llvm::TableGenMain(const char *argv0, MultiFileTableGenMainFn MainFn) { |
| 242 | ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr = |
| 243 | MemoryBuffer::getFileOrSTDIN(Filename: InputFilename, /*IsText=*/true); |
| 244 | if (std::error_code EC = FileOrErr.getError()) |
| 245 | return reportError(ProgName: argv0, Msg: "Could not open input file '" + InputFilename + |
| 246 | "': " + EC.message() + "\n" ); |
| 247 | |
| 248 | // Tell SrcMgr about this buffer, which is what TGParser will pick up. |
| 249 | SrcMgr.AddNewSourceBuffer(F: std::move(*FileOrErr), IncludeLoc: SMLoc()); |
| 250 | |
| 251 | // Record the location of the include directory so that the lexer can find |
| 252 | // it later. |
| 253 | SrcMgr.setIncludeDirs(IncludeDirs); |
| 254 | SrcMgr.setVirtualFileSystem(vfs::getRealFileSystem()); |
| 255 | |
| 256 | if (Preprocess) { |
| 257 | TGLexer Lex(SrcMgr, MacroNames); |
| 258 | std::string OutString; |
| 259 | raw_string_ostream Out(OutString); |
| 260 | if (int Ret = preprocessInput(Lex, OS&: Out)) |
| 261 | return Ret; |
| 262 | // Return early if the lexer printed errors but still reached EOF (e.g., a |
| 263 | // NUL character in the source). |
| 264 | if (ErrorsPrinted > 0) |
| 265 | return reportError(ProgName: argv0, Msg: Twine(ErrorsPrinted) + " errors.\n" ); |
| 266 | if (!DependFilename.empty()) { |
| 267 | if (int Ret = createDependencyFile(argv0)) |
| 268 | return Ret; |
| 269 | } |
| 270 | return WriteOutput(argv0, Filename: OutputFilename, Content: OutString); |
| 271 | } |
| 272 | |
| 273 | RecordKeeper Records; |
| 274 | Records.saveInputFilename(Filename: InputFilename); |
| 275 | |
| 276 | TGTimer &Timer = Records.getTimer(); |
| 277 | if (TimePhases) |
| 278 | Timer.startPhaseTiming(); |
| 279 | |
| 280 | // Parse the input file. |
| 281 | |
| 282 | Timer.startTimer(Name: "Parse, build records" ); |
| 283 | |
| 284 | TGParser Parser(SrcMgr, MacroNames, Records, NoWarnOnUnusedTemplateArgs); |
| 285 | |
| 286 | if (Parser.ParseFile()) |
| 287 | return 1; |
| 288 | Timer.stopTimer(); |
| 289 | |
| 290 | // Return early if any other errors were generated during parsing |
| 291 | // (e.g., assert failures). |
| 292 | if (ErrorsPrinted > 0) |
| 293 | return reportError(ProgName: argv0, Msg: Twine(ErrorsPrinted) + " errors.\n" ); |
| 294 | |
| 295 | // Write output to memory. |
| 296 | Timer.startBackendTimer(Name: "Backend overall" ); |
| 297 | TableGenOutputFiles OutFiles; |
| 298 | unsigned status = 0; |
| 299 | // ApplyCallback will return true if it did not apply any callback. In that |
| 300 | // case, attempt to apply the MainFn. |
| 301 | StringRef FilenamePrefix(sys::path::stem(path: OutputFilename)); |
| 302 | if (TableGen::Emitter::ApplyCallback(Records, OutFiles, FilenamePrefix)) |
| 303 | status = MainFn ? MainFn(OutFiles, Records) : 1; |
| 304 | Timer.stopBackendTimer(); |
| 305 | if (status) |
| 306 | return 1; |
| 307 | |
| 308 | // Always write the depfile, even if the main output hasn't changed. |
| 309 | // If it's missing, Ninja considers the output dirty. If this was below |
| 310 | // the early exit below and someone deleted the .inc.d file but not the .inc |
| 311 | // file, tablegen would never write the depfile. |
| 312 | if (!DependFilename.empty()) { |
| 313 | if (int Ret = createDependencyFile(argv0)) |
| 314 | return Ret; |
| 315 | } |
| 316 | |
| 317 | Timer.startTimer(Name: "Write output" ); |
| 318 | if (int Ret = WriteOutput(argv0, Filename: OutputFilename, Content: OutFiles.MainFile)) |
| 319 | return Ret; |
| 320 | for (auto [Suffix, Content] : OutFiles.AdditionalFiles) { |
| 321 | SmallString<128> Filename(OutputFilename); |
| 322 | // TODO: Format using the split-file convention when writing to stdout? |
| 323 | if (Filename != "-" ) { |
| 324 | sys::path::replace_extension(path&: Filename, extension: "" ); |
| 325 | Filename.append(RHS: Suffix); |
| 326 | } |
| 327 | if (int Ret = WriteOutput(argv0, Filename, Content)) |
| 328 | return Ret; |
| 329 | } |
| 330 | |
| 331 | Timer.stopTimer(); |
| 332 | Timer.stopPhaseTiming(); |
| 333 | |
| 334 | if (ErrorsPrinted > 0) |
| 335 | return reportError(ProgName: argv0, Msg: Twine(ErrorsPrinted) + " errors.\n" ); |
| 336 | return 0; |
| 337 | } |
| 338 | |
| 339 | int llvm::TableGenMain(const char *argv0, TableGenMainFn MainFn) { |
| 340 | return TableGenMain(argv0, MainFn: [&MainFn](TableGenOutputFiles &OutFiles, |
| 341 | const RecordKeeper &Records) { |
| 342 | std::string S; |
| 343 | raw_string_ostream OS(S); |
| 344 | int Res = MainFn(OS, Records); |
| 345 | OutFiles = {.MainFile: std::move(S), .AdditionalFiles: {}}; |
| 346 | return Res; |
| 347 | }); |
| 348 | } |
| 349 | |