1//===- FormatGen.cpp - Utilities for custom assembly formats ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "FormatGen.h"
10#include "llvm/ADT/StringSwitch.h"
11#include "llvm/Support/SourceMgr.h"
12#include "llvm/TableGen/Error.h"
13
14using namespace mlir;
15using namespace mlir::tblgen;
16using llvm::SourceMgr;
17
18//===----------------------------------------------------------------------===//
19// FormatToken
20//===----------------------------------------------------------------------===//
21
22SMLoc FormatToken::getLoc() const {
23 return SMLoc::getFromPointer(Ptr: spelling.data());
24}
25
26//===----------------------------------------------------------------------===//
27// FormatLexer
28//===----------------------------------------------------------------------===//
29
30FormatLexer::FormatLexer(SourceMgr &mgr, SMLoc loc)
31 : mgr(mgr), loc(loc),
32 curBuffer(mgr.getMemoryBuffer(i: mgr.getMainFileID())->getBuffer()),
33 curPtr(curBuffer.begin()) {}
34
35FormatToken FormatLexer::emitError(SMLoc loc, const Twine &msg) {
36 mgr.PrintMessage(Loc: loc, Kind: SourceMgr::DK_Error, Msg: msg);
37 llvm::SrcMgr.PrintMessage(Loc: this->loc, Kind: SourceMgr::DK_Note,
38 Msg: "in custom assembly format for this operation");
39 return formToken(kind: FormatToken::error, tokStart: loc.getPointer());
40}
41
42FormatToken FormatLexer::emitError(const char *loc, const Twine &msg) {
43 return emitError(loc: SMLoc::getFromPointer(Ptr: loc), msg);
44}
45
46FormatToken FormatLexer::emitErrorAndNote(SMLoc loc, const Twine &msg,
47 const Twine &note) {
48 mgr.PrintMessage(Loc: loc, Kind: SourceMgr::DK_Error, Msg: msg);
49 llvm::SrcMgr.PrintMessage(Loc: this->loc, Kind: SourceMgr::DK_Note,
50 Msg: "in custom assembly format for this operation");
51 mgr.PrintMessage(Loc: loc, Kind: SourceMgr::DK_Note, Msg: note);
52 return formToken(kind: FormatToken::error, tokStart: loc.getPointer());
53}
54
55int FormatLexer::getNextChar() {
56 char curChar = *curPtr++;
57 switch (curChar) {
58 default:
59 return (unsigned char)curChar;
60 case 0: {
61 // A nul character in the stream is either the end of the current buffer or
62 // a random nul in the file. Disambiguate that here.
63 if (curPtr - 1 != curBuffer.end())
64 return 0;
65
66 // Otherwise, return end of file.
67 --curPtr;
68 return EOF;
69 }
70 case '\n':
71 case '\r':
72 // Handle the newline character by ignoring it and incrementing the line
73 // count. However, be careful about 'dos style' files with \n\r in them.
74 // Only treat a \n\r or \r\n as a single line.
75 if ((*curPtr == '\n' || (*curPtr == '\r')) && *curPtr != curChar)
76 ++curPtr;
77 return '\n';
78 }
79}
80
81FormatToken FormatLexer::lexToken() {
82 const char *tokStart = curPtr;
83
84 // This always consumes at least one character.
85 int curChar = getNextChar();
86 switch (curChar) {
87 default:
88 // Handle identifiers: [a-zA-Z_]
89 if (isalpha(curChar) || curChar == '_')
90 return lexIdentifier(tokStart);
91
92 // Unknown character, emit an error.
93 return emitError(loc: tokStart, msg: "unexpected character");
94 case EOF:
95 // Return EOF denoting the end of lexing.
96 return formToken(kind: FormatToken::eof, tokStart);
97
98 // Lex punctuation.
99 case '^':
100 return formToken(kind: FormatToken::caret, tokStart);
101 case ':':
102 return formToken(kind: FormatToken::colon, tokStart);
103 case ',':
104 return formToken(kind: FormatToken::comma, tokStart);
105 case '=':
106 return formToken(kind: FormatToken::equal, tokStart);
107 case '<':
108 return formToken(kind: FormatToken::less, tokStart);
109 case '>':
110 return formToken(kind: FormatToken::greater, tokStart);
111 case '?':
112 return formToken(kind: FormatToken::question, tokStart);
113 case '(':
114 return formToken(kind: FormatToken::l_paren, tokStart);
115 case ')':
116 return formToken(kind: FormatToken::r_paren, tokStart);
117 case '*':
118 return formToken(kind: FormatToken::star, tokStart);
119 case '|':
120 return formToken(kind: FormatToken::pipe, tokStart);
121
122 // Ignore whitespace characters.
123 case 0:
124 case ' ':
125 case '\t':
126 case '\n':
127 return lexToken();
128
129 case '`':
130 return lexLiteral(tokStart);
131 case '$':
132 return lexVariable(tokStart);
133 case '"':
134 return lexString(tokStart);
135 }
136}
137
138FormatToken FormatLexer::lexLiteral(const char *tokStart) {
139 assert(curPtr[-1] == '`');
140
141 // Lex a literal surrounded by ``.
142 while (const char curChar = *curPtr++) {
143 if (curChar == '`')
144 return formToken(kind: FormatToken::literal, tokStart);
145 }
146 return emitError(loc: curPtr - 1, msg: "unexpected end of file in literal");
147}
148
149FormatToken FormatLexer::lexVariable(const char *tokStart) {
150 if (!isalpha(curPtr[0]) && curPtr[0] != '_')
151 return emitError(loc: curPtr - 1, msg: "expected variable name");
152
153 // Otherwise, consume the rest of the characters.
154 while (isalnum(*curPtr) || *curPtr == '_')
155 ++curPtr;
156 return formToken(kind: FormatToken::variable, tokStart);
157}
158
159FormatToken FormatLexer::lexString(const char *tokStart) {
160 // Lex until another quote, respecting escapes.
161 bool escape = false;
162 while (const char curChar = *curPtr++) {
163 if (!escape && curChar == '"')
164 return formToken(kind: FormatToken::string, tokStart);
165 escape = curChar == '\\';
166 }
167 return emitError(loc: curPtr - 1, msg: "unexpected end of file in string");
168}
169
170FormatToken FormatLexer::lexIdentifier(const char *tokStart) {
171 // Match the rest of the identifier regex: [0-9a-zA-Z_\-]*
172 while (isalnum(*curPtr) || *curPtr == '_' || *curPtr == '-')
173 ++curPtr;
174
175 // Check to see if this identifier is a keyword.
176 StringRef str(tokStart, curPtr - tokStart);
177 auto kind =
178 StringSwitch<FormatToken::Kind>(str)
179 .Case(S: "attr-dict", Value: FormatToken::kw_attr_dict)
180 .Case(S: "attr-dict-with-keyword", Value: FormatToken::kw_attr_dict_w_keyword)
181 .Case(S: "prop-dict", Value: FormatToken::kw_prop_dict)
182 .Case(S: "custom", Value: FormatToken::kw_custom)
183 .Case(S: "enum", Value: FormatToken::kw_enum)
184 .Case(S: "functional-type", Value: FormatToken::kw_functional_type)
185 .Case(S: "oilist", Value: FormatToken::kw_oilist)
186 .Case(S: "operands", Value: FormatToken::kw_operands)
187 .Case(S: "params", Value: FormatToken::kw_params)
188 .Case(S: "ref", Value: FormatToken::kw_ref)
189 .Case(S: "regions", Value: FormatToken::kw_regions)
190 .Case(S: "results", Value: FormatToken::kw_results)
191 .Case(S: "struct", Value: FormatToken::kw_struct)
192 .Case(S: "successors", Value: FormatToken::kw_successors)
193 .Case(S: "type", Value: FormatToken::kw_type)
194 .Case(S: "qualified", Value: FormatToken::kw_qualified)
195 .Default(Value: FormatToken::identifier);
196 return FormatToken(kind, str);
197}
198
199//===----------------------------------------------------------------------===//
200// FormatParser
201//===----------------------------------------------------------------------===//
202
203FormatElement::~FormatElement() = default;
204
205FormatParser::~FormatParser() = default;
206
207FailureOr<std::vector<FormatElement *>> FormatParser::parse() {
208 SMLoc loc = curToken.getLoc();
209
210 // Parse each of the format elements into the main format.
211 std::vector<FormatElement *> elements;
212 while (curToken.getKind() != FormatToken::eof) {
213 FailureOr<FormatElement *> element = parseElement(ctx: TopLevelContext);
214 if (failed(Result: element))
215 return failure();
216 elements.push_back(x: *element);
217 }
218
219 // Verify the format.
220 if (failed(Result: verify(loc, elements)))
221 return failure();
222 return elements;
223}
224
225//===----------------------------------------------------------------------===//
226// Element Parsing
227//===----------------------------------------------------------------------===//
228
229FailureOr<FormatElement *> FormatParser::parseElement(Context ctx) {
230 if (curToken.is(kind: FormatToken::literal))
231 return parseLiteral(ctx);
232 if (curToken.is(kind: FormatToken::string))
233 return parseString(ctx);
234 if (curToken.is(kind: FormatToken::variable))
235 return parseVariable(ctx);
236 if (curToken.isKeyword())
237 return parseDirective(ctx);
238 if (curToken.is(kind: FormatToken::l_paren))
239 return parseOptionalGroup(ctx);
240 return emitError(loc: curToken.getLoc(),
241 msg: "expected literal, variable, directive, or optional group");
242}
243
244FailureOr<FormatElement *> FormatParser::parseLiteral(Context ctx) {
245 FormatToken tok = curToken;
246 SMLoc loc = tok.getLoc();
247 consumeToken();
248
249 if (ctx != TopLevelContext) {
250 return emitError(
251 loc,
252 msg: "literals may only be used in the top-level section of the format");
253 }
254 // Get the spelling without the surrounding backticks.
255 StringRef value = tok.getSpelling();
256 // Prevents things like `$arg0` or empty literals (when a literal is expected
257 // but not found) from getting segmentation faults.
258 if (value.size() < 2 || value[0] != '`' || value[value.size() - 1] != '`')
259 return emitError(loc: tok.getLoc(), msg: "expected literal, but got '" + value + "'");
260 value = value.drop_front().drop_back();
261
262 // The parsed literal is a space element (`` or ` `) or a newline.
263 if (value.empty() || value == " " || value == "\\n")
264 return create<WhitespaceElement>(args&: value);
265
266 // Check that the parsed literal is valid.
267 if (!isValidLiteral(value, emitError: [&](Twine msg) {
268 (void)emitError(loc, msg: "expected valid literal but got '" + value +
269 "': " + msg);
270 }))
271 return failure();
272 return create<LiteralElement>(args&: value);
273}
274
275FailureOr<FormatElement *> FormatParser::parseString(Context ctx) {
276 FormatToken tok = curToken;
277 SMLoc loc = tok.getLoc();
278 consumeToken();
279
280 if (ctx != CustomDirectiveContext) {
281 return emitError(
282 loc, msg: "strings may only be used as 'custom' directive arguments");
283 }
284 // Escape the string.
285 std::string value;
286 StringRef contents = tok.getSpelling().drop_front().drop_back();
287 value.reserve(res_arg: contents.size());
288 bool escape = false;
289 for (char c : contents) {
290 escape = c == '\\';
291 if (!escape)
292 value.push_back(c: c);
293 }
294 return create<StringElement>(args: std::move(value));
295}
296
297FailureOr<FormatElement *> FormatParser::parseVariable(Context ctx) {
298 FormatToken tok = curToken;
299 SMLoc loc = tok.getLoc();
300 consumeToken();
301
302 // Get the name of the variable without the leading `$`.
303 StringRef name = tok.getSpelling().drop_front();
304 return parseVariableImpl(loc, name, ctx);
305}
306
307FailureOr<FormatElement *> FormatParser::parseDirective(Context ctx) {
308 FormatToken tok = curToken;
309 SMLoc loc = tok.getLoc();
310 consumeToken();
311
312 if (tok.is(kind: FormatToken::kw_custom))
313 return parseCustomDirective(loc, ctx);
314 if (tok.is(kind: FormatToken::kw_ref))
315 return parseRefDirective(loc, context: ctx);
316 if (tok.is(kind: FormatToken::kw_qualified))
317 return parseQualifiedDirective(loc, ctx);
318 return parseDirectiveImpl(loc, kind: tok.getKind(), ctx);
319}
320
321FailureOr<FormatElement *> FormatParser::parseOptionalGroup(Context ctx) {
322 SMLoc loc = curToken.getLoc();
323 consumeToken();
324 if (ctx != TopLevelContext) {
325 return emitError(loc,
326 msg: "optional groups can only be used as top-level elements");
327 }
328
329 // Parse the child elements for this optional group.
330 std::vector<FormatElement *> thenElements, elseElements;
331 FormatElement *anchor = nullptr;
332 auto parseChildElements =
333 [this, &anchor](std::vector<FormatElement *> &elements) -> LogicalResult {
334 do {
335 FailureOr<FormatElement *> element = parseElement(ctx: TopLevelContext);
336 if (failed(Result: element))
337 return failure();
338 // Check for an anchor.
339 if (curToken.is(kind: FormatToken::caret)) {
340 if (anchor) {
341 return emitError(loc: curToken.getLoc(),
342 msg: "only one element can be marked as the anchor of an "
343 "optional group");
344 }
345 anchor = *element;
346 consumeToken();
347 }
348 elements.push_back(x: *element);
349 } while (!curToken.is(kind: FormatToken::r_paren));
350 return success();
351 };
352
353 // Parse the 'then' elements. If the anchor was found in this group, then the
354 // optional is not inverted.
355 if (failed(Result: parseChildElements(thenElements)))
356 return failure();
357 consumeToken();
358 bool inverted = !anchor;
359
360 // Parse the `else` elements of this optional group.
361 if (curToken.is(kind: FormatToken::colon)) {
362 consumeToken();
363 if (failed(Result: parseToken(
364 kind: FormatToken::l_paren,
365 msg: "expected '(' to start else branch of optional group")) ||
366 failed(Result: parseChildElements(elseElements)))
367 return failure();
368 consumeToken();
369 }
370 if (failed(Result: parseToken(kind: FormatToken::question,
371 msg: "expected '?' after optional group")))
372 return failure();
373
374 // The optional group is required to have an anchor.
375 if (!anchor)
376 return emitError(loc, msg: "optional group has no anchor element");
377
378 // Verify the child elements.
379 if (failed(Result: verifyOptionalGroupElements(loc, elements: thenElements, anchor)) ||
380 failed(Result: verifyOptionalGroupElements(loc, elements: elseElements, anchor: nullptr)))
381 return failure();
382
383 // Get the first parsable element. It must be an element that can be
384 // optionally-parsed.
385 auto isWhitespace = [](FormatElement *element) {
386 return isa<WhitespaceElement>(Val: element);
387 };
388 auto thenParseBegin = llvm::find_if_not(Range&: thenElements, P: isWhitespace);
389 auto elseParseBegin = llvm::find_if_not(Range&: elseElements, P: isWhitespace);
390 unsigned thenParseStart = std::distance(first: thenElements.begin(), last: thenParseBegin);
391 unsigned elseParseStart = std::distance(first: elseElements.begin(), last: elseParseBegin);
392
393 if (!isa<LiteralElement, VariableElement, CustomDirective>(Val: *thenParseBegin)) {
394 return emitError(loc, msg: "first parsable element of an optional group must be "
395 "a literal, variable, or custom directive");
396 }
397 return create<OptionalElement>(args: std::move(thenElements),
398 args: std::move(elseElements), args&: thenParseStart,
399 args&: elseParseStart, args&: anchor, args&: inverted);
400}
401
402FailureOr<FormatElement *> FormatParser::parseCustomDirective(SMLoc loc,
403 Context ctx) {
404 if (ctx != TopLevelContext && ctx != StructDirectiveContext) {
405 return emitError(loc, msg: "`custom` can only be used at the top-level context "
406 "or within a `struct` directive");
407 }
408
409 if (failed(Result: parseToken(kind: FormatToken::less,
410 msg: "expected '<' before custom directive name")))
411 return failure();
412 // Preserve `custom<enum>` now that `enum` is also a directive keyword.
413 if (!curToken.is(kind: FormatToken::identifier) &&
414 !curToken.is(kind: FormatToken::kw_enum))
415 return emitError(loc: curToken.getLoc(),
416 msg: "expected custom directive name identifier");
417 FormatToken nameTok = curToken;
418 consumeToken();
419 if (failed(Result: parseToken(kind: FormatToken::greater,
420 msg: "expected '>' after custom directive name")) ||
421 failed(Result: parseToken(kind: FormatToken::l_paren,
422 msg: "expected '(' before custom directive parameters")))
423 return failure();
424
425 // Parse the arguments.
426 std::vector<FormatElement *> arguments;
427 while (true) {
428 FailureOr<FormatElement *> argument = parseElement(ctx: CustomDirectiveContext);
429 if (failed(Result: argument))
430 return failure();
431 arguments.push_back(x: *argument);
432 if (!curToken.is(kind: FormatToken::comma))
433 break;
434 consumeToken();
435 }
436
437 if (failed(Result: parseToken(kind: FormatToken::r_paren,
438 msg: "expected ')' after custom directive parameters")))
439 return failure();
440
441 if (failed(Result: verifyCustomDirectiveArguments(loc, arguments)))
442 return failure();
443 return create<CustomDirective>(args: nameTok.getSpelling(), args: std::move(arguments));
444}
445
446FailureOr<FormatElement *> FormatParser::parseRefDirective(SMLoc loc,
447 Context context) {
448 if (context != CustomDirectiveContext)
449 return emitError(loc, msg: "'ref' is only valid within a `custom` directive");
450
451 FailureOr<FormatElement *> arg;
452 if (failed(Result: parseToken(kind: FormatToken::l_paren,
453 msg: "expected '(' before argument list")) ||
454 failed(Result: arg = parseElement(ctx: RefDirectiveContext)) ||
455 failed(
456 Result: parseToken(kind: FormatToken::r_paren, msg: "expected ')' after argument list")))
457 return failure();
458
459 return create<RefDirective>(args&: *arg);
460}
461
462FailureOr<FormatElement *> FormatParser::parseQualifiedDirective(SMLoc loc,
463 Context ctx) {
464 if (failed(Result: parseToken(kind: FormatToken::l_paren,
465 msg: "expected '(' before argument list")))
466 return failure();
467 FailureOr<FormatElement *> var = parseElement(ctx);
468 if (failed(Result: var))
469 return var;
470 if (failed(Result: markQualified(loc, element: *var)))
471 return failure();
472 if (failed(
473 Result: parseToken(kind: FormatToken::r_paren, msg: "expected ')' after argument list")))
474 return failure();
475 return var;
476}
477
478//===----------------------------------------------------------------------===//
479// Utility Functions
480//===----------------------------------------------------------------------===//
481
482bool mlir::tblgen::shouldEmitSpaceBefore(StringRef value,
483 bool lastWasPunctuation) {
484 if (value.size() != 1 && value != "->")
485 return true;
486 if (lastWasPunctuation)
487 return !StringRef(">)}],").contains(C: value.front());
488 return !StringRef("<>(){}[],").contains(C: value.front());
489}
490
491bool mlir::tblgen::canFormatStringAsKeyword(
492 StringRef value, function_ref<void(Twine)> emitError) {
493 if (value.empty()) {
494 if (emitError)
495 emitError("keywords cannot be empty");
496 return false;
497 }
498 if (!isalpha(value.front()) && value.front() != '_') {
499 if (emitError)
500 emitError("valid keyword starts with a letter or '_'");
501 return false;
502 }
503 if (!llvm::all_of(Range: value.drop_front(), P: [](char c) {
504 return isalnum(c) || c == '_' || c == '$' || c == '.';
505 })) {
506 if (emitError)
507 emitError(
508 "keywords should contain only alphanum, '_', '$', or '.' characters");
509 return false;
510 }
511 return true;
512}
513
514bool mlir::tblgen::isValidLiteral(StringRef value,
515 function_ref<void(Twine)> emitError) {
516 if (value.empty()) {
517 if (emitError)
518 emitError("literal can't be empty");
519 return false;
520 }
521 char front = value.front();
522
523 // If there is only one character, this must either be punctuation or a
524 // single character bare identifier.
525 if (value.size() == 1) {
526 StringRef bare = "_:,=<>()[]{}?+-*";
527 if (isalpha(front) || bare.contains(C: front))
528 return true;
529 if (emitError)
530 emitError("single character literal must be a letter or one of '" + bare +
531 "'");
532 return false;
533 }
534 // Check the punctuation that are larger than a single character.
535 if (value == "->")
536 return true;
537 if (value == "...")
538 return true;
539
540 // Otherwise, this must be an identifier.
541 return canFormatStringAsKeyword(value, emitError);
542}
543
544//===----------------------------------------------------------------------===//
545// Commandline Options
546//===----------------------------------------------------------------------===//
547
548llvm::cl::opt<bool> mlir::tblgen::formatErrorIsFatal(
549 "asmformat-error-is-fatal",
550 llvm::cl::desc("Emit a fatal error if format parsing fails"),
551 llvm::cl::init(Val: true));
552