1//===- FormatGen.h - Utilities for custom assembly formats ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains common classes for building custom assembly format parsers
10// and generators.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_
15#define MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_
16
17#include "mlir/Support/LLVM.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSet.h"
20#include "llvm/Support/Allocator.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/SMLoc.h"
23#include <vector>
24
25namespace llvm {
26class SourceMgr;
27} // namespace llvm
28
29namespace mlir {
30namespace tblgen {
31
32//===----------------------------------------------------------------------===//
33// FormatToken
34//===----------------------------------------------------------------------===//
35
36/// This class represents a specific token in the input format.
37class FormatToken {
38public:
39 /// Basic token kinds.
40 enum Kind {
41 // Markers.
42 eof,
43 error,
44
45 // Tokens with no info.
46 l_paren,
47 r_paren,
48 caret,
49 colon,
50 comma,
51 equal,
52 less,
53 greater,
54 question,
55 star,
56 pipe,
57
58 // Keywords.
59 keyword_start,
60 kw_attr_dict,
61 kw_attr_dict_w_keyword,
62 kw_prop_dict,
63 kw_custom,
64 kw_enum,
65 kw_functional_type,
66 kw_oilist,
67 kw_operands,
68 kw_params,
69 kw_qualified,
70 kw_ref,
71 kw_regions,
72 kw_results,
73 kw_struct,
74 kw_successors,
75 kw_type,
76 keyword_end,
77
78 // String valued tokens.
79 identifier,
80 literal,
81 variable,
82 string,
83 };
84
85 FormatToken(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}
86
87 /// Return the bytes that make up this token.
88 StringRef getSpelling() const { return spelling; }
89
90 /// Return the kind of this token.
91 Kind getKind() const { return kind; }
92
93 /// Return a location for this token.
94 SMLoc getLoc() const;
95
96 /// Returns true if the token is of the given kind.
97 bool is(Kind kind) { return getKind() == kind; }
98
99 /// Return if this token is a keyword.
100 bool isKeyword() const {
101 return getKind() > Kind::keyword_start && getKind() < Kind::keyword_end;
102 }
103
104private:
105 /// Discriminator that indicates the kind of token this is.
106 Kind kind;
107
108 /// A reference to the entire token contents; this is always a pointer into
109 /// a memory buffer owned by the source manager.
110 StringRef spelling;
111};
112
113//===----------------------------------------------------------------------===//
114// FormatLexer
115//===----------------------------------------------------------------------===//
116
117/// This class implements a simple lexer for operation assembly format strings.
118class FormatLexer {
119public:
120 FormatLexer(llvm::SourceMgr &mgr, SMLoc loc);
121
122 /// Lex the next token and return it.
123 FormatToken lexToken();
124
125 /// Emit an error to the lexer with the given location and message.
126 FormatToken emitError(SMLoc loc, const Twine &msg);
127 FormatToken emitError(const char *loc, const Twine &msg);
128
129 FormatToken emitErrorAndNote(SMLoc loc, const Twine &msg, const Twine &note);
130
131private:
132 /// Return the next character in the stream.
133 int getNextChar();
134
135 /// Lex an identifier, literal, variable, or string.
136 FormatToken lexIdentifier(const char *tokStart);
137 FormatToken lexLiteral(const char *tokStart);
138 FormatToken lexVariable(const char *tokStart);
139 FormatToken lexString(const char *tokStart);
140
141 /// Create a token with the current pointer and a start pointer.
142 FormatToken formToken(FormatToken::Kind kind, const char *tokStart) {
143 return FormatToken(kind, StringRef(tokStart, curPtr - tokStart));
144 }
145
146 /// The source manager containing the format string.
147 llvm::SourceMgr &mgr;
148 /// Location of the format string.
149 SMLoc loc;
150 /// Buffer containing the format string.
151 StringRef curBuffer;
152 /// Current pointer in the buffer.
153 const char *curPtr;
154};
155
156//===----------------------------------------------------------------------===//
157// FormatElement
158//===----------------------------------------------------------------------===//
159
160/// This class represents a single format element.
161///
162/// If you squint and take a close look, you can see the outline of a `Format`
163/// dialect.
164class FormatElement {
165public:
166 virtual ~FormatElement();
167
168 // The top-level kinds of format elements.
169 enum Kind { Literal, String, Variable, Whitespace, Directive, Optional };
170
171 /// Support LLVM-style RTTI.
172 static bool classof(const FormatElement *el) { return true; }
173
174 /// Get the element kind.
175 Kind getKind() const { return kind; }
176
177protected:
178 /// Create a format element with the given kind.
179 FormatElement(Kind kind) : kind(kind) {}
180
181private:
182 /// The kind of the element.
183 Kind kind;
184};
185
186/// The base class for all format elements. This class implements common methods
187/// for LLVM-style RTTI.
188template <FormatElement::Kind ElementKind>
189class FormatElementBase : public FormatElement {
190public:
191 /// Support LLVM-style RTTI.
192 static bool classof(const FormatElement *el) {
193 return ElementKind == el->getKind();
194 }
195
196protected:
197 /// Create a format element with the given kind.
198 FormatElementBase() : FormatElement(ElementKind) {}
199};
200
201/// This class represents a literal element. A literal is either one of the
202/// supported punctuation characters (e.g. `(` or `,`) or a string literal (e.g.
203/// `literal`).
204class LiteralElement : public FormatElementBase<FormatElement::Literal> {
205public:
206 /// Create a literal element with the given spelling.
207 explicit LiteralElement(StringRef spelling) : spelling(spelling) {}
208
209 /// Get the spelling of the literal.
210 StringRef getSpelling() const { return spelling; }
211
212private:
213 /// The spelling of the variable, i.e. the string contained within the
214 /// backticks.
215 StringRef spelling;
216};
217
218/// This class represents a raw string that can contain arbitrary C++ code.
219class StringElement : public FormatElementBase<FormatElement::String> {
220public:
221 /// Create a string element with the given contents.
222 explicit StringElement(std::string value) : value(std::move(value)) {}
223
224 /// Get the value of the string element.
225 StringRef getValue() const { return value; }
226
227private:
228 /// The contents of the string.
229 std::string value;
230};
231
232/// This class represents a variable element. A variable refers to some part of
233/// the object being parsed, e.g. an attribute or operand on an operation or a
234/// parameter on an attribute.
235class VariableElement : public FormatElementBase<FormatElement::Variable> {
236public:
237 /// These are the kinds of variables.
238 enum Kind {
239 Attribute,
240 Operand,
241 Region,
242 Result,
243 Successor,
244 Parameter,
245 Property
246 };
247
248 /// Get the kind of variable.
249 Kind getKind() const { return kind; }
250
251protected:
252 /// Create a variable with a kind.
253 VariableElement(Kind kind) : kind(kind) {}
254
255private:
256 /// The kind of variable.
257 Kind kind;
258};
259
260/// Base class for variable elements. This class implements common methods for
261/// LLVM-style RTTI.
262template <VariableElement::Kind VariableKind>
263class VariableElementBase : public VariableElement {
264public:
265 /// An element is of this class if it is a variable and has the same variable
266 /// type.
267 static bool classof(const FormatElement *el) {
268 if (auto *varEl = dyn_cast<VariableElement>(Val: el))
269 return VariableKind == varEl->getKind();
270 return false;
271 }
272
273protected:
274 /// Create a variable element with the given variable kind.
275 VariableElementBase() : VariableElement(VariableKind) {}
276};
277
278/// This class represents a whitespace element, e.g. a newline or space. It is a
279/// literal that is printed but never parsed. When the value is empty, i.e. ``,
280/// a space is elided where one would have been printed automatically.
281class WhitespaceElement : public FormatElementBase<FormatElement::Whitespace> {
282public:
283 /// Create a whitespace element.
284 explicit WhitespaceElement(StringRef value) : value(value) {}
285
286 /// Get the whitespace value.
287 StringRef getValue() const { return value; }
288
289private:
290 /// The value of the whitespace element. Can be empty.
291 StringRef value;
292};
293
294class DirectiveElement : public FormatElementBase<FormatElement::Directive> {
295public:
296 /// These are the kinds of directives.
297 enum Kind {
298 AttrDict,
299 PropDict,
300 Custom,
301 FunctionalType,
302 OIList,
303 Operands,
304 Ref,
305 Regions,
306 Results,
307 Successors,
308 Type,
309 Params,
310 Struct
311 };
312
313 /// Get the directive kind.
314 Kind getKind() const { return kind; }
315
316protected:
317 /// Create a directive element with a kind.
318 DirectiveElement(Kind kind) : kind(kind) {}
319
320private:
321 /// The directive kind.
322 Kind kind;
323};
324
325/// Base class for directive elements. This class implements common methods for
326/// LLVM-style RTTI.
327template <DirectiveElement::Kind DirectiveKind>
328class DirectiveElementBase : public DirectiveElement {
329public:
330 /// Create a directive element with the specified kind.
331 DirectiveElementBase() : DirectiveElement(DirectiveKind) {}
332
333 /// A format element is of this class if it is a directive element and has the
334 /// same kind.
335 static bool classof(const FormatElement *el) {
336 if (auto *directiveEl = dyn_cast<DirectiveElement>(Val: el))
337 return DirectiveKind == directiveEl->getKind();
338 return false;
339 }
340};
341
342/// Base class for a directive that contains references to elements of type `T`
343/// in a vector.
344template <DirectiveElement::Kind DirectiveKind, typename T>
345class VectorDirectiveBase : public DirectiveElementBase<DirectiveKind> {
346public:
347 using Base = VectorDirectiveBase<DirectiveKind, T>;
348
349 VectorDirectiveBase(std::vector<T> &&elems) : elems(std::move(elems)) {}
350
351 /// Get the elements contained in this directive.
352 ArrayRef<T> getElements() const { return elems; }
353
354 /// Get the number of elements.
355 unsigned getNumElements() const { return elems.size(); }
356
357 /// Take all of the elements from this directive.
358 std::vector<T> takeElements() { return std::move(elems); }
359
360protected:
361 /// The elements captured by this directive.
362 std::vector<T> elems;
363};
364
365/// This class represents a custom format directive that is implemented by the
366/// user in C++. The directive accepts a list of arguments that is passed to the
367/// C++ function.
368class CustomDirective
369 : public VectorDirectiveBase<DirectiveElement::Custom, FormatElement *> {
370public:
371 using Base::Base;
372 /// Create a custom directive with a name and list of arguments.
373 CustomDirective(StringRef name, std::vector<FormatElement *> &&arguments)
374 : Base(std::move(arguments)), name(name) {}
375
376 /// Get the custom directive name.
377 StringRef getName() const { return name; }
378
379 template <typename T>
380 FailureOr<T *> getFrontAs() const {
381 if (getNumElements() != 1)
382 return failure();
383 if (T *elem = dyn_cast<T>(getElements()[0]))
384 return elem;
385 return failure();
386 }
387
388private:
389 /// The name of the custom directive. The name is used to call two C++
390 /// methods: `parse{name}` and `print{name}` with the given arguments.
391 StringRef name;
392};
393
394/// This class represents a reference directive. This directive can be used to
395/// reference but not bind a previously bound variable or format object. Its
396/// current only use is to pass variables as arguments to the custom directive.
397class RefDirective : public DirectiveElementBase<DirectiveElement::Ref> {
398public:
399 /// Create a reference directive with the single referenced child.
400 RefDirective(FormatElement *arg) : arg(arg) {}
401
402 /// Get the reference argument.
403 FormatElement *getArg() const { return arg; }
404
405private:
406 /// The referenced argument.
407 FormatElement *arg;
408};
409
410/// This class represents a group of elements that are optionally emitted based
411/// on an optional variable "anchor" and a group of elements that are emitted
412/// when the anchor element is not present.
413class OptionalElement : public FormatElementBase<FormatElement::Optional> {
414public:
415 /// Create an optional group with the given child elements.
416 OptionalElement(std::vector<FormatElement *> &&thenElements,
417 std::vector<FormatElement *> &&elseElements,
418 unsigned thenParseStart, unsigned elseParseStart,
419 FormatElement *anchor, bool inverted)
420 : thenElements(std::move(thenElements)),
421 elseElements(std::move(elseElements)), thenParseStart(thenParseStart),
422 elseParseStart(elseParseStart), anchor(anchor), inverted(inverted) {}
423
424 /// Return the `then` elements of the optional group. Drops the first
425 /// `thenParseStart` whitespace elements if `parseable` is true.
426 ArrayRef<FormatElement *> getThenElements(bool parseable = false) const {
427 return llvm::ArrayRef(thenElements)
428 .drop_front(N: parseable ? thenParseStart : 0);
429 }
430
431 /// Return the `else` elements of the optional group. Drops the first
432 /// `elseParseStart` whitespace elements if `parseable` is true.
433 ArrayRef<FormatElement *> getElseElements(bool parseable = false) const {
434 return llvm::ArrayRef(elseElements)
435 .drop_front(N: parseable ? elseParseStart : 0);
436 }
437
438 /// Return the anchor of the optional group.
439 FormatElement *getAnchor() const { return anchor; }
440
441 /// Return true if the optional group is inverted.
442 bool isInverted() const { return inverted; }
443
444private:
445 /// The child elements emitted when the anchor is present.
446 std::vector<FormatElement *> thenElements;
447 /// The child elements emitted when the anchor is not present.
448 std::vector<FormatElement *> elseElements;
449 /// The index of the first element that is parsed in `thenElements`. That is,
450 /// the first non-whitespace element.
451 unsigned thenParseStart;
452 /// The index of the first element that is parsed in `elseElements`. That is,
453 /// the first non-whitespace element.
454 unsigned elseParseStart;
455 /// The anchor element of the optional group.
456 FormatElement *anchor;
457 /// Whether the optional group condition is inverted and the anchor element is
458 /// in the else group.
459 bool inverted;
460};
461
462//===----------------------------------------------------------------------===//
463// FormatParserBase
464//===----------------------------------------------------------------------===//
465
466/// Base class for a parser that implements an assembly format. This class
467/// defines a common assembly format syntax and the creation of format elements.
468/// Subclasses will need to implement parsing for the format elements they
469/// support.
470class FormatParser {
471public:
472 /// Vtable anchor.
473 virtual ~FormatParser();
474
475 /// Parse the assembly format.
476 FailureOr<std::vector<FormatElement *>> parse();
477
478protected:
479 /// The current context of the parser when parsing an element.
480 enum Context {
481 /// The element is being parsed in a "top-level" context, i.e. at the top of
482 /// the format or in an optional group.
483 TopLevelContext,
484 /// The element is being parsed as a custom directive child.
485 CustomDirectiveContext,
486 /// The element is being parsed as a type directive child.
487 TypeDirectiveContext,
488 /// The element is being parsed as a reference directive child.
489 RefDirectiveContext,
490 /// The element is being parsed as a struct directive child.
491 StructDirectiveContext
492 };
493
494 /// Create a format parser with the given source manager and a location.
495 explicit FormatParser(llvm::SourceMgr &mgr, llvm::SMLoc loc)
496 : lexer(mgr, loc), curToken(lexer.lexToken()) {}
497
498 /// Allocate and construct a format element.
499 template <typename FormatElementT, typename... Args>
500 FormatElementT *create(Args &&...args) {
501 // FormatElementT *ptr = allocator.Allocate<FormatElementT>();
502 // ::new (ptr) FormatElementT(std::forward<Args>(args)...);
503 // return ptr;
504 auto mem = std::make_unique<FormatElementT>(std::forward<Args>(args)...);
505 FormatElementT *ptr = mem.get();
506 allocator.push_back(std::move(mem));
507 return ptr;
508 }
509
510 //===--------------------------------------------------------------------===//
511 // Element Parsing
512
513 /// Parse a single element of any kind.
514 FailureOr<FormatElement *> parseElement(Context ctx);
515 /// Parse a literal.
516 FailureOr<FormatElement *> parseLiteral(Context ctx);
517 /// Parse a string.
518 FailureOr<FormatElement *> parseString(Context ctx);
519 /// Parse a variable.
520 FailureOr<FormatElement *> parseVariable(Context ctx);
521 /// Parse a directive.
522 FailureOr<FormatElement *> parseDirective(Context ctx);
523 /// Parse an optional group.
524 FailureOr<FormatElement *> parseOptionalGroup(Context ctx);
525 /// Parse a custom directive.
526 FailureOr<FormatElement *> parseCustomDirective(llvm::SMLoc loc, Context ctx);
527 /// Parse a ref directive.
528 FailureOr<FormatElement *> parseRefDirective(SMLoc loc, Context context);
529 /// Parse a qualified directive.
530 FailureOr<FormatElement *> parseQualifiedDirective(SMLoc loc, Context ctx);
531
532 /// Parse a format-specific variable kind.
533 virtual FailureOr<FormatElement *>
534 parseVariableImpl(llvm::SMLoc loc, StringRef name, Context ctx) = 0;
535 /// Parse a format-specific directive kind.
536 virtual FailureOr<FormatElement *>
537 parseDirectiveImpl(llvm::SMLoc loc, FormatToken::Kind kind, Context ctx) = 0;
538
539 //===--------------------------------------------------------------------===//
540 // Format Verification
541
542 /// Verify that the format is well-formed.
543 virtual LogicalResult verify(llvm::SMLoc loc,
544 ArrayRef<FormatElement *> elements) = 0;
545 /// Verify the arguments to a custom directive.
546 virtual LogicalResult
547 verifyCustomDirectiveArguments(llvm::SMLoc loc,
548 ArrayRef<FormatElement *> arguments) = 0;
549 /// Verify the elements of an optional group.
550 virtual LogicalResult
551 verifyOptionalGroupElements(llvm::SMLoc loc,
552 ArrayRef<FormatElement *> elements,
553 FormatElement *anchor) = 0;
554
555 /// Mark 'element' as qualified. If 'element' cannot be qualified an error
556 /// should be emitted and failure returned.
557 virtual LogicalResult markQualified(llvm::SMLoc loc,
558 FormatElement *element) = 0;
559
560 //===--------------------------------------------------------------------===//
561 // Lexer Utilities
562
563 /// Emit an error at the given location.
564 LogicalResult emitError(llvm::SMLoc loc, const Twine &msg) {
565 lexer.emitError(loc, msg);
566 return failure();
567 }
568
569 /// Emit an error and a note at the given notation.
570 LogicalResult emitErrorAndNote(llvm::SMLoc loc, const Twine &msg,
571 const Twine &note) {
572 lexer.emitErrorAndNote(loc, msg, note);
573 return failure();
574 }
575
576 /// Parse a single token of the expected kind.
577 FailureOr<FormatToken> parseToken(FormatToken::Kind kind, const Twine &msg) {
578 if (!curToken.is(kind))
579 return emitError(loc: curToken.getLoc(), msg);
580 FormatToken tok = curToken;
581 consumeToken();
582 return tok;
583 }
584
585 /// Advance the lexer to the next token.
586 void consumeToken() {
587 assert(!curToken.is(FormatToken::eof) && !curToken.is(FormatToken::error) &&
588 "shouldn't advance past EOF or errors");
589 curToken = lexer.lexToken();
590 }
591
592 /// Get the current token.
593 FormatToken peekToken() { return curToken; }
594
595private:
596 /// The format parser retains ownership of the format elements in a bump
597 /// pointer allocator.
598 // FIXME: FormatElement with `std::vector` need to be converted to use
599 // trailing objects.
600 // llvm::BumpPtrAllocator allocator;
601 std::vector<std::unique_ptr<FormatElement>> allocator;
602 /// The format lexer to use.
603 FormatLexer lexer;
604 /// The current token in the lexer.
605 FormatToken curToken;
606};
607
608//===----------------------------------------------------------------------===//
609// Utility Functions
610//===----------------------------------------------------------------------===//
611
612/// Whether a space needs to be emitted before a literal. E.g., two keywords
613/// back-to-back require a space separator, but a keyword followed by '<' does
614/// not require a space.
615bool shouldEmitSpaceBefore(StringRef value, bool lastWasPunctuation);
616
617/// Returns true if the given string can be formatted as a keyword.
618bool canFormatStringAsKeyword(StringRef value,
619 function_ref<void(Twine)> emitError = nullptr);
620
621/// Returns true if the given string is valid format literal element.
622/// If `emitError` is provided, it is invoked with the reason for the failure.
623bool isValidLiteral(StringRef value,
624 function_ref<void(Twine)> emitError = nullptr);
625
626/// Whether a failure in parsing the assembly format should be a fatal error.
627extern llvm::cl::opt<bool> formatErrorIsFatal;
628
629} // namespace tblgen
630} // namespace mlir
631
632#endif // MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_
633