1//===- DialectSymbolParser.cpp - MLIR Dialect Symbol Parser --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the parser for the dialect symbols, such as extended
10// attributes and types.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AsmParserImpl.h"
15#include "Parser.h"
16#include "mlir/AsmParser/AsmParserState.h"
17#include "mlir/IR/AsmState.h"
18#include "mlir/IR/Attributes.h"
19#include "mlir/IR/BuiltinAttributeInterfaces.h"
20#include "mlir/IR/BuiltinAttributes.h"
21#include "mlir/IR/BuiltinTypes.h"
22#include "mlir/IR/Dialect.h"
23#include "mlir/IR/DialectImplementation.h"
24#include "mlir/IR/MLIRContext.h"
25#include "mlir/Support/LLVM.h"
26#include "llvm/Support/MemoryBuffer.h"
27#include "llvm/Support/SourceMgr.h"
28#include <cassert>
29#include <cstddef>
30#include <utility>
31
32using namespace mlir;
33using namespace mlir::detail;
34using llvm::MemoryBuffer;
35using llvm::SourceMgr;
36
37namespace {
38/// This class provides the main implementation of the DialectAsmParser that
39/// allows for dialects to parse attributes and types. This allows for dialect
40/// hooking into the main MLIR parsing logic.
41class CustomDialectAsmParser : public AsmParserImpl<DialectAsmParser> {
42public:
43 CustomDialectAsmParser(StringRef fullSpec, Parser &parser)
44 : AsmParserImpl<DialectAsmParser>(parser.getToken().getLoc(), parser),
45 fullSpec(fullSpec) {}
46 ~CustomDialectAsmParser() override = default;
47
48 /// Returns the full specification of the symbol being parsed. This allows
49 /// for using a separate parser if necessary.
50 StringRef getFullSymbolSpec() const override { return fullSpec; }
51
52private:
53 /// The full symbol specification.
54 StringRef fullSpec;
55};
56} // namespace
57
58///
59/// pretty-dialect-sym-body ::= '<' pretty-dialect-sym-contents+ '>'
60/// pretty-dialect-sym-contents ::= pretty-dialect-sym-body
61/// | '(' pretty-dialect-sym-contents+ ')'
62/// | '[' pretty-dialect-sym-contents+ ']'
63/// | '{' pretty-dialect-sym-contents+ '}'
64/// | '[^[<({>\])}\0]+'
65///
66ParseResult Parser::parseDialectSymbolBody(StringRef &body,
67 bool &isCodeCompletion) {
68 // Symbol bodies are a relatively unstructured format that contains a series
69 // of properly nested punctuation, with anything else in the middle. Scan
70 // ahead to find it and consume it if successful, otherwise emit an error.
71 const char *curPtr = getTokenSpelling().data();
72
73 // Scan over the nested punctuation, bailing out on error and consuming until
74 // we find the end. We know that we're currently looking at the '<', so we can
75 // go until we find the matching '>' character.
76 assert(*curPtr == '<');
77 SmallVector<char, 8> nestedPunctuation;
78 const char *codeCompleteLoc = state.lex.getCodeCompleteLoc();
79
80 // Functor used to emit an unbalanced punctuation error.
81 auto emitPunctError = [&] {
82 return emitError() << "unbalanced '" << nestedPunctuation.back()
83 << "' character in pretty dialect name";
84 };
85 // Functor used to check for unbalanced punctuation.
86 auto checkNestedPunctuation = [&](char expectedToken) -> ParseResult {
87 if (nestedPunctuation.back() != expectedToken)
88 return emitPunctError();
89 nestedPunctuation.pop_back();
90 return success();
91 };
92 const char *curBufferEnd = state.lex.getBufferEnd();
93 do {
94 // Handle code completions, which may appear in the middle of the symbol
95 // body.
96 if (curPtr == codeCompleteLoc) {
97 isCodeCompletion = true;
98 nestedPunctuation.clear();
99 break;
100 }
101
102 if (curBufferEnd == curPtr) {
103 if (!nestedPunctuation.empty())
104 return emitPunctError();
105 return emitError(message: "unexpected nul or EOF in pretty dialect name");
106 }
107
108 char c = *curPtr++;
109 switch (c) {
110 case '\0':
111 if (!nestedPunctuation.empty())
112 return emitPunctError();
113 return emitError(message: "unexpected nul or EOF in pretty dialect name");
114 case '<':
115 case '[':
116 case '(':
117 case '{':
118 nestedPunctuation.push_back(Elt: c);
119 continue;
120
121 case '-':
122 // The sequence `->` is treated as special token.
123 if (curPtr != curBufferEnd && *curPtr == '>')
124 ++curPtr;
125 continue;
126
127 case '>':
128 if (failed(Result: checkNestedPunctuation('<')))
129 return failure();
130 break;
131 case ']':
132 if (failed(Result: checkNestedPunctuation('[')))
133 return failure();
134 break;
135 case ')':
136 if (failed(Result: checkNestedPunctuation('(')))
137 return failure();
138 break;
139 case '}':
140 if (failed(Result: checkNestedPunctuation('{')))
141 return failure();
142 break;
143 case '"': {
144 // Dispatch to the lexer to lex past strings.
145 resetToken(tokPos: curPtr - 1);
146 curPtr = state.curToken.getEndLoc().getPointer();
147
148 // Handle code completions, which may appear in the middle of the symbol
149 // body.
150 if (state.curToken.isCodeCompletion()) {
151 isCodeCompletion = true;
152 nestedPunctuation.clear();
153 break;
154 }
155
156 // Otherwise, ensure this token was actually a string.
157 if (state.curToken.isNot(k: Token::string))
158 return failure();
159 break;
160 }
161
162 default:
163 continue;
164 }
165 } while (!nestedPunctuation.empty());
166
167 // Ok, we succeeded, remember where we stopped, reset the lexer to know it is
168 // consuming all this stuff, and return.
169 resetToken(tokPos: curPtr);
170
171 unsigned length = curPtr - body.begin();
172 body = StringRef(body.data(), length);
173 return success();
174}
175
176/// Parse an extended dialect symbol.
177template <typename Symbol, typename SymbolAliasMap, typename CreateFn>
178static Symbol parseExtendedSymbol(Parser &p, AsmParserState *asmState,
179 SymbolAliasMap &aliases,
180 CreateFn &&createSymbol) {
181 Token tok = p.getToken();
182
183 // Handle code completion of the extended symbol.
184 StringRef identifier = tok.getSpelling().drop_front();
185 if (tok.isCodeCompletion() && identifier.empty())
186 return p.codeCompleteDialectSymbol(aliases);
187
188 // Parse the dialect namespace.
189 SMRange range = p.getToken().getLocRange();
190 SMLoc loc = p.getToken().getLoc();
191 p.consumeToken();
192
193 // Check to see if this is a pretty name.
194 auto [dialectName, symbolData] = identifier.split(Separator: '.');
195 bool isPrettyName = !symbolData.empty() || identifier.back() == '.';
196
197 // Check to see if the symbol has trailing data, i.e. has an immediately
198 // following '<'.
199 bool hasTrailingData =
200 p.getToken().is(k: Token::less) &&
201 identifier.bytes_end() == p.getTokenSpelling().bytes_begin();
202
203 // If there is no '<' token following this, and if the typename contains no
204 // dot, then we are parsing a symbol alias.
205 if (!hasTrailingData && !isPrettyName) {
206 // Check for an alias for this type.
207 auto aliasIt = aliases.find(identifier);
208 if (aliasIt == aliases.end())
209 return (p.emitWrongTokenError(message: "undefined symbol alias id '" + identifier +
210 "'"),
211 nullptr);
212 if (asmState) {
213 if constexpr (std::is_same_v<Symbol, Type>)
214 asmState->addTypeAliasUses(name: identifier, locations: range);
215 else
216 asmState->addAttrAliasUses(name: identifier, locations: range);
217 }
218 return aliasIt->second;
219 }
220
221 // If this isn't an alias, we are parsing a dialect-specific symbol. If the
222 // name contains a dot, then this is the "pretty" form. If not, it is the
223 // verbose form that looks like <...>.
224 if (!isPrettyName) {
225 // Point the symbol data to the end of the dialect name to start.
226 symbolData = StringRef(dialectName.end(), 0);
227
228 // Parse the body of the symbol.
229 bool isCodeCompletion = false;
230 if (p.parseDialectSymbolBody(body&: symbolData, isCodeCompletion))
231 return nullptr;
232 symbolData = symbolData.drop_front();
233
234 // If the body contained a code completion it won't have the trailing `>`
235 // token, so don't drop it.
236 if (!isCodeCompletion)
237 symbolData = symbolData.drop_back();
238 } else {
239 loc = SMLoc::getFromPointer(Ptr: symbolData.data());
240
241 // If the dialect's symbol is followed immediately by a <, then lex the body
242 // of it into prettyName.
243 if (hasTrailingData && p.parseDialectSymbolBody(body&: symbolData))
244 return nullptr;
245 }
246
247 return createSymbol(dialectName, symbolData, loc);
248}
249
250/// Parse an extended attribute.
251///
252/// extended-attribute ::= (dialect-attribute | attribute-alias)
253/// dialect-attribute ::= `#` dialect-namespace `<` attr-data `>`
254/// (`:` type)?
255/// | `#` alias-name pretty-dialect-sym-body? (`:` type)?
256/// attribute-alias ::= `#` alias-name
257///
258Attribute Parser::parseExtendedAttr(Type type) {
259 MLIRContext *ctx = getContext();
260 Attribute attr = parseExtendedSymbol<Attribute>(
261 p&: *this, asmState: state.asmState, aliases&: state.symbols.attributeAliasDefinitions,
262 createSymbol: [&](StringRef dialectName, StringRef symbolData, SMLoc loc) -> Attribute {
263 // Parse an optional trailing colon type.
264 Type attrType = type;
265 if (consumeIf(kind: Token::colon) && !(attrType = parseType()))
266 return Attribute();
267
268 // If we found a registered dialect, then ask it to parse the attribute.
269 if (Dialect *dialect =
270 builder.getContext()->getOrLoadDialect(name: dialectName)) {
271 // Temporarily reset the lexer to let the dialect parse the attribute.
272 const char *curLexerPos = getToken().getLoc().getPointer();
273 resetToken(tokPos: symbolData.data());
274
275 // Parse the attribute.
276 CustomDialectAsmParser customParser(symbolData, *this);
277 Attribute attr = dialect->parseAttribute(parser&: customParser, type: attrType);
278 resetToken(tokPos: curLexerPos);
279 return attr;
280 }
281
282 // Otherwise, form a new opaque attribute.
283 return OpaqueAttr::getChecked(
284 emitError: [&] { return emitError(loc); }, dialect: StringAttr::get(context: ctx, bytes: dialectName),
285 attrData: symbolData, type: attrType ? attrType : NoneType::get(context: ctx));
286 });
287
288 // Ensure that the attribute has the same type as requested.
289 auto typedAttr = dyn_cast_or_null<TypedAttr>(Val&: attr);
290 if (type && typedAttr && typedAttr.getType() != type) {
291 emitError(message: "attribute type different than expected: expected ")
292 << type << ", but got " << typedAttr.getType();
293 return nullptr;
294 }
295 return attr;
296}
297
298/// Parse an extended type.
299///
300/// extended-type ::= (dialect-type | type-alias)
301/// dialect-type ::= `!` dialect-namespace `<` `"` type-data `"` `>`
302/// dialect-type ::= `!` alias-name pretty-dialect-attribute-body?
303/// type-alias ::= `!` alias-name
304///
305Type Parser::parseExtendedType() {
306 MLIRContext *ctx = getContext();
307 return parseExtendedSymbol<Type>(
308 p&: *this, asmState: state.asmState, aliases&: state.symbols.typeAliasDefinitions,
309 createSymbol: [&](StringRef dialectName, StringRef symbolData, SMLoc loc) -> Type {
310 // If we found a registered dialect, then ask it to parse the type.
311 if (auto *dialect = ctx->getOrLoadDialect(name: dialectName)) {
312 // Temporarily reset the lexer to let the dialect parse the type.
313 const char *curLexerPos = getToken().getLoc().getPointer();
314 resetToken(tokPos: symbolData.data());
315
316 // Parse the type.
317 CustomDialectAsmParser customParser(symbolData, *this);
318 Type type = dialect->parseType(parser&: customParser);
319 resetToken(tokPos: curLexerPos);
320 return type;
321 }
322
323 // Otherwise, form a new opaque type.
324 return OpaqueType::getChecked(emitError: [&] { return emitError(loc); },
325 dialectNamespace: StringAttr::get(context: ctx, bytes: dialectName),
326 typeData: symbolData);
327 });
328}
329
330//===----------------------------------------------------------------------===//
331// mlir::parseAttribute/parseType
332//===----------------------------------------------------------------------===//
333
334/// Parses a symbol, of type 'T', and returns it if parsing was successful. If
335/// parsing failed, nullptr is returned.
336template <typename T, typename ParserFn>
337static T parseSymbol(StringRef inputStr, MLIRContext *context,
338 size_t *numReadOut, bool isKnownNullTerminated,
339 ParserFn &&parserFn) {
340 // Set the buffer name to the string being parsed, so that it appears in error
341 // diagnostics.
342 auto memBuffer =
343 isKnownNullTerminated
344 ? MemoryBuffer::getMemBuffer(InputData: inputStr,
345 /*BufferName=*/inputStr)
346 : MemoryBuffer::getMemBufferCopy(InputData: inputStr, /*BufferName=*/inputStr);
347 SourceMgr sourceMgr;
348 sourceMgr.AddNewSourceBuffer(F: std::move(memBuffer), IncludeLoc: SMLoc());
349 SymbolState aliasState;
350 ParserConfig config(context);
351 ParserState state(sourceMgr, config, aliasState, /*asmState=*/nullptr,
352 /*codeCompleteContext=*/nullptr);
353 Parser parser(state);
354
355 Token startTok = parser.getToken();
356 T symbol = parserFn(parser);
357 if (!symbol)
358 return T();
359
360 // Provide the number of bytes that were read.
361 Token endTok = parser.getToken();
362 size_t numRead =
363 endTok.getLoc().getPointer() - startTok.getLoc().getPointer();
364 if (numReadOut) {
365 *numReadOut = numRead;
366 } else if (numRead != inputStr.size()) {
367 parser.emitError(loc: endTok.getLoc()) << "found trailing characters: '"
368 << inputStr.drop_front(N: numRead) << "'";
369 return T();
370 }
371 return symbol;
372}
373
374Attribute mlir::parseAttribute(StringRef attrStr, MLIRContext *context,
375 Type type, size_t *numRead,
376 bool isKnownNullTerminated) {
377 return parseSymbol<Attribute>(
378 inputStr: attrStr, context, numReadOut: numRead, isKnownNullTerminated,
379 parserFn: [type](Parser &parser) { return parser.parseAttribute(type); });
380}
381Type mlir::parseType(StringRef typeStr, MLIRContext *context, size_t *numRead,
382 bool isKnownNullTerminated) {
383 return parseSymbol<Type>(inputStr: typeStr, context, numReadOut: numRead, isKnownNullTerminated,
384 parserFn: [](Parser &parser) { return parser.parseType(); });
385}
386