1//===- TGLexer.h - Lexer for TableGen Files ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This class represents the Lexer for tablegen files.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TABLEGEN_TGLEXER_H
14#define LLVM_LIB_TABLEGEN_TGLEXER_H
15
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/StringSet.h"
19#include "llvm/Support/DataTypes.h"
20#include "llvm/Support/SMLoc.h"
21#include <cassert>
22#include <string>
23
24namespace llvm {
25template <typename T> class ArrayRef;
26class SourceMgr;
27class Twine;
28
29namespace tgtok {
30enum TokKind {
31 // Markers
32 Eof,
33 Error,
34
35 // Tokens with no info.
36 minus, // -
37 plus, // +
38 l_square, // [
39 r_square, // ]
40 l_brace, // {
41 r_brace, // }
42 l_paren, // (
43 r_paren, // )
44 less, // <
45 greater, // >
46 colon, // :
47 semi, // ;
48 comma, // ,
49 dot, // .
50 equal, // =
51 question, // ?
52 paste, // #
53 dotdotdot, // ...
54
55 // Boolean literals.
56 TrueVal,
57 FalseVal,
58
59 // Integer value.
60 IntVal,
61
62 // Binary constant. Note that these are sized according to the number of
63 // bits given.
64 BinaryIntVal,
65
66 // Preprocessing tokens for internal usage by the lexer.
67 // They are never returned as a result of Lex().
68 Ifdef,
69 Ifndef,
70 Else,
71 Endif,
72 Define,
73
74 // Reserved keywords. ('ElseKW' is named to distinguish it from the
75 // existing 'Else' that means the preprocessor #else.)
76 Bit,
77 Bits,
78 Code,
79 Dag,
80 ElseKW,
81 Field,
82 In,
83 Include,
84 Int,
85 List,
86 String,
87 Then,
88
89 // Object start tokens.
90 OBJECT_START_FIRST,
91 Assert = OBJECT_START_FIRST,
92 Class,
93 Def,
94 Defm,
95 Defset,
96 Deftype,
97 Defvar,
98 Dump,
99 Foreach,
100 If,
101 Let,
102 MultiClass,
103 OBJECT_START_LAST = MultiClass,
104
105 // Bang operators.
106 BANG_OPERATOR_FIRST,
107 XConcat = BANG_OPERATOR_FIRST,
108 XADD,
109 XSUB,
110 XMUL,
111 XDIV,
112 XNOT,
113 XLOG2,
114 XAND,
115 XOR,
116 XXOR,
117 XSRA,
118 XSRL,
119 XSHL,
120 XListConcat,
121 XListFlatten,
122 XListSplat,
123 XStrConcat,
124 XInterleave,
125 XSubstr,
126 XFind,
127 XMatch,
128 XCast,
129 XSubst,
130 XForEach,
131 XFilter,
132 XFoldl,
133 XHead,
134 XTail,
135 XSize,
136 XEmpty,
137 XInitialized,
138 XInstances,
139 XIf,
140 XCond,
141 XSwitch,
142 XEq,
143 XIsA,
144 XDag,
145 XNe,
146 XLe,
147 XLt,
148 XGe,
149 XGt,
150 XSetDagOp,
151 XGetDagOp,
152 XSetDagOpName,
153 XGetDagOpName,
154 XExists,
155 XListRemove,
156 XToLower,
157 XToUpper,
158 XRange,
159 XSort,
160 XGetDagArg,
161 XGetDagName,
162 XSetDagArg,
163 XSetDagName,
164 XRepr,
165 BANG_OPERATOR_LAST = XRepr,
166
167 // String valued tokens.
168 STRING_VALUE_FIRST,
169 Id = STRING_VALUE_FIRST,
170 StrVal,
171 VarName,
172 CodeFragment,
173 STRING_VALUE_LAST = CodeFragment,
174};
175
176/// isBangOperator - Return true if this is a bang operator.
177static inline bool isBangOperator(tgtok::TokKind Kind) {
178 return tgtok::BANG_OPERATOR_FIRST <= Kind && Kind <= BANG_OPERATOR_LAST;
179}
180
181/// isObjectStart - Return true if this is a valid first token for a statement.
182static inline bool isObjectStart(tgtok::TokKind Kind) {
183 return tgtok::OBJECT_START_FIRST <= Kind && Kind <= OBJECT_START_LAST;
184}
185
186/// isStringValue - Return true if this is a string value.
187static inline bool isStringValue(tgtok::TokKind Kind) {
188 return tgtok::STRING_VALUE_FIRST <= Kind && Kind <= STRING_VALUE_LAST;
189}
190} // namespace tgtok
191
192/// TGLexer - TableGen Lexer class.
193class TGLexer {
194 SourceMgr &SrcMgr;
195
196 const char *CurPtr = nullptr;
197 StringRef CurBuf;
198
199 // Information about the current token.
200 const char *TokStart = nullptr;
201 tgtok::TokKind CurCode = tgtok::TokKind::Eof;
202 std::string CurStrVal; // This is valid for Id, StrVal, VarName, CodeFragment
203 int64_t CurIntVal = 0; // This is valid for IntVal.
204
205 /// CurBuffer - This is the current buffer index we're lexing from as managed
206 /// by the SourceMgr object.
207 unsigned CurBuffer = 0;
208
209public:
210 TGLexer(SourceMgr &SrcMgr, ArrayRef<std::string> Macros);
211
212 tgtok::TokKind Lex() { return CurCode = LexToken(FileOrLineStart: CurPtr == CurBuf.begin()); }
213
214 tgtok::TokKind getCode() const { return CurCode; }
215
216 const std::string &getCurStrVal() const {
217 assert(tgtok::isStringValue(CurCode) &&
218 "This token doesn't have a string value");
219 return CurStrVal;
220 }
221 int64_t getCurIntVal() const {
222 assert(CurCode == tgtok::IntVal && "This token isn't an integer");
223 return CurIntVal;
224 }
225 std::pair<int64_t, unsigned> getCurBinaryIntVal() const {
226 assert(CurCode == tgtok::BinaryIntVal &&
227 "This token isn't a binary integer");
228 return {CurIntVal, (CurPtr - TokStart) - 2};
229 }
230
231 SMLoc getLoc() const;
232 SMRange getLocRange() const;
233
234private:
235 /// LexToken - Read the next token and return its code.
236 tgtok::TokKind LexToken(bool FileOrLineStart = false);
237
238 tgtok::TokKind ReturnError(SMLoc Loc, const Twine &Msg);
239 tgtok::TokKind ReturnError(const char *Loc, const Twine &Msg);
240
241 int getNextChar();
242 int peekNextChar(int Index) const;
243 void SkipBCPLComment();
244 bool SkipCComment();
245 tgtok::TokKind LexIdentifier();
246 bool LexInclude();
247 tgtok::TokKind LexString();
248 tgtok::TokKind LexVarName();
249 tgtok::TokKind LexNumber();
250 tgtok::TokKind LexBracket();
251 tgtok::TokKind LexExclaim();
252
253 // Process EOF encountered in LexToken().
254 // If EOF is met in an include file, then the method will update
255 // CurPtr, CurBuf and preprocessing include stack, and return true.
256 // If EOF is met in the top-level file, then the method will
257 // update and check the preprocessing include stack, and return false.
258 bool processEOF();
259
260 // *** Structures and methods for preprocessing support ***
261
262 // A set of macro names that are defined either via command line or
263 // by using:
264 // #define NAME
265 StringSet<> DefinedMacros;
266
267 // Each of #ifdef and #else directives has a descriptor associated
268 // with it.
269 //
270 // An ordered list of preprocessing controls defined by #ifdef/#else
271 // directives that are in effect currently is called preprocessing
272 // control stack. It is represented as a vector of PreprocessorControlDesc's.
273 //
274 // The control stack is updated according to the following rules:
275 //
276 // For each #ifdef we add an element to the control stack.
277 // For each #else we replace the top element with a descriptor
278 // with an inverted IsDefined value.
279 // For each #endif we pop the top element from the control stack.
280 //
281 // When CurPtr reaches the current buffer's end, the control stack
282 // must be empty, i.e. #ifdef and the corresponding #endif
283 // must be located in the same file.
284 struct PreprocessorControlDesc {
285 // Either tgtok::Ifdef or tgtok::Else.
286 tgtok::TokKind Kind;
287
288 // True, if the condition for this directive is true, false - otherwise.
289 // Examples:
290 // #ifdef NAME : true, if NAME is defined, false - otherwise.
291 // ...
292 // #else : false, if NAME is defined, true - otherwise.
293 bool IsDefined;
294
295 // Pointer into CurBuf to the beginning of the preprocessing directive
296 // word, e.g.:
297 // #ifdef NAME
298 // ^ - SrcPos
299 SMLoc SrcPos;
300 };
301
302 // We want to disallow code like this:
303 // file1.td:
304 // #define NAME
305 // #ifdef NAME
306 // include "file2.td"
307 // EOF
308 // file2.td:
309 // #endif
310 // EOF
311 //
312 // To do this, we clear the preprocessing control stack on entry
313 // to each of the included file. PrepIncludeStack is used to store
314 // preprocessing control stacks for the current file and all its
315 // parent files. The back() element is the preprocessing control
316 // stack for the current file.
317 SmallVector<SmallVector<PreprocessorControlDesc>> PrepIncludeStack;
318
319 // Validate that the current preprocessing control stack is empty,
320 // since we are about to exit a file, and pop the include stack.
321 //
322 // If IncludeStackMustBeEmpty is true, the include stack must be empty
323 // after the popping, otherwise, the include stack must not be empty
324 // after the popping. Basically, the include stack must be empty
325 // only if we exit the "top-level" file (i.e. finish lexing).
326 //
327 // The method returns false, if the current preprocessing control stack
328 // is not empty (e.g. there is an unterminated #ifdef/#else),
329 // true - otherwise.
330 bool prepExitInclude(bool IncludeStackMustBeEmpty);
331
332 // Look ahead for a preprocessing directive starting from CurPtr. The caller
333 // must only call this method, if *(CurPtr - 1) is '#'. If the method matches
334 // a preprocessing directive word followed by a whitespace, then it returns
335 // one of the internal token kinds, i.e. Ifdef, Else, Endif, Define.
336 //
337 // CurPtr is not adjusted by this method.
338 tgtok::TokKind prepIsDirective() const;
339
340 // Given a preprocessing token kind, adjusts CurPtr to the end
341 // of the preprocessing directive word.
342 //
343 // We use look-ahead prepIsDirective() and prepEatPreprocessorDirective()
344 // to avoid adjusting CurPtr before we are sure that '#' is followed
345 // by a preprocessing directive. If it is not, then we fall back to
346 // tgtok::paste interpretation of '#'.
347 void prepEatPreprocessorDirective(tgtok::TokKind Kind);
348
349 // The main "exit" point from the token parsing to preprocessor.
350 //
351 // The method is called for CurPtr, when prepIsDirective() returns
352 // true. The first parameter matches the result of prepIsDirective(),
353 // denoting the actual preprocessor directive to be processed.
354 //
355 // If the preprocessing directive disables the tokens processing, e.g.:
356 // #ifdef NAME // NAME is undefined
357 // then lexPreprocessor() enters the lines-skipping mode.
358 // In this mode, it does not parse any tokens, because the code under
359 // the #ifdef may not even be a correct tablegen code. The preprocessor
360 // looks for lines containing other preprocessing directives, which
361 // may be prepended with whitespaces and C-style comments. If the line
362 // does not contain a preprocessing directive, it is skipped completely.
363 // Otherwise, the preprocessing directive is processed by recursively
364 // calling lexPreprocessor(). The processing of the encountered
365 // preprocessing directives includes updating preprocessing control stack
366 // and adding new macros into DefinedMacros set.
367 //
368 // The second parameter controls whether lexPreprocessor() is called from
369 // LexToken() (true) or recursively from lexPreprocessor() (false).
370 //
371 // If ReturnNextLiveToken is true, the method returns the next
372 // LEX token following the current directive or following the end
373 // of the disabled preprocessing region corresponding to this directive.
374 // If ReturnNextLiveToken is false, the method returns the first parameter,
375 // unless there were errors encountered in the disabled preprocessing
376 // region - in this case, it returns tgtok::Error.
377 tgtok::TokKind lexPreprocessor(tgtok::TokKind Kind,
378 bool ReturnNextLiveToken = true);
379
380 // Worker method for lexPreprocessor() to skip lines after some
381 // preprocessing directive up to the buffer end or to the directive
382 // that re-enables token processing. The method returns true
383 // upon processing the next directive that re-enables tokens
384 // processing. False is returned if an error was encountered.
385 //
386 // Note that prepSkipRegion() calls lexPreprocessor() to process
387 // encountered preprocessing directives. In this case, the second
388 // parameter to lexPreprocessor() is set to false. Being passed
389 // false ReturnNextLiveToken, lexPreprocessor() must never call
390 // prepSkipRegion(). We assert this by passing ReturnNextLiveToken
391 // to prepSkipRegion() and checking that it is never set to false.
392 bool prepSkipRegion(bool MustNeverBeFalse);
393
394 // Lex name of the macro after either #ifdef or #define. We could have used
395 // LexIdentifier(), but it has special handling of "include" word, which
396 // could result in awkward diagnostic errors. Consider:
397 // ----
398 // #ifdef include
399 // class ...
400 // ----
401 // LexIdentifier() will engage LexInclude(), which will complain about
402 // missing file with name "class". Instead, prepLexMacroName() will treat
403 // "include" as a normal macro name.
404 //
405 // On entry, CurPtr points to the end of a preprocessing directive word.
406 // The method allows for whitespaces between the preprocessing directive
407 // and the macro name. The allowed whitespaces are ' ' and '\t'.
408 //
409 // If the first non-whitespace symbol after the preprocessing directive
410 // is a valid start symbol for an identifier (i.e. [a-zA-Z_]), then
411 // the method updates TokStart to the position of the first non-whitespace
412 // symbol, sets CurPtr to the position of the macro name's last symbol,
413 // and returns a string reference to the macro name. Otherwise,
414 // TokStart is set to the first non-whitespace symbol after the preprocessing
415 // directive, and the method returns an empty string reference.
416 //
417 // In all cases, TokStart may be used to point to the word following
418 // the preprocessing directive.
419 StringRef prepLexMacroName();
420
421 // Skip any whitespaces starting from CurPtr. The method is used
422 // only in the lines-skipping mode to find the first non-whitespace
423 // symbol after or at CurPtr. Allowed whitespaces are ' ', '\t', '\n'
424 // and '\r'. The method skips C-style comments as well, because
425 // it is used to find the beginning of the preprocessing directive.
426 // If we do not handle C-style comments the following code would
427 // result in incorrect detection of a preprocessing directive:
428 // /*
429 // #ifdef NAME
430 // */
431 // As long as we skip C-style comments, the following code is correctly
432 // recognized as a preprocessing directive:
433 // /* first line comment
434 // second line comment */ #ifdef NAME
435 //
436 // The method returns true upon reaching the first non-whitespace symbol
437 // or EOF, CurPtr is set to point to this symbol. The method returns false,
438 // if an error occurred during skipping of a C-style comment.
439 bool prepSkipLineBegin();
440
441 // Skip any whitespaces or comments after a preprocessing directive.
442 // The method returns true upon reaching either end of the line
443 // or end of the file. If there is a multiline C-style comment
444 // after the preprocessing directive, the method skips
445 // the comment, so the final CurPtr may point to one of the next lines.
446 // The method returns false, if an error occurred during skipping
447 // C- or C++-style comment, or a non-whitespace symbol appears
448 // after the preprocessing directive.
449 //
450 // The method maybe called both during lines-skipping and tokens
451 // processing. It actually verifies that only whitespaces or/and
452 // comments follow a preprocessing directive.
453 //
454 // After the execution of this mehod, CurPtr points either to new line
455 // symbol, buffer end or non-whitespace symbol following the preprocesing
456 // directive.
457 bool prepSkipDirectiveEnd();
458
459 // Return true, if the current preprocessor control stack is such that
460 // we should allow lexer to process the next token, false - otherwise.
461 //
462 // In particular, the method returns true, if all the #ifdef/#else
463 // controls on the stack have their IsDefined member set to true.
464 bool prepIsProcessingEnabled();
465
466 // Report an error, if we reach EOF with non-empty preprocessing control
467 // stack. This means there is no matching #endif for the previous
468 // #ifdef/#else.
469 void prepReportPreprocessorStackError();
470};
471
472} // end namespace llvm
473
474#endif
475