1//===- AsmLexer.cpp - Lexer for Assembly Files ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This class implements the lexer for assembly files.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/MC/MCParser/AsmLexer.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/MC/MCAsmInfo.h"
19#include "llvm/Support/Compiler.h"
20#include "llvm/Support/SMLoc.h"
21#include "llvm/Support/SaveAndRestore.h"
22#include "llvm/Support/raw_ostream.h"
23#include <cassert>
24#include <cctype>
25#include <cstdio>
26#include <cstring>
27#include <string>
28
29using namespace llvm;
30
31SMLoc AsmToken::getLoc() const { return SMLoc::getFromPointer(Ptr: Str.data()); }
32
33SMLoc AsmToken::getEndLoc() const {
34 return SMLoc::getFromPointer(Ptr: Str.data() + Str.size());
35}
36
37SMRange AsmToken::getLocRange() const { return SMRange(getLoc(), getEndLoc()); }
38
39void AsmToken::dump(raw_ostream &OS) const {
40 switch (Kind) {
41 case AsmToken::Error:
42 OS << "error";
43 break;
44 case AsmToken::Identifier:
45 OS << "identifier: " << getString();
46 break;
47 case AsmToken::Integer:
48 OS << "int: " << getString();
49 break;
50 case AsmToken::Real:
51 OS << "real: " << getString();
52 break;
53 case AsmToken::String:
54 OS << "string: " << getString();
55 break;
56
57 // clang-format off
58 case AsmToken::Amp: OS << "Amp"; break;
59 case AsmToken::AmpAmp: OS << "AmpAmp"; break;
60 case AsmToken::At: OS << "At"; break;
61 case AsmToken::BackSlash: OS << "BackSlash"; break;
62 case AsmToken::BigNum: OS << "BigNum"; break;
63 case AsmToken::Caret: OS << "Caret"; break;
64 case AsmToken::Colon: OS << "Colon"; break;
65 case AsmToken::Comma: OS << "Comma"; break;
66 case AsmToken::Comment: OS << "Comment"; break;
67 case AsmToken::Dollar: OS << "Dollar"; break;
68 case AsmToken::Dot: OS << "Dot"; break;
69 case AsmToken::EndOfStatement: OS << "EndOfStatement"; break;
70 case AsmToken::Eof: OS << "Eof"; break;
71 case AsmToken::Equal: OS << "Equal"; break;
72 case AsmToken::EqualEqual: OS << "EqualEqual"; break;
73 case AsmToken::Exclaim: OS << "Exclaim"; break;
74 case AsmToken::ExclaimEqual: OS << "ExclaimEqual"; break;
75 case AsmToken::Greater: OS << "Greater"; break;
76 case AsmToken::GreaterEqual: OS << "GreaterEqual"; break;
77 case AsmToken::GreaterGreater: OS << "GreaterGreater"; break;
78 case AsmToken::Hash: OS << "Hash"; break;
79 case AsmToken::HashDirective: OS << "HashDirective"; break;
80 case AsmToken::LBrac: OS << "LBrac"; break;
81 case AsmToken::LCurly: OS << "LCurly"; break;
82 case AsmToken::LParen: OS << "LParen"; break;
83 case AsmToken::Less: OS << "Less"; break;
84 case AsmToken::LessEqual: OS << "LessEqual"; break;
85 case AsmToken::LessGreater: OS << "LessGreater"; break;
86 case AsmToken::LessLess: OS << "LessLess"; break;
87 case AsmToken::Minus: OS << "Minus"; break;
88 case AsmToken::MinusGreater: OS << "MinusGreater"; break;
89 case AsmToken::Percent: OS << "Percent"; break;
90 case AsmToken::Pipe: OS << "Pipe"; break;
91 case AsmToken::PipePipe: OS << "PipePipe"; break;
92 case AsmToken::Plus: OS << "Plus"; break;
93 case AsmToken::Question: OS << "Question"; break;
94 case AsmToken::RBrac: OS << "RBrac"; break;
95 case AsmToken::RCurly: OS << "RCurly"; break;
96 case AsmToken::RParen: OS << "RParen"; break;
97 case AsmToken::Slash: OS << "Slash"; break;
98 case AsmToken::Space: OS << "Space"; break;
99 case AsmToken::Star: OS << "Star"; break;
100 case AsmToken::Tilde: OS << "Tilde"; break;
101 // clang-format on
102 }
103
104 // Print the token string.
105 OS << " (\"";
106 OS.write_escaped(Str: getString());
107 OS << "\")";
108}
109
110AsmLexer::AsmLexer(const MCAsmInfo &MAI) : MAI(MAI) {
111 // For COFF targets, this is true, while for ELF targets, it should be false.
112 // Currently, @specifier parsing depends on '@' being included in the token.
113 AllowAtInIdentifier = !StringRef(MAI.getCommentString()).starts_with(Prefix: "@") &&
114 MAI.useAtForSpecifier();
115 LexMotorolaIntegers = MAI.shouldUseMotorolaIntegers();
116
117 CurTok.emplace_back(Args: AsmToken::Space, Args: StringRef());
118}
119
120void AsmLexer::setBuffer(StringRef Buf, const char *ptr,
121 bool EndStatementAtEOF) {
122 // Buffer must be NULL-terminated. NULL terminator must reside at `Buf.end()`.
123 // It must be safe to dereference `Buf.end()`.
124 assert(*Buf.end() == '\0' &&
125 "Buffer provided to AsmLexer lacks null terminator.");
126
127 CurBuf = Buf;
128
129 if (ptr)
130 CurPtr = ptr;
131 else
132 CurPtr = CurBuf.begin();
133
134 TokStart = nullptr;
135 this->EndStatementAtEOF = EndStatementAtEOF;
136}
137
138/// ReturnError - Set the error to the specified string at the specified
139/// location. This is defined to always return AsmToken::Error.
140AsmToken AsmLexer::ReturnError(const char *Loc, const std::string &Msg) {
141 SetError(errLoc: SMLoc::getFromPointer(Ptr: Loc), err: Msg);
142
143 return AsmToken(AsmToken::Error, StringRef(Loc, CurPtr - Loc));
144}
145
146int AsmLexer::getNextChar() {
147 if (CurPtr == CurBuf.end())
148 return EOF;
149 return (unsigned char)*CurPtr++;
150}
151
152int AsmLexer::peekNextChar() {
153 if (CurPtr == CurBuf.end())
154 return EOF;
155 return (unsigned char)*CurPtr;
156}
157
158/// The leading integral digit sequence and dot should have already been
159/// consumed, some or all of the fractional digit sequence *can* have been
160/// consumed.
161AsmToken AsmLexer::LexFloatLiteral() {
162 // Skip the fractional digit sequence.
163 while (isDigit(C: *CurPtr))
164 ++CurPtr;
165
166 if (*CurPtr == '-' || *CurPtr == '+')
167 return ReturnError(Loc: CurPtr, Msg: "invalid sign in float literal");
168
169 // Check for exponent
170 if ((*CurPtr == 'e' || *CurPtr == 'E')) {
171 ++CurPtr;
172
173 if (*CurPtr == '-' || *CurPtr == '+')
174 ++CurPtr;
175
176 while (isDigit(C: *CurPtr))
177 ++CurPtr;
178 }
179
180 return AsmToken(AsmToken::Real,
181 StringRef(TokStart, CurPtr - TokStart));
182}
183
184/// LexHexFloatLiteral matches essentially (.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+
185/// while making sure there are enough actual digits around for the constant to
186/// be valid.
187///
188/// The leading "0x[0-9a-fA-F]*" (i.e. integer part) has already been consumed
189/// before we get here.
190AsmToken AsmLexer::LexHexFloatLiteral(bool NoIntDigits) {
191 assert((*CurPtr == 'p' || *CurPtr == 'P' || *CurPtr == '.') &&
192 "unexpected parse state in floating hex");
193 bool NoFracDigits = true;
194
195 // Skip the fractional part if there is one
196 if (*CurPtr == '.') {
197 ++CurPtr;
198
199 const char *FracStart = CurPtr;
200 while (isHexDigit(C: *CurPtr))
201 ++CurPtr;
202
203 NoFracDigits = CurPtr == FracStart;
204 }
205
206 if (NoIntDigits && NoFracDigits)
207 return ReturnError(Loc: TokStart, Msg: "invalid hexadecimal floating-point constant: "
208 "expected at least one significand digit");
209
210 // Make sure we do have some kind of proper exponent part
211 if (*CurPtr != 'p' && *CurPtr != 'P')
212 return ReturnError(Loc: TokStart, Msg: "invalid hexadecimal floating-point constant: "
213 "expected exponent part 'p'");
214 ++CurPtr;
215
216 if (*CurPtr == '+' || *CurPtr == '-')
217 ++CurPtr;
218
219 // N.b. exponent digits are *not* hex
220 const char *ExpStart = CurPtr;
221 while (isDigit(C: *CurPtr))
222 ++CurPtr;
223
224 if (CurPtr == ExpStart)
225 return ReturnError(Loc: TokStart, Msg: "invalid hexadecimal floating-point constant: "
226 "expected at least one exponent digit");
227
228 return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
229}
230
231AsmToken AsmLexer::LexIdentifier() {
232 // Check for floating point literals.
233 if (CurPtr[-1] == '.' && isDigit(C: *CurPtr)) {
234 // Disambiguate a .1243foo identifier from a floating literal.
235 while (isDigit(C: *CurPtr))
236 ++CurPtr;
237
238 if (!isIdentifierChar(C: *CurPtr, AllowAt: AllowAtInIdentifier,
239 AllowHash: AllowHashInIdentifier) ||
240 *CurPtr == 'e' || *CurPtr == 'E')
241 return LexFloatLiteral();
242 }
243
244 while (isIdentifierChar(C: *CurPtr, AllowAt: AllowAtInIdentifier, AllowHash: AllowHashInIdentifier))
245 ++CurPtr;
246
247 // Handle . as a special case.
248 if (CurPtr == TokStart+1 && TokStart[0] == '.')
249 return AsmToken(AsmToken::Dot, StringRef(TokStart, 1));
250
251 return AsmToken(AsmToken::Identifier, StringRef(TokStart, CurPtr - TokStart));
252}
253
254/// LexSlash: Slash: /
255/// C-Style Comment: /* ... */
256/// C-style Comment: // ...
257AsmToken AsmLexer::LexSlash() {
258 if (!MAI.shouldAllowAdditionalComments()) {
259 IsAtStartOfStatement = false;
260 return AsmToken(AsmToken::Slash, StringRef(TokStart, 1));
261 }
262
263 switch (*CurPtr) {
264 case '*':
265 IsAtStartOfStatement = false;
266 break; // C style comment.
267 case '/':
268 ++CurPtr;
269 return LexLineComment();
270 default:
271 IsAtStartOfStatement = false;
272 return AsmToken(AsmToken::Slash, StringRef(TokStart, 1));
273 }
274
275 // C Style comment.
276 ++CurPtr; // skip the star.
277 const char *CommentTextStart = CurPtr;
278 while (CurPtr != CurBuf.end()) {
279 switch (*CurPtr++) {
280 case '*':
281 // End of the comment?
282 if (*CurPtr != '/')
283 break;
284 // If we have a CommentConsumer, notify it about the comment.
285 if (CommentConsumer) {
286 CommentConsumer->HandleComment(
287 Loc: SMLoc::getFromPointer(Ptr: CommentTextStart),
288 CommentText: StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
289 }
290 ++CurPtr; // End the */.
291 return AsmToken(AsmToken::Comment,
292 StringRef(TokStart, CurPtr - TokStart));
293 }
294 }
295 return ReturnError(Loc: TokStart, Msg: "unterminated comment");
296}
297
298/// LexLineComment: Comment: #[^\n]*
299/// : //[^\n]*
300AsmToken AsmLexer::LexLineComment() {
301 // Mark This as an end of statement with a body of the
302 // comment. While it would be nicer to leave this two tokens,
303 // backwards compatability with TargetParsers makes keeping this in this form
304 // better.
305 const char *CommentTextStart = CurPtr;
306 int CurChar = getNextChar();
307 while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
308 CurChar = getNextChar();
309 const char *NewlinePtr = CurPtr;
310 if (CurChar == '\r' && CurPtr != CurBuf.end() && *CurPtr == '\n')
311 ++CurPtr;
312
313 // If we have a CommentConsumer, notify it about the comment.
314 if (CommentConsumer) {
315 CommentConsumer->HandleComment(
316 Loc: SMLoc::getFromPointer(Ptr: CommentTextStart),
317 CommentText: StringRef(CommentTextStart, NewlinePtr - 1 - CommentTextStart));
318 }
319
320 IsAtStartOfLine = true;
321 // This is a whole line comment. leave newline
322 if (IsAtStartOfStatement)
323 return AsmToken(AsmToken::EndOfStatement,
324 StringRef(TokStart, CurPtr - TokStart));
325 IsAtStartOfStatement = true;
326
327 return AsmToken(AsmToken::EndOfStatement,
328 StringRef(TokStart, CurPtr - 1 - TokStart));
329}
330
331static void SkipIgnoredIntegerSuffix(const char *&CurPtr) {
332 // Skip case-insensitive ULL, UL, U, L and LL suffixes.
333 if (CurPtr[0] == 'U' || CurPtr[0] == 'u')
334 ++CurPtr;
335 if (CurPtr[0] == 'L' || CurPtr[0] == 'l')
336 ++CurPtr;
337 if (CurPtr[0] == 'L' || CurPtr[0] == 'l')
338 ++CurPtr;
339}
340
341// Look ahead to search for first non-hex digit, if it's [hH], then we treat the
342// integer as a hexadecimal, possibly with leading zeroes.
343static unsigned doHexLookAhead(const char *&CurPtr, unsigned DefaultRadix,
344 bool LexHex) {
345 const char *FirstNonDec = nullptr;
346 const char *LookAhead = CurPtr;
347 while (true) {
348 if (isDigit(C: *LookAhead)) {
349 ++LookAhead;
350 } else {
351 if (!FirstNonDec)
352 FirstNonDec = LookAhead;
353
354 // Keep going if we are looking for a 'h' suffix.
355 if (LexHex && isHexDigit(C: *LookAhead))
356 ++LookAhead;
357 else
358 break;
359 }
360 }
361 bool isHex = LexHex && (*LookAhead == 'h' || *LookAhead == 'H');
362 CurPtr = isHex || !FirstNonDec ? LookAhead : FirstNonDec;
363 if (isHex)
364 return 16;
365 return DefaultRadix;
366}
367
368static const char *findLastDigit(const char *CurPtr, unsigned DefaultRadix) {
369 while (hexDigitValue(C: *CurPtr) < DefaultRadix) {
370 ++CurPtr;
371 }
372 return CurPtr;
373}
374
375static AsmToken intToken(StringRef Ref, APInt &Value) {
376 if (Value.isIntN(N: 64))
377 return AsmToken(AsmToken::Integer, Ref, Value);
378 return AsmToken(AsmToken::BigNum, Ref, Value);
379}
380
381static std::string radixName(unsigned Radix) {
382 switch (Radix) {
383 case 2:
384 return "binary";
385 case 8:
386 return "octal";
387 case 10:
388 return "decimal";
389 case 16:
390 return "hexadecimal";
391 default:
392 return "base-" + std::to_string(val: Radix);
393 }
394}
395
396/// LexDigit: First character is [0-9].
397/// Local Label: [0-9][:]
398/// Forward/Backward Label: [0-9][fb]
399/// Binary integer: 0b[01]+
400/// Octal integer: 0[0-7]+
401/// Hex integer: 0x[0-9a-fA-F]+ or [0x]?[0-9][0-9a-fA-F]*[hH]
402/// Decimal integer: [1-9][0-9]*
403AsmToken AsmLexer::LexDigit() {
404 // MASM-flavor binary integer: [01]+[yY] (if DefaultRadix < 16, [bByY])
405 // MASM-flavor octal integer: [0-7]+[oOqQ]
406 // MASM-flavor decimal integer: [0-9]+[tT] (if DefaultRadix < 16, [dDtT])
407 // MASM-flavor hexadecimal integer: [0-9][0-9a-fA-F]*[hH]
408 if (LexMasmIntegers && isdigit(CurPtr[-1])) {
409 const char *FirstNonBinary =
410 (CurPtr[-1] != '0' && CurPtr[-1] != '1') ? CurPtr - 1 : nullptr;
411 const char *FirstNonDecimal =
412 (CurPtr[-1] < '0' || CurPtr[-1] > '9') ? CurPtr - 1 : nullptr;
413 const char *OldCurPtr = CurPtr;
414 while (isHexDigit(C: *CurPtr)) {
415 switch (*CurPtr) {
416 default:
417 if (!FirstNonDecimal) {
418 FirstNonDecimal = CurPtr;
419 }
420 [[fallthrough]];
421 case '9':
422 case '8':
423 case '7':
424 case '6':
425 case '5':
426 case '4':
427 case '3':
428 case '2':
429 if (!FirstNonBinary) {
430 FirstNonBinary = CurPtr;
431 }
432 break;
433 case '1':
434 case '0':
435 break;
436 }
437 ++CurPtr;
438 }
439 if (*CurPtr == '.') {
440 // MASM float literals (other than hex floats) always contain a ".", and
441 // are always written in decimal.
442 ++CurPtr;
443 return LexFloatLiteral();
444 }
445
446 if (LexMasmHexFloats && (*CurPtr == 'r' || *CurPtr == 'R')) {
447 ++CurPtr;
448 return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
449 }
450
451 unsigned Radix = 0;
452 if (*CurPtr == 'h' || *CurPtr == 'H') {
453 // hexadecimal number
454 ++CurPtr;
455 Radix = 16;
456 } else if (*CurPtr == 't' || *CurPtr == 'T') {
457 // decimal number
458 ++CurPtr;
459 Radix = 10;
460 } else if (*CurPtr == 'o' || *CurPtr == 'O' || *CurPtr == 'q' ||
461 *CurPtr == 'Q') {
462 // octal number
463 ++CurPtr;
464 Radix = 8;
465 } else if (*CurPtr == 'y' || *CurPtr == 'Y') {
466 // binary number
467 ++CurPtr;
468 Radix = 2;
469 } else if (FirstNonDecimal && FirstNonDecimal + 1 == CurPtr &&
470 DefaultRadix < 14 &&
471 (*FirstNonDecimal == 'd' || *FirstNonDecimal == 'D')) {
472 Radix = 10;
473 } else if (FirstNonBinary && FirstNonBinary + 1 == CurPtr &&
474 DefaultRadix < 12 &&
475 (*FirstNonBinary == 'b' || *FirstNonBinary == 'B')) {
476 Radix = 2;
477 }
478
479 if (Radix) {
480 StringRef Result(TokStart, CurPtr - TokStart);
481 APInt Value(128, 0, true);
482
483 if (Result.drop_back().getAsInteger(Radix, Result&: Value))
484 return ReturnError(Loc: TokStart, Msg: "invalid " + radixName(Radix) + " number");
485
486 // MSVC accepts and ignores type suffices on integer literals.
487 SkipIgnoredIntegerSuffix(CurPtr);
488
489 return intToken(Ref: Result, Value);
490 }
491
492 // default-radix integers, or floating point numbers, fall through
493 CurPtr = OldCurPtr;
494 }
495
496 // MASM default-radix integers: [0-9a-fA-F]+
497 // (All other integer literals have a radix specifier.)
498 if (LexMasmIntegers && UseMasmDefaultRadix) {
499 CurPtr = findLastDigit(CurPtr, DefaultRadix: 16);
500 StringRef Result(TokStart, CurPtr - TokStart);
501
502 APInt Value(128, 0, true);
503 if (Result.getAsInteger(Radix: DefaultRadix, Result&: Value)) {
504 return ReturnError(Loc: TokStart,
505 Msg: "invalid " + radixName(Radix: DefaultRadix) + " number");
506 }
507
508 return intToken(Ref: Result, Value);
509 }
510
511 // Motorola hex integers: $[0-9a-fA-F]+
512 if (LexMotorolaIntegers && CurPtr[-1] == '$') {
513 const char *NumStart = CurPtr;
514 while (isHexDigit(C: CurPtr[0]))
515 ++CurPtr;
516
517 APInt Result(128, 0);
518 if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(Radix: 16, Result))
519 return ReturnError(Loc: TokStart, Msg: "invalid hexadecimal number");
520
521 return intToken(Ref: StringRef(TokStart, CurPtr - TokStart), Value&: Result);
522 }
523
524 // Motorola binary integers: %[01]+
525 if (LexMotorolaIntegers && CurPtr[-1] == '%') {
526 const char *NumStart = CurPtr;
527 while (*CurPtr == '0' || *CurPtr == '1')
528 ++CurPtr;
529
530 APInt Result(128, 0);
531 if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(Radix: 2, Result))
532 return ReturnError(Loc: TokStart, Msg: "invalid binary number");
533
534 return intToken(Ref: StringRef(TokStart, CurPtr - TokStart), Value&: Result);
535 }
536
537 // Decimal integer: [1-9][0-9]*
538 // HLASM-flavour decimal integer: [0-9][0-9]*
539 // FIXME: Later on, support for fb for HLASM has to be added in
540 // as they probably would be needed for asm goto
541 if (LexHLASMIntegers || CurPtr[-1] != '0' || CurPtr[0] == '.') {
542 unsigned Radix = doHexLookAhead(CurPtr, DefaultRadix: 10, LexHex: LexMasmIntegers);
543
544 if (!LexHLASMIntegers) {
545 bool IsHex = Radix == 16;
546 // Check for floating point literals.
547 if (!IsHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
548 if (*CurPtr == '.')
549 ++CurPtr;
550 return LexFloatLiteral();
551 }
552 }
553
554 StringRef Result(TokStart, CurPtr - TokStart);
555
556 APInt Value(128, 0, true);
557 if (Result.getAsInteger(Radix, Result&: Value))
558 return ReturnError(Loc: TokStart, Msg: "invalid " + radixName(Radix) + " number");
559
560 if (!LexHLASMIntegers)
561 // The darwin/x86 (and x86-64) assembler accepts and ignores type
562 // suffices on integer literals.
563 SkipIgnoredIntegerSuffix(CurPtr);
564
565 return intToken(Ref: Result, Value);
566 }
567
568 if (!LexMasmIntegers && ((*CurPtr == 'b') || (*CurPtr == 'B'))) {
569 ++CurPtr;
570 // See if we actually have "0b" as part of something like "jmp 0b\n"
571 if (!isDigit(C: CurPtr[0])) {
572 --CurPtr;
573 StringRef Result(TokStart, CurPtr - TokStart);
574 return AsmToken(AsmToken::Integer, Result, 0);
575 }
576 const char *NumStart = CurPtr;
577 while (CurPtr[0] == '0' || CurPtr[0] == '1')
578 ++CurPtr;
579
580 // Requires at least one binary digit.
581 if (CurPtr == NumStart)
582 return ReturnError(Loc: TokStart, Msg: "invalid binary number");
583
584 StringRef Result(TokStart, CurPtr - TokStart);
585
586 APInt Value(128, 0, true);
587 if (Result.substr(Start: 2).getAsInteger(Radix: 2, Result&: Value))
588 return ReturnError(Loc: TokStart, Msg: "invalid binary number");
589
590 // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
591 // suffixes on integer literals.
592 SkipIgnoredIntegerSuffix(CurPtr);
593
594 return intToken(Ref: Result, Value);
595 }
596
597 if ((*CurPtr == 'x') || (*CurPtr == 'X')) {
598 ++CurPtr;
599 const char *NumStart = CurPtr;
600 while (isHexDigit(C: CurPtr[0]))
601 ++CurPtr;
602
603 // "0x.0p0" is valid, and "0x0p0" (but not "0xp0" for example, which will be
604 // diagnosed by LexHexFloatLiteral).
605 if (CurPtr[0] == '.' || CurPtr[0] == 'p' || CurPtr[0] == 'P')
606 return LexHexFloatLiteral(NoIntDigits: NumStart == CurPtr);
607
608 // Otherwise requires at least one hex digit.
609 if (CurPtr == NumStart)
610 return ReturnError(Loc: CurPtr-2, Msg: "invalid hexadecimal number");
611
612 APInt Result(128, 0);
613 if (StringRef(TokStart, CurPtr - TokStart).getAsInteger(Radix: 0, Result))
614 return ReturnError(Loc: TokStart, Msg: "invalid hexadecimal number");
615
616 // Consume the optional [hH].
617 if (LexMasmIntegers && (*CurPtr == 'h' || *CurPtr == 'H'))
618 ++CurPtr;
619
620 // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
621 // suffixes on integer literals.
622 SkipIgnoredIntegerSuffix(CurPtr);
623
624 return intToken(Ref: StringRef(TokStart, CurPtr - TokStart), Value&: Result);
625 }
626
627 // Either octal or hexadecimal.
628 APInt Value(128, 0, true);
629 unsigned Radix = doHexLookAhead(CurPtr, DefaultRadix: 8, LexHex: LexMasmIntegers);
630 StringRef Result(TokStart, CurPtr - TokStart);
631 if (Result.getAsInteger(Radix, Result&: Value))
632 return ReturnError(Loc: TokStart, Msg: "invalid " + radixName(Radix) + " number");
633
634 // Consume the [hH].
635 if (Radix == 16)
636 ++CurPtr;
637
638 // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
639 // suffixes on integer literals.
640 SkipIgnoredIntegerSuffix(CurPtr);
641
642 return intToken(Ref: Result, Value);
643}
644
645/// LexSingleQuote: Integer: 'b'
646AsmToken AsmLexer::LexSingleQuote() {
647 int CurChar = getNextChar();
648
649 if (LexHLASMStrings)
650 return ReturnError(Loc: TokStart, Msg: "invalid usage of character literals");
651
652 if (LexMasmStrings) {
653 while (CurChar != EOF) {
654 if (CurChar != '\'') {
655 CurChar = getNextChar();
656 } else if (peekNextChar() == '\'') {
657 // In MASM single-quote strings, doubled single-quotes mean an escaped
658 // single quote, so should be lexed in.
659 (void)getNextChar();
660 CurChar = getNextChar();
661 } else {
662 break;
663 }
664 }
665 if (CurChar == EOF)
666 return ReturnError(Loc: TokStart, Msg: "unterminated string constant");
667 return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
668 }
669
670 if (CurChar == '\\')
671 CurChar = getNextChar();
672
673 if (CurChar == EOF)
674 return ReturnError(Loc: TokStart, Msg: "unterminated single quote");
675
676 CurChar = getNextChar();
677
678 if (CurChar != '\'')
679 return ReturnError(Loc: TokStart, Msg: "single quote way too long");
680
681 // The idea here being that 'c' is basically just an integral
682 // constant.
683 StringRef Res = StringRef(TokStart,CurPtr - TokStart);
684 long long Value;
685
686 if (Res.starts_with(Prefix: "\'\\")) {
687 char theChar = Res[2];
688 switch (theChar) {
689 default: Value = theChar; break;
690 case '\'': Value = '\''; break;
691 case 't': Value = '\t'; break;
692 case 'n': Value = '\n'; break;
693 case 'b': Value = '\b'; break;
694 case 'f': Value = '\f'; break;
695 case 'r': Value = '\r'; break;
696 }
697 } else
698 Value = TokStart[1];
699
700 return AsmToken(AsmToken::Integer, Res, Value);
701}
702
703/// LexQuote: String: "..."
704AsmToken AsmLexer::LexQuote() {
705 int CurChar = getNextChar();
706 if (LexHLASMStrings)
707 return ReturnError(Loc: TokStart, Msg: "invalid usage of string literals");
708
709 if (LexMasmStrings) {
710 while (CurChar != EOF) {
711 if (CurChar != '"') {
712 CurChar = getNextChar();
713 } else if (peekNextChar() == '"') {
714 // In MASM double-quoted strings, doubled double-quotes mean an escaped
715 // double quote, so should be lexed in.
716 (void)getNextChar();
717 CurChar = getNextChar();
718 } else {
719 break;
720 }
721 }
722 if (CurChar == EOF)
723 return ReturnError(Loc: TokStart, Msg: "unterminated string constant");
724 return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
725 }
726
727 while (CurChar != '"') {
728 if (CurChar == '\\') {
729 // Allow \", etc.
730 CurChar = getNextChar();
731 }
732
733 if (CurChar == EOF)
734 return ReturnError(Loc: TokStart, Msg: "unterminated string constant");
735
736 CurChar = getNextChar();
737 }
738
739 return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
740}
741
742StringRef AsmLexer::LexUntilEndOfStatement() {
743 TokStart = CurPtr;
744
745 while (!isAtStartOfComment(Ptr: CurPtr) && // Start of line comment.
746 !isAtStatementSeparator(Ptr: CurPtr) && // End of statement marker.
747 *CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
748 ++CurPtr;
749 }
750 return StringRef(TokStart, CurPtr-TokStart);
751}
752
753StringRef AsmLexer::LexUntilEndOfLine() {
754 TokStart = CurPtr;
755
756 while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
757 ++CurPtr;
758 }
759 return StringRef(TokStart, CurPtr-TokStart);
760}
761
762size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
763 bool ShouldSkipSpace) {
764 SaveAndRestore SavedTokenStart(TokStart);
765 SaveAndRestore SavedCurPtr(CurPtr);
766 SaveAndRestore SavedAtStartOfLine(IsAtStartOfLine);
767 SaveAndRestore SavedAtStartOfStatement(IsAtStartOfStatement);
768 SaveAndRestore SavedSkipSpace(SkipSpace, ShouldSkipSpace);
769 SaveAndRestore SavedIsPeeking(IsPeeking, true);
770 std::string SavedErr = getErr();
771 SMLoc SavedErrLoc = getErrLoc();
772
773 size_t ReadCount;
774 for (ReadCount = 0; ReadCount < Buf.size(); ++ReadCount) {
775 AsmToken Token = LexToken();
776
777 Buf[ReadCount] = Token;
778
779 if (Token.is(K: AsmToken::Eof)) {
780 ReadCount++;
781 break;
782 }
783 }
784
785 SetError(errLoc: SavedErrLoc, err: SavedErr);
786 return ReadCount;
787}
788
789bool AsmLexer::isAtStartOfComment(const char *Ptr) {
790 if (MAI.isHLASM() && !IsAtStartOfStatement)
791 return false;
792
793 StringRef CommentString = MAI.getCommentString();
794
795 if (CommentString.size() == 1)
796 return CommentString[0] == Ptr[0];
797
798 // Allow # preprocessor comments also be counted as comments for "##" cases
799 if (CommentString[1] == '#')
800 return CommentString[0] == Ptr[0];
801
802 return strncmp(s1: Ptr, s2: CommentString.data(), n: CommentString.size()) == 0;
803}
804
805bool AsmLexer::isAtStatementSeparator(const char *Ptr) {
806 return strncmp(s1: Ptr, s2: MAI.getSeparatorString(),
807 n: strlen(s: MAI.getSeparatorString())) == 0;
808}
809
810AsmToken AsmLexer::LexToken() {
811 TokStart = CurPtr;
812 // This always consumes at least one character.
813 int CurChar = getNextChar();
814
815 if (!IsPeeking && CurChar == '#' && IsAtStartOfStatement) {
816 // If this starts with a '#', this may be a cpp
817 // hash directive and otherwise a line comment.
818 AsmToken TokenBuf[2];
819 MutableArrayRef<AsmToken> Buf(TokenBuf, 2);
820 size_t num = peekTokens(Buf, ShouldSkipSpace: true);
821 // There cannot be a space preceding this
822 if (IsAtStartOfLine && num == 2 && TokenBuf[0].is(K: AsmToken::Integer) &&
823 TokenBuf[1].is(K: AsmToken::String)) {
824 CurPtr = TokStart; // reset curPtr;
825 StringRef s = LexUntilEndOfLine();
826 UnLex(Token: TokenBuf[1]);
827 UnLex(Token: TokenBuf[0]);
828 return AsmToken(AsmToken::HashDirective, s);
829 }
830
831 if (MAI.shouldAllowAdditionalComments())
832 return LexLineComment();
833 }
834
835 if (isAtStartOfComment(Ptr: TokStart)) {
836 StringRef CommentString = MAI.getCommentString();
837 // For multi-char comment strings, advance CurPtr only if we matched the
838 // full string. This stops us from accidentally eating the newline if the
839 // current line ends in a single comment char.
840 if (CommentString.size() > 1 &&
841 StringRef(TokStart, CommentString.size()) == CommentString) {
842 CurPtr += CommentString.size() - 1;
843 }
844 return LexLineComment();
845 }
846
847 if (isAtStatementSeparator(Ptr: TokStart)) {
848 CurPtr += strlen(s: MAI.getSeparatorString()) - 1;
849 IsAtStartOfLine = true;
850 IsAtStartOfStatement = true;
851 return AsmToken(AsmToken::EndOfStatement,
852 StringRef(TokStart, strlen(s: MAI.getSeparatorString())));
853 }
854
855 // If we're missing a newline at EOF, make sure we still get an
856 // EndOfStatement token before the Eof token.
857 if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
858 IsAtStartOfLine = true;
859 IsAtStartOfStatement = true;
860 return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
861 }
862 IsAtStartOfLine = false;
863 bool OldIsAtStartOfStatement = IsAtStartOfStatement;
864 IsAtStartOfStatement = false;
865 switch (CurChar) {
866 default:
867 // Handle identifier: [a-zA-Z_.$@#?][a-zA-Z0-9_.$@#?]*
868 // Whether or not the lexer accepts '$', '@', '#' and '?' at the start of
869 // an identifier is target-dependent. These characters are handled in the
870 // respective switch cases.
871 if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
872 return LexIdentifier();
873
874 // Unknown character, emit an error.
875 return ReturnError(Loc: TokStart, Msg: "invalid character in input");
876 case EOF:
877 if (EndStatementAtEOF) {
878 IsAtStartOfLine = true;
879 IsAtStartOfStatement = true;
880 }
881 return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
882 case 0:
883 case ' ':
884 case '\t':
885 IsAtStartOfStatement = OldIsAtStartOfStatement;
886 while (*CurPtr == ' ' || *CurPtr == '\t')
887 CurPtr++;
888 if (SkipSpace)
889 return LexToken(); // Ignore whitespace.
890 else
891 return AsmToken(AsmToken::Space, StringRef(TokStart, CurPtr - TokStart));
892 case '\r': {
893 IsAtStartOfLine = true;
894 IsAtStartOfStatement = true;
895 // If this is a CR followed by LF, treat that as one token.
896 if (CurPtr != CurBuf.end() && *CurPtr == '\n')
897 ++CurPtr;
898 return AsmToken(AsmToken::EndOfStatement,
899 StringRef(TokStart, CurPtr - TokStart));
900 }
901 case '\n':
902 IsAtStartOfLine = true;
903 IsAtStartOfStatement = true;
904 return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
905 case ':': return AsmToken(AsmToken::Colon, StringRef(TokStart, 1));
906 case '+': return AsmToken(AsmToken::Plus, StringRef(TokStart, 1));
907 case '~': return AsmToken(AsmToken::Tilde, StringRef(TokStart, 1));
908 case '(': return AsmToken(AsmToken::LParen, StringRef(TokStart, 1));
909 case ')': return AsmToken(AsmToken::RParen, StringRef(TokStart, 1));
910 case '[': return AsmToken(AsmToken::LBrac, StringRef(TokStart, 1));
911 case ']': return AsmToken(AsmToken::RBrac, StringRef(TokStart, 1));
912 case '{': return AsmToken(AsmToken::LCurly, StringRef(TokStart, 1));
913 case '}': return AsmToken(AsmToken::RCurly, StringRef(TokStart, 1));
914 case '*': return AsmToken(AsmToken::Star, StringRef(TokStart, 1));
915 case ',': return AsmToken(AsmToken::Comma, StringRef(TokStart, 1));
916 case '$': {
917 if (LexMotorolaIntegers && isHexDigit(C: *CurPtr))
918 return LexDigit();
919 if (MAI.doesAllowDollarAtStartOfIdentifier())
920 return LexIdentifier();
921 return AsmToken(AsmToken::Dollar, StringRef(TokStart, 1));
922 }
923 case '@':
924 if (MAI.doesAllowAtAtStartOfIdentifier())
925 return LexIdentifier();
926 return AsmToken(AsmToken::At, StringRef(TokStart, 1));
927 case '#':
928 if (MAI.isHLASM())
929 return LexIdentifier();
930 return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
931 case '?':
932 if (MAI.doesAllowQuestionAtStartOfIdentifier())
933 return LexIdentifier();
934 return AsmToken(AsmToken::Question, StringRef(TokStart, 1));
935 case '\\': return AsmToken(AsmToken::BackSlash, StringRef(TokStart, 1));
936 case '=':
937 if (*CurPtr == '=') {
938 ++CurPtr;
939 return AsmToken(AsmToken::EqualEqual, StringRef(TokStart, 2));
940 }
941 return AsmToken(AsmToken::Equal, StringRef(TokStart, 1));
942 case '-':
943 if (*CurPtr == '>') {
944 ++CurPtr;
945 return AsmToken(AsmToken::MinusGreater, StringRef(TokStart, 2));
946 }
947 return AsmToken(AsmToken::Minus, StringRef(TokStart, 1));
948 case '|':
949 if (*CurPtr == '|') {
950 ++CurPtr;
951 return AsmToken(AsmToken::PipePipe, StringRef(TokStart, 2));
952 }
953 return AsmToken(AsmToken::Pipe, StringRef(TokStart, 1));
954 case '^': return AsmToken(AsmToken::Caret, StringRef(TokStart, 1));
955 case '&':
956 if (*CurPtr == '&') {
957 ++CurPtr;
958 return AsmToken(AsmToken::AmpAmp, StringRef(TokStart, 2));
959 }
960 return AsmToken(AsmToken::Amp, StringRef(TokStart, 1));
961 case '!':
962 if (*CurPtr == '=') {
963 ++CurPtr;
964 return AsmToken(AsmToken::ExclaimEqual, StringRef(TokStart, 2));
965 }
966 return AsmToken(AsmToken::Exclaim, StringRef(TokStart, 1));
967 case '%':
968 if (LexMotorolaIntegers && (*CurPtr == '0' || *CurPtr == '1')) {
969 return LexDigit();
970 }
971 return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
972 case '/':
973 IsAtStartOfStatement = OldIsAtStartOfStatement;
974 return LexSlash();
975 case '\'': return LexSingleQuote();
976 case '"': return LexQuote();
977 case '0': case '1': case '2': case '3': case '4':
978 case '5': case '6': case '7': case '8': case '9':
979 return LexDigit();
980 case '<':
981 switch (*CurPtr) {
982 case '<':
983 ++CurPtr;
984 return AsmToken(AsmToken::LessLess, StringRef(TokStart, 2));
985 case '=':
986 ++CurPtr;
987 return AsmToken(AsmToken::LessEqual, StringRef(TokStart, 2));
988 case '>':
989 ++CurPtr;
990 return AsmToken(AsmToken::LessGreater, StringRef(TokStart, 2));
991 default:
992 return AsmToken(AsmToken::Less, StringRef(TokStart, 1));
993 }
994 case '>':
995 switch (*CurPtr) {
996 case '>':
997 ++CurPtr;
998 return AsmToken(AsmToken::GreaterGreater, StringRef(TokStart, 2));
999 case '=':
1000 ++CurPtr;
1001 return AsmToken(AsmToken::GreaterEqual, StringRef(TokStart, 2));
1002 default:
1003 return AsmToken(AsmToken::Greater, StringRef(TokStart, 1));
1004 }
1005
1006 // TODO: Quoted identifiers (objc methods etc)
1007 // local labels: [0-9][:]
1008 // Forward/backward labels: [0-9][fb]
1009 // Integers, fp constants, character constants.
1010 }
1011}
1012