1//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Lexer and Token interfaces.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Lex/Lexer.h"
14#include "UnicodeCharSets.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/Diagnostic.h"
17#include "clang/Basic/IdentifierTable.h"
18#include "clang/Basic/LLVM.h"
19#include "clang/Basic/LangOptions.h"
20#include "clang/Basic/SourceLocation.h"
21#include "clang/Basic/SourceManager.h"
22#include "clang/Basic/TokenKinds.h"
23#include "clang/Lex/LexDiagnostic.h"
24#include "clang/Lex/LiteralSupport.h"
25#include "clang/Lex/MultipleIncludeOpt.h"
26#include "clang/Lex/Preprocessor.h"
27#include "clang/Lex/PreprocessorOptions.h"
28#include "clang/Lex/Token.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/StringSwitch.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/ConvertUTF.h"
35#include "llvm/Support/MemoryBufferRef.h"
36#include "llvm/Support/NativeFormatting.h"
37#include "llvm/Support/Unicode.h"
38#include "llvm/Support/UnicodeCharRanges.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <cstring>
44#include <optional>
45#include <string>
46#include <tuple>
47
48#ifdef __SSE4_2__
49#include <nmmintrin.h>
50#endif
51
52using namespace clang;
53
54//===----------------------------------------------------------------------===//
55// Token Class Implementation
56//===----------------------------------------------------------------------===//
57
58/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
59bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
60 if (isAnnotation())
61 return false;
62 if (const IdentifierInfo *II = getIdentifierInfo())
63 return II->getObjCKeywordID() == objcKey;
64 return false;
65}
66
67/// getObjCKeywordID - Return the ObjC keyword kind.
68tok::ObjCKeywordKind Token::getObjCKeywordID() const {
69 if (isAnnotation())
70 return tok::objc_not_keyword;
71 const IdentifierInfo *specId = getIdentifierInfo();
72 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
73}
74
75/// Determine whether the token kind starts a simple-type-specifier.
76bool Token::isSimpleTypeSpecifier(const LangOptions &LangOpts) const {
77 switch (getKind()) {
78 case tok::annot_typename:
79 case tok::annot_decltype:
80 case tok::annot_pack_indexing_type:
81 return true;
82
83 case tok::kw_short:
84 case tok::kw_long:
85 case tok::kw___int64:
86 case tok::kw___int128:
87 case tok::kw_signed:
88 case tok::kw_unsigned:
89 case tok::kw_void:
90 case tok::kw_char:
91 case tok::kw_int:
92 case tok::kw_half:
93 case tok::kw_float:
94 case tok::kw_double:
95 case tok::kw___bf16:
96 case tok::kw__Float16:
97 case tok::kw___float128:
98 case tok::kw___ibm128:
99 case tok::kw_wchar_t:
100 case tok::kw_bool:
101 case tok::kw__Bool:
102 case tok::kw__Accum:
103 case tok::kw__Fract:
104 case tok::kw__Sat:
105#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
106#include "clang/Basic/TransformTypeTraits.def"
107 case tok::kw___auto_type:
108 case tok::kw_char16_t:
109 case tok::kw_char32_t:
110 case tok::kw_typeof:
111 case tok::kw_decltype:
112 case tok::kw_char8_t:
113 return getIdentifierInfo()->isKeyword(LangOpts);
114
115 default:
116 return false;
117 }
118}
119
120//===----------------------------------------------------------------------===//
121// Lexer Class Implementation
122//===----------------------------------------------------------------------===//
123
124void Lexer::anchor() {}
125
126void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
127 const char *BufEnd) {
128 BufferStart = BufStart;
129 BufferPtr = BufPtr;
130 BufferEnd = BufEnd;
131
132 assert(BufEnd[0] == 0 &&
133 "We assume that the input buffer has a null character at the end"
134 " to simplify lexing!");
135
136 // Check whether we have a BOM in the beginning of the buffer. If yes - act
137 // accordingly. Right now we support only UTF-8 with and without BOM, so, just
138 // skip the UTF-8 BOM if it's present.
139 if (BufferStart == BufferPtr) {
140 // Determine the size of the BOM.
141 StringRef Buf(BufferStart, BufferEnd - BufferStart);
142 size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
143 .StartsWith(S: "\xEF\xBB\xBF", Value: 3) // UTF-8 BOM
144 .Default(Value: 0);
145
146 // Skip the BOM.
147 BufferPtr += BOMLength;
148 }
149
150 Is_PragmaLexer = false;
151 CurrentConflictMarkerState = CMK_None;
152
153 // Start of the file is a start of line.
154 IsAtStartOfLine = true;
155 IsAtPhysicalStartOfLine = true;
156
157 HasLeadingSpace = false;
158 HasLeadingEmptyMacro = false;
159
160 // We are not after parsing a #.
161 ParsingPreprocessorDirective = false;
162
163 // We are not after parsing #include.
164 ParsingFilename = false;
165
166 // We are not in raw mode. Raw mode disables diagnostics and interpretation
167 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
168 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
169 // or otherwise skipping over tokens.
170 LexingRawMode = false;
171
172 // Default to not keeping comments.
173 ExtendedTokenMode = 0;
174
175 NewLinePtr = nullptr;
176
177 IsFirstPPToken = true;
178}
179
180/// Lexer constructor - Create a new lexer object for the specified buffer
181/// with the specified preprocessor managing the lexing process. This lexer
182/// assumes that the associated file buffer and Preprocessor objects will
183/// outlive it, so it doesn't take ownership of either of them.
184Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
185 Preprocessor &PP, bool IsFirstIncludeOfFile)
186 : PreprocessorLexer(&PP, FID),
187 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
188 LangOpts(PP.getLangOpts()), LineComment(LangOpts.LineComment),
189 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
190 InitLexer(BufStart: InputFile.getBufferStart(), BufPtr: InputFile.getBufferStart(),
191 BufEnd: InputFile.getBufferEnd());
192
193 resetExtendedTokenMode();
194}
195
196/// Lexer constructor - Create a new raw lexer object. This object is only
197/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
198/// range will outlive it, so it doesn't take ownership of it.
199Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
200 const char *BufStart, const char *BufPtr, const char *BufEnd,
201 bool IsFirstIncludeOfFile)
202 : FileLoc(fileloc), LangOpts(langOpts), LineComment(LangOpts.LineComment),
203 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
204 InitLexer(BufStart, BufPtr, BufEnd);
205
206 // We *are* in raw mode.
207 LexingRawMode = true;
208}
209
210/// Lexer constructor - Create a new raw lexer object. This object is only
211/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
212/// range will outlive it, so it doesn't take ownership of it.
213Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
214 const SourceManager &SM, const LangOptions &langOpts,
215 bool IsFirstIncludeOfFile)
216 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
217 FromFile.getBufferStart(), FromFile.getBufferEnd(),
218 IsFirstIncludeOfFile) {}
219
220void Lexer::resetExtendedTokenMode() {
221 assert(PP && "Cannot reset token mode without a preprocessor");
222 if (LangOpts.TraditionalCPP)
223 SetKeepWhitespaceMode(true);
224 else
225 SetCommentRetentionState(PP->getCommentRetentionState());
226}
227
228/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
229/// _Pragma expansion. This has a variety of magic semantics that this method
230/// sets up. It returns a new'd Lexer that must be delete'd when done.
231///
232/// On entrance to this routine, TokStartLoc is a macro location which has a
233/// spelling loc that indicates the bytes to be lexed for the token and an
234/// expansion location that indicates where all lexed tokens should be
235/// "expanded from".
236///
237/// TODO: It would really be nice to make _Pragma just be a wrapper around a
238/// normal lexer that remaps tokens as they fly by. This would require making
239/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
240/// interface that could handle this stuff. This would pull GetMappedTokenLoc
241/// out of the critical path of the lexer!
242///
243Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
244 SourceLocation ExpansionLocStart,
245 SourceLocation ExpansionLocEnd,
246 unsigned TokLen, Preprocessor &PP) {
247 SourceManager &SM = PP.getSourceManager();
248
249 // Create the lexer as if we were going to lex the file normally.
250 FileID SpellingFID = SM.getFileID(SpellingLoc);
251 llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(FID: SpellingFID);
252 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
253
254 // Now that the lexer is created, change the start/end locations so that we
255 // just lex the subsection of the file that we want. This is lexing from a
256 // scratch buffer.
257 const char *StrData = SM.getCharacterData(SL: SpellingLoc);
258
259 L->BufferPtr = StrData;
260 L->BufferEnd = StrData+TokLen;
261 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
262
263 // Set the SourceLocation with the remapping information. This ensures that
264 // GetMappedTokenLoc will remap the tokens as they are lexed.
265 L->FileLoc = SM.createExpansionLoc(SpellingLoc: SM.getLocForStartOfFile(FID: SpellingFID),
266 ExpansionLocStart,
267 ExpansionLocEnd, Length: TokLen);
268
269 // Ensure that the lexer thinks it is inside a directive, so that end \n will
270 // return an EOD token.
271 L->ParsingPreprocessorDirective = true;
272
273 // This lexer really is for _Pragma.
274 L->Is_PragmaLexer = true;
275 return L;
276}
277
278void Lexer::seek(unsigned Offset, bool IsAtStartOfLine) {
279 this->IsAtPhysicalStartOfLine = IsAtStartOfLine;
280 this->IsAtStartOfLine = IsAtStartOfLine;
281 assert((BufferStart + Offset) <= BufferEnd);
282 BufferPtr = BufferStart + Offset;
283}
284
285template <typename T> static void StringifyImpl(T &Str, char Quote) {
286 typename T::size_type i = 0, e = Str.size();
287 while (i < e) {
288 if (Str[i] == '\\' || Str[i] == Quote) {
289 Str.insert(Str.begin() + i, '\\');
290 i += 2;
291 ++e;
292 } else if (Str[i] == '\n' || Str[i] == '\r') {
293 // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
294 if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') &&
295 Str[i] != Str[i + 1]) {
296 Str[i] = '\\';
297 Str[i + 1] = 'n';
298 } else {
299 // Replace '\n' and '\r' to '\\' followed by 'n'.
300 Str[i] = '\\';
301 Str.insert(Str.begin() + i + 1, 'n');
302 ++e;
303 }
304 i += 2;
305 } else
306 ++i;
307 }
308}
309
310std::string Lexer::Stringify(StringRef Str, bool Charify) {
311 std::string Result = std::string(Str);
312 char Quote = Charify ? '\'' : '"';
313 StringifyImpl(Str&: Result, Quote);
314 return Result;
315}
316
317void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, Quote: '"'); }
318
319//===----------------------------------------------------------------------===//
320// Token Spelling
321//===----------------------------------------------------------------------===//
322
323/// Slow case of getSpelling. Extract the characters comprising the
324/// spelling of this token from the provided input buffer.
325static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
326 const LangOptions &LangOpts, char *Spelling) {
327 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
328
329 size_t Length = 0;
330 const char *BufEnd = BufPtr + Tok.getLength();
331
332 if (tok::isStringLiteral(K: Tok.getKind())) {
333 // Munch the encoding-prefix and opening double-quote.
334 while (BufPtr < BufEnd) {
335 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
336 Spelling[Length++] = CharAndSize.Char;
337 BufPtr += CharAndSize.Size;
338
339 if (Spelling[Length - 1] == '"')
340 break;
341 }
342
343 // Raw string literals need special handling; trigraph expansion and line
344 // splicing do not occur within their d-char-sequence nor within their
345 // r-char-sequence.
346 if (Length >= 2 &&
347 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') {
348 // Search backwards from the end of the token to find the matching closing
349 // quote.
350 const char *RawEnd = BufEnd;
351 do --RawEnd; while (*RawEnd != '"');
352 size_t RawLength = RawEnd - BufPtr + 1;
353
354 // Everything between the quotes is included verbatim in the spelling.
355 memcpy(dest: Spelling + Length, src: BufPtr, n: RawLength);
356 Length += RawLength;
357 BufPtr += RawLength;
358
359 // The rest of the token is lexed normally.
360 }
361 }
362
363 while (BufPtr < BufEnd) {
364 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
365 Spelling[Length++] = CharAndSize.Char;
366 BufPtr += CharAndSize.Size;
367 }
368
369 assert(Length < Tok.getLength() &&
370 "NeedsCleaning flag set on token that didn't need cleaning!");
371 return Length;
372}
373
374/// getSpelling() - Return the 'spelling' of this token. The spelling of a
375/// token are the characters used to represent the token in the source file
376/// after trigraph expansion and escaped-newline folding. In particular, this
377/// wants to get the true, uncanonicalized, spelling of things like digraphs
378/// UCNs, etc.
379StringRef Lexer::getSpelling(SourceLocation loc,
380 SmallVectorImpl<char> &buffer,
381 const SourceManager &SM,
382 const LangOptions &options,
383 bool *invalid) {
384 // Break down the source location.
385 FileIDAndOffset locInfo = SM.getDecomposedLoc(Loc: loc);
386
387 // Try to the load the file buffer.
388 bool invalidTemp = false;
389 StringRef file = SM.getBufferData(FID: locInfo.first, Invalid: &invalidTemp);
390 if (invalidTemp) {
391 if (invalid) *invalid = true;
392 return {};
393 }
394
395 const char *tokenBegin = file.data() + locInfo.second;
396
397 // Lex from the start of the given location.
398 Lexer lexer(SM.getLocForStartOfFile(FID: locInfo.first), options,
399 file.begin(), tokenBegin, file.end());
400 Token token;
401 lexer.LexFromRawLexer(Result&: token);
402
403 unsigned length = token.getLength();
404
405 // Common case: no need for cleaning.
406 if (!token.needsCleaning())
407 return StringRef(tokenBegin, length);
408
409 // Hard case, we need to relex the characters into the string.
410 buffer.resize(N: length);
411 buffer.resize(N: getSpellingSlow(Tok: token, BufPtr: tokenBegin, LangOpts: options, Spelling: buffer.data()));
412 return StringRef(buffer.data(), buffer.size());
413}
414
415/// getSpelling() - Return the 'spelling' of this token. The spelling of a
416/// token are the characters used to represent the token in the source file
417/// after trigraph expansion and escaped-newline folding. In particular, this
418/// wants to get the true, uncanonicalized, spelling of things like digraphs
419/// UCNs, etc.
420std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
421 const LangOptions &LangOpts, bool *Invalid) {
422 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
423
424 bool CharDataInvalid = false;
425 const char *TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(),
426 Invalid: &CharDataInvalid);
427 if (Invalid)
428 *Invalid = CharDataInvalid;
429 if (CharDataInvalid)
430 return {};
431
432 // If this token contains nothing interesting, return it directly.
433 if (!Tok.needsCleaning())
434 return std::string(TokStart, TokStart + Tok.getLength());
435
436 std::string Result;
437 Result.resize(n: Tok.getLength());
438 Result.resize(n: getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: &*Result.begin()));
439 return Result;
440}
441
442/// getSpelling - This method is used to get the spelling of a token into a
443/// preallocated buffer, instead of as an std::string. The caller is required
444/// to allocate enough space for the token, which is guaranteed to be at least
445/// Tok.getLength() bytes long. The actual length of the token is returned.
446///
447/// Note that this method may do two possible things: it may either fill in
448/// the buffer specified with characters, or it may *change the input pointer*
449/// to point to a constant buffer with the data already in it (avoiding a
450/// copy). The caller is not allowed to modify the returned buffer pointer
451/// if an internal buffer is returned.
452unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
453 const SourceManager &SourceMgr,
454 const LangOptions &LangOpts, bool *Invalid) {
455 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
456
457 const char *TokStart = nullptr;
458 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
459 if (Tok.is(K: tok::raw_identifier))
460 TokStart = Tok.getRawIdentifier().data();
461 else if (!Tok.hasUCN()) {
462 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
463 // Just return the string from the identifier table, which is very quick.
464 Buffer = II->getNameStart();
465 return II->getLength();
466 }
467 }
468
469 // NOTE: this can be checked even after testing for an IdentifierInfo.
470 if (Tok.isLiteral())
471 TokStart = Tok.getLiteralData();
472
473 if (!TokStart) {
474 // Compute the start of the token in the input lexer buffer.
475 bool CharDataInvalid = false;
476 TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(), Invalid: &CharDataInvalid);
477 if (Invalid)
478 *Invalid = CharDataInvalid;
479 if (CharDataInvalid) {
480 Buffer = "";
481 return 0;
482 }
483 }
484
485 // If this token contains nothing interesting, return it directly.
486 if (!Tok.needsCleaning()) {
487 Buffer = TokStart;
488 return Tok.getLength();
489 }
490
491 // Otherwise, hard case, relex the characters into the string.
492 return getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: const_cast<char*>(Buffer));
493}
494
495/// MeasureTokenLength - Relex the token at the specified location and return
496/// its length in bytes in the input file. If the token needs cleaning (e.g.
497/// includes a trigraph or an escaped newline) then this count includes bytes
498/// that are part of that.
499unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
500 const SourceManager &SM,
501 const LangOptions &LangOpts) {
502 Token TheTok;
503 if (getRawToken(Loc, Result&: TheTok, SM, LangOpts))
504 return 0;
505 return TheTok.getLength();
506}
507
508/// Relex the token at the specified location.
509/// \returns true if there was a failure, false on success.
510bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
511 const SourceManager &SM,
512 const LangOptions &LangOpts,
513 bool IgnoreWhiteSpace) {
514 // TODO: this could be special cased for common tokens like identifiers, ')',
515 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
516 // all obviously single-char tokens. This could use
517 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
518 // something.
519
520 // If this comes from a macro expansion, we really do want the macro name, not
521 // the token this macro expanded to.
522 Loc = SM.getExpansionLoc(Loc);
523 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
524 bool Invalid = false;
525 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
526 if (Invalid)
527 return true;
528
529 const char *StrData = Buffer.data()+LocInfo.second;
530
531 if (!IgnoreWhiteSpace && isWhitespace(c: SkipEscapedNewLines(P: StrData)[0]))
532 return true;
533
534 // Create a lexer starting at the beginning of this token.
535 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
536 Buffer.begin(), StrData, Buffer.end());
537 TheLexer.SetCommentRetentionState(true);
538 TheLexer.LexFromRawLexer(Result);
539 return false;
540}
541
542/// Returns the pointer that points to the beginning of line that contains
543/// the given offset, or null if the offset if invalid.
544static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
545 const char *BufStart = Buffer.data();
546 if (Offset >= Buffer.size())
547 return nullptr;
548
549 const char *LexStart = BufStart + Offset;
550 for (; LexStart != BufStart; --LexStart) {
551 if (isVerticalWhitespace(c: LexStart[0]) &&
552 !Lexer::isNewLineEscaped(BufferStart: BufStart, Str: LexStart)) {
553 // LexStart should point at first character of logical line.
554 ++LexStart;
555 break;
556 }
557 }
558 return LexStart;
559}
560
561static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
562 const SourceManager &SM,
563 const LangOptions &LangOpts) {
564 assert(Loc.isFileID());
565 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
566 if (LocInfo.first.isInvalid())
567 return Loc;
568
569 bool Invalid = false;
570 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
571 if (Invalid)
572 return Loc;
573
574 // Back up from the current location until we hit the beginning of a line
575 // (or the buffer). We'll relex from that point.
576 const char *StrData = Buffer.data() + LocInfo.second;
577 const char *LexStart = findBeginningOfLine(Buffer, Offset: LocInfo.second);
578 if (!LexStart || LexStart == StrData)
579 return Loc;
580
581 // Create a lexer starting at the beginning of this token.
582 SourceLocation LexerStartLoc = Loc.getLocWithOffset(Offset: -LocInfo.second);
583 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
584 Buffer.end());
585 TheLexer.SetCommentRetentionState(true);
586
587 // Lex tokens until we find the token that contains the source location.
588 Token TheTok;
589 do {
590 TheLexer.LexFromRawLexer(Result&: TheTok);
591
592 if (TheLexer.getBufferLocation() > StrData) {
593 // Lexing this token has taken the lexer past the source location we're
594 // looking for. If the current token encompasses our source location,
595 // return the beginning of that token.
596 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
597 return TheTok.getLocation();
598
599 // We ended up skipping over the source location entirely, which means
600 // that it points into whitespace. We're done here.
601 break;
602 }
603 } while (TheTok.getKind() != tok::eof);
604
605 // We've passed our source location; just return the original source location.
606 return Loc;
607}
608
609SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
610 const SourceManager &SM,
611 const LangOptions &LangOpts) {
612 if (Loc.isFileID())
613 return getBeginningOfFileToken(Loc, SM, LangOpts);
614
615 if (!SM.isMacroArgExpansion(Loc))
616 return Loc;
617
618 SourceLocation FileLoc = SM.getSpellingLoc(Loc);
619 SourceLocation BeginFileLoc = getBeginningOfFileToken(Loc: FileLoc, SM, LangOpts);
620 FileIDAndOffset FileLocInfo = SM.getDecomposedLoc(Loc: FileLoc);
621 FileIDAndOffset BeginFileLocInfo = SM.getDecomposedLoc(Loc: BeginFileLoc);
622 assert(FileLocInfo.first == BeginFileLocInfo.first &&
623 FileLocInfo.second >= BeginFileLocInfo.second);
624 return Loc.getLocWithOffset(Offset: BeginFileLocInfo.second - FileLocInfo.second);
625}
626
627namespace {
628
629enum PreambleDirectiveKind {
630 PDK_Skipped,
631 PDK_Unknown
632};
633
634} // namespace
635
636PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
637 const LangOptions &LangOpts,
638 unsigned MaxLines) {
639 // Create a lexer starting at the beginning of the file. Note that we use a
640 // "fake" file source location at offset 1 so that the lexer will track our
641 // position within the file.
642 const SourceLocation::UIntTy StartOffset = 1;
643 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(Encoding: StartOffset);
644 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
645 Buffer.end());
646 TheLexer.SetCommentRetentionState(true);
647
648 bool InPreprocessorDirective = false;
649 Token TheTok;
650 SourceLocation ActiveCommentLoc;
651
652 unsigned MaxLineOffset = 0;
653 if (MaxLines) {
654 const char *CurPtr = Buffer.begin();
655 unsigned CurLine = 0;
656 while (CurPtr != Buffer.end()) {
657 char ch = *CurPtr++;
658 if (ch == '\n') {
659 ++CurLine;
660 if (CurLine == MaxLines)
661 break;
662 }
663 }
664 if (CurPtr != Buffer.end())
665 MaxLineOffset = CurPtr - Buffer.begin();
666 }
667
668 do {
669 TheLexer.LexFromRawLexer(Result&: TheTok);
670
671 if (InPreprocessorDirective) {
672 // If we've hit the end of the file, we're done.
673 if (TheTok.getKind() == tok::eof) {
674 break;
675 }
676
677 // If we haven't hit the end of the preprocessor directive, skip this
678 // token.
679 if (!TheTok.isAtStartOfLine())
680 continue;
681
682 // We've passed the end of the preprocessor directive, and will look
683 // at this token again below.
684 InPreprocessorDirective = false;
685 }
686
687 // Keep track of the # of lines in the preamble.
688 if (TheTok.isAtStartOfLine()) {
689 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
690
691 // If we were asked to limit the number of lines in the preamble,
692 // and we're about to exceed that limit, we're done.
693 if (MaxLineOffset && TokOffset >= MaxLineOffset)
694 break;
695 }
696
697 // Comments are okay; skip over them.
698 if (TheTok.getKind() == tok::comment) {
699 if (ActiveCommentLoc.isInvalid())
700 ActiveCommentLoc = TheTok.getLocation();
701 continue;
702 }
703
704 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
705 // This is the start of a preprocessor directive.
706 Token HashTok = TheTok;
707 InPreprocessorDirective = true;
708 ActiveCommentLoc = SourceLocation();
709
710 // Figure out which directive this is. Since we're lexing raw tokens,
711 // we don't have an identifier table available. Instead, just look at
712 // the raw identifier to recognize and categorize preprocessor directives.
713 TheLexer.LexFromRawLexer(Result&: TheTok);
714 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
715 StringRef Keyword = TheTok.getRawIdentifier();
716 PreambleDirectiveKind PDK
717 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
718 .Case(S: "include", Value: PDK_Skipped)
719 .Case(S: "__include_macros", Value: PDK_Skipped)
720 .Case(S: "define", Value: PDK_Skipped)
721 .Case(S: "undef", Value: PDK_Skipped)
722 .Case(S: "line", Value: PDK_Skipped)
723 .Case(S: "error", Value: PDK_Skipped)
724 .Case(S: "pragma", Value: PDK_Skipped)
725 .Case(S: "import", Value: PDK_Skipped)
726 .Case(S: "include_next", Value: PDK_Skipped)
727 .Case(S: "warning", Value: PDK_Skipped)
728 .Case(S: "ident", Value: PDK_Skipped)
729 .Case(S: "sccs", Value: PDK_Skipped)
730 .Case(S: "assert", Value: PDK_Skipped)
731 .Case(S: "unassert", Value: PDK_Skipped)
732 .Case(S: "if", Value: PDK_Skipped)
733 .Case(S: "ifdef", Value: PDK_Skipped)
734 .Case(S: "ifndef", Value: PDK_Skipped)
735 .Case(S: "elif", Value: PDK_Skipped)
736 .Case(S: "elifdef", Value: PDK_Skipped)
737 .Case(S: "elifndef", Value: PDK_Skipped)
738 .Case(S: "else", Value: PDK_Skipped)
739 .Case(S: "endif", Value: PDK_Skipped)
740 .Default(Value: PDK_Unknown);
741
742 switch (PDK) {
743 case PDK_Skipped:
744 continue;
745
746 case PDK_Unknown:
747 // We don't know what this directive is; stop at the '#'.
748 break;
749 }
750 }
751
752 // We only end up here if we didn't recognize the preprocessor
753 // directive or it was one that can't occur in the preamble at this
754 // point. Roll back the current token to the location of the '#'.
755 TheTok = HashTok;
756 } else if (TheTok.isAtStartOfLine() &&
757 TheTok.getKind() == tok::raw_identifier &&
758 TheTok.getRawIdentifier() == "module" &&
759 LangOpts.CPlusPlusModules) {
760 // The initial global module fragment introducer "module;" is part of
761 // the preamble, which runs up to the module declaration "module foo;".
762 Token ModuleTok = TheTok;
763 do {
764 TheLexer.LexFromRawLexer(Result&: TheTok);
765 } while (TheTok.getKind() == tok::comment);
766 if (TheTok.getKind() != tok::semi) {
767 // Not global module fragment, roll back.
768 TheTok = ModuleTok;
769 break;
770 }
771 continue;
772 }
773
774 // We hit a token that we don't recognize as being in the
775 // "preprocessing only" part of the file, so we're no longer in
776 // the preamble.
777 break;
778 } while (true);
779
780 SourceLocation End;
781 if (ActiveCommentLoc.isValid())
782 End = ActiveCommentLoc; // don't truncate a decl comment.
783 else
784 End = TheTok.getLocation();
785
786 return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
787 TheTok.isAtStartOfLine());
788}
789
790unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
791 const SourceManager &SM,
792 const LangOptions &LangOpts) {
793 // Figure out how many physical characters away the specified expansion
794 // character is. This needs to take into consideration newlines and
795 // trigraphs.
796 bool Invalid = false;
797 const char *TokPtr = SM.getCharacterData(SL: TokStart, Invalid: &Invalid);
798
799 // If they request the first char of the token, we're trivially done.
800 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(C: *TokPtr)))
801 return 0;
802
803 unsigned PhysOffset = 0;
804
805 // The usual case is that tokens don't contain anything interesting. Skip
806 // over the uninteresting characters. If a token only consists of simple
807 // chars, this method is extremely fast.
808 while (Lexer::isObviouslySimpleCharacter(C: *TokPtr)) {
809 if (CharNo == 0)
810 return PhysOffset;
811 ++TokPtr;
812 --CharNo;
813 ++PhysOffset;
814 }
815
816 // If we have a character that may be a trigraph or escaped newline, use a
817 // lexer to parse it correctly.
818 for (; CharNo; --CharNo) {
819 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: TokPtr, LangOpts);
820 TokPtr += CharAndSize.Size;
821 PhysOffset += CharAndSize.Size;
822 }
823
824 // Final detail: if we end up on an escaped newline, we want to return the
825 // location of the actual byte of the token. For example foo\<newline>bar
826 // advanced by 3 should return the location of b, not of \\. One compounding
827 // detail of this is that the escape may be made by a trigraph.
828 if (!Lexer::isObviouslySimpleCharacter(C: *TokPtr))
829 PhysOffset += Lexer::SkipEscapedNewLines(P: TokPtr)-TokPtr;
830
831 return PhysOffset;
832}
833
834/// Computes the source location just past the end of the
835/// token at this source location.
836///
837/// This routine can be used to produce a source location that
838/// points just past the end of the token referenced by \p Loc, and
839/// is generally used when a diagnostic needs to point just after a
840/// token where it expected something different that it received. If
841/// the returned source location would not be meaningful (e.g., if
842/// it points into a macro), this routine returns an invalid
843/// source location.
844///
845/// \param Offset an offset from the end of the token, where the source
846/// location should refer to. The default offset (0) produces a source
847/// location pointing just past the end of the token; an offset of 1 produces
848/// a source location pointing to the last character in the token, etc.
849SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
850 const SourceManager &SM,
851 const LangOptions &LangOpts) {
852 if (Loc.isInvalid())
853 return {};
854
855 if (Loc.isMacroID()) {
856 if (Offset > 0 || !isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
857 return {}; // Points inside the macro expansion.
858 }
859
860 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
861 if (Len > Offset)
862 Len = Len - Offset;
863 else
864 return Loc;
865
866 return Loc.getLocWithOffset(Offset: Len);
867}
868
869/// Returns true if the given MacroID location points at the first
870/// token of the macro expansion.
871bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
872 const SourceManager &SM,
873 const LangOptions &LangOpts,
874 SourceLocation *MacroBegin) {
875 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
876
877 SourceLocation expansionLoc;
878 if (!SM.isAtStartOfImmediateMacroExpansion(Loc: loc, MacroBegin: &expansionLoc))
879 return false;
880
881 if (expansionLoc.isFileID()) {
882 // No other macro expansions, this is the first.
883 if (MacroBegin)
884 *MacroBegin = expansionLoc;
885 return true;
886 }
887
888 return isAtStartOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroBegin);
889}
890
891/// Returns true if the given MacroID location points at the last
892/// token of the macro expansion.
893bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
894 const SourceManager &SM,
895 const LangOptions &LangOpts,
896 SourceLocation *MacroEnd) {
897 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
898
899 SourceLocation spellLoc = SM.getSpellingLoc(Loc: loc);
900 unsigned tokLen = MeasureTokenLength(Loc: spellLoc, SM, LangOpts);
901 if (tokLen == 0)
902 return false;
903
904 SourceLocation afterLoc = loc.getLocWithOffset(Offset: tokLen);
905 SourceLocation expansionLoc;
906 if (!SM.isAtEndOfImmediateMacroExpansion(Loc: afterLoc, MacroEnd: &expansionLoc))
907 return false;
908
909 if (expansionLoc.isFileID()) {
910 // No other macro expansions.
911 if (MacroEnd)
912 *MacroEnd = expansionLoc;
913 return true;
914 }
915
916 return isAtEndOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroEnd);
917}
918
919static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
920 const SourceManager &SM,
921 const LangOptions &LangOpts) {
922 SourceLocation Begin = Range.getBegin();
923 SourceLocation End = Range.getEnd();
924 assert(Begin.isFileID() && End.isFileID());
925 if (Range.isTokenRange()) {
926 End = Lexer::getLocForEndOfToken(Loc: End, Offset: 0, SM,LangOpts);
927 if (End.isInvalid())
928 return {};
929 }
930
931 // Break down the source locations.
932 FileID FID;
933 unsigned BeginOffs;
934 std::tie(args&: FID, args&: BeginOffs) = SM.getDecomposedLoc(Loc: Begin);
935 if (FID.isInvalid())
936 return {};
937
938 unsigned EndOffs;
939 if (!SM.isInFileID(Loc: End, FID, RelativeOffset: &EndOffs) ||
940 BeginOffs > EndOffs)
941 return {};
942
943 return CharSourceRange::getCharRange(B: Begin, E: End);
944}
945
946// Assumes that `Loc` is in an expansion.
947static bool isInExpansionTokenRange(const SourceLocation Loc,
948 const SourceManager &SM) {
949 return SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Loc))
950 .getExpansion()
951 .isExpansionTokenRange();
952}
953
954CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
955 const SourceManager &SM,
956 const LangOptions &LangOpts) {
957 SourceLocation Begin = Range.getBegin();
958 SourceLocation End = Range.getEnd();
959 if (Begin.isInvalid() || End.isInvalid())
960 return {};
961
962 if (Begin.isFileID() && End.isFileID())
963 return makeRangeFromFileLocs(Range, SM, LangOpts);
964
965 if (Begin.isMacroID() && End.isFileID()) {
966 if (!isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &Begin))
967 return {};
968 Range.setBegin(Begin);
969 return makeRangeFromFileLocs(Range, SM, LangOpts);
970 }
971
972 if (Begin.isFileID() && End.isMacroID()) {
973 if (Range.isTokenRange()) {
974 if (!isAtEndOfMacroExpansion(loc: End, SM, LangOpts, MacroEnd: &End))
975 return {};
976 // Use the *original* end, not the expanded one in `End`.
977 Range.setTokenRange(isInExpansionTokenRange(Loc: Range.getEnd(), SM));
978 } else if (!isAtStartOfMacroExpansion(loc: End, SM, LangOpts, MacroBegin: &End))
979 return {};
980 Range.setEnd(End);
981 return makeRangeFromFileLocs(Range, SM, LangOpts);
982 }
983
984 assert(Begin.isMacroID() && End.isMacroID());
985 SourceLocation MacroBegin, MacroEnd;
986 if (isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &MacroBegin) &&
987 ((Range.isTokenRange() && isAtEndOfMacroExpansion(loc: End, SM, LangOpts,
988 MacroEnd: &MacroEnd)) ||
989 (Range.isCharRange() && isAtStartOfMacroExpansion(loc: End, SM, LangOpts,
990 MacroBegin: &MacroEnd)))) {
991 Range.setBegin(MacroBegin);
992 Range.setEnd(MacroEnd);
993 // Use the *original* `End`, not the expanded one in `MacroEnd`.
994 if (Range.isTokenRange())
995 Range.setTokenRange(isInExpansionTokenRange(Loc: End, SM));
996 return makeRangeFromFileLocs(Range, SM, LangOpts);
997 }
998
999 bool Invalid = false;
1000 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Begin),
1001 Invalid: &Invalid);
1002 if (Invalid)
1003 return {};
1004
1005 if (BeginEntry.getExpansion().isMacroArgExpansion()) {
1006 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: End),
1007 Invalid: &Invalid);
1008 if (Invalid)
1009 return {};
1010
1011 if (EndEntry.getExpansion().isMacroArgExpansion() &&
1012 BeginEntry.getExpansion().getExpansionLocStart() ==
1013 EndEntry.getExpansion().getExpansionLocStart()) {
1014 Range.setBegin(SM.getImmediateSpellingLoc(Loc: Begin));
1015 Range.setEnd(SM.getImmediateSpellingLoc(Loc: End));
1016 return makeFileCharRange(Range, SM, LangOpts);
1017 }
1018 }
1019
1020 return {};
1021}
1022
1023StringRef Lexer::getSourceText(CharSourceRange Range,
1024 const SourceManager &SM,
1025 const LangOptions &LangOpts,
1026 bool *Invalid) {
1027 Range = makeFileCharRange(Range, SM, LangOpts);
1028 if (Range.isInvalid()) {
1029 if (Invalid) *Invalid = true;
1030 return {};
1031 }
1032
1033 // Break down the source location.
1034 FileIDAndOffset beginInfo = SM.getDecomposedLoc(Loc: Range.getBegin());
1035 if (beginInfo.first.isInvalid()) {
1036 if (Invalid) *Invalid = true;
1037 return {};
1038 }
1039
1040 unsigned EndOffs;
1041 if (!SM.isInFileID(Loc: Range.getEnd(), FID: beginInfo.first, RelativeOffset: &EndOffs) ||
1042 beginInfo.second > EndOffs) {
1043 if (Invalid) *Invalid = true;
1044 return {};
1045 }
1046
1047 // Try to the load the file buffer.
1048 bool invalidTemp = false;
1049 StringRef file = SM.getBufferData(FID: beginInfo.first, Invalid: &invalidTemp);
1050 if (invalidTemp) {
1051 if (Invalid) *Invalid = true;
1052 return {};
1053 }
1054
1055 if (Invalid) *Invalid = false;
1056 return file.substr(Start: beginInfo.second, N: EndOffs - beginInfo.second);
1057}
1058
1059StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1060 const SourceManager &SM,
1061 const LangOptions &LangOpts) {
1062 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1063
1064 // Find the location of the immediate macro expansion.
1065 while (true) {
1066 FileID FID = SM.getFileID(SpellingLoc: Loc);
1067 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
1068 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
1069 Loc = Expansion.getExpansionLocStart();
1070 if (!Expansion.isMacroArgExpansion())
1071 break;
1072
1073 // For macro arguments we need to check that the argument did not come
1074 // from an inner macro, e.g: "MAC1( MAC2(foo) )"
1075
1076 // Loc points to the argument id of the macro definition, move to the
1077 // macro expansion.
1078 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1079 SourceLocation SpellLoc = Expansion.getSpellingLoc();
1080 if (SpellLoc.isFileID())
1081 break; // No inner macro.
1082
1083 // If spelling location resides in the same FileID as macro expansion
1084 // location, it means there is no inner macro.
1085 FileID MacroFID = SM.getFileID(SpellingLoc: Loc);
1086 if (SM.isInFileID(Loc: SpellLoc, FID: MacroFID))
1087 break;
1088
1089 // Argument came from inner macro.
1090 Loc = SpellLoc;
1091 }
1092
1093 // Find the spelling location of the start of the non-argument expansion
1094 // range. This is where the macro name was spelled in order to begin
1095 // expanding this macro.
1096 Loc = SM.getSpellingLoc(Loc);
1097
1098 // Dig out the buffer where the macro name was spelled and the extents of the
1099 // name so that we can render it into the expansion note.
1100 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1101 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1102 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1103 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1104}
1105
1106StringRef Lexer::getImmediateMacroNameForDiagnostics(
1107 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
1108 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1109 // Walk past macro argument expansions.
1110 while (SM.isMacroArgExpansion(Loc))
1111 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1112
1113 // If the macro's spelling isn't FileID or from scratch space, then it's
1114 // actually a token paste or stringization (or similar) and not a macro at
1115 // all.
1116 SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
1117 if (!SpellLoc.isFileID() || SM.isWrittenInScratchSpace(Loc: SpellLoc))
1118 return {};
1119
1120 // Find the spelling location of the start of the non-argument expansion
1121 // range. This is where the macro name was spelled in order to begin
1122 // expanding this macro.
1123 Loc = SM.getSpellingLoc(Loc: SM.getImmediateExpansionRange(Loc).getBegin());
1124
1125 // Dig out the buffer where the macro name was spelled and the extents of the
1126 // name so that we can render it into the expansion note.
1127 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1128 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1129 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1130 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1131}
1132
1133bool Lexer::isAsciiIdentifierContinueChar(char c, const LangOptions &LangOpts) {
1134 return isAsciiIdentifierContinue(c, AllowDollar: LangOpts.DollarIdents);
1135}
1136
1137bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1138 assert(isVerticalWhitespace(Str[0]));
1139 if (Str - 1 < BufferStart)
1140 return false;
1141
1142 if ((Str[0] == '\n' && Str[-1] == '\r') ||
1143 (Str[0] == '\r' && Str[-1] == '\n')) {
1144 if (Str - 2 < BufferStart)
1145 return false;
1146 --Str;
1147 }
1148 --Str;
1149
1150 // Rewind to first non-space character:
1151 while (Str > BufferStart && isHorizontalWhitespace(c: *Str))
1152 --Str;
1153
1154 return *Str == '\\';
1155}
1156
1157StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1158 const SourceManager &SM) {
1159 if (Loc.isInvalid() || Loc.isMacroID())
1160 return {};
1161 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1162 if (LocInfo.first.isInvalid())
1163 return {};
1164 bool Invalid = false;
1165 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
1166 if (Invalid)
1167 return {};
1168 const char *Line = findBeginningOfLine(Buffer, Offset: LocInfo.second);
1169 if (!Line)
1170 return {};
1171 StringRef Rest = Buffer.substr(Start: Line - Buffer.data());
1172 size_t NumWhitespaceChars = Rest.find_first_not_of(Chars: " \t");
1173 return NumWhitespaceChars == StringRef::npos
1174 ? ""
1175 : Rest.take_front(N: NumWhitespaceChars);
1176}
1177
1178//===----------------------------------------------------------------------===//
1179// Diagnostics forwarding code.
1180//===----------------------------------------------------------------------===//
1181
1182/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1183/// lexer buffer was all expanded at a single point, perform the mapping.
1184/// This is currently only used for _Pragma implementation, so it is the slow
1185/// path of the hot getSourceLocation method. Do not allow it to be inlined.
1186static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1187 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1188static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1189 SourceLocation FileLoc,
1190 unsigned CharNo, unsigned TokLen) {
1191 assert(FileLoc.isMacroID() && "Must be a macro expansion");
1192
1193 // Otherwise, we're lexing "mapped tokens". This is used for things like
1194 // _Pragma handling. Combine the expansion location of FileLoc with the
1195 // spelling location.
1196 SourceManager &SM = PP.getSourceManager();
1197
1198 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
1199 // characters come from spelling(FileLoc)+Offset.
1200 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc: FileLoc);
1201 SpellingLoc = SpellingLoc.getLocWithOffset(Offset: CharNo);
1202
1203 // Figure out the expansion loc range, which is the range covered by the
1204 // original _Pragma(...) sequence.
1205 CharSourceRange II = SM.getImmediateExpansionRange(Loc: FileLoc);
1206
1207 return SM.createExpansionLoc(SpellingLoc, ExpansionLocStart: II.getBegin(), ExpansionLocEnd: II.getEnd(), Length: TokLen);
1208}
1209
1210/// getSourceLocation - Return a source location identifier for the specified
1211/// offset in the current file.
1212SourceLocation Lexer::getSourceLocation(const char *Loc,
1213 unsigned TokLen) const {
1214 assert(Loc >= BufferStart && Loc <= BufferEnd &&
1215 "Location out of range for this buffer!");
1216
1217 // In the normal case, we're just lexing from a simple file buffer, return
1218 // the file id from FileLoc with the offset specified.
1219 unsigned CharNo = Loc-BufferStart;
1220 if (FileLoc.isFileID())
1221 return FileLoc.getLocWithOffset(Offset: CharNo);
1222
1223 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1224 // tokens are lexed from where the _Pragma was defined.
1225 assert(PP && "This doesn't work on raw lexers");
1226 return GetMappedTokenLoc(PP&: *PP, FileLoc, CharNo, TokLen);
1227}
1228
1229/// Diag - Forwarding function for diagnostics. This translate a source
1230/// position in the current buffer into a SourceLocation object for rendering.
1231DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1232 return PP->Diag(Loc: getSourceLocation(Loc), DiagID);
1233}
1234
1235//===----------------------------------------------------------------------===//
1236// Trigraph and Escaped Newline Handling Code.
1237//===----------------------------------------------------------------------===//
1238
1239/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1240/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1241static char GetTrigraphCharForLetter(char Letter) {
1242 switch (Letter) {
1243 default: return 0;
1244 case '=': return '#';
1245 case ')': return ']';
1246 case '(': return '[';
1247 case '!': return '|';
1248 case '\'': return '^';
1249 case '>': return '}';
1250 case '/': return '\\';
1251 case '<': return '{';
1252 case '-': return '~';
1253 }
1254}
1255
1256/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1257/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
1258/// return the result character. Finally, emit a warning about trigraph use
1259/// whether trigraphs are enabled or not.
1260static char DecodeTrigraphChar(const char *CP, Lexer *L, bool Trigraphs) {
1261 char Res = GetTrigraphCharForLetter(Letter: *CP);
1262 if (!Res)
1263 return Res;
1264
1265 if (!Trigraphs) {
1266 if (L && !L->isLexingRawMode())
1267 L->Diag(Loc: CP-2, DiagID: diag::trigraph_ignored);
1268 return 0;
1269 }
1270
1271 if (L && !L->isLexingRawMode())
1272 L->Diag(Loc: CP-2, DiagID: diag::trigraph_converted) << StringRef(&Res, 1);
1273 return Res;
1274}
1275
1276/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1277/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1278/// trigraph equivalent on entry to this function.
1279unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1280 unsigned Size = 0;
1281 while (isWhitespace(c: Ptr[Size])) {
1282 ++Size;
1283
1284 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
1285 continue;
1286
1287 // If this is a \r\n or \n\r, skip the other half.
1288 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
1289 Ptr[Size-1] != Ptr[Size])
1290 ++Size;
1291
1292 return Size;
1293 }
1294
1295 // Not an escaped newline, must be a \t or something else.
1296 return 0;
1297}
1298
1299/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1300/// them), skip over them and return the first non-escaped-newline found,
1301/// otherwise return P.
1302const char *Lexer::SkipEscapedNewLines(const char *P) {
1303 while (true) {
1304 const char *AfterEscape;
1305 if (*P == '\\') {
1306 AfterEscape = P+1;
1307 } else if (*P == '?') {
1308 // If not a trigraph for escape, bail out.
1309 if (P[1] != '?' || P[2] != '/')
1310 return P;
1311 // FIXME: Take LangOpts into account; the language might not
1312 // support trigraphs.
1313 AfterEscape = P+3;
1314 } else {
1315 return P;
1316 }
1317
1318 unsigned NewLineSize = Lexer::getEscapedNewLineSize(Ptr: AfterEscape);
1319 if (NewLineSize == 0) return P;
1320 P = AfterEscape+NewLineSize;
1321 }
1322}
1323
1324std::optional<Token> Lexer::findNextToken(SourceLocation Loc,
1325 const SourceManager &SM,
1326 const LangOptions &LangOpts,
1327 bool IncludeComments) {
1328 if (Loc.isMacroID()) {
1329 if (!Lexer::isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
1330 return std::nullopt;
1331 }
1332 Loc = Lexer::getLocForEndOfToken(Loc, Offset: 0, SM, LangOpts);
1333
1334 // Break down the source location.
1335 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1336
1337 // Try to load the file buffer.
1338 bool InvalidTemp = false;
1339 StringRef File = SM.getBufferData(FID: LocInfo.first, Invalid: &InvalidTemp);
1340 if (InvalidTemp)
1341 return std::nullopt;
1342
1343 const char *TokenBegin = File.data() + LocInfo.second;
1344
1345 // Lex from the start of the given location.
1346 Lexer lexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts, File.begin(),
1347 TokenBegin, File.end());
1348 lexer.SetCommentRetentionState(IncludeComments);
1349 // Find the token.
1350 Token Tok;
1351 lexer.LexFromRawLexer(Result&: Tok);
1352 return Tok;
1353}
1354
1355std::optional<Token> Lexer::findPreviousToken(SourceLocation Loc,
1356 const SourceManager &SM,
1357 const LangOptions &LangOpts,
1358 bool IncludeComments) {
1359 const auto StartOfFile = SM.getLocForStartOfFile(FID: SM.getFileID(SpellingLoc: Loc));
1360 while (Loc != StartOfFile) {
1361 Loc = Loc.getLocWithOffset(Offset: -1);
1362 if (Loc.isInvalid())
1363 return std::nullopt;
1364
1365 Loc = GetBeginningOfToken(Loc, SM, LangOpts);
1366 Token Tok;
1367 if (getRawToken(Loc, Result&: Tok, SM, LangOpts))
1368 continue; // Not a token, go to prev location.
1369 if (!Tok.is(K: tok::comment) || IncludeComments) {
1370 return Tok;
1371 }
1372 }
1373 return std::nullopt;
1374}
1375
1376/// Checks that the given token is the first token that occurs after the
1377/// given location (this excludes comments and whitespace). Returns the location
1378/// immediately after the specified token. If the token is not found or the
1379/// location is inside a macro, the returned source location will be invalid.
1380SourceLocation Lexer::findLocationAfterToken(
1381 SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
1382 const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
1383 std::optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
1384 if (!Tok || Tok->isNot(K: TKind))
1385 return {};
1386 SourceLocation TokenLoc = Tok->getLocation();
1387
1388 // Calculate how much whitespace needs to be skipped if any.
1389 unsigned NumWhitespaceChars = 0;
1390 if (SkipTrailingWhitespaceAndNewLine) {
1391 const char *TokenEnd = SM.getCharacterData(SL: TokenLoc) + Tok->getLength();
1392 unsigned char C = *TokenEnd;
1393 while (isHorizontalWhitespace(c: C)) {
1394 C = *(++TokenEnd);
1395 NumWhitespaceChars++;
1396 }
1397
1398 // Skip \r, \n, \r\n, or \n\r
1399 if (C == '\n' || C == '\r') {
1400 char PrevC = C;
1401 C = *(++TokenEnd);
1402 NumWhitespaceChars++;
1403 if ((C == '\n' || C == '\r') && C != PrevC)
1404 NumWhitespaceChars++;
1405 }
1406 }
1407
1408 return TokenLoc.getLocWithOffset(Offset: Tok->getLength() + NumWhitespaceChars);
1409}
1410
1411/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1412/// get its size, and return it. This is tricky in several cases:
1413/// 1. If currently at the start of a trigraph, we warn about the trigraph,
1414/// then either return the trigraph (skipping 3 chars) or the '?',
1415/// depending on whether trigraphs are enabled or not.
1416/// 2. If this is an escaped newline (potentially with whitespace between
1417/// the backslash and newline), implicitly skip the newline and return
1418/// the char after it.
1419///
1420/// This handles the slow/uncommon case of the getCharAndSize method. Here we
1421/// know that we can accumulate into Size, and that we have already incremented
1422/// Ptr by Size bytes.
1423///
1424/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1425/// be updated to match.
1426Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
1427 unsigned Size = 0;
1428 // If we have a slash, look for an escaped newline.
1429 if (Ptr[0] == '\\') {
1430 ++Size;
1431 ++Ptr;
1432Slash:
1433 // Common case, backslash-char where the char is not whitespace.
1434 if (!isWhitespace(c: Ptr[0]))
1435 return {.Char: '\\', .Size: Size};
1436
1437 // See if we have optional whitespace characters between the slash and
1438 // newline.
1439 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1440 // Remember that this token needs to be cleaned.
1441 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1442
1443 // Warn if there was whitespace between the backslash and newline.
1444 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1445 Diag(Loc: Ptr, DiagID: diag::backslash_newline_space);
1446
1447 // Found backslash<whitespace><newline>. Parse the char after it.
1448 Size += EscapedNewLineSize;
1449 Ptr += EscapedNewLineSize;
1450
1451 // Use slow version to accumulate a correct size field.
1452 auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
1453 CharAndSize.Size += Size;
1454 return CharAndSize;
1455 }
1456
1457 // Otherwise, this is not an escaped newline, just return the slash.
1458 return {.Char: '\\', .Size: Size};
1459 }
1460
1461 // If this is a trigraph, process it.
1462 if (Ptr[0] == '?' && Ptr[1] == '?') {
1463 // If this is actually a legal trigraph (not something like "??x"), emit
1464 // a trigraph warning. If so, and if trigraphs are enabled, return it.
1465 if (char C = DecodeTrigraphChar(CP: Ptr + 2, L: Tok ? this : nullptr,
1466 Trigraphs: LangOpts.Trigraphs)) {
1467 // Remember that this token needs to be cleaned.
1468 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1469
1470 Ptr += 3;
1471 Size += 3;
1472 if (C == '\\') goto Slash;
1473 return {.Char: C, .Size: Size};
1474 }
1475 }
1476
1477 // If this is neither, return a single character.
1478 return {.Char: *Ptr, .Size: Size + 1u};
1479}
1480
1481/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1482/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
1483/// and that we have already incremented Ptr by Size bytes.
1484///
1485/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1486/// be updated to match.
1487Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
1488 const LangOptions &LangOpts) {
1489
1490 unsigned Size = 0;
1491 // If we have a slash, look for an escaped newline.
1492 if (Ptr[0] == '\\') {
1493 ++Size;
1494 ++Ptr;
1495Slash:
1496 // Common case, backslash-char where the char is not whitespace.
1497 if (!isWhitespace(c: Ptr[0]))
1498 return {.Char: '\\', .Size: Size};
1499
1500 // See if we have optional whitespace characters followed by a newline.
1501 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1502 // Found backslash<whitespace><newline>. Parse the char after it.
1503 Size += EscapedNewLineSize;
1504 Ptr += EscapedNewLineSize;
1505
1506 // Use slow version to accumulate a correct size field.
1507 auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
1508 CharAndSize.Size += Size;
1509 return CharAndSize;
1510 }
1511
1512 // Otherwise, this is not an escaped newline, just return the slash.
1513 return {.Char: '\\', .Size: Size};
1514 }
1515
1516 // If this is a trigraph, process it.
1517 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1518 // If this is actually a legal trigraph (not something like "??x"), return
1519 // it.
1520 if (char C = GetTrigraphCharForLetter(Letter: Ptr[2])) {
1521 Ptr += 3;
1522 Size += 3;
1523 if (C == '\\') goto Slash;
1524 return {.Char: C, .Size: Size};
1525 }
1526 }
1527
1528 // If this is neither, return a single character.
1529 return {.Char: *Ptr, .Size: Size + 1u};
1530}
1531
1532//===----------------------------------------------------------------------===//
1533// Helper methods for lexing.
1534//===----------------------------------------------------------------------===//
1535
1536/// Routine that indiscriminately sets the offset into the source file.
1537void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1538 BufferPtr = BufferStart + Offset;
1539 if (BufferPtr > BufferEnd)
1540 BufferPtr = BufferEnd;
1541 // FIXME: What exactly does the StartOfLine bit mean? There are two
1542 // possible meanings for the "start" of the line: the first token on the
1543 // unexpanded line, or the first token on the expanded line.
1544 IsAtStartOfLine = StartOfLine;
1545 IsAtPhysicalStartOfLine = StartOfLine;
1546}
1547
1548static bool isUnicodeWhitespace(uint32_t Codepoint) {
1549 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
1550 UnicodeWhitespaceCharRanges);
1551 return UnicodeWhitespaceChars.contains(C: Codepoint);
1552}
1553
1554static llvm::SmallString<5> codepointAsHexString(uint32_t C) {
1555 llvm::SmallString<5> CharBuf;
1556 llvm::raw_svector_ostream CharOS(CharBuf);
1557 llvm::write_hex(S&: CharOS, N: C, Style: llvm::HexPrintStyle::Upper, Width: 4);
1558 return CharBuf;
1559}
1560
1561// To mitigate https://github.com/llvm/llvm-project/issues/54732,
1562// we allow "Mathematical Notation Characters" in identifiers.
1563// This is a proposed profile that extends the XID_Start/XID_continue
1564// with mathematical symbols, superscipts and subscripts digits
1565// found in some production software.
1566// https://www.unicode.org/L2/L2022/22230-math-profile.pdf
1567static bool isMathematicalExtensionID(uint32_t C, const LangOptions &LangOpts,
1568 bool IsStart, bool &IsExtension) {
1569 static const llvm::sys::UnicodeCharSet MathStartChars(
1570 MathematicalNotationProfileIDStartRanges);
1571 static const llvm::sys::UnicodeCharSet MathContinueChars(
1572 MathematicalNotationProfileIDContinueRanges);
1573 if (MathStartChars.contains(C) ||
1574 (!IsStart && MathContinueChars.contains(C))) {
1575 IsExtension = true;
1576 return true;
1577 }
1578 return false;
1579}
1580
1581static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts,
1582 bool &IsExtension) {
1583 if (LangOpts.AsmPreprocessor) {
1584 return false;
1585 } else if (LangOpts.DollarIdents && '$' == C) {
1586 return true;
1587 } else if (LangOpts.CPlusPlus || LangOpts.C23) {
1588 // A non-leading codepoint must have the XID_Continue property.
1589 // XIDContinueRanges doesn't contains characters also in XIDStartRanges,
1590 // so we need to check both tables.
1591 // '_' doesn't have the XID_Continue property but is allowed in C and C++.
1592 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1593 static const llvm::sys::UnicodeCharSet XIDContinueChars(XIDContinueRanges);
1594 if (C == '_' || XIDStartChars.contains(C) || XIDContinueChars.contains(C))
1595 return true;
1596 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/false,
1597 IsExtension);
1598 } else if (LangOpts.C11) {
1599 static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
1600 C11AllowedIDCharRanges);
1601 return C11AllowedIDChars.contains(C);
1602 } else {
1603 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1604 C99AllowedIDCharRanges);
1605 return C99AllowedIDChars.contains(C);
1606 }
1607}
1608
1609static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts,
1610 bool &IsExtension) {
1611 assert(C > 0x7F && "isAllowedInitiallyIDChar called with an ASCII codepoint");
1612 IsExtension = false;
1613 if (LangOpts.AsmPreprocessor) {
1614 return false;
1615 }
1616 if (LangOpts.CPlusPlus || LangOpts.C23) {
1617 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1618 if (XIDStartChars.contains(C))
1619 return true;
1620 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/true,
1621 IsExtension);
1622 }
1623 if (!isAllowedIDChar(C, LangOpts, IsExtension))
1624 return false;
1625 if (LangOpts.C11) {
1626 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
1627 C11DisallowedInitialIDCharRanges);
1628 return !C11DisallowedInitialIDChars.contains(C);
1629 }
1630 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1631 C99DisallowedInitialIDCharRanges);
1632 return !C99DisallowedInitialIDChars.contains(C);
1633}
1634
1635static void diagnoseExtensionInIdentifier(DiagnosticsEngine &Diags, uint32_t C,
1636 CharSourceRange Range) {
1637
1638 static const llvm::sys::UnicodeCharSet MathStartChars(
1639 MathematicalNotationProfileIDStartRanges);
1640 static const llvm::sys::UnicodeCharSet MathContinueChars(
1641 MathematicalNotationProfileIDContinueRanges);
1642
1643 (void)MathStartChars;
1644 (void)MathContinueChars;
1645 assert((MathStartChars.contains(C) || MathContinueChars.contains(C)) &&
1646 "Unexpected mathematical notation codepoint");
1647 Diags.Report(Loc: Range.getBegin(), DiagID: diag::ext_mathematical_notation)
1648 << codepointAsHexString(C) << Range;
1649}
1650
1651static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1652 const char *End) {
1653 return CharSourceRange::getCharRange(B: L.getSourceLocation(Loc: Begin),
1654 E: L.getSourceLocation(Loc: End));
1655}
1656
1657static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
1658 CharSourceRange Range, bool IsFirst) {
1659 // Check C99 compatibility.
1660 if (!Diags.isIgnored(DiagID: diag::warn_c99_compat_unicode_id, Loc: Range.getBegin())) {
1661 enum {
1662 CannotAppearInIdentifier = 0,
1663 CannotStartIdentifier
1664 };
1665
1666 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1667 C99AllowedIDCharRanges);
1668 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1669 C99DisallowedInitialIDCharRanges);
1670 if (!C99AllowedIDChars.contains(C)) {
1671 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1672 << Range
1673 << CannotAppearInIdentifier;
1674 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) {
1675 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1676 << Range
1677 << CannotStartIdentifier;
1678 }
1679 }
1680}
1681
1682/// After encountering UTF-8 character C and interpreting it as an identifier
1683/// character, check whether it's a homoglyph for a common non-identifier
1684/// source character that is unlikely to be an intentional identifier
1685/// character and warn if so.
1686static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
1687 CharSourceRange Range) {
1688 // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
1689 struct HomoglyphPair {
1690 uint32_t Character;
1691 char LooksLike;
1692 bool operator<(HomoglyphPair R) const { return Character < R.Character; }
1693 };
1694 static constexpr HomoglyphPair SortedHomoglyphs[] = {
1695 {.Character: U'\u00ad', .LooksLike: 0}, // SOFT HYPHEN
1696 {.Character: U'\u01c3', .LooksLike: '!'}, // LATIN LETTER RETROFLEX CLICK
1697 {.Character: U'\u037e', .LooksLike: ';'}, // GREEK QUESTION MARK
1698 {.Character: U'\u200b', .LooksLike: 0}, // ZERO WIDTH SPACE
1699 {.Character: U'\u200c', .LooksLike: 0}, // ZERO WIDTH NON-JOINER
1700 {.Character: U'\u200d', .LooksLike: 0}, // ZERO WIDTH JOINER
1701 {.Character: U'\u2060', .LooksLike: 0}, // WORD JOINER
1702 {.Character: U'\u2061', .LooksLike: 0}, // FUNCTION APPLICATION
1703 {.Character: U'\u2062', .LooksLike: 0}, // INVISIBLE TIMES
1704 {.Character: U'\u2063', .LooksLike: 0}, // INVISIBLE SEPARATOR
1705 {.Character: U'\u2064', .LooksLike: 0}, // INVISIBLE PLUS
1706 {.Character: U'\u2212', .LooksLike: '-'}, // MINUS SIGN
1707 {.Character: U'\u2215', .LooksLike: '/'}, // DIVISION SLASH
1708 {.Character: U'\u2216', .LooksLike: '\\'}, // SET MINUS
1709 {.Character: U'\u2217', .LooksLike: '*'}, // ASTERISK OPERATOR
1710 {.Character: U'\u2223', .LooksLike: '|'}, // DIVIDES
1711 {.Character: U'\u2227', .LooksLike: '^'}, // LOGICAL AND
1712 {.Character: U'\u2236', .LooksLike: ':'}, // RATIO
1713 {.Character: U'\u223c', .LooksLike: '~'}, // TILDE OPERATOR
1714 {.Character: U'\ua789', .LooksLike: ':'}, // MODIFIER LETTER COLON
1715 {.Character: U'\ufeff', .LooksLike: 0}, // ZERO WIDTH NO-BREAK SPACE
1716 {.Character: U'\uff01', .LooksLike: '!'}, // FULLWIDTH EXCLAMATION MARK
1717 {.Character: U'\uff03', .LooksLike: '#'}, // FULLWIDTH NUMBER SIGN
1718 {.Character: U'\uff04', .LooksLike: '$'}, // FULLWIDTH DOLLAR SIGN
1719 {.Character: U'\uff05', .LooksLike: '%'}, // FULLWIDTH PERCENT SIGN
1720 {.Character: U'\uff06', .LooksLike: '&'}, // FULLWIDTH AMPERSAND
1721 {.Character: U'\uff08', .LooksLike: '('}, // FULLWIDTH LEFT PARENTHESIS
1722 {.Character: U'\uff09', .LooksLike: ')'}, // FULLWIDTH RIGHT PARENTHESIS
1723 {.Character: U'\uff0a', .LooksLike: '*'}, // FULLWIDTH ASTERISK
1724 {.Character: U'\uff0b', .LooksLike: '+'}, // FULLWIDTH ASTERISK
1725 {.Character: U'\uff0c', .LooksLike: ','}, // FULLWIDTH COMMA
1726 {.Character: U'\uff0d', .LooksLike: '-'}, // FULLWIDTH HYPHEN-MINUS
1727 {.Character: U'\uff0e', .LooksLike: '.'}, // FULLWIDTH FULL STOP
1728 {.Character: U'\uff0f', .LooksLike: '/'}, // FULLWIDTH SOLIDUS
1729 {.Character: U'\uff1a', .LooksLike: ':'}, // FULLWIDTH COLON
1730 {.Character: U'\uff1b', .LooksLike: ';'}, // FULLWIDTH SEMICOLON
1731 {.Character: U'\uff1c', .LooksLike: '<'}, // FULLWIDTH LESS-THAN SIGN
1732 {.Character: U'\uff1d', .LooksLike: '='}, // FULLWIDTH EQUALS SIGN
1733 {.Character: U'\uff1e', .LooksLike: '>'}, // FULLWIDTH GREATER-THAN SIGN
1734 {.Character: U'\uff1f', .LooksLike: '?'}, // FULLWIDTH QUESTION MARK
1735 {.Character: U'\uff20', .LooksLike: '@'}, // FULLWIDTH COMMERCIAL AT
1736 {.Character: U'\uff3b', .LooksLike: '['}, // FULLWIDTH LEFT SQUARE BRACKET
1737 {.Character: U'\uff3c', .LooksLike: '\\'}, // FULLWIDTH REVERSE SOLIDUS
1738 {.Character: U'\uff3d', .LooksLike: ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
1739 {.Character: U'\uff3e', .LooksLike: '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
1740 {.Character: U'\uff5b', .LooksLike: '{'}, // FULLWIDTH LEFT CURLY BRACKET
1741 {.Character: U'\uff5c', .LooksLike: '|'}, // FULLWIDTH VERTICAL LINE
1742 {.Character: U'\uff5d', .LooksLike: '}'}, // FULLWIDTH RIGHT CURLY BRACKET
1743 {.Character: U'\uff5e', .LooksLike: '~'}, // FULLWIDTH TILDE
1744 {.Character: 0, .LooksLike: 0}
1745 };
1746 auto Homoglyph =
1747 std::lower_bound(first: std::begin(arr: SortedHomoglyphs),
1748 last: std::end(arr: SortedHomoglyphs) - 1, val: HomoglyphPair{.Character: C, .LooksLike: '\0'});
1749 if (Homoglyph->Character == C) {
1750 if (Homoglyph->LooksLike) {
1751 const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
1752 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_homoglyph)
1753 << Range << codepointAsHexString(C) << LooksLikeStr;
1754 } else {
1755 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_zero_width)
1756 << Range << codepointAsHexString(C);
1757 }
1758 }
1759}
1760
1761static void diagnoseInvalidUnicodeCodepointInIdentifier(
1762 DiagnosticsEngine &Diags, const LangOptions &LangOpts, uint32_t CodePoint,
1763 CharSourceRange Range, bool IsFirst) {
1764 if (isASCII(c: CodePoint))
1765 return;
1766
1767 bool IsExtension;
1768 bool IsIDStart = isAllowedInitiallyIDChar(C: CodePoint, LangOpts, IsExtension);
1769 bool IsIDContinue =
1770 IsIDStart || isAllowedIDChar(C: CodePoint, LangOpts, IsExtension);
1771
1772 if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
1773 return;
1774
1775 bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
1776
1777 if (!IsFirst || InvalidOnlyAtStart) {
1778 Diags.Report(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed_identifier)
1779 << Range << codepointAsHexString(C: CodePoint) << int(InvalidOnlyAtStart)
1780 << FixItHint::CreateRemoval(RemoveRange: Range);
1781 } else {
1782 Diags.Report(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed)
1783 << Range << codepointAsHexString(C: CodePoint)
1784 << FixItHint::CreateRemoval(RemoveRange: Range);
1785 }
1786}
1787
1788bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1789 Token &Result) {
1790 const char *UCNPtr = CurPtr + Size;
1791 uint32_t CodePoint = tryReadUCN(StartPtr&: UCNPtr, SlashLoc: CurPtr, /*Token=*/Result: nullptr);
1792 if (CodePoint == 0) {
1793 return false;
1794 }
1795 bool IsExtension = false;
1796 if (!isAllowedIDChar(C: CodePoint, LangOpts, IsExtension)) {
1797 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1798 return false;
1799 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1800 !PP->isPreprocessedOutput())
1801 diagnoseInvalidUnicodeCodepointInIdentifier(
1802 Diags&: PP->getDiagnostics(), LangOpts, CodePoint,
1803 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1804 /*IsFirst=*/false);
1805
1806 // We got a unicode codepoint that is neither a space nor a
1807 // a valid identifier part.
1808 // Carry on as if the codepoint was valid for recovery purposes.
1809 } else if (!isLexingRawMode()) {
1810 if (IsExtension)
1811 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C: CodePoint,
1812 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr));
1813
1814 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1815 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1816 /*IsFirst=*/false);
1817 }
1818
1819 Result.setFlag(Token::HasUCN);
1820 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') ||
1821 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
1822 CurPtr = UCNPtr;
1823 else
1824 while (CurPtr != UCNPtr)
1825 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
1826 return true;
1827}
1828
1829bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
1830 llvm::UTF32 CodePoint;
1831
1832 // If a UTF-8 codepoint appears immediately after an escaped new line,
1833 // CurPtr may point to the splicing \ on the preceding line,
1834 // so we need to skip it.
1835 unsigned FirstCodeUnitSize;
1836 getCharAndSize(Ptr: CurPtr, Size&: FirstCodeUnitSize);
1837 const char *CharStart = CurPtr + FirstCodeUnitSize - 1;
1838 const char *UnicodePtr = CharStart;
1839
1840 llvm::ConversionResult ConvResult = llvm::convertUTF8Sequence(
1841 source: (const llvm::UTF8 **)&UnicodePtr, sourceEnd: (const llvm::UTF8 *)BufferEnd,
1842 target: &CodePoint, flags: llvm::strictConversion);
1843 if (ConvResult != llvm::conversionOK)
1844 return false;
1845
1846 bool IsExtension = false;
1847 if (!isAllowedIDChar(C: static_cast<uint32_t>(CodePoint), LangOpts,
1848 IsExtension)) {
1849 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1850 return false;
1851
1852 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1853 !PP->isPreprocessedOutput())
1854 diagnoseInvalidUnicodeCodepointInIdentifier(
1855 Diags&: PP->getDiagnostics(), LangOpts, CodePoint,
1856 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr), /*IsFirst=*/false);
1857 // We got a unicode codepoint that is neither a space nor a
1858 // a valid identifier part. Carry on as if the codepoint was
1859 // valid for recovery purposes.
1860 } else if (!isLexingRawMode()) {
1861 if (IsExtension)
1862 diagnoseExtensionInIdentifier(
1863 Diags&: PP->getDiagnostics(), C: CodePoint,
1864 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1865 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1866 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr),
1867 /*IsFirst=*/false);
1868 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C: CodePoint,
1869 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1870 }
1871
1872 // Once we sucessfully parsed some UTF-8,
1873 // calling ConsumeChar ensures the NeedsCleaning flag is set on the token
1874 // being lexed, and that warnings about trailing spaces are emitted.
1875 ConsumeChar(Ptr: CurPtr, Size: FirstCodeUnitSize, Tok&: Result);
1876 CurPtr = UnicodePtr;
1877 return true;
1878}
1879
1880bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
1881 const char *CurPtr) {
1882 bool IsExtension = false;
1883 if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
1884 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1885 !PP->isPreprocessedOutput()) {
1886 if (IsExtension)
1887 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C,
1888 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1889 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C,
1890 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1891 /*IsFirst=*/true);
1892 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C,
1893 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1894 }
1895
1896 MIOpt.ReadToken();
1897 return LexIdentifierContinue(Result, CurPtr);
1898 }
1899
1900 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1901 !PP->isPreprocessedOutput() && !isASCII(c: *BufferPtr) &&
1902 !isUnicodeWhitespace(Codepoint: C)) {
1903 // Non-ASCII characters tend to creep into source code unintentionally.
1904 // Instead of letting the parser complain about the unknown token,
1905 // just drop the character.
1906 // Note that we can /only/ do this when the non-ASCII character is actually
1907 // spelled as Unicode, not written as a UCN. The standard requires that
1908 // we not throw away any possible preprocessor tokens, but there's a
1909 // loophole in the mapping of Unicode characters to basic character set
1910 // characters that allows us to map these particular characters to, say,
1911 // whitespace.
1912 diagnoseInvalidUnicodeCodepointInIdentifier(
1913 Diags&: PP->getDiagnostics(), LangOpts, CodePoint: C,
1914 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr), /*IsStart*/ IsFirst: true);
1915 BufferPtr = CurPtr;
1916 return false;
1917 }
1918
1919 // Otherwise, we have an explicit UCN or a character that's unlikely to show
1920 // up by accident.
1921 MIOpt.ReadToken();
1922 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
1923 return true;
1924}
1925
1926static const char *
1927fastParseASCIIIdentifier(const char *CurPtr,
1928 [[maybe_unused]] const char *BufferEnd) {
1929#ifdef __SSE4_2__
1930 alignas(16) static constexpr char AsciiIdentifierRange[16] = {
1931 '_', '_', 'A', 'Z', 'a', 'z', '0', '9',
1932 };
1933 constexpr ssize_t BytesPerRegister = 16;
1934
1935 __m128i AsciiIdentifierRangeV =
1936 _mm_load_si128((const __m128i *)AsciiIdentifierRange);
1937
1938 while (LLVM_LIKELY(BufferEnd - CurPtr >= BytesPerRegister)) {
1939 __m128i Cv = _mm_loadu_si128((const __m128i *)(CurPtr));
1940
1941 int Consumed = _mm_cmpistri(AsciiIdentifierRangeV, Cv,
1942 _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES |
1943 _SIDD_UBYTE_OPS | _SIDD_NEGATIVE_POLARITY);
1944 CurPtr += Consumed;
1945 if (Consumed == BytesPerRegister)
1946 continue;
1947 return CurPtr;
1948 }
1949#endif
1950
1951 unsigned char C = *CurPtr;
1952 while (isAsciiIdentifierContinue(c: C))
1953 C = *++CurPtr;
1954 return CurPtr;
1955}
1956
1957bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
1958 // Match [_A-Za-z0-9]*, we have already matched an identifier start.
1959
1960 while (true) {
1961
1962 CurPtr = fastParseASCIIIdentifier(CurPtr, BufferEnd);
1963
1964 unsigned Size;
1965 // Slow path: handle trigraph, unicode codepoints, UCNs.
1966 unsigned char C = getCharAndSize(Ptr: CurPtr, Size);
1967 if (isAsciiIdentifierContinue(c: C)) {
1968 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
1969 continue;
1970 }
1971 if (C == '$') {
1972 // If we hit a $ and they are not supported in identifiers, we are done.
1973 if (!LangOpts.DollarIdents)
1974 break;
1975 // Otherwise, emit a diagnostic and continue.
1976 if (!isLexingRawMode())
1977 Diag(Loc: CurPtr, DiagID: diag::ext_dollar_in_identifier);
1978 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
1979 continue;
1980 }
1981 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
1982 continue;
1983 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
1984 continue;
1985 // Neither an expected Unicode codepoint nor a UCN.
1986 break;
1987 }
1988
1989 const char *IdStart = BufferPtr;
1990 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::raw_identifier);
1991 Result.setRawIdentifierData(IdStart);
1992
1993 // If we are in raw mode, return this identifier raw. There is no need to
1994 // look up identifier information or attempt to macro expand it.
1995 if (LexingRawMode)
1996 return true;
1997
1998 // Fill in Result.IdentifierInfo and update the token kind,
1999 // looking up the identifier in the identifier table.
2000 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
2001 // Note that we have to call PP->LookUpIdentifierInfo() even for code
2002 // completion, it writes IdentifierInfo into Result, and callers rely on it.
2003
2004 // If the completion point is at the end of an identifier, we want to treat
2005 // the identifier as incomplete even if it resolves to a macro or a keyword.
2006 // This allows e.g. 'class^' to complete to 'classifier'.
2007 if (isCodeCompletionPoint(CurPtr)) {
2008 // Return the code-completion token.
2009 Result.setKind(tok::code_completion);
2010 // Skip the code-completion char and all immediate identifier characters.
2011 // This ensures we get consistent behavior when completing at any point in
2012 // an identifier (i.e. at the start, in the middle, at the end). Note that
2013 // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
2014 // simpler.
2015 assert(*CurPtr == 0 && "Completion character must be 0");
2016 ++CurPtr;
2017 // Note that code completion token is not added as a separate character
2018 // when the completion point is at the end of the buffer. Therefore, we need
2019 // to check if the buffer has ended.
2020 if (CurPtr < BufferEnd) {
2021 while (isAsciiIdentifierContinue(c: *CurPtr))
2022 ++CurPtr;
2023 }
2024 BufferPtr = CurPtr;
2025 return true;
2026 }
2027
2028 // Finally, now that we know we have an identifier, pass this off to the
2029 // preprocessor, which may macro expand it or something.
2030 if (II->isHandleIdentifierCase())
2031 return PP->HandleIdentifier(Identifier&: Result);
2032
2033 return true;
2034}
2035
2036/// isHexaLiteral - Return true if Start points to a hex constant.
2037/// in microsoft mode (where this is supposed to be several different tokens).
2038bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
2039 auto CharAndSize1 = Lexer::getCharAndSizeNoWarn(Ptr: Start, LangOpts);
2040 char C1 = CharAndSize1.Char;
2041 if (C1 != '0')
2042 return false;
2043
2044 auto CharAndSize2 =
2045 Lexer::getCharAndSizeNoWarn(Ptr: Start + CharAndSize1.Size, LangOpts);
2046 char C2 = CharAndSize2.Char;
2047 return (C2 == 'x' || C2 == 'X');
2048}
2049
2050/// LexNumericConstant - Lex the remainder of a integer or floating point
2051/// constant. From[-1] is the first character lexed. Return the end of the
2052/// constant.
2053bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
2054 unsigned Size;
2055 char C = getCharAndSize(Ptr: CurPtr, Size);
2056 char PrevCh = 0;
2057 while (isPreprocessingNumberBody(c: C)) {
2058 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2059 PrevCh = C;
2060 if (LangOpts.HLSL && C == '.' && (*CurPtr == 'x' || *CurPtr == 'r')) {
2061 CurPtr -= Size;
2062 break;
2063 }
2064 C = getCharAndSize(Ptr: CurPtr, Size);
2065 }
2066
2067 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
2068 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
2069 // If we are in Microsoft mode, don't continue if the constant is hex.
2070 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
2071 if (!LangOpts.MicrosoftExt || !isHexaLiteral(Start: BufferPtr, LangOpts))
2072 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2073 }
2074
2075 // If we have a hex FP constant, continue.
2076 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
2077 // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
2078 // not-quite-conforming extension. Only do so if this looks like it's
2079 // actually meant to be a hexfloat, and not if it has a ud-suffix.
2080 bool IsHexFloat = true;
2081 if (!LangOpts.C99) {
2082 if (!isHexaLiteral(Start: BufferPtr, LangOpts))
2083 IsHexFloat = false;
2084 else if (!LangOpts.CPlusPlus17 &&
2085 std::find(first: BufferPtr, last: CurPtr, val: '_') != CurPtr)
2086 IsHexFloat = false;
2087 }
2088 if (IsHexFloat)
2089 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2090 }
2091
2092 // If we have a digit separator, continue.
2093 if (C == '\'' && (LangOpts.CPlusPlus14 || LangOpts.C23)) {
2094 auto [Next, NextSize] = getCharAndSizeNoWarn(Ptr: CurPtr + Size, LangOpts);
2095 if (isAsciiIdentifierContinue(c: Next)) {
2096 if (!isLexingRawMode())
2097 Diag(Loc: CurPtr, DiagID: LangOpts.CPlusPlus
2098 ? diag::warn_cxx11_compat_digit_separator
2099 : diag::warn_c23_compat_digit_separator);
2100 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2101 CurPtr = ConsumeChar(Ptr: CurPtr, Size: NextSize, Tok&: Result);
2102 return LexNumericConstant(Result, CurPtr);
2103 }
2104 }
2105
2106 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
2107 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2108 return LexNumericConstant(Result, CurPtr);
2109 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2110 return LexNumericConstant(Result, CurPtr);
2111
2112 // Update the location of token as well as BufferPtr.
2113 const char *TokStart = BufferPtr;
2114 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::numeric_constant);
2115 Result.setLiteralData(TokStart);
2116 return true;
2117}
2118
2119/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
2120/// in C++11, or warn on a ud-suffix in C++98.
2121const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
2122 bool IsStringLiteral) {
2123 assert(LangOpts.CPlusPlus);
2124
2125 // Maximally munch an identifier.
2126 unsigned Size;
2127 char C = getCharAndSize(Ptr: CurPtr, Size);
2128 bool Consumed = false;
2129
2130 if (!isAsciiIdentifierStart(c: C)) {
2131 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2132 Consumed = true;
2133 else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2134 Consumed = true;
2135 else
2136 return CurPtr;
2137 }
2138
2139 if (!LangOpts.CPlusPlus11) {
2140 if (!isLexingRawMode())
2141 Diag(Loc: CurPtr,
2142 DiagID: C == '_' ? diag::warn_cxx11_compat_user_defined_literal
2143 : diag::warn_cxx11_compat_reserved_user_defined_literal)
2144 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2145 return CurPtr;
2146 }
2147
2148 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
2149 // that does not start with an underscore is ill-formed. As a conforming
2150 // extension, we treat all such suffixes as if they had whitespace before
2151 // them. We assume a suffix beginning with a UCN or UTF-8 character is more
2152 // likely to be a ud-suffix than a macro, however, and accept that.
2153 if (!Consumed) {
2154 bool IsUDSuffix = false;
2155 if (C == '_')
2156 IsUDSuffix = true;
2157 else if (IsStringLiteral && LangOpts.CPlusPlus14) {
2158 // In C++1y, we need to look ahead a few characters to see if this is a
2159 // valid suffix for a string literal or a numeric literal (this could be
2160 // the 'operator""if' defining a numeric literal operator).
2161 const unsigned MaxStandardSuffixLength = 3;
2162 char Buffer[MaxStandardSuffixLength] = { C };
2163 unsigned Consumed = Size;
2164 unsigned Chars = 1;
2165 while (true) {
2166 auto [Next, NextSize] =
2167 getCharAndSizeNoWarn(Ptr: CurPtr + Consumed, LangOpts);
2168 if (!isAsciiIdentifierContinue(c: Next)) {
2169 // End of suffix. Check whether this is on the allowed list.
2170 const StringRef CompleteSuffix(Buffer, Chars);
2171 IsUDSuffix =
2172 StringLiteralParser::isValidUDSuffix(LangOpts, Suffix: CompleteSuffix);
2173 break;
2174 }
2175
2176 if (Chars == MaxStandardSuffixLength)
2177 // Too long: can't be a standard suffix.
2178 break;
2179
2180 Buffer[Chars++] = Next;
2181 Consumed += NextSize;
2182 }
2183 }
2184
2185 if (!IsUDSuffix) {
2186 if (!isLexingRawMode())
2187 Diag(Loc: CurPtr, DiagID: LangOpts.MSVCCompat
2188 ? diag::ext_ms_reserved_user_defined_literal
2189 : diag::ext_reserved_user_defined_literal)
2190 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2191 return CurPtr;
2192 }
2193
2194 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2195 }
2196
2197 Result.setFlag(Token::HasUDSuffix);
2198 while (true) {
2199 C = getCharAndSize(Ptr: CurPtr, Size);
2200 if (isAsciiIdentifierContinue(c: C)) {
2201 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2202 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
2203 } else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result)) {
2204 } else
2205 break;
2206 }
2207
2208 return CurPtr;
2209}
2210
2211/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
2212/// either " or L" or u8" or u" or U".
2213bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
2214 tok::TokenKind Kind) {
2215 const char *AfterQuote = CurPtr;
2216 // Does this string contain the \0 character?
2217 const char *NulCharacter = nullptr;
2218
2219 if (!isLexingRawMode() &&
2220 (Kind == tok::utf8_string_literal ||
2221 Kind == tok::utf16_string_literal ||
2222 Kind == tok::utf32_string_literal))
2223 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
2224 : diag::warn_c99_compat_unicode_literal);
2225
2226 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2227 while (C != '"') {
2228 // Skip escaped characters. Escaped newlines will already be processed by
2229 // getAndAdvanceChar.
2230 if (C == '\\')
2231 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2232
2233 if (C == '\n' || C == '\r' || // Newline.
2234 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2235 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2236 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 1;
2237 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2238 return true;
2239 }
2240
2241 if (C == 0) {
2242 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2243 if (ParsingFilename)
2244 codeCompleteIncludedFile(PathStart: AfterQuote, CompletionPoint: CurPtr - 1, /*IsAngled=*/false);
2245 else
2246 PP->CodeCompleteNaturalLanguage();
2247 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2248 cutOffLexing();
2249 return true;
2250 }
2251
2252 NulCharacter = CurPtr-1;
2253 }
2254 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2255 }
2256
2257 // If we are in C++11, lex the optional ud-suffix.
2258 if (LangOpts.CPlusPlus)
2259 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2260
2261 // If a nul character existed in the string, warn about it.
2262 if (NulCharacter && !isLexingRawMode())
2263 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2264
2265 // Update the location of the token as well as the BufferPtr instance var.
2266 const char *TokStart = BufferPtr;
2267 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2268 Result.setLiteralData(TokStart);
2269 return true;
2270}
2271
2272/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
2273/// having lexed R", LR", u8R", uR", or UR".
2274bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2275 tok::TokenKind Kind) {
2276 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
2277 // Between the initial and final double quote characters of the raw string,
2278 // any transformations performed in phases 1 and 2 (trigraphs,
2279 // universal-character-names, and line splicing) are reverted.
2280
2281 if (!isLexingRawMode())
2282 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_raw_string_literal);
2283
2284 unsigned PrefixLen = 0;
2285
2286 while (PrefixLen != 16 && isRawStringDelimBody(c: CurPtr[PrefixLen])) {
2287 if (!isLexingRawMode() &&
2288 llvm::is_contained(Set: {'$', '@', '`'}, Element: CurPtr[PrefixLen])) {
2289 const char *Pos = &CurPtr[PrefixLen];
2290 Diag(Loc: Pos, DiagID: LangOpts.CPlusPlus26
2291 ? diag::warn_cxx26_compat_raw_string_literal_character_set
2292 : diag::ext_cxx26_raw_string_literal_character_set)
2293 << StringRef(Pos, 1);
2294 }
2295 ++PrefixLen;
2296 }
2297
2298 // If the last character was not a '(', then we didn't lex a valid delimiter.
2299 if (CurPtr[PrefixLen] != '(') {
2300 if (!isLexingRawMode()) {
2301 const char *PrefixEnd = &CurPtr[PrefixLen];
2302 if (PrefixLen == 16) {
2303 Diag(Loc: PrefixEnd, DiagID: diag::err_raw_delim_too_long);
2304 } else if (*PrefixEnd == '\n') {
2305 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_newline_raw_delim);
2306 } else {
2307 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_char_raw_delim)
2308 << StringRef(PrefixEnd, 1);
2309 }
2310 }
2311
2312 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2313 // it's possible the '"' was intended to be part of the raw string, but
2314 // there's not much we can do about that.
2315 while (true) {
2316 char C = *CurPtr++;
2317
2318 if (C == '"')
2319 break;
2320 if (C == 0 && CurPtr-1 == BufferEnd) {
2321 --CurPtr;
2322 break;
2323 }
2324 }
2325
2326 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2327 return true;
2328 }
2329
2330 // Save prefix and move CurPtr past it
2331 const char *Prefix = CurPtr;
2332 CurPtr += PrefixLen + 1; // skip over prefix and '('
2333
2334 while (true) {
2335 char C = *CurPtr++;
2336
2337 if (C == ')') {
2338 // Check for prefix match and closing quote.
2339 if (strncmp(s1: CurPtr, s2: Prefix, n: PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
2340 CurPtr += PrefixLen + 1; // skip over prefix and '"'
2341 break;
2342 }
2343 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
2344 if (!isLexingRawMode())
2345 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_raw_string)
2346 << StringRef(Prefix, PrefixLen);
2347 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2348 return true;
2349 }
2350 }
2351
2352 // If we are in C++11, lex the optional ud-suffix.
2353 if (LangOpts.CPlusPlus)
2354 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2355
2356 // Update the location of token as well as BufferPtr.
2357 const char *TokStart = BufferPtr;
2358 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2359 Result.setLiteralData(TokStart);
2360 return true;
2361}
2362
2363/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2364/// after having lexed the '<' character. This is used for #include filenames.
2365bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2366 // Does this string contain the \0 character?
2367 const char *NulCharacter = nullptr;
2368 const char *AfterLessPos = CurPtr;
2369 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2370 while (C != '>') {
2371 // Skip escaped characters. Escaped newlines will already be processed by
2372 // getAndAdvanceChar.
2373 if (C == '\\')
2374 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2375
2376 if (isVerticalWhitespace(c: C) || // Newline.
2377 (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
2378 // If the filename is unterminated, then it must just be a lone <
2379 // character. Return this as such.
2380 FormTokenWithChars(Result, TokEnd: AfterLessPos, Kind: tok::less);
2381 return true;
2382 }
2383
2384 if (C == 0) {
2385 if (isCodeCompletionPoint(CurPtr: CurPtr - 1)) {
2386 codeCompleteIncludedFile(PathStart: AfterLessPos, CompletionPoint: CurPtr - 1, /*IsAngled=*/true);
2387 cutOffLexing();
2388 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2389 return true;
2390 }
2391 NulCharacter = CurPtr-1;
2392 }
2393 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2394 }
2395
2396 // If a nul character existed in the string, warn about it.
2397 if (NulCharacter && !isLexingRawMode())
2398 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2399
2400 // Update the location of token as well as BufferPtr.
2401 const char *TokStart = BufferPtr;
2402 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::header_name);
2403 Result.setLiteralData(TokStart);
2404 return true;
2405}
2406
2407void Lexer::codeCompleteIncludedFile(const char *PathStart,
2408 const char *CompletionPoint,
2409 bool IsAngled) {
2410 // Completion only applies to the filename, after the last slash.
2411 StringRef PartialPath(PathStart, CompletionPoint - PathStart);
2412 llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
2413 auto Slash = PartialPath.find_last_of(Chars: SlashChars);
2414 StringRef Dir =
2415 (Slash == StringRef::npos) ? "" : PartialPath.take_front(N: Slash);
2416 const char *StartOfFilename =
2417 (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
2418 // Code completion filter range is the filename only, up to completion point.
2419 PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
2420 Name: StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
2421 // We should replace the characters up to the closing quote or closest slash,
2422 // if any.
2423 while (CompletionPoint < BufferEnd) {
2424 char Next = *(CompletionPoint + 1);
2425 if (Next == 0 || Next == '\r' || Next == '\n')
2426 break;
2427 ++CompletionPoint;
2428 if (Next == (IsAngled ? '>' : '"'))
2429 break;
2430 if (SlashChars.contains(C: Next))
2431 break;
2432 }
2433
2434 PP->setCodeCompletionTokenRange(
2435 Start: FileLoc.getLocWithOffset(Offset: StartOfFilename - BufferStart),
2436 End: FileLoc.getLocWithOffset(Offset: CompletionPoint - BufferStart));
2437 PP->CodeCompleteIncludedFile(Dir, IsAngled);
2438}
2439
2440/// LexCharConstant - Lex the remainder of a character constant, after having
2441/// lexed either ' or L' or u8' or u' or U'.
2442bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2443 tok::TokenKind Kind) {
2444 // Does this character contain the \0 character?
2445 const char *NulCharacter = nullptr;
2446
2447 if (!isLexingRawMode()) {
2448 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
2449 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2450 ? diag::warn_cxx98_compat_unicode_literal
2451 : diag::warn_c99_compat_unicode_literal);
2452 else if (Kind == tok::utf8_char_constant)
2453 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2454 ? diag::warn_cxx14_compat_u8_character_literal
2455 : diag::warn_c17_compat_u8_character_literal);
2456 }
2457
2458 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2459 if (C == '\'') {
2460 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2461 Diag(Loc: BufferPtr, DiagID: diag::ext_empty_character);
2462 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2463 return true;
2464 }
2465
2466 while (C != '\'') {
2467 // Skip escaped characters.
2468 if (C == '\\')
2469 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2470
2471 if (C == '\n' || C == '\r' || // Newline.
2472 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2473 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2474 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 0;
2475 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2476 return true;
2477 }
2478
2479 if (C == 0) {
2480 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2481 PP->CodeCompleteNaturalLanguage();
2482 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2483 cutOffLexing();
2484 return true;
2485 }
2486
2487 NulCharacter = CurPtr-1;
2488 }
2489 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2490 }
2491
2492 // If we are in C++11, lex the optional ud-suffix.
2493 if (LangOpts.CPlusPlus)
2494 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: false);
2495
2496 // If a nul character existed in the character, warn about it.
2497 if (NulCharacter && !isLexingRawMode())
2498 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 0;
2499
2500 // Update the location of token as well as BufferPtr.
2501 const char *TokStart = BufferPtr;
2502 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2503 Result.setLiteralData(TokStart);
2504 return true;
2505}
2506
2507/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2508/// Update BufferPtr to point to the next non-whitespace character and return.
2509///
2510/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2511bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
2512 bool &TokAtPhysicalStartOfLine) {
2513 // Whitespace - Skip it, then return the token after the whitespace.
2514 bool SawNewline = isVerticalWhitespace(c: CurPtr[-1]);
2515
2516 unsigned char Char = *CurPtr;
2517
2518 const char *lastNewLine = nullptr;
2519 auto setLastNewLine = [&](const char *Ptr) {
2520 lastNewLine = Ptr;
2521 if (!NewLinePtr)
2522 NewLinePtr = Ptr;
2523 };
2524 if (SawNewline)
2525 setLastNewLine(CurPtr - 1);
2526
2527 // Skip consecutive spaces efficiently.
2528 while (true) {
2529 // Skip horizontal whitespace very aggressively.
2530 while (isHorizontalWhitespace(c: Char))
2531 Char = *++CurPtr;
2532
2533 // Otherwise if we have something other than whitespace, we're done.
2534 if (!isVerticalWhitespace(c: Char))
2535 break;
2536
2537 if (ParsingPreprocessorDirective) {
2538 // End of preprocessor directive line, let LexTokenInternal handle this.
2539 BufferPtr = CurPtr;
2540 return false;
2541 }
2542
2543 // OK, but handle newline.
2544 if (*CurPtr == '\n')
2545 setLastNewLine(CurPtr);
2546 SawNewline = true;
2547 Char = *++CurPtr;
2548 }
2549
2550 // If the client wants us to return whitespace, return it now.
2551 if (isKeepWhitespaceMode()) {
2552 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2553 if (SawNewline) {
2554 IsAtStartOfLine = true;
2555 IsAtPhysicalStartOfLine = true;
2556 }
2557 // FIXME: The next token will not have LeadingSpace set.
2558 return true;
2559 }
2560
2561 // If this isn't immediately after a newline, there is leading space.
2562 char PrevChar = CurPtr[-1];
2563 bool HasLeadingSpace = !isVerticalWhitespace(c: PrevChar);
2564
2565 Result.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace);
2566 if (SawNewline) {
2567 Result.setFlag(Token::StartOfLine);
2568 TokAtPhysicalStartOfLine = true;
2569
2570 if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) {
2571 if (auto *Handler = PP->getEmptylineHandler())
2572 Handler->HandleEmptyline(Range: SourceRange(getSourceLocation(Loc: NewLinePtr + 1),
2573 getSourceLocation(Loc: lastNewLine)));
2574 }
2575 }
2576
2577 BufferPtr = CurPtr;
2578 return false;
2579}
2580
2581/// We have just read the // characters from input. Skip until we find the
2582/// newline character that terminates the comment. Then update BufferPtr and
2583/// return.
2584///
2585/// If we're in KeepCommentMode or any CommentHandler has inserted
2586/// some tokens, this will store the first token and return true.
2587bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
2588 bool &TokAtPhysicalStartOfLine) {
2589 // If Line comments aren't explicitly enabled for this language, emit an
2590 // extension warning.
2591 if (!LineComment) {
2592 if (!isLexingRawMode()) // There's no PP in raw mode, so can't emit diags.
2593 Diag(Loc: BufferPtr, DiagID: diag::ext_line_comment);
2594
2595 // Mark them enabled so we only emit one warning for this translation
2596 // unit.
2597 LineComment = true;
2598 }
2599
2600 // Scan over the body of the comment. The common case, when scanning, is that
2601 // the comment contains normal ascii characters with nothing interesting in
2602 // them. As such, optimize for this case with the inner loop.
2603 //
2604 // This loop terminates with CurPtr pointing at the newline (or end of buffer)
2605 // character that ends the line comment.
2606
2607 // C++23 [lex.phases] p1
2608 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2609 // diagnostic only once per entire ill-formed subsequence to avoid
2610 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2611 bool UnicodeDecodingAlreadyDiagnosed = false;
2612
2613 char C;
2614 while (true) {
2615 C = *CurPtr;
2616 // Skip over characters in the fast loop.
2617 while (isASCII(c: C) && C != 0 && // Potentially EOF.
2618 C != '\n' && C != '\r') { // Newline or DOS-style newline.
2619 C = *++CurPtr;
2620 UnicodeDecodingAlreadyDiagnosed = false;
2621 }
2622
2623 if (!isASCII(c: C)) {
2624 unsigned Length = llvm::getUTF8SequenceSize(
2625 source: (const llvm::UTF8 *)CurPtr, sourceEnd: (const llvm::UTF8 *)BufferEnd);
2626 if (Length == 0) {
2627 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
2628 Diag(Loc: CurPtr, DiagID: diag::warn_invalid_utf8_in_comment);
2629 UnicodeDecodingAlreadyDiagnosed = true;
2630 ++CurPtr;
2631 } else {
2632 UnicodeDecodingAlreadyDiagnosed = false;
2633 CurPtr += Length;
2634 }
2635 continue;
2636 }
2637
2638 const char *NextLine = CurPtr;
2639 if (C != 0) {
2640 // We found a newline, see if it's escaped.
2641 const char *EscapePtr = CurPtr-1;
2642 bool HasSpace = false;
2643 while (isHorizontalWhitespace(c: *EscapePtr)) { // Skip whitespace.
2644 --EscapePtr;
2645 HasSpace = true;
2646 }
2647
2648 if (*EscapePtr == '\\')
2649 // Escaped newline.
2650 CurPtr = EscapePtr;
2651 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
2652 EscapePtr[-2] == '?' && LangOpts.Trigraphs)
2653 // Trigraph-escaped newline.
2654 CurPtr = EscapePtr-2;
2655 else
2656 break; // This is a newline, we're done.
2657
2658 // If there was space between the backslash and newline, warn about it.
2659 if (HasSpace && !isLexingRawMode())
2660 Diag(Loc: EscapePtr, DiagID: diag::backslash_newline_space);
2661 }
2662
2663 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
2664 // properly decode the character. Read it in raw mode to avoid emitting
2665 // diagnostics about things like trigraphs. If we see an escaped newline,
2666 // we'll handle it below.
2667 const char *OldPtr = CurPtr;
2668 bool OldRawMode = isLexingRawMode();
2669 LexingRawMode = true;
2670 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2671 LexingRawMode = OldRawMode;
2672
2673 // If we only read only one character, then no special handling is needed.
2674 // We're done and can skip forward to the newline.
2675 if (C != 0 && CurPtr == OldPtr+1) {
2676 CurPtr = NextLine;
2677 break;
2678 }
2679
2680 // If we read multiple characters, and one of those characters was a \r or
2681 // \n, then we had an escaped newline within the comment. Emit diagnostic
2682 // unless the next line is also a // comment.
2683 if (CurPtr != OldPtr + 1 && C != '/' &&
2684 (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) {
2685 for (; OldPtr != CurPtr; ++OldPtr)
2686 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
2687 // Okay, we found a // comment that ends in a newline, if the next
2688 // line is also a // comment, but has spaces, don't emit a diagnostic.
2689 if (isWhitespace(c: C)) {
2690 const char *ForwardPtr = CurPtr;
2691 while (isWhitespace(c: *ForwardPtr)) // Skip whitespace.
2692 ++ForwardPtr;
2693 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
2694 break;
2695 }
2696
2697 if (!isLexingRawMode())
2698 Diag(Loc: OldPtr-1, DiagID: diag::ext_multi_line_line_comment);
2699 break;
2700 }
2701 }
2702
2703 if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
2704 --CurPtr;
2705 break;
2706 }
2707
2708 if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2709 PP->CodeCompleteNaturalLanguage();
2710 cutOffLexing();
2711 return false;
2712 }
2713 }
2714
2715 // Found but did not consume the newline. Notify comment handlers about the
2716 // comment unless we're in a #if 0 block.
2717 if (PP && !isLexingRawMode() &&
2718 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
2719 getSourceLocation(Loc: CurPtr)))) {
2720 BufferPtr = CurPtr;
2721 return true; // A token has to be returned.
2722 }
2723
2724 // If we are returning comments as tokens, return this comment as a token.
2725 if (inKeepCommentMode())
2726 return SaveLineComment(Result, CurPtr);
2727
2728 // If we are inside a preprocessor directive and we see the end of line,
2729 // return immediately, so that the lexer can return this as an EOD token.
2730 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
2731 BufferPtr = CurPtr;
2732 return false;
2733 }
2734
2735 // Otherwise, eat the \n character. We don't care if this is a \n\r or
2736 // \r\n sequence. This is an efficiency hack (because we know the \n can't
2737 // contribute to another token), it isn't needed for correctness. Note that
2738 // this is ok even in KeepWhitespaceMode, because we would have returned the
2739 // comment above in that mode.
2740 NewLinePtr = CurPtr++;
2741
2742 // The next returned token is at the start of the line.
2743 Result.setFlag(Token::StartOfLine);
2744 TokAtPhysicalStartOfLine = true;
2745 // No leading whitespace seen so far.
2746 Result.clearFlag(Flag: Token::LeadingSpace);
2747 BufferPtr = CurPtr;
2748 return false;
2749}
2750
2751/// If in save-comment mode, package up this Line comment in an appropriate
2752/// way and return it.
2753bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2754 // If we're not in a preprocessor directive, just return the // comment
2755 // directly.
2756 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
2757
2758 if (!ParsingPreprocessorDirective || LexingRawMode)
2759 return true;
2760
2761 // If this Line-style comment is in a macro definition, transmogrify it into
2762 // a C-style block comment.
2763 bool Invalid = false;
2764 std::string Spelling = PP->getSpelling(Tok: Result, Invalid: &Invalid);
2765 if (Invalid)
2766 return true;
2767
2768 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
2769 Spelling[1] = '*'; // Change prefix to "/*".
2770 Spelling += "*/"; // add suffix.
2771
2772 Result.setKind(tok::comment);
2773 PP->CreateString(Str: Spelling, Tok&: Result,
2774 ExpansionLocStart: Result.getLocation(), ExpansionLocEnd: Result.getLocation());
2775 return true;
2776}
2777
2778/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2779/// character (either \\n or \\r) is part of an escaped newline sequence. Issue
2780/// a diagnostic if so. We know that the newline is inside of a block comment.
2781static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, Lexer *L,
2782 bool Trigraphs) {
2783 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
2784
2785 // Position of the first trigraph in the ending sequence.
2786 const char *TrigraphPos = nullptr;
2787 // Position of the first whitespace after a '\' in the ending sequence.
2788 const char *SpacePos = nullptr;
2789
2790 while (true) {
2791 // Back up off the newline.
2792 --CurPtr;
2793
2794 // If this is a two-character newline sequence, skip the other character.
2795 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
2796 // \n\n or \r\r -> not escaped newline.
2797 if (CurPtr[0] == CurPtr[1])
2798 return false;
2799 // \n\r or \r\n -> skip the newline.
2800 --CurPtr;
2801 }
2802
2803 // If we have horizontal whitespace, skip over it. We allow whitespace
2804 // between the slash and newline.
2805 while (isHorizontalWhitespace(c: *CurPtr) || *CurPtr == 0) {
2806 SpacePos = CurPtr;
2807 --CurPtr;
2808 }
2809
2810 // If we have a slash, this is an escaped newline.
2811 if (*CurPtr == '\\') {
2812 --CurPtr;
2813 } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') {
2814 // This is a trigraph encoding of a slash.
2815 TrigraphPos = CurPtr - 2;
2816 CurPtr -= 3;
2817 } else {
2818 return false;
2819 }
2820
2821 // If the character preceding the escaped newline is a '*', then after line
2822 // splicing we have a '*/' ending the comment.
2823 if (*CurPtr == '*')
2824 break;
2825
2826 if (*CurPtr != '\n' && *CurPtr != '\r')
2827 return false;
2828 }
2829
2830 if (TrigraphPos) {
2831 // If no trigraphs are enabled, warn that we ignored this trigraph and
2832 // ignore this * character.
2833 if (!Trigraphs) {
2834 if (!L->isLexingRawMode())
2835 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ignored_block_comment);
2836 return false;
2837 }
2838 if (!L->isLexingRawMode())
2839 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ends_block_comment);
2840 }
2841
2842 // Warn about having an escaped newline between the */ characters.
2843 if (!L->isLexingRawMode())
2844 L->Diag(Loc: CurPtr + 1, DiagID: diag::escaped_newline_block_comment_end);
2845
2846 // If there was space between the backslash and newline, warn about it.
2847 if (SpacePos && !L->isLexingRawMode())
2848 L->Diag(Loc: SpacePos, DiagID: diag::backslash_newline_space);
2849
2850 return true;
2851}
2852
2853#ifdef __SSE2__
2854#include <emmintrin.h>
2855#elif __ALTIVEC__
2856#include <altivec.h>
2857#undef bool
2858#endif
2859
2860/// We have just read from input the / and * characters that started a comment.
2861/// Read until we find the * and / characters that terminate the comment.
2862/// Note that we don't bother decoding trigraphs or escaped newlines in block
2863/// comments, because they cannot cause the comment to end. The only thing
2864/// that can happen is the comment could end with an escaped newline between
2865/// the terminating * and /.
2866///
2867/// If we're in KeepCommentMode or any CommentHandler has inserted
2868/// some tokens, this will store the first token and return true.
2869bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
2870 bool &TokAtPhysicalStartOfLine) {
2871 // Scan one character past where we should, looking for a '/' character. Once
2872 // we find it, check to see if it was preceded by a *. This common
2873 // optimization helps people who like to put a lot of * characters in their
2874 // comments.
2875
2876 // The first character we get with newlines and trigraphs skipped to handle
2877 // the degenerate /*/ case below correctly if the * has an escaped newline
2878 // after it.
2879 unsigned CharSize;
2880 unsigned char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
2881 CurPtr += CharSize;
2882 if (C == 0 && CurPtr == BufferEnd+1) {
2883 if (!isLexingRawMode())
2884 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
2885 --CurPtr;
2886
2887 // KeepWhitespaceMode should return this broken comment as a token. Since
2888 // it isn't a well formed comment, just return it as an 'unknown' token.
2889 if (isKeepWhitespaceMode()) {
2890 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2891 return true;
2892 }
2893
2894 BufferPtr = CurPtr;
2895 return false;
2896 }
2897
2898 // Check to see if the first character after the '/*' is another /. If so,
2899 // then this slash does not end the block comment, it is part of it.
2900 if (C == '/')
2901 C = *CurPtr++;
2902
2903 // C++23 [lex.phases] p1
2904 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2905 // diagnostic only once per entire ill-formed subsequence to avoid
2906 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2907 bool UnicodeDecodingAlreadyDiagnosed = false;
2908
2909 while (true) {
2910 // Skip over all non-interesting characters until we find end of buffer or a
2911 // (probably ending) '/' character.
2912 if (CurPtr + 24 < BufferEnd &&
2913 // If there is a code-completion point avoid the fast scan because it
2914 // doesn't check for '\0'.
2915 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
2916 // While not aligned to a 16-byte boundary.
2917 while (C != '/' && (intptr_t)CurPtr % 16 != 0) {
2918 if (!isASCII(c: C))
2919 goto MultiByteUTF8;
2920 C = *CurPtr++;
2921 }
2922 if (C == '/') goto FoundSlash;
2923
2924#ifdef __SSE2__
2925 __m128i Slashes = _mm_set1_epi8(b: '/');
2926 while (CurPtr + 16 < BufferEnd) {
2927 int Mask = _mm_movemask_epi8(a: *(const __m128i *)CurPtr);
2928 if (LLVM_UNLIKELY(Mask != 0)) {
2929 goto MultiByteUTF8;
2930 }
2931 // look for slashes
2932 int cmp = _mm_movemask_epi8(a: _mm_cmpeq_epi8(a: *(const __m128i*)CurPtr,
2933 b: Slashes));
2934 if (cmp != 0) {
2935 // Adjust the pointer to point directly after the first slash. It's
2936 // not necessary to set C here, it will be overwritten at the end of
2937 // the outer loop.
2938 CurPtr += llvm::countr_zero<unsigned>(Val: cmp) + 1;
2939 goto FoundSlash;
2940 }
2941 CurPtr += 16;
2942 }
2943#elif __ALTIVEC__
2944 __vector unsigned char LongUTF = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
2945 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
2946 0x80, 0x80, 0x80, 0x80};
2947 __vector unsigned char Slashes = {
2948 '/', '/', '/', '/', '/', '/', '/', '/',
2949 '/', '/', '/', '/', '/', '/', '/', '/'
2950 };
2951 while (CurPtr + 16 < BufferEnd) {
2952 if (LLVM_UNLIKELY(
2953 vec_any_ge(*(const __vector unsigned char *)CurPtr, LongUTF)))
2954 goto MultiByteUTF8;
2955 if (vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) {
2956 break;
2957 }
2958 CurPtr += 16;
2959 }
2960
2961#else
2962 while (CurPtr + 16 < BufferEnd) {
2963 bool HasNonASCII = false;
2964 for (unsigned I = 0; I < 16; ++I)
2965 HasNonASCII |= !isASCII(CurPtr[I]);
2966
2967 if (LLVM_UNLIKELY(HasNonASCII))
2968 goto MultiByteUTF8;
2969
2970 bool HasSlash = false;
2971 for (unsigned I = 0; I < 16; ++I)
2972 HasSlash |= CurPtr[I] == '/';
2973 if (HasSlash)
2974 break;
2975 CurPtr += 16;
2976 }
2977#endif
2978
2979 // It has to be one of the bytes scanned, increment to it and read one.
2980 C = *CurPtr++;
2981 }
2982
2983 // Loop to scan the remainder, warning on invalid UTF-8
2984 // if the corresponding warning is enabled, emitting a diagnostic only once
2985 // per sequence that cannot be decoded.
2986 while (C != '/' && C != '\0') {
2987 if (isASCII(c: C)) {
2988 UnicodeDecodingAlreadyDiagnosed = false;
2989 C = *CurPtr++;
2990 continue;
2991 }
2992 MultiByteUTF8:
2993 // CurPtr is 1 code unit past C, so to decode
2994 // the codepoint, we need to read from the previous position.
2995 unsigned Length = llvm::getUTF8SequenceSize(
2996 source: (const llvm::UTF8 *)CurPtr - 1, sourceEnd: (const llvm::UTF8 *)BufferEnd);
2997 if (Length == 0) {
2998 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
2999 Diag(Loc: CurPtr - 1, DiagID: diag::warn_invalid_utf8_in_comment);
3000 UnicodeDecodingAlreadyDiagnosed = true;
3001 } else {
3002 UnicodeDecodingAlreadyDiagnosed = false;
3003 CurPtr += Length - 1;
3004 }
3005 C = *CurPtr++;
3006 }
3007
3008 if (C == '/') {
3009 FoundSlash:
3010 if (CurPtr[-2] == '*') // We found the final */. We're done!
3011 break;
3012
3013 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
3014 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr: CurPtr - 2, L: this,
3015 Trigraphs: LangOpts.Trigraphs)) {
3016 // We found the final */, though it had an escaped newline between the
3017 // * and /. We're done!
3018 break;
3019 }
3020 }
3021 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
3022 // If this is a /* inside of the comment, emit a warning. Don't do this
3023 // if this is a /*/, which will end the comment. This misses cases with
3024 // embedded escaped newlines, but oh well.
3025 if (!isLexingRawMode())
3026 Diag(Loc: CurPtr-1, DiagID: diag::warn_nested_block_comment);
3027 }
3028 } else if (C == 0 && CurPtr == BufferEnd+1) {
3029 if (!isLexingRawMode())
3030 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
3031 // Note: the user probably forgot a */. We could continue immediately
3032 // after the /*, but this would involve lexing a lot of what really is the
3033 // comment, which surely would confuse the parser.
3034 --CurPtr;
3035
3036 // KeepWhitespaceMode should return this broken comment as a token. Since
3037 // it isn't a well formed comment, just return it as an 'unknown' token.
3038 if (isKeepWhitespaceMode()) {
3039 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3040 return true;
3041 }
3042
3043 BufferPtr = CurPtr;
3044 return false;
3045 } else if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3046 PP->CodeCompleteNaturalLanguage();
3047 cutOffLexing();
3048 return false;
3049 }
3050
3051 C = *CurPtr++;
3052 }
3053
3054 // Notify comment handlers about the comment unless we're in a #if 0 block.
3055 if (PP && !isLexingRawMode() &&
3056 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
3057 getSourceLocation(Loc: CurPtr)))) {
3058 BufferPtr = CurPtr;
3059 return true; // A token has to be returned.
3060 }
3061
3062 // If we are returning comments as tokens, return this comment as a token.
3063 if (inKeepCommentMode()) {
3064 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
3065 return true;
3066 }
3067
3068 // It is common for the tokens immediately after a /**/ comment to be
3069 // whitespace. Instead of going through the big switch, handle it
3070 // efficiently now. This is safe even in KeepWhitespaceMode because we would
3071 // have already returned above with the comment as a token.
3072 if (isHorizontalWhitespace(c: *CurPtr)) {
3073 SkipWhitespace(Result, CurPtr: CurPtr+1, TokAtPhysicalStartOfLine);
3074 return false;
3075 }
3076
3077 // Otherwise, just return so that the next character will be lexed as a token.
3078 BufferPtr = CurPtr;
3079 Result.setFlag(Token::LeadingSpace);
3080 return false;
3081}
3082
3083//===----------------------------------------------------------------------===//
3084// Primary Lexing Entry Points
3085//===----------------------------------------------------------------------===//
3086
3087/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
3088/// uninterpreted string. This switches the lexer out of directive mode.
3089void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
3090 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
3091 "Must be in a preprocessing directive!");
3092 Token Tmp;
3093 Tmp.startToken();
3094
3095 // CurPtr - Cache BufferPtr in an automatic variable.
3096 const char *CurPtr = BufferPtr;
3097 while (true) {
3098 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Tmp);
3099 switch (Char) {
3100 default:
3101 if (Result)
3102 Result->push_back(Elt: Char);
3103 break;
3104 case 0: // Null.
3105 // Found end of file?
3106 if (CurPtr-1 != BufferEnd) {
3107 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3108 PP->CodeCompleteNaturalLanguage();
3109 cutOffLexing();
3110 return;
3111 }
3112
3113 // Nope, normal character, continue.
3114 if (Result)
3115 Result->push_back(Elt: Char);
3116 break;
3117 }
3118 // FALL THROUGH.
3119 [[fallthrough]];
3120 case '\r':
3121 case '\n':
3122 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
3123 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
3124 BufferPtr = CurPtr-1;
3125
3126 // Next, lex the character, which should handle the EOD transition.
3127 Lex(Result&: Tmp);
3128 if (Tmp.is(K: tok::code_completion)) {
3129 if (PP)
3130 PP->CodeCompleteNaturalLanguage();
3131 Lex(Result&: Tmp);
3132 }
3133 assert(Tmp.is(tok::eod) && "Unexpected token!");
3134
3135 // Finally, we're done;
3136 return;
3137 }
3138 }
3139}
3140
3141/// LexEndOfFile - CurPtr points to the end of this file. Handle this
3142/// condition, reporting diagnostics and handling other edge cases as required.
3143/// This returns true if Result contains a token, false if PP.Lex should be
3144/// called again.
3145bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
3146 // If we hit the end of the file while parsing a preprocessor directive,
3147 // end the preprocessor directive first. The next token returned will
3148 // then be the end of file.
3149 if (ParsingPreprocessorDirective) {
3150 // Done parsing the "line".
3151 ParsingPreprocessorDirective = false;
3152 // Update the location of token as well as BufferPtr.
3153 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::eod);
3154
3155 // Restore comment saving mode, in case it was disabled for directive.
3156 if (PP)
3157 resetExtendedTokenMode();
3158 return true; // Have a token.
3159 }
3160
3161 // If we are in raw mode, return this event as an EOF token. Let the caller
3162 // that put us in raw mode handle the event.
3163 if (isLexingRawMode()) {
3164 Result.startToken();
3165 BufferPtr = BufferEnd;
3166 FormTokenWithChars(Result, TokEnd: BufferEnd, Kind: tok::eof);
3167 return true;
3168 }
3169
3170 if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) {
3171 PP->setRecordedPreambleConditionalStack(ConditionalStack);
3172 // If the preamble cuts off the end of a header guard, consider it guarded.
3173 // The guard is valid for the preamble content itself, and for tools the
3174 // most useful answer is "yes, this file has a header guard".
3175 if (!ConditionalStack.empty())
3176 MIOpt.ExitTopLevelConditional();
3177 ConditionalStack.clear();
3178 }
3179
3180 // Issue diagnostics for unterminated #if and missing newline.
3181
3182 // If we are in a #if directive, emit an error.
3183 while (!ConditionalStack.empty()) {
3184 if (PP->getCodeCompletionFileLoc() != FileLoc)
3185 PP->Diag(Loc: ConditionalStack.back().IfLoc,
3186 DiagID: diag::err_pp_unterminated_conditional);
3187 ConditionalStack.pop_back();
3188 }
3189
3190 // Before C++11 and C2y, a file not ending with a newline was UB. Both
3191 // standards changed this behavior (as a DR or equivalent), but we still have
3192 // an opt-in diagnostic to warn about it.
3193 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
3194 Diag(Loc: BufferEnd, DiagID: diag::warn_no_newline_eof)
3195 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: BufferEnd), Code: "\n");
3196
3197 BufferPtr = CurPtr;
3198
3199 // Finally, let the preprocessor handle this.
3200 return PP->HandleEndOfFile(Result, isEndOfMacro: isPragmaLexer());
3201}
3202
3203/// peekNextPPToken - Return std::nullopt if there are no more tokens in the
3204/// buffer controlled by this lexer, otherwise return the next unexpanded
3205/// token.
3206std::optional<Token> Lexer::peekNextPPToken() {
3207 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
3208
3209 if (isDependencyDirectivesLexer()) {
3210 if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size())
3211 return std::nullopt;
3212 Token Result;
3213 (void)convertDependencyDirectiveToken(
3214 DDTok: DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result);
3215 return Result;
3216 }
3217
3218 // Switch to 'skipping' mode. This will ensure that we can lex a token
3219 // without emitting diagnostics, disables macro expansion, and will cause EOF
3220 // to return an EOF token instead of popping the include stack.
3221 LexingRawMode = true;
3222
3223 // Save state that can be changed while lexing so that we can restore it.
3224 const char *TmpBufferPtr = BufferPtr;
3225 bool inPPDirectiveMode = ParsingPreprocessorDirective;
3226 bool atStartOfLine = IsAtStartOfLine;
3227 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3228 bool leadingSpace = HasLeadingSpace;
3229 bool isFirstPPToken = IsFirstPPToken;
3230
3231 Token Tok;
3232 Lex(Result&: Tok);
3233
3234 // Restore state that may have changed.
3235 BufferPtr = TmpBufferPtr;
3236 ParsingPreprocessorDirective = inPPDirectiveMode;
3237 HasLeadingSpace = leadingSpace;
3238 IsAtStartOfLine = atStartOfLine;
3239 IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
3240 IsFirstPPToken = isFirstPPToken;
3241 // Restore the lexer back to non-skipping mode.
3242 LexingRawMode = false;
3243
3244 if (Tok.is(K: tok::eof))
3245 return std::nullopt;
3246 return Tok;
3247}
3248
3249/// Find the end of a version control conflict marker.
3250static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
3251 ConflictMarkerKind CMK) {
3252 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
3253 size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
3254 auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(Start: TermLen);
3255 size_t Pos = RestOfBuffer.find(Str: Terminator);
3256 while (Pos != StringRef::npos) {
3257 // Must occur at start of line.
3258 if (Pos == 0 ||
3259 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) {
3260 RestOfBuffer = RestOfBuffer.substr(Start: Pos+TermLen);
3261 Pos = RestOfBuffer.find(Str: Terminator);
3262 continue;
3263 }
3264 return RestOfBuffer.data()+Pos;
3265 }
3266 return nullptr;
3267}
3268
3269/// IsStartOfConflictMarker - If the specified pointer is the start of a version
3270/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
3271/// and recover nicely. This returns true if it is a conflict marker and false
3272/// if not.
3273bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
3274 // Only a conflict marker if it starts at the beginning of a line.
3275 if (CurPtr != BufferStart &&
3276 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3277 return false;
3278
3279 // Check to see if we have <<<<<<< or >>>>.
3280 if (!StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: "<<<<<<<") &&
3281 !StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: ">>>> "))
3282 return false;
3283
3284 // If we have a situation where we don't care about conflict markers, ignore
3285 // it.
3286 if (CurrentConflictMarkerState || isLexingRawMode())
3287 return false;
3288
3289 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
3290
3291 // Check to see if there is an ending marker somewhere in the buffer at the
3292 // start of a line to terminate this conflict marker.
3293 if (FindConflictEnd(CurPtr, BufferEnd, CMK: Kind)) {
3294 // We found a match. We are really in a conflict marker.
3295 // Diagnose this, and ignore to the end of line.
3296 Diag(Loc: CurPtr, DiagID: diag::err_conflict_marker);
3297 CurrentConflictMarkerState = Kind;
3298
3299 // Skip ahead to the end of line. We know this exists because the
3300 // end-of-conflict marker starts with \r or \n.
3301 while (*CurPtr != '\r' && *CurPtr != '\n') {
3302 assert(CurPtr != BufferEnd && "Didn't find end of line");
3303 ++CurPtr;
3304 }
3305 BufferPtr = CurPtr;
3306 return true;
3307 }
3308
3309 // No end of conflict marker found.
3310 return false;
3311}
3312
3313/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
3314/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
3315/// is the end of a conflict marker. Handle it by ignoring up until the end of
3316/// the line. This returns true if it is a conflict marker and false if not.
3317bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
3318 // Only a conflict marker if it starts at the beginning of a line.
3319 if (CurPtr != BufferStart &&
3320 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3321 return false;
3322
3323 // If we have a situation where we don't care about conflict markers, ignore
3324 // it.
3325 if (!CurrentConflictMarkerState || isLexingRawMode())
3326 return false;
3327
3328 // Check to see if we have the marker (4 characters in a row).
3329 for (unsigned i = 1; i != 4; ++i)
3330 if (CurPtr[i] != CurPtr[0])
3331 return false;
3332
3333 // If we do have it, search for the end of the conflict marker. This could
3334 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
3335 // be the end of conflict marker.
3336 if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
3337 CMK: CurrentConflictMarkerState)) {
3338 CurPtr = End;
3339
3340 // Skip ahead to the end of line.
3341 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
3342 ++CurPtr;
3343
3344 BufferPtr = CurPtr;
3345
3346 // No longer in the conflict marker.
3347 CurrentConflictMarkerState = CMK_None;
3348 return true;
3349 }
3350
3351 return false;
3352}
3353
3354static const char *findPlaceholderEnd(const char *CurPtr,
3355 const char *BufferEnd) {
3356 if (CurPtr == BufferEnd)
3357 return nullptr;
3358 BufferEnd -= 1; // Scan until the second last character.
3359 for (; CurPtr != BufferEnd; ++CurPtr) {
3360 if (CurPtr[0] == '#' && CurPtr[1] == '>')
3361 return CurPtr + 2;
3362 }
3363 return nullptr;
3364}
3365
3366bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
3367 assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!");
3368 if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
3369 return false;
3370 const char *End = findPlaceholderEnd(CurPtr: CurPtr + 1, BufferEnd);
3371 if (!End)
3372 return false;
3373 const char *Start = CurPtr - 1;
3374 if (!LangOpts.AllowEditorPlaceholders)
3375 Diag(Loc: Start, DiagID: diag::err_placeholder_in_source);
3376 Result.startToken();
3377 FormTokenWithChars(Result, TokEnd: End, Kind: tok::raw_identifier);
3378 Result.setRawIdentifierData(Start);
3379 PP->LookUpIdentifierInfo(Identifier&: Result);
3380 Result.setFlag(Token::IsEditorPlaceholder);
3381 BufferPtr = End;
3382 return true;
3383}
3384
3385bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
3386 if (PP && PP->isCodeCompletionEnabled()) {
3387 SourceLocation Loc = FileLoc.getLocWithOffset(Offset: CurPtr-BufferStart);
3388 return Loc == PP->getCodeCompletionLoc();
3389 }
3390
3391 return false;
3392}
3393
3394void Lexer::DiagnoseDelimitedOrNamedEscapeSequence(SourceLocation Loc,
3395 bool Named,
3396 const LangOptions &Opts,
3397 DiagnosticsEngine &Diags) {
3398 unsigned DiagId;
3399 if (Opts.CPlusPlus23)
3400 DiagId = diag::warn_cxx23_delimited_escape_sequence;
3401 else if (Opts.C2y && !Named)
3402 DiagId = diag::warn_c2y_delimited_escape_sequence;
3403 else
3404 DiagId = diag::ext_delimited_escape_sequence;
3405
3406 // The trailing arguments are only used by the extension warning; either this
3407 // is a C2y extension or a C++23 extension, unless it's a named escape
3408 // sequence in C, then it's a Clang extension.
3409 unsigned Ext;
3410 if (!Opts.CPlusPlus)
3411 Ext = Named ? 2 /* Clang extension */ : 1 /* C2y extension */;
3412 else
3413 Ext = 0; // C++23 extension
3414
3415 Diags.Report(Loc, DiagID: DiagId) << Named << Ext;
3416}
3417
3418std::optional<uint32_t> Lexer::tryReadNumericUCN(const char *&StartPtr,
3419 const char *SlashLoc,
3420 Token *Result) {
3421 unsigned CharSize;
3422 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3423 assert((Kind == 'u' || Kind == 'U') && "expected a UCN");
3424
3425 unsigned NumHexDigits;
3426 if (Kind == 'u')
3427 NumHexDigits = 4;
3428 else if (Kind == 'U')
3429 NumHexDigits = 8;
3430
3431 bool Delimited = false;
3432 bool FoundEndDelimiter = false;
3433 unsigned Count = 0;
3434 bool Diagnose = Result && !isLexingRawMode();
3435
3436 if (!LangOpts.CPlusPlus && !LangOpts.C99) {
3437 if (Diagnose)
3438 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_not_valid_in_c89);
3439 return std::nullopt;
3440 }
3441
3442 const char *CurPtr = StartPtr + CharSize;
3443 const char *KindLoc = &CurPtr[-1];
3444
3445 uint32_t CodePoint = 0;
3446 while (Count != NumHexDigits || Delimited) {
3447 char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3448 if (!Delimited && Count == 0 && C == '{') {
3449 Delimited = true;
3450 CurPtr += CharSize;
3451 continue;
3452 }
3453
3454 if (Delimited && C == '}') {
3455 CurPtr += CharSize;
3456 FoundEndDelimiter = true;
3457 break;
3458 }
3459
3460 unsigned Value = llvm::hexDigitValue(C);
3461 if (Value == -1U) {
3462 if (!Delimited)
3463 break;
3464 if (Diagnose)
3465 Diag(Loc: SlashLoc, DiagID: diag::warn_delimited_ucn_incomplete)
3466 << StringRef(KindLoc, 1);
3467 return std::nullopt;
3468 }
3469
3470 if (CodePoint & 0xF000'0000) {
3471 if (Diagnose)
3472 Diag(Loc: KindLoc, DiagID: diag::err_escape_too_large) << 0;
3473 return std::nullopt;
3474 }
3475
3476 CodePoint <<= 4;
3477 CodePoint |= Value;
3478 CurPtr += CharSize;
3479 Count++;
3480 }
3481
3482 if (Count == 0) {
3483 if (Diagnose)
3484 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3485 : diag::warn_ucn_escape_no_digits)
3486 << StringRef(KindLoc, 1);
3487 return std::nullopt;
3488 }
3489
3490 if (Delimited && Kind == 'U') {
3491 if (Diagnose)
3492 Diag(Loc: SlashLoc, DiagID: diag::err_hex_escape_no_digits) << StringRef(KindLoc, 1);
3493 return std::nullopt;
3494 }
3495
3496 if (!Delimited && Count != NumHexDigits) {
3497 if (Diagnose) {
3498 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3499 // If the user wrote \U1234, suggest a fixit to \u.
3500 if (Count == 4 && NumHexDigits == 8) {
3501 CharSourceRange URange = makeCharRange(L&: *this, Begin: KindLoc, End: KindLoc + 1);
3502 Diag(Loc: KindLoc, DiagID: diag::note_ucn_four_not_eight)
3503 << FixItHint::CreateReplacement(RemoveRange: URange, Code: "u");
3504 }
3505 }
3506 return std::nullopt;
3507 }
3508
3509 if (Delimited && PP)
3510 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: false,
3511 Opts: PP->getLangOpts(),
3512 Diags&: PP->getDiagnostics());
3513
3514 if (Result) {
3515 Result->setFlag(Token::HasUCN);
3516 // If the UCN contains either a trigraph or a line splicing,
3517 // we need to call getAndAdvanceChar again to set the appropriate flags
3518 // on Result.
3519 if (CurPtr - StartPtr == (ptrdiff_t)(Count + 1 + (Delimited ? 2 : 0)))
3520 StartPtr = CurPtr;
3521 else
3522 while (StartPtr != CurPtr)
3523 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3524 } else {
3525 StartPtr = CurPtr;
3526 }
3527 return CodePoint;
3528}
3529
3530std::optional<uint32_t> Lexer::tryReadNamedUCN(const char *&StartPtr,
3531 const char *SlashLoc,
3532 Token *Result) {
3533 unsigned CharSize;
3534 bool Diagnose = Result && !isLexingRawMode();
3535
3536 char C = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3537 assert(C == 'N' && "expected \\N{...}");
3538
3539 const char *CurPtr = StartPtr + CharSize;
3540 const char *KindLoc = &CurPtr[-1];
3541
3542 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3543 if (C != '{') {
3544 if (Diagnose)
3545 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3546 return std::nullopt;
3547 }
3548 CurPtr += CharSize;
3549 const char *StartName = CurPtr;
3550 bool FoundEndDelimiter = false;
3551 llvm::SmallVector<char, 30> Buffer;
3552 while (C) {
3553 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3554 CurPtr += CharSize;
3555 if (C == '}') {
3556 FoundEndDelimiter = true;
3557 break;
3558 }
3559
3560 if (isVerticalWhitespace(c: C))
3561 break;
3562 Buffer.push_back(Elt: C);
3563 }
3564
3565 if (!FoundEndDelimiter || Buffer.empty()) {
3566 if (Diagnose)
3567 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3568 : diag::warn_delimited_ucn_incomplete)
3569 << StringRef(KindLoc, 1);
3570 return std::nullopt;
3571 }
3572
3573 StringRef Name(Buffer.data(), Buffer.size());
3574 std::optional<char32_t> Match =
3575 llvm::sys::unicode::nameToCodepointStrict(Name);
3576 std::optional<llvm::sys::unicode::LooseMatchingResult> LooseMatch;
3577 if (!Match) {
3578 LooseMatch = llvm::sys::unicode::nameToCodepointLooseMatching(Name);
3579 if (Diagnose) {
3580 Diag(Loc: StartName, DiagID: diag::err_invalid_ucn_name)
3581 << StringRef(Buffer.data(), Buffer.size())
3582 << makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize);
3583 if (LooseMatch) {
3584 Diag(Loc: StartName, DiagID: diag::note_invalid_ucn_name_loose_matching)
3585 << FixItHint::CreateReplacement(
3586 RemoveRange: makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize),
3587 Code: LooseMatch->Name);
3588 }
3589 }
3590 // We do not offer misspelled character names suggestions here
3591 // as the set of what would be a valid suggestion depends on context,
3592 // and we should not make invalid suggestions.
3593 }
3594
3595 if (Diagnose && Match)
3596 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: true,
3597 Opts: PP->getLangOpts(),
3598 Diags&: PP->getDiagnostics());
3599
3600 // If no diagnostic has been emitted yet, likely because we are doing a
3601 // tentative lexing, we do not want to recover here to make sure the token
3602 // will not be incorrectly considered valid. This function will be called
3603 // again and a diagnostic emitted then.
3604 if (LooseMatch && Diagnose)
3605 Match = LooseMatch->CodePoint;
3606
3607 if (Result) {
3608 Result->setFlag(Token::HasUCN);
3609 // If the UCN contains either a trigraph or a line splicing,
3610 // we need to call getAndAdvanceChar again to set the appropriate flags
3611 // on Result.
3612 if (CurPtr - StartPtr == (ptrdiff_t)(Buffer.size() + 3))
3613 StartPtr = CurPtr;
3614 else
3615 while (StartPtr != CurPtr)
3616 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3617 } else {
3618 StartPtr = CurPtr;
3619 }
3620 return Match ? std::optional<uint32_t>(*Match) : std::nullopt;
3621}
3622
3623uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3624 Token *Result) {
3625
3626 unsigned CharSize;
3627 std::optional<uint32_t> CodePointOpt;
3628 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3629 if (Kind == 'u' || Kind == 'U')
3630 CodePointOpt = tryReadNumericUCN(StartPtr, SlashLoc, Result);
3631 else if (Kind == 'N')
3632 CodePointOpt = tryReadNamedUCN(StartPtr, SlashLoc, Result);
3633
3634 if (!CodePointOpt)
3635 return 0;
3636
3637 uint32_t CodePoint = *CodePointOpt;
3638
3639 // Don't apply C family restrictions to UCNs in assembly mode
3640 if (LangOpts.AsmPreprocessor)
3641 return CodePoint;
3642
3643 // C23 6.4.3p2: A universal character name shall not designate a code point
3644 // where the hexadecimal value is:
3645 // - in the range D800 through DFFF inclusive; or
3646 // - greater than 10FFFF.
3647 // A universal-character-name outside the c-char-sequence of a character
3648 // constant, or the s-char-sequence of a string-literal shall not designate
3649 // a control character or a character in the basic character set.
3650
3651 // C++11 [lex.charset]p2: If the hexadecimal value for a
3652 // universal-character-name corresponds to a surrogate code point (in the
3653 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
3654 // if the hexadecimal value for a universal-character-name outside the
3655 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or
3656 // string literal corresponds to a control character (in either of the
3657 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
3658 // basic source character set, the program is ill-formed.
3659 if (CodePoint < 0xA0) {
3660 // We don't use isLexingRawMode() here because we need to warn about bad
3661 // UCNs even when skipping preprocessing tokens in a #if block.
3662 if (Result && PP) {
3663 if (CodePoint < 0x20 || CodePoint >= 0x7F)
3664 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_control_character);
3665 else {
3666 char C = static_cast<char>(CodePoint);
3667 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
3668 }
3669 }
3670
3671 return 0;
3672 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) {
3673 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
3674 // We don't use isLexingRawMode() here because we need to diagnose bad
3675 // UCNs even when skipping preprocessing tokens in a #if block.
3676 if (Result && PP) {
3677 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11)
3678 Diag(Loc: BufferPtr, DiagID: diag::warn_ucn_escape_surrogate);
3679 else
3680 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_invalid);
3681 }
3682 return 0;
3683 }
3684
3685 return CodePoint;
3686}
3687
3688bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3689 const char *CurPtr) {
3690 if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
3691 isUnicodeWhitespace(Codepoint: C)) {
3692 Diag(Loc: BufferPtr, DiagID: diag::ext_unicode_whitespace)
3693 << makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr);
3694
3695 Result.setFlag(Token::LeadingSpace);
3696 return true;
3697 }
3698 return false;
3699}
3700
3701void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3702 IsAtStartOfLine = Result.isAtStartOfLine();
3703 HasLeadingSpace = Result.hasLeadingSpace();
3704 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
3705 // Note that this doesn't affect IsAtPhysicalStartOfLine.
3706}
3707
3708bool Lexer::Lex(Token &Result) {
3709 assert(!isDependencyDirectivesLexer());
3710
3711 // Start a new token.
3712 Result.startToken();
3713
3714 // Set up misc whitespace flags for LexTokenInternal.
3715 if (IsAtStartOfLine) {
3716 Result.setFlag(Token::StartOfLine);
3717 IsAtStartOfLine = false;
3718 }
3719
3720 if (HasLeadingSpace) {
3721 Result.setFlag(Token::LeadingSpace);
3722 HasLeadingSpace = false;
3723 }
3724
3725 if (HasLeadingEmptyMacro) {
3726 Result.setFlag(Token::LeadingEmptyMacro);
3727 HasLeadingEmptyMacro = false;
3728 }
3729
3730 if (IsFirstPPToken) {
3731 Result.setFlag(Token::FirstPPToken);
3732 IsFirstPPToken = false;
3733 }
3734
3735 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3736 IsAtPhysicalStartOfLine = false;
3737 bool isRawLex = isLexingRawMode();
3738 (void) isRawLex;
3739 bool returnedToken = LexTokenInternal(Result, TokAtPhysicalStartOfLine: atPhysicalStartOfLine);
3740 // (After the LexTokenInternal call, the lexer might be destroyed.)
3741 assert((returnedToken || !isRawLex) && "Raw lex must succeed");
3742 return returnedToken;
3743}
3744
3745/// LexTokenInternal - This implements a simple C family lexer. It is an
3746/// extremely performance critical piece of code. This assumes that the buffer
3747/// has a null character at the end of the file. This returns a preprocessing
3748/// token, not a normal token, as such, it is an internal interface. It assumes
3749/// that the Flags of result have been cleared before calling this.
3750bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
3751LexStart:
3752 assert(!Result.needsCleaning() && "Result needs cleaning");
3753 assert(!Result.hasPtrData() && "Result has not been reset");
3754
3755 // CurPtr - Cache BufferPtr in an automatic variable.
3756 const char *CurPtr = BufferPtr;
3757
3758 // Small amounts of horizontal whitespace is very common between tokens.
3759 if (isHorizontalWhitespace(c: *CurPtr)) {
3760 do {
3761 ++CurPtr;
3762 } while (isHorizontalWhitespace(c: *CurPtr));
3763
3764 // If we are keeping whitespace and other tokens, just return what we just
3765 // skipped. The next lexer invocation will return the token after the
3766 // whitespace.
3767 if (isKeepWhitespaceMode()) {
3768 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3769 // FIXME: The next token will not have LeadingSpace set.
3770 return true;
3771 }
3772
3773 BufferPtr = CurPtr;
3774 Result.setFlag(Token::LeadingSpace);
3775 }
3776
3777 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
3778
3779 // Read a character, advancing over it.
3780 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3781 tok::TokenKind Kind;
3782
3783 if (!isVerticalWhitespace(c: Char))
3784 NewLinePtr = nullptr;
3785
3786 switch (Char) {
3787 case 0: // Null.
3788 // Found end of file?
3789 if (CurPtr-1 == BufferEnd)
3790 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3791
3792 // Check if we are performing code completion.
3793 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3794 // Return the code-completion token.
3795 Result.startToken();
3796 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::code_completion);
3797 return true;
3798 }
3799
3800 if (!isLexingRawMode())
3801 Diag(Loc: CurPtr-1, DiagID: diag::null_in_file);
3802 Result.setFlag(Token::LeadingSpace);
3803 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3804 return true; // KeepWhitespaceMode
3805
3806 // We know the lexer hasn't changed, so just try again with this lexer.
3807 // (We manually eliminate the tail call to avoid recursion.)
3808 goto LexNextToken;
3809
3810 case 26: // DOS & CP/M EOF: "^Z".
3811 // If we're in Microsoft extensions mode, treat this as end of file.
3812 if (LangOpts.MicrosoftExt) {
3813 if (!isLexingRawMode())
3814 Diag(Loc: CurPtr-1, DiagID: diag::ext_ctrl_z_eof_microsoft);
3815 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3816 }
3817
3818 // If Microsoft extensions are disabled, this is just random garbage.
3819 Kind = tok::unknown;
3820 break;
3821
3822 case '\r':
3823 if (CurPtr[0] == '\n')
3824 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3825 [[fallthrough]];
3826 case '\n':
3827 // If we are inside a preprocessor directive and we see the end of line,
3828 // we know we are done with the directive, so return an EOD token.
3829 if (ParsingPreprocessorDirective) {
3830 // Done parsing the "line".
3831 ParsingPreprocessorDirective = false;
3832
3833 // Restore comment saving mode, in case it was disabled for directive.
3834 if (PP)
3835 resetExtendedTokenMode();
3836
3837 // Since we consumed a newline, we are back at the start of a line.
3838 IsAtStartOfLine = true;
3839 IsAtPhysicalStartOfLine = true;
3840 NewLinePtr = CurPtr - 1;
3841
3842 Kind = tok::eod;
3843 break;
3844 }
3845
3846 // No leading whitespace seen so far.
3847 Result.clearFlag(Flag: Token::LeadingSpace);
3848
3849 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3850 return true; // KeepWhitespaceMode
3851
3852 // We only saw whitespace, so just try again with this lexer.
3853 // (We manually eliminate the tail call to avoid recursion.)
3854 goto LexNextToken;
3855 case ' ':
3856 case '\t':
3857 case '\f':
3858 case '\v':
3859 SkipHorizontalWhitespace:
3860 Result.setFlag(Token::LeadingSpace);
3861 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3862 return true; // KeepWhitespaceMode
3863
3864 SkipIgnoredUnits:
3865 CurPtr = BufferPtr;
3866
3867 // If the next token is obviously a // or /* */ comment, skip it efficiently
3868 // too (without going through the big switch stmt).
3869 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
3870 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
3871 if (SkipLineComment(Result, CurPtr: CurPtr+2, TokAtPhysicalStartOfLine))
3872 return true; // There is a token to return.
3873 goto SkipIgnoredUnits;
3874 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
3875 if (SkipBlockComment(Result, CurPtr: CurPtr+2, TokAtPhysicalStartOfLine))
3876 return true; // There is a token to return.
3877 goto SkipIgnoredUnits;
3878 } else if (isHorizontalWhitespace(c: *CurPtr)) {
3879 goto SkipHorizontalWhitespace;
3880 }
3881 // We only saw whitespace, so just try again with this lexer.
3882 // (We manually eliminate the tail call to avoid recursion.)
3883 goto LexNextToken;
3884
3885 // C99 6.4.4.1: Integer Constants.
3886 // C99 6.4.4.2: Floating Constants.
3887 case '0': case '1': case '2': case '3': case '4':
3888 case '5': case '6': case '7': case '8': case '9':
3889 // Notify MIOpt that we read a non-whitespace/non-comment token.
3890 MIOpt.ReadToken();
3891 return LexNumericConstant(Result, CurPtr);
3892
3893 // Identifier (e.g., uber), or
3894 // UTF-8 (C23/C++17) or UTF-16 (C11/C++11) character literal, or
3895 // UTF-8 or UTF-16 string literal (C11/C++11).
3896 case 'u':
3897 // Notify MIOpt that we read a non-whitespace/non-comment token.
3898 MIOpt.ReadToken();
3899
3900 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
3901 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3902
3903 // UTF-16 string literal
3904 if (Char == '"')
3905 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3906 Kind: tok::utf16_string_literal);
3907
3908 // UTF-16 character constant
3909 if (Char == '\'')
3910 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3911 Kind: tok::utf16_char_constant);
3912
3913 // UTF-16 raw string literal
3914 if (Char == 'R' && LangOpts.RawStringLiterals &&
3915 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
3916 return LexRawStringLiteral(Result,
3917 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3918 Size: SizeTmp2, Tok&: Result),
3919 Kind: tok::utf16_string_literal);
3920
3921 if (Char == '8') {
3922 char Char2 = getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2);
3923
3924 // UTF-8 string literal
3925 if (Char2 == '"')
3926 return LexStringLiteral(Result,
3927 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3928 Size: SizeTmp2, Tok&: Result),
3929 Kind: tok::utf8_string_literal);
3930 if (Char2 == '\'' && (LangOpts.CPlusPlus17 || LangOpts.C23))
3931 return LexCharConstant(
3932 Result, CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3933 Size: SizeTmp2, Tok&: Result),
3934 Kind: tok::utf8_char_constant);
3935
3936 if (Char2 == 'R' && LangOpts.RawStringLiterals) {
3937 unsigned SizeTmp3;
3938 char Char3 = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
3939 // UTF-8 raw string literal
3940 if (Char3 == '"') {
3941 return LexRawStringLiteral(Result,
3942 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3943 Size: SizeTmp2, Tok&: Result),
3944 Size: SizeTmp3, Tok&: Result),
3945 Kind: tok::utf8_string_literal);
3946 }
3947 }
3948 }
3949 }
3950
3951 // treat u like the start of an identifier.
3952 return LexIdentifierContinue(Result, CurPtr);
3953
3954 case 'U': // Identifier (e.g. Uber) or C11/C++11 UTF-32 string literal
3955 // Notify MIOpt that we read a non-whitespace/non-comment token.
3956 MIOpt.ReadToken();
3957
3958 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
3959 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3960
3961 // UTF-32 string literal
3962 if (Char == '"')
3963 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3964 Kind: tok::utf32_string_literal);
3965
3966 // UTF-32 character constant
3967 if (Char == '\'')
3968 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3969 Kind: tok::utf32_char_constant);
3970
3971 // UTF-32 raw string literal
3972 if (Char == 'R' && LangOpts.RawStringLiterals &&
3973 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
3974 return LexRawStringLiteral(Result,
3975 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3976 Size: SizeTmp2, Tok&: Result),
3977 Kind: tok::utf32_string_literal);
3978 }
3979
3980 // treat U like the start of an identifier.
3981 return LexIdentifierContinue(Result, CurPtr);
3982
3983 case 'R': // Identifier or C++0x raw string literal
3984 // Notify MIOpt that we read a non-whitespace/non-comment token.
3985 MIOpt.ReadToken();
3986
3987 if (LangOpts.RawStringLiterals) {
3988 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3989
3990 if (Char == '"')
3991 return LexRawStringLiteral(Result,
3992 CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3993 Kind: tok::string_literal);
3994 }
3995
3996 // treat R like the start of an identifier.
3997 return LexIdentifierContinue(Result, CurPtr);
3998
3999 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
4000 // Notify MIOpt that we read a non-whitespace/non-comment token.
4001 MIOpt.ReadToken();
4002 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4003
4004 // Wide string literal.
4005 if (Char == '"')
4006 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4007 Kind: tok::wide_string_literal);
4008
4009 // Wide raw string literal.
4010 if (LangOpts.RawStringLiterals && Char == 'R' &&
4011 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4012 return LexRawStringLiteral(Result,
4013 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4014 Size: SizeTmp2, Tok&: Result),
4015 Kind: tok::wide_string_literal);
4016
4017 // Wide character constant.
4018 if (Char == '\'')
4019 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4020 Kind: tok::wide_char_constant);
4021 // FALL THROUGH, treating L like the start of an identifier.
4022 [[fallthrough]];
4023
4024 // C99 6.4.2: Identifiers.
4025 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
4026 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
4027 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
4028 case 'V': case 'W': case 'X': case 'Y': case 'Z':
4029 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
4030 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
4031 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
4032 case 'v': case 'w': case 'x': case 'y': case 'z':
4033 case '_':
4034 // Notify MIOpt that we read a non-whitespace/non-comment token.
4035 MIOpt.ReadToken();
4036 return LexIdentifierContinue(Result, CurPtr);
4037
4038 case '$': // $ in identifiers.
4039 if (LangOpts.DollarIdents) {
4040 if (!isLexingRawMode())
4041 Diag(Loc: CurPtr-1, DiagID: diag::ext_dollar_in_identifier);
4042 // Notify MIOpt that we read a non-whitespace/non-comment token.
4043 MIOpt.ReadToken();
4044 return LexIdentifierContinue(Result, CurPtr);
4045 }
4046
4047 Kind = tok::unknown;
4048 break;
4049
4050 // C99 6.4.4: Character Constants.
4051 case '\'':
4052 // Notify MIOpt that we read a non-whitespace/non-comment token.
4053 MIOpt.ReadToken();
4054 return LexCharConstant(Result, CurPtr, Kind: tok::char_constant);
4055
4056 // C99 6.4.5: String Literals.
4057 case '"':
4058 // Notify MIOpt that we read a non-whitespace/non-comment token.
4059 MIOpt.ReadToken();
4060 return LexStringLiteral(Result, CurPtr,
4061 Kind: ParsingFilename ? tok::header_name
4062 : tok::string_literal);
4063
4064 // C99 6.4.6: Punctuators.
4065 case '?':
4066 Kind = tok::question;
4067 break;
4068 case '[':
4069 Kind = tok::l_square;
4070 break;
4071 case ']':
4072 Kind = tok::r_square;
4073 break;
4074 case '(':
4075 Kind = tok::l_paren;
4076 break;
4077 case ')':
4078 Kind = tok::r_paren;
4079 break;
4080 case '{':
4081 Kind = tok::l_brace;
4082 break;
4083 case '}':
4084 Kind = tok::r_brace;
4085 break;
4086 case '.':
4087 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4088 if (Char >= '0' && Char <= '9') {
4089 // Notify MIOpt that we read a non-whitespace/non-comment token.
4090 MIOpt.ReadToken();
4091
4092 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result));
4093 } else if (LangOpts.CPlusPlus && Char == '*') {
4094 Kind = tok::periodstar;
4095 CurPtr += SizeTmp;
4096 } else if (Char == '.' &&
4097 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '.') {
4098 Kind = tok::ellipsis;
4099 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4100 Size: SizeTmp2, Tok&: Result);
4101 } else {
4102 Kind = tok::period;
4103 }
4104 break;
4105 case '&':
4106 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4107 if (Char == '&') {
4108 Kind = tok::ampamp;
4109 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4110 } else if (Char == '=') {
4111 Kind = tok::ampequal;
4112 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4113 } else {
4114 Kind = tok::amp;
4115 }
4116 break;
4117 case '*':
4118 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4119 Kind = tok::starequal;
4120 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4121 } else {
4122 Kind = tok::star;
4123 }
4124 break;
4125 case '+':
4126 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4127 if (Char == '+') {
4128 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4129 Kind = tok::plusplus;
4130 } else if (Char == '=') {
4131 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4132 Kind = tok::plusequal;
4133 } else {
4134 Kind = tok::plus;
4135 }
4136 break;
4137 case '-':
4138 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4139 if (Char == '-') { // --
4140 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4141 Kind = tok::minusminus;
4142 } else if (Char == '>' && LangOpts.CPlusPlus &&
4143 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '*') { // C++ ->*
4144 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4145 Size: SizeTmp2, Tok&: Result);
4146 Kind = tok::arrowstar;
4147 } else if (Char == '>') { // ->
4148 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4149 Kind = tok::arrow;
4150 } else if (Char == '=') { // -=
4151 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4152 Kind = tok::minusequal;
4153 } else {
4154 Kind = tok::minus;
4155 }
4156 break;
4157 case '~':
4158 Kind = tok::tilde;
4159 break;
4160 case '!':
4161 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4162 Kind = tok::exclaimequal;
4163 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4164 } else {
4165 Kind = tok::exclaim;
4166 }
4167 break;
4168 case '/':
4169 // 6.4.9: Comments
4170 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4171 if (Char == '/') { // Line comment.
4172 // Even if Line comments are disabled (e.g. in C89 mode), we generally
4173 // want to lex this as a comment. There is one problem with this though,
4174 // that in one particular corner case, this can change the behavior of the
4175 // resultant program. For example, In "foo //**/ bar", C89 would lex
4176 // this as "foo / bar" and languages with Line comments would lex it as
4177 // "foo". Check to see if the character after the second slash is a '*'.
4178 // If so, we will lex that as a "/" instead of the start of a comment.
4179 // However, we never do this if we are just preprocessing.
4180 bool TreatAsComment =
4181 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP);
4182 if (!TreatAsComment)
4183 if (!(PP && PP->isPreprocessedOutput()))
4184 TreatAsComment = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) != '*';
4185
4186 if (TreatAsComment) {
4187 if (SkipLineComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4188 TokAtPhysicalStartOfLine))
4189 return true; // There is a token to return.
4190
4191 // It is common for the tokens immediately after a // comment to be
4192 // whitespace (indentation for the next line). Instead of going through
4193 // the big switch, handle it efficiently now.
4194 goto SkipIgnoredUnits;
4195 }
4196 }
4197
4198 if (Char == '*') { // /**/ comment.
4199 if (SkipBlockComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4200 TokAtPhysicalStartOfLine))
4201 return true; // There is a token to return.
4202
4203 // We only saw whitespace, so just try again with this lexer.
4204 // (We manually eliminate the tail call to avoid recursion.)
4205 goto LexNextToken;
4206 }
4207
4208 if (Char == '=') {
4209 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4210 Kind = tok::slashequal;
4211 } else {
4212 Kind = tok::slash;
4213 }
4214 break;
4215 case '%':
4216 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4217 if (Char == '=') {
4218 Kind = tok::percentequal;
4219 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4220 } else if (LangOpts.Digraphs && Char == '>') {
4221 Kind = tok::r_brace; // '%>' -> '}'
4222 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4223 } else if (LangOpts.Digraphs && Char == ':') {
4224 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4225 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4226 if (Char == '%' && getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == ':') {
4227 Kind = tok::hashhash; // '%:%:' -> '##'
4228 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4229 Size: SizeTmp2, Tok&: Result);
4230 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
4231 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4232 if (!isLexingRawMode())
4233 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4234 Kind = tok::hashat;
4235 } else { // '%:' -> '#'
4236 // We parsed a # character. If this occurs at the start of the line,
4237 // it's actually the start of a preprocessing directive. Callback to
4238 // the preprocessor to handle it.
4239 // TODO: -fpreprocessed mode??
4240 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
4241 goto HandleDirective;
4242
4243 Kind = tok::hash;
4244 }
4245 } else {
4246 Kind = tok::percent;
4247 }
4248 break;
4249 case '<':
4250 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4251 if (ParsingFilename) {
4252 return LexAngledStringLiteral(Result, CurPtr);
4253 } else if (Char == '<') {
4254 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4255 if (After == '=') {
4256 Kind = tok::lesslessequal;
4257 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4258 Size: SizeTmp2, Tok&: Result);
4259 } else if (After == '<' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4260 // If this is actually a '<<<<<<<' version control conflict marker,
4261 // recognize it as such and recover nicely.
4262 goto LexNextToken;
4263 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4264 // If this is '<<<<' and we're in a Perforce-style conflict marker,
4265 // ignore it.
4266 goto LexNextToken;
4267 } else if (LangOpts.CUDA && After == '<') {
4268 Kind = tok::lesslessless;
4269 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4270 Size: SizeTmp2, Tok&: Result);
4271 } else {
4272 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4273 Kind = tok::lessless;
4274 }
4275 } else if (Char == '=') {
4276 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4277 if (After == '>') {
4278 if (LangOpts.CPlusPlus20) {
4279 if (!isLexingRawMode())
4280 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx17_compat_spaceship);
4281 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4282 Size: SizeTmp2, Tok&: Result);
4283 Kind = tok::spaceship;
4284 break;
4285 }
4286 // Suggest adding a space between the '<=' and the '>' to avoid a
4287 // change in semantics if this turns up in C++ <=17 mode.
4288 if (LangOpts.CPlusPlus && !isLexingRawMode()) {
4289 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx20_compat_spaceship)
4290 << FixItHint::CreateInsertion(
4291 InsertionLoc: getSourceLocation(Loc: CurPtr + SizeTmp, TokLen: SizeTmp2), Code: " ");
4292 }
4293 }
4294 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4295 Kind = tok::lessequal;
4296 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '['
4297 if (LangOpts.CPlusPlus11 &&
4298 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == ':') {
4299 // C++0x [lex.pptoken]p3:
4300 // Otherwise, if the next three characters are <:: and the subsequent
4301 // character is neither : nor >, the < is treated as a preprocessor
4302 // token by itself and not as the first character of the alternative
4303 // token <:.
4304 unsigned SizeTmp3;
4305 char After = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4306 if (After != ':' && After != '>') {
4307 Kind = tok::less;
4308 if (!isLexingRawMode())
4309 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_less_colon_colon);
4310 break;
4311 }
4312 }
4313
4314 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4315 Kind = tok::l_square;
4316 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{'
4317 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4318 Kind = tok::l_brace;
4319 } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 &&
4320 lexEditorPlaceholder(Result, CurPtr)) {
4321 return true;
4322 } else {
4323 Kind = tok::less;
4324 }
4325 break;
4326 case '>':
4327 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4328 if (Char == '=') {
4329 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4330 Kind = tok::greaterequal;
4331 } else if (Char == '>') {
4332 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4333 if (After == '=') {
4334 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4335 Size: SizeTmp2, Tok&: Result);
4336 Kind = tok::greatergreaterequal;
4337 } else if (After == '>' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4338 // If this is actually a '>>>>' conflict marker, recognize it as such
4339 // and recover nicely.
4340 goto LexNextToken;
4341 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4342 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
4343 goto LexNextToken;
4344 } else if (LangOpts.CUDA && After == '>') {
4345 Kind = tok::greatergreatergreater;
4346 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4347 Size: SizeTmp2, Tok&: Result);
4348 } else {
4349 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4350 Kind = tok::greatergreater;
4351 }
4352 } else {
4353 Kind = tok::greater;
4354 }
4355 break;
4356 case '^':
4357 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4358 if (Char == '=') {
4359 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4360 Kind = tok::caretequal;
4361 } else {
4362 if (LangOpts.OpenCL && Char == '^')
4363 Diag(Loc: CurPtr, DiagID: diag::err_opencl_logical_exclusive_or);
4364 Kind = tok::caret;
4365 }
4366 break;
4367 case '|':
4368 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4369 if (Char == '=') {
4370 Kind = tok::pipeequal;
4371 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4372 } else if (Char == '|') {
4373 // If this is '|||||||' and we're in a conflict marker, ignore it.
4374 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4375 goto LexNextToken;
4376 Kind = tok::pipepipe;
4377 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4378 } else {
4379 Kind = tok::pipe;
4380 }
4381 break;
4382 case ':':
4383 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4384 if (LangOpts.Digraphs && Char == '>') {
4385 Kind = tok::r_square; // ':>' -> ']'
4386 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4387 } else if (Char == ':') {
4388 Kind = tok::coloncolon;
4389 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4390 } else {
4391 Kind = tok::colon;
4392 }
4393 break;
4394 case ';':
4395 Kind = tok::semi;
4396 break;
4397 case '=':
4398 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4399 if (Char == '=') {
4400 // If this is '====' and we're in a conflict marker, ignore it.
4401 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4402 goto LexNextToken;
4403
4404 Kind = tok::equalequal;
4405 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4406 } else {
4407 Kind = tok::equal;
4408 }
4409 break;
4410 case ',':
4411 Kind = tok::comma;
4412 break;
4413 case '#':
4414 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4415 if (Char == '#') {
4416 Kind = tok::hashhash;
4417 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4418 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize
4419 Kind = tok::hashat;
4420 if (!isLexingRawMode())
4421 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4422 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4423 } else {
4424 // We parsed a # character. If this occurs at the start of the line,
4425 // it's actually the start of a preprocessing directive. Callback to
4426 // the preprocessor to handle it.
4427 // TODO: -fpreprocessed mode??
4428 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
4429 goto HandleDirective;
4430
4431 Kind = tok::hash;
4432 }
4433 break;
4434
4435 case '@':
4436 // Objective C support.
4437 if (CurPtr[-1] == '@' && LangOpts.ObjC)
4438 Kind = tok::at;
4439 else
4440 Kind = tok::unknown;
4441 break;
4442
4443 // UCNs (C99 6.4.3, C++11 [lex.charset]p2)
4444 case '\\':
4445 if (!LangOpts.AsmPreprocessor) {
4446 if (uint32_t CodePoint = tryReadUCN(StartPtr&: CurPtr, SlashLoc: BufferPtr, Result: &Result)) {
4447 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4448 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4449 return true; // KeepWhitespaceMode
4450
4451 // We only saw whitespace, so just try again with this lexer.
4452 // (We manually eliminate the tail call to avoid recursion.)
4453 goto LexNextToken;
4454 }
4455
4456 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4457 }
4458 }
4459
4460 Kind = tok::unknown;
4461 break;
4462
4463 default: {
4464 if (isASCII(c: Char)) {
4465 Kind = tok::unknown;
4466 break;
4467 }
4468
4469 llvm::UTF32 CodePoint;
4470
4471 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
4472 // an escaped newline.
4473 --CurPtr;
4474 llvm::ConversionResult Status =
4475 llvm::convertUTF8Sequence(source: (const llvm::UTF8 **)&CurPtr,
4476 sourceEnd: (const llvm::UTF8 *)BufferEnd,
4477 target: &CodePoint,
4478 flags: llvm::strictConversion);
4479 if (Status == llvm::conversionOK) {
4480 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4481 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4482 return true; // KeepWhitespaceMode
4483
4484 // We only saw whitespace, so just try again with this lexer.
4485 // (We manually eliminate the tail call to avoid recursion.)
4486 goto LexNextToken;
4487 }
4488 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4489 }
4490
4491 if (isLexingRawMode() || ParsingPreprocessorDirective ||
4492 PP->isPreprocessedOutput()) {
4493 ++CurPtr;
4494 Kind = tok::unknown;
4495 break;
4496 }
4497
4498 // Non-ASCII characters tend to creep into source code unintentionally.
4499 // Instead of letting the parser complain about the unknown token,
4500 // just diagnose the invalid UTF-8, then drop the character.
4501 Diag(Loc: CurPtr, DiagID: diag::err_invalid_utf8);
4502
4503 BufferPtr = CurPtr+1;
4504 // We're pretending the character didn't exist, so just try again with
4505 // this lexer.
4506 // (We manually eliminate the tail call to avoid recursion.)
4507 goto LexNextToken;
4508 }
4509 }
4510
4511 // Notify MIOpt that we read a non-whitespace/non-comment token.
4512 MIOpt.ReadToken();
4513
4514 // Update the location of token as well as BufferPtr.
4515 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
4516 return true;
4517
4518HandleDirective:
4519 // We parsed a # character and it's the start of a preprocessing directive.
4520
4521 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::hash);
4522 PP->HandleDirective(Result);
4523
4524 if (PP->hadModuleLoaderFatalFailure())
4525 // With a fatal failure in the module loader, we abort parsing.
4526 return true;
4527
4528 // We parsed the directive; lex a token with the new state.
4529 return false;
4530
4531LexNextToken:
4532 Result.clearFlag(Flag: Token::NeedsCleaning);
4533 goto LexStart;
4534}
4535
4536const char *Lexer::convertDependencyDirectiveToken(
4537 const dependency_directives_scan::Token &DDTok, Token &Result) {
4538 const char *TokPtr = BufferStart + DDTok.Offset;
4539 Result.startToken();
4540 Result.setLocation(getSourceLocation(Loc: TokPtr));
4541 Result.setKind(DDTok.Kind);
4542 Result.setFlag((Token::TokenFlags)DDTok.Flags);
4543 Result.setLength(DDTok.Length);
4544 BufferPtr = TokPtr + DDTok.Length;
4545 return TokPtr;
4546}
4547
4548bool Lexer::LexDependencyDirectiveToken(Token &Result) {
4549 assert(isDependencyDirectivesLexer());
4550
4551 using namespace dependency_directives_scan;
4552
4553 if (BufferPtr == BufferEnd)
4554 return LexEndOfFile(Result, CurPtr: BufferPtr);
4555
4556 while (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) {
4557 if (DepDirectives.front().Kind == pp_eof)
4558 return LexEndOfFile(Result, CurPtr: BufferEnd);
4559 if (DepDirectives.front().Kind == tokens_present_before_eof)
4560 MIOpt.ReadToken();
4561 NextDepDirectiveTokenIndex = 0;
4562 DepDirectives = DepDirectives.drop_front();
4563 }
4564
4565 const dependency_directives_scan::Token &DDTok =
4566 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex++];
4567 if (NextDepDirectiveTokenIndex > 1 || DDTok.Kind != tok::hash) {
4568 // Read something other than a preprocessor directive hash.
4569 MIOpt.ReadToken();
4570 }
4571
4572 if (ParsingFilename && DDTok.is(K: tok::less)) {
4573 BufferPtr = BufferStart + DDTok.Offset;
4574 LexAngledStringLiteral(Result, CurPtr: BufferPtr + 1);
4575 if (Result.isNot(K: tok::header_name))
4576 return true;
4577 // Advance the index of lexed tokens.
4578 while (true) {
4579 const dependency_directives_scan::Token &NextTok =
4580 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex];
4581 if (BufferStart + NextTok.Offset >= BufferPtr)
4582 break;
4583 ++NextDepDirectiveTokenIndex;
4584 }
4585 return true;
4586 }
4587
4588 const char *TokPtr = convertDependencyDirectiveToken(DDTok, Result);
4589
4590 if (Result.is(K: tok::hash) && Result.isAtStartOfLine()) {
4591 PP->HandleDirective(Result);
4592 return false;
4593 }
4594 if (Result.is(K: tok::raw_identifier)) {
4595 Result.setRawIdentifierData(TokPtr);
4596 if (!isLexingRawMode()) {
4597 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
4598 if (II->isHandleIdentifierCase())
4599 return PP->HandleIdentifier(Identifier&: Result);
4600 }
4601 return true;
4602 }
4603 if (Result.isLiteral()) {
4604 Result.setLiteralData(TokPtr);
4605 return true;
4606 }
4607 if (Result.is(K: tok::colon)) {
4608 // Convert consecutive colons to 'tok::coloncolon'.
4609 if (*BufferPtr == ':') {
4610 assert(DepDirectives.front().Tokens[NextDepDirectiveTokenIndex].is(
4611 tok::colon));
4612 ++NextDepDirectiveTokenIndex;
4613 Result.setKind(tok::coloncolon);
4614 }
4615 return true;
4616 }
4617 if (Result.is(K: tok::eod))
4618 ParsingPreprocessorDirective = false;
4619
4620 return true;
4621}
4622
4623bool Lexer::LexDependencyDirectiveTokenWhileSkipping(Token &Result) {
4624 assert(isDependencyDirectivesLexer());
4625
4626 using namespace dependency_directives_scan;
4627
4628 bool Stop = false;
4629 unsigned NestedIfs = 0;
4630 do {
4631 DepDirectives = DepDirectives.drop_front();
4632 switch (DepDirectives.front().Kind) {
4633 case pp_none:
4634 llvm_unreachable("unexpected 'pp_none'");
4635 case pp_include:
4636 case pp___include_macros:
4637 case pp_define:
4638 case pp_undef:
4639 case pp_import:
4640 case pp_pragma_import:
4641 case pp_pragma_once:
4642 case pp_pragma_push_macro:
4643 case pp_pragma_pop_macro:
4644 case pp_pragma_include_alias:
4645 case pp_pragma_system_header:
4646 case pp_include_next:
4647 case decl_at_import:
4648 case cxx_module_decl:
4649 case cxx_import_decl:
4650 case cxx_export_module_decl:
4651 case cxx_export_import_decl:
4652 case tokens_present_before_eof:
4653 break;
4654 case pp_if:
4655 case pp_ifdef:
4656 case pp_ifndef:
4657 ++NestedIfs;
4658 break;
4659 case pp_elif:
4660 case pp_elifdef:
4661 case pp_elifndef:
4662 case pp_else:
4663 if (!NestedIfs) {
4664 Stop = true;
4665 }
4666 break;
4667 case pp_endif:
4668 if (!NestedIfs) {
4669 Stop = true;
4670 } else {
4671 --NestedIfs;
4672 }
4673 break;
4674 case pp_eof:
4675 NextDepDirectiveTokenIndex = 0;
4676 return LexEndOfFile(Result, CurPtr: BufferEnd);
4677 }
4678 } while (!Stop);
4679
4680 const dependency_directives_scan::Token &DDTok =
4681 DepDirectives.front().Tokens.front();
4682 assert(DDTok.is(tok::hash));
4683 NextDepDirectiveTokenIndex = 1;
4684
4685 convertDependencyDirectiveToken(DDTok, Result);
4686 return false;
4687}
4688