1//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Lexer and Token interfaces.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Lex/Lexer.h"
14#include "UnicodeCharSets.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/Diagnostic.h"
17#include "clang/Basic/IdentifierTable.h"
18#include "clang/Basic/LLVM.h"
19#include "clang/Basic/LangOptions.h"
20#include "clang/Basic/SourceLocation.h"
21#include "clang/Basic/SourceManager.h"
22#include "clang/Basic/TokenKinds.h"
23#include "clang/Lex/LexDiagnostic.h"
24#include "clang/Lex/LiteralSupport.h"
25#include "clang/Lex/MultipleIncludeOpt.h"
26#include "clang/Lex/Preprocessor.h"
27#include "clang/Lex/PreprocessorOptions.h"
28#include "clang/Lex/Token.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/StringSwitch.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/ConvertUTF.h"
35#include "llvm/Support/MemoryBufferRef.h"
36#include "llvm/Support/NativeFormatting.h"
37#include "llvm/Support/Unicode.h"
38#include "llvm/Support/UnicodeCharRanges.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <cstring>
44#include <limits>
45#include <optional>
46#include <string>
47
48#ifdef __SSE4_2__
49#include <nmmintrin.h>
50#endif
51
52using namespace clang;
53
54//===----------------------------------------------------------------------===//
55// Token Class Implementation
56//===----------------------------------------------------------------------===//
57
58/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
59bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
60 if (isAnnotation())
61 return false;
62 if (const IdentifierInfo *II = getIdentifierInfo())
63 return II->getObjCKeywordID() == objcKey;
64 return false;
65}
66
67/// getObjCKeywordID - Return the ObjC keyword kind.
68tok::ObjCKeywordKind Token::getObjCKeywordID() const {
69 if (isAnnotation())
70 return tok::objc_not_keyword;
71 const IdentifierInfo *specId = getIdentifierInfo();
72 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
73}
74
75bool Token::isModuleContextualKeyword(bool AllowExport) const {
76 if (AllowExport && is(K: tok::kw_export))
77 return true;
78 if (isOneOf(Ks: tok::kw_import, Ks: tok::kw_module))
79 return true;
80 if (isNot(K: tok::identifier))
81 return false;
82 const auto *II = getIdentifierInfo();
83 return II->isImportKeyword() || II->isModuleKeyword();
84}
85
86/// Determine whether the token kind starts a simple-type-specifier.
87bool Token::isSimpleTypeSpecifier(const LangOptions &LangOpts) const {
88 switch (getKind()) {
89 case tok::annot_typename:
90 case tok::annot_decltype:
91 case tok::annot_pack_indexing_type:
92 return true;
93
94 case tok::kw_short:
95 case tok::kw_long:
96 case tok::kw___int64:
97 case tok::kw___int128:
98 case tok::kw_signed:
99 case tok::kw_unsigned:
100 case tok::kw_void:
101 case tok::kw_char:
102 case tok::kw_int:
103 case tok::kw_half:
104 case tok::kw_float:
105 case tok::kw_double:
106 case tok::kw___bf16:
107 case tok::kw__Float16:
108 case tok::kw___float128:
109 case tok::kw___ibm128:
110 case tok::kw_wchar_t:
111 case tok::kw_bool:
112 case tok::kw__Bool:
113 case tok::kw__Accum:
114 case tok::kw__Fract:
115 case tok::kw__Sat:
116#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
117#include "clang/Basic/TransformTypeTraits.def"
118 case tok::kw___auto_type:
119 case tok::kw_char16_t:
120 case tok::kw_char32_t:
121 case tok::kw_typeof:
122 case tok::kw_decltype:
123 case tok::kw_char8_t:
124 return getIdentifierInfo()->isKeyword(LangOpts);
125
126 default:
127 return false;
128 }
129}
130
131//===----------------------------------------------------------------------===//
132// Lexer Class Implementation
133//===----------------------------------------------------------------------===//
134
135void Lexer::anchor() {}
136
137void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
138 const char *BufEnd) {
139 BufferStart = BufStart;
140 BufferPtr = BufPtr;
141 BufferEnd = BufEnd;
142
143 assert(BufEnd[0] == 0 &&
144 "We assume that the input buffer has a null character at the end"
145 " to simplify lexing!");
146
147 // Check whether we have a BOM in the beginning of the buffer. If yes - act
148 // accordingly. Right now we support only UTF-8 with and without BOM, so, just
149 // skip the UTF-8 BOM if it's present.
150 if (BufferStart == BufferPtr) {
151 // Determine the size of the BOM.
152 StringRef Buf(BufferStart, BufferEnd - BufferStart);
153 size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
154 .StartsWith(S: "\xEF\xBB\xBF", Value: 3) // UTF-8 BOM
155 .Default(Value: 0);
156
157 // Skip the BOM.
158 BufferPtr += BOMLength;
159 }
160
161 Is_PragmaLexer = false;
162 CurrentConflictMarkerState = CMK_None;
163
164 // Start of the file is a start of line.
165 IsAtStartOfLine = true;
166 IsAtPhysicalStartOfLine = true;
167
168 HasLeadingSpace = false;
169 HasLeadingEmptyMacro = false;
170
171 // We are not after parsing a #.
172 ParsingPreprocessorDirective = false;
173
174 // We are not after parsing #include.
175 ParsingFilename = false;
176
177 // We are not in raw mode. Raw mode disables diagnostics and interpretation
178 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
179 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
180 // or otherwise skipping over tokens.
181 LexingRawMode = false;
182
183 // Default to not keeping comments.
184 ExtendedTokenMode = 0;
185
186 NewLinePtr = nullptr;
187}
188
189/// Lexer constructor - Create a new lexer object for the specified buffer
190/// with the specified preprocessor managing the lexing process. This lexer
191/// assumes that the associated file buffer and Preprocessor objects will
192/// outlive it, so it doesn't take ownership of either of them.
193Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
194 Preprocessor &PP, bool IsFirstIncludeOfFile)
195 : PreprocessorLexer(&PP, FID),
196 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
197 LangOpts(PP.getLangOpts()), LineComment(LangOpts.LineComment),
198 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
199 InitLexer(BufStart: InputFile.getBufferStart(), BufPtr: InputFile.getBufferStart(),
200 BufEnd: InputFile.getBufferEnd());
201
202 resetExtendedTokenMode();
203}
204
205/// Lexer constructor - Create a new raw lexer object. This object is only
206/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
207/// range will outlive it, so it doesn't take ownership of it.
208Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
209 const char *BufStart, const char *BufPtr, const char *BufEnd,
210 bool IsFirstIncludeOfFile)
211 : FileLoc(fileloc), LangOpts(langOpts), LineComment(LangOpts.LineComment),
212 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
213 InitLexer(BufStart, BufPtr, BufEnd);
214
215 // We *are* in raw mode.
216 LexingRawMode = true;
217}
218
219/// Lexer constructor - Create a new raw lexer object. This object is only
220/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
221/// range will outlive it, so it doesn't take ownership of it.
222Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
223 const SourceManager &SM, const LangOptions &langOpts,
224 bool IsFirstIncludeOfFile)
225 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
226 FromFile.getBufferStart(), FromFile.getBufferEnd(),
227 IsFirstIncludeOfFile) {}
228
229void Lexer::resetExtendedTokenMode() {
230 assert(PP && "Cannot reset token mode without a preprocessor");
231 if (LangOpts.TraditionalCPP)
232 SetKeepWhitespaceMode(true);
233 else
234 SetCommentRetentionState(PP->getCommentRetentionState());
235}
236
237/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
238/// _Pragma expansion. This has a variety of magic semantics that this method
239/// sets up. It returns a new'd Lexer that must be delete'd when done.
240///
241/// On entrance to this routine, TokStartLoc is a macro location which has a
242/// spelling loc that indicates the bytes to be lexed for the token and an
243/// expansion location that indicates where all lexed tokens should be
244/// "expanded from".
245///
246/// TODO: It would really be nice to make _Pragma just be a wrapper around a
247/// normal lexer that remaps tokens as they fly by. This would require making
248/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
249/// interface that could handle this stuff. This would pull GetMappedTokenLoc
250/// out of the critical path of the lexer!
251///
252Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
253 SourceLocation ExpansionLocStart,
254 SourceLocation ExpansionLocEnd,
255 unsigned TokLen, Preprocessor &PP) {
256 SourceManager &SM = PP.getSourceManager();
257
258 // Create the lexer as if we were going to lex the file normally.
259 FileID SpellingFID = SM.getFileID(SpellingLoc);
260 llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(FID: SpellingFID);
261 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
262
263 // Now that the lexer is created, change the start/end locations so that we
264 // just lex the subsection of the file that we want. This is lexing from a
265 // scratch buffer.
266 const char *StrData = SM.getCharacterData(SL: SpellingLoc);
267
268 L->BufferPtr = StrData;
269 L->BufferEnd = StrData+TokLen;
270 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
271
272 // Set the SourceLocation with the remapping information. This ensures that
273 // GetMappedTokenLoc will remap the tokens as they are lexed.
274 L->FileLoc = SM.createExpansionLoc(SpellingLoc: SM.getLocForStartOfFile(FID: SpellingFID),
275 ExpansionLocStart,
276 ExpansionLocEnd, Length: TokLen);
277
278 // Ensure that the lexer thinks it is inside a directive, so that end \n will
279 // return an EOD token.
280 L->ParsingPreprocessorDirective = true;
281
282 // This lexer really is for _Pragma.
283 L->Is_PragmaLexer = true;
284 return L;
285}
286
287void Lexer::seek(unsigned Offset, bool IsAtStartOfLine) {
288 this->IsAtPhysicalStartOfLine = IsAtStartOfLine;
289 this->IsAtStartOfLine = IsAtStartOfLine;
290 assert((BufferStart + Offset) <= BufferEnd);
291 BufferPtr = BufferStart + Offset;
292}
293
294template <typename T> static void StringifyImpl(T &Str, char Quote) {
295 typename T::size_type i = 0, e = Str.size();
296 while (i < e) {
297 if (Str[i] == '\\' || Str[i] == Quote) {
298 Str.insert(Str.begin() + i, '\\');
299 i += 2;
300 ++e;
301 } else if (Str[i] == '\n' || Str[i] == '\r') {
302 // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
303 if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') &&
304 Str[i] != Str[i + 1]) {
305 Str[i] = '\\';
306 Str[i + 1] = 'n';
307 } else {
308 // Replace '\n' and '\r' to '\\' followed by 'n'.
309 Str[i] = '\\';
310 Str.insert(Str.begin() + i + 1, 'n');
311 ++e;
312 }
313 i += 2;
314 } else
315 ++i;
316 }
317}
318
319std::string Lexer::Stringify(StringRef Str, bool Charify) {
320 std::string Result = std::string(Str);
321 char Quote = Charify ? '\'' : '"';
322 StringifyImpl(Str&: Result, Quote);
323 return Result;
324}
325
326void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, Quote: '"'); }
327
328//===----------------------------------------------------------------------===//
329// Token Spelling
330//===----------------------------------------------------------------------===//
331
332/// Slow case of getSpelling. Extract the characters comprising the
333/// spelling of this token from the provided input buffer.
334static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
335 const LangOptions &LangOpts, char *Spelling) {
336 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
337
338 size_t Length = 0;
339 const char *BufEnd = BufPtr + Tok.getLength();
340
341 if (tok::isStringLiteral(K: Tok.getKind())) {
342 // Munch the encoding-prefix and opening double-quote.
343 while (BufPtr < BufEnd) {
344 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
345 Spelling[Length++] = CharAndSize.Char;
346 BufPtr += CharAndSize.Size;
347
348 if (Spelling[Length - 1] == '"')
349 break;
350 }
351
352 // Raw string literals need special handling; trigraph expansion and line
353 // splicing do not occur within their d-char-sequence nor within their
354 // r-char-sequence.
355 if (Length >= 2 &&
356 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') {
357 // Search backwards from the end of the token to find the matching closing
358 // quote.
359 const char *RawEnd = BufEnd;
360 do --RawEnd; while (*RawEnd != '"');
361 size_t RawLength = RawEnd - BufPtr + 1;
362
363 // Everything between the quotes is included verbatim in the spelling.
364 memcpy(dest: Spelling + Length, src: BufPtr, n: RawLength);
365 Length += RawLength;
366 BufPtr += RawLength;
367
368 // The rest of the token is lexed normally.
369 }
370 }
371
372 while (BufPtr < BufEnd) {
373 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
374 Spelling[Length++] = CharAndSize.Char;
375 BufPtr += CharAndSize.Size;
376 }
377
378 assert(Length < Tok.getLength() &&
379 "NeedsCleaning flag set on token that didn't need cleaning!");
380 return Length;
381}
382
383/// getSpelling() - Return the 'spelling' of this token. The spelling of a
384/// token are the characters used to represent the token in the source file
385/// after trigraph expansion and escaped-newline folding. In particular, this
386/// wants to get the true, uncanonicalized, spelling of things like digraphs
387/// UCNs, etc.
388StringRef Lexer::getSpelling(SourceLocation loc,
389 SmallVectorImpl<char> &buffer,
390 const SourceManager &SM,
391 const LangOptions &options,
392 bool *invalid) {
393 // Break down the source location.
394 FileIDAndOffset locInfo = SM.getDecomposedLoc(Loc: loc);
395
396 // Try to the load the file buffer.
397 bool invalidTemp = false;
398 StringRef file = SM.getBufferData(FID: locInfo.first, Invalid: &invalidTemp);
399 if (invalidTemp) {
400 if (invalid) *invalid = true;
401 return {};
402 }
403
404 const char *tokenBegin = file.data() + locInfo.second;
405
406 // Lex from the start of the given location.
407 Lexer lexer(SM.getLocForStartOfFile(FID: locInfo.first), options,
408 file.begin(), tokenBegin, file.end());
409 Token token;
410 lexer.LexFromRawLexer(Result&: token);
411
412 unsigned length = token.getLength();
413
414 // Common case: no need for cleaning.
415 if (!token.needsCleaning())
416 return StringRef(tokenBegin, length);
417
418 // Hard case, we need to relex the characters into the string.
419 buffer.resize(N: length);
420 buffer.resize(N: getSpellingSlow(Tok: token, BufPtr: tokenBegin, LangOpts: options, Spelling: buffer.data()));
421 return StringRef(buffer.data(), buffer.size());
422}
423
424/// getSpelling() - Return the 'spelling' of this token. The spelling of a
425/// token are the characters used to represent the token in the source file
426/// after trigraph expansion and escaped-newline folding. In particular, this
427/// wants to get the true, uncanonicalized, spelling of things like digraphs
428/// UCNs, etc.
429std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
430 const LangOptions &LangOpts, bool *Invalid) {
431 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
432
433 bool CharDataInvalid = false;
434 const char *TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(),
435 Invalid: &CharDataInvalid);
436 if (Invalid)
437 *Invalid = CharDataInvalid;
438 if (CharDataInvalid)
439 return {};
440
441 // If this token contains nothing interesting, return it directly.
442 if (!Tok.needsCleaning())
443 return std::string(TokStart, TokStart + Tok.getLength());
444
445 std::string Result;
446 Result.resize(n: Tok.getLength());
447 Result.resize(n: getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: &*Result.begin()));
448 return Result;
449}
450
451/// getSpelling - This method is used to get the spelling of a token into a
452/// preallocated buffer, instead of as an std::string. The caller is required
453/// to allocate enough space for the token, which is guaranteed to be at least
454/// Tok.getLength() bytes long. The actual length of the token is returned.
455///
456/// Note that this method may do two possible things: it may either fill in
457/// the buffer specified with characters, or it may *change the input pointer*
458/// to point to a constant buffer with the data already in it (avoiding a
459/// copy). The caller is not allowed to modify the returned buffer pointer
460/// if an internal buffer is returned.
461unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
462 const SourceManager &SourceMgr,
463 const LangOptions &LangOpts, bool *Invalid) {
464 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
465
466 const char *TokStart = nullptr;
467 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
468 if (Tok.is(K: tok::raw_identifier))
469 TokStart = Tok.getRawIdentifier().data();
470 else if (!Tok.hasUCN()) {
471 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
472 // Just return the string from the identifier table, which is very quick.
473 Buffer = II->getNameStart();
474 return II->getLength();
475 }
476 }
477
478 // NOTE: this can be checked even after testing for an IdentifierInfo.
479 if (Tok.isLiteral())
480 TokStart = Tok.getLiteralData();
481
482 if (!TokStart) {
483 // Compute the start of the token in the input lexer buffer.
484 bool CharDataInvalid = false;
485 TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(), Invalid: &CharDataInvalid);
486 if (Invalid)
487 *Invalid = CharDataInvalid;
488 if (CharDataInvalid) {
489 Buffer = "";
490 return 0;
491 }
492 }
493
494 // If this token contains nothing interesting, return it directly.
495 if (!Tok.needsCleaning()) {
496 Buffer = TokStart;
497 return Tok.getLength();
498 }
499
500 // Otherwise, hard case, relex the characters into the string.
501 return getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: const_cast<char*>(Buffer));
502}
503
504/// MeasureTokenLength - Relex the token at the specified location and return
505/// its length in bytes in the input file. If the token needs cleaning (e.g.
506/// includes a trigraph or an escaped newline) then this count includes bytes
507/// that are part of that.
508unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
509 const SourceManager &SM,
510 const LangOptions &LangOpts) {
511 Token TheTok;
512 if (getRawToken(Loc, Result&: TheTok, SM, LangOpts))
513 return 0;
514 return TheTok.getLength();
515}
516
517/// Relex the token at the specified location.
518/// \returns true if there was a failure, false on success.
519bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
520 const SourceManager &SM,
521 const LangOptions &LangOpts,
522 bool IgnoreWhiteSpace) {
523 // TODO: this could be special cased for common tokens like identifiers, ')',
524 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
525 // all obviously single-char tokens. This could use
526 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
527 // something.
528
529 // If this comes from a macro expansion, we really do want the macro name, not
530 // the token this macro expanded to.
531 Loc = SM.getExpansionLoc(Loc);
532 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
533 bool Invalid = false;
534 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
535 if (Invalid)
536 return true;
537
538 const char *StrData = Buffer.data()+LocInfo.second;
539
540 if (!IgnoreWhiteSpace && isWhitespace(c: SkipEscapedNewLines(P: StrData)[0]))
541 return true;
542
543 // Create a lexer starting at the beginning of this token.
544 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
545 Buffer.begin(), StrData, Buffer.end());
546 TheLexer.SetCommentRetentionState(true);
547 TheLexer.LexFromRawLexer(Result);
548 return false;
549}
550
551/// Returns the pointer that points to the beginning of line that contains
552/// the given offset, or null if the offset if invalid.
553static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
554 const char *BufStart = Buffer.data();
555 if (Offset >= Buffer.size())
556 return nullptr;
557
558 const char *LexStart = BufStart + Offset;
559 for (; LexStart != BufStart; --LexStart) {
560 if (isVerticalWhitespace(c: LexStart[0]) &&
561 !Lexer::isNewLineEscaped(BufferStart: BufStart, Str: LexStart)) {
562 // LexStart should point at first character of logical line.
563 ++LexStart;
564 break;
565 }
566 }
567 return LexStart;
568}
569
570static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
571 const SourceManager &SM,
572 const LangOptions &LangOpts) {
573 assert(Loc.isFileID());
574 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
575 if (LocInfo.first.isInvalid())
576 return Loc;
577
578 bool Invalid = false;
579 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
580 if (Invalid)
581 return Loc;
582
583 // Back up from the current location until we hit the beginning of a line
584 // (or the buffer). We'll relex from that point.
585 const char *StrData = Buffer.data() + LocInfo.second;
586 const char *LexStart = findBeginningOfLine(Buffer, Offset: LocInfo.second);
587 if (!LexStart || LexStart == StrData)
588 return Loc;
589
590 // Create a lexer starting at the beginning of this token.
591 SourceLocation LexerStartLoc = Loc.getLocWithOffset(Offset: -LocInfo.second);
592 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
593 Buffer.end());
594 TheLexer.SetCommentRetentionState(true);
595
596 // Lex tokens until we find the token that contains the source location.
597 Token TheTok;
598 do {
599 TheLexer.LexFromRawLexer(Result&: TheTok);
600
601 if (TheLexer.getBufferLocation() > StrData) {
602 // Lexing this token has taken the lexer past the source location we're
603 // looking for. If the current token encompasses our source location,
604 // return the beginning of that token.
605 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
606 return TheTok.getLocation();
607
608 // We ended up skipping over the source location entirely, which means
609 // that it points into whitespace. We're done here.
610 break;
611 }
612 } while (TheTok.getKind() != tok::eof);
613
614 // We've passed our source location; just return the original source location.
615 return Loc;
616}
617
618SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
619 const SourceManager &SM,
620 const LangOptions &LangOpts) {
621 if (Loc.isFileID())
622 return getBeginningOfFileToken(Loc, SM, LangOpts);
623
624 if (!SM.isMacroArgExpansion(Loc))
625 return Loc;
626
627 SourceLocation FileLoc = SM.getSpellingLoc(Loc);
628 SourceLocation BeginFileLoc = getBeginningOfFileToken(Loc: FileLoc, SM, LangOpts);
629 FileIDAndOffset FileLocInfo = SM.getDecomposedLoc(Loc: FileLoc);
630 FileIDAndOffset BeginFileLocInfo = SM.getDecomposedLoc(Loc: BeginFileLoc);
631 assert(FileLocInfo.first == BeginFileLocInfo.first &&
632 FileLocInfo.second >= BeginFileLocInfo.second);
633 return Loc.getLocWithOffset(Offset: BeginFileLocInfo.second - FileLocInfo.second);
634}
635
636namespace {
637
638enum PreambleDirectiveKind {
639 PDK_Skipped,
640 PDK_Unknown
641};
642
643} // namespace
644
645PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
646 const LangOptions &LangOpts,
647 unsigned MaxLines) {
648 // Create a lexer starting at the beginning of the file. Note that we use a
649 // "fake" file source location at offset 1 so that the lexer will track our
650 // position within the file.
651 const SourceLocation::UIntTy StartOffset = 1;
652 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(Encoding: StartOffset);
653 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
654 Buffer.end());
655 TheLexer.SetCommentRetentionState(true);
656
657 bool InPreprocessorDirective = false;
658 Token TheTok;
659 SourceLocation ActiveCommentLoc;
660
661 unsigned MaxLineOffset = 0;
662 if (MaxLines) {
663 const char *CurPtr = Buffer.begin();
664 unsigned CurLine = 0;
665 while (CurPtr != Buffer.end()) {
666 char ch = *CurPtr++;
667 if (ch == '\n') {
668 ++CurLine;
669 if (CurLine == MaxLines)
670 break;
671 }
672 }
673 if (CurPtr != Buffer.end())
674 MaxLineOffset = CurPtr - Buffer.begin();
675 }
676
677 do {
678 TheLexer.LexFromRawLexer(Result&: TheTok);
679
680 if (InPreprocessorDirective) {
681 // If we've hit the end of the file, we're done.
682 if (TheTok.getKind() == tok::eof) {
683 break;
684 }
685
686 // If we haven't hit the end of the preprocessor directive, skip this
687 // token.
688 if (!TheTok.isAtStartOfLine())
689 continue;
690
691 // We've passed the end of the preprocessor directive, and will look
692 // at this token again below.
693 InPreprocessorDirective = false;
694 }
695
696 // Keep track of the # of lines in the preamble.
697 if (TheTok.isAtStartOfLine()) {
698 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
699
700 // If we were asked to limit the number of lines in the preamble,
701 // and we're about to exceed that limit, we're done.
702 if (MaxLineOffset && TokOffset >= MaxLineOffset)
703 break;
704 }
705
706 // Comments are okay; skip over them.
707 if (TheTok.getKind() == tok::comment) {
708 if (ActiveCommentLoc.isInvalid())
709 ActiveCommentLoc = TheTok.getLocation();
710 continue;
711 }
712
713 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
714 // This is the start of a preprocessor directive.
715 Token HashTok = TheTok;
716 InPreprocessorDirective = true;
717 ActiveCommentLoc = SourceLocation();
718
719 // Figure out which directive this is. Since we're lexing raw tokens,
720 // we don't have an identifier table available. Instead, just look at
721 // the raw identifier to recognize and categorize preprocessor directives.
722 TheLexer.LexFromRawLexer(Result&: TheTok);
723 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
724 StringRef Keyword = TheTok.getRawIdentifier();
725 PreambleDirectiveKind PDK
726 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
727 .Case(S: "include", Value: PDK_Skipped)
728 .Case(S: "__include_macros", Value: PDK_Skipped)
729 .Case(S: "define", Value: PDK_Skipped)
730 .Case(S: "undef", Value: PDK_Skipped)
731 .Case(S: "line", Value: PDK_Skipped)
732 .Case(S: "error", Value: PDK_Skipped)
733 .Case(S: "pragma", Value: PDK_Skipped)
734 .Case(S: "import", Value: PDK_Skipped)
735 .Case(S: "include_next", Value: PDK_Skipped)
736 .Case(S: "warning", Value: PDK_Skipped)
737 .Case(S: "ident", Value: PDK_Skipped)
738 .Case(S: "sccs", Value: PDK_Skipped)
739 .Case(S: "assert", Value: PDK_Skipped)
740 .Case(S: "unassert", Value: PDK_Skipped)
741 .Case(S: "if", Value: PDK_Skipped)
742 .Case(S: "ifdef", Value: PDK_Skipped)
743 .Case(S: "ifndef", Value: PDK_Skipped)
744 .Case(S: "elif", Value: PDK_Skipped)
745 .Case(S: "elifdef", Value: PDK_Skipped)
746 .Case(S: "elifndef", Value: PDK_Skipped)
747 .Case(S: "else", Value: PDK_Skipped)
748 .Case(S: "endif", Value: PDK_Skipped)
749 .Default(Value: PDK_Unknown);
750
751 switch (PDK) {
752 case PDK_Skipped:
753 continue;
754
755 case PDK_Unknown:
756 // We don't know what this directive is; stop at the '#'.
757 break;
758 }
759 }
760
761 // We only end up here if we didn't recognize the preprocessor
762 // directive or it was one that can't occur in the preamble at this
763 // point. Roll back the current token to the location of the '#'.
764 TheTok = HashTok;
765 } else if (TheTok.isAtStartOfLine() &&
766 TheTok.getKind() == tok::raw_identifier &&
767 TheTok.getRawIdentifier() == "module" &&
768 LangOpts.CPlusPlusModules) {
769 // The initial global module fragment introducer "module;" is part of
770 // the preamble, which runs up to the module declaration "module foo;".
771 Token ModuleTok = TheTok;
772 do {
773 TheLexer.LexFromRawLexer(Result&: TheTok);
774 } while (TheTok.getKind() == tok::comment);
775 if (TheTok.getKind() != tok::semi) {
776 // Not global module fragment, roll back.
777 TheTok = ModuleTok;
778 break;
779 }
780 continue;
781 }
782
783 // We hit a token that we don't recognize as being in the
784 // "preprocessing only" part of the file, so we're no longer in
785 // the preamble.
786 break;
787 } while (true);
788
789 SourceLocation End;
790 if (ActiveCommentLoc.isValid())
791 End = ActiveCommentLoc; // don't truncate a decl comment.
792 else
793 End = TheTok.getLocation();
794
795 return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
796 TheTok.isAtStartOfLine());
797}
798
799unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
800 const SourceManager &SM,
801 const LangOptions &LangOpts) {
802 // Figure out how many physical characters away the specified expansion
803 // character is. This needs to take into consideration newlines and
804 // trigraphs.
805 bool Invalid = false;
806 const char *TokPtr = SM.getCharacterData(SL: TokStart, Invalid: &Invalid);
807
808 // If they request the first char of the token, we're trivially done.
809 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(C: *TokPtr)))
810 return 0;
811
812 unsigned PhysOffset = 0;
813
814 // The usual case is that tokens don't contain anything interesting. Skip
815 // over the uninteresting characters. If a token only consists of simple
816 // chars, this method is extremely fast.
817 while (Lexer::isObviouslySimpleCharacter(C: *TokPtr)) {
818 if (CharNo == 0)
819 return PhysOffset;
820 ++TokPtr;
821 --CharNo;
822 ++PhysOffset;
823 }
824
825 // If we have a character that may be a trigraph or escaped newline, use a
826 // lexer to parse it correctly.
827 for (; CharNo; --CharNo) {
828 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: TokPtr, LangOpts);
829 TokPtr += CharAndSize.Size;
830 PhysOffset += CharAndSize.Size;
831 }
832
833 // Final detail: if we end up on an escaped newline, we want to return the
834 // location of the actual byte of the token. For example foo\<newline>bar
835 // advanced by 3 should return the location of b, not of \\. One compounding
836 // detail of this is that the escape may be made by a trigraph.
837 if (!Lexer::isObviouslySimpleCharacter(C: *TokPtr))
838 PhysOffset += Lexer::SkipEscapedNewLines(P: TokPtr)-TokPtr;
839
840 return PhysOffset;
841}
842
843/// Computes the source location just past the end of the
844/// token at this source location.
845///
846/// This routine can be used to produce a source location that
847/// points just past the end of the token referenced by \p Loc, and
848/// is generally used when a diagnostic needs to point just after a
849/// token where it expected something different that it received. If
850/// the returned source location would not be meaningful (e.g., if
851/// it points into a macro), this routine returns an invalid
852/// source location.
853///
854/// \param Offset an offset from the end of the token, where the source
855/// location should refer to. The default offset (0) produces a source
856/// location pointing just past the end of the token; an offset of 1 produces
857/// a source location pointing to the last character in the token, etc.
858SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
859 const SourceManager &SM,
860 const LangOptions &LangOpts) {
861 if (Loc.isInvalid())
862 return {};
863
864 if (Loc.isMacroID()) {
865 if (Offset > 0 || !isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
866 return {}; // Points inside the macro expansion.
867 }
868
869 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
870 if (Len > Offset)
871 Len = Len - Offset;
872 else
873 return Loc;
874
875 return Loc.getLocWithOffset(Offset: Len);
876}
877
878/// Returns true if the given MacroID location points at the first
879/// token of the macro expansion.
880bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
881 const SourceManager &SM,
882 const LangOptions &LangOpts,
883 SourceLocation *MacroBegin) {
884 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
885
886 SourceLocation expansionLoc;
887 if (!SM.isAtStartOfImmediateMacroExpansion(Loc: loc, MacroBegin: &expansionLoc))
888 return false;
889
890 if (expansionLoc.isFileID()) {
891 // No other macro expansions, this is the first.
892 if (MacroBegin)
893 *MacroBegin = expansionLoc;
894 return true;
895 }
896
897 return isAtStartOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroBegin);
898}
899
900/// Returns true if the given MacroID location points at the last
901/// token of the macro expansion.
902bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
903 const SourceManager &SM,
904 const LangOptions &LangOpts,
905 SourceLocation *MacroEnd) {
906 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
907
908 SourceLocation spellLoc = SM.getSpellingLoc(Loc: loc);
909 unsigned tokLen = MeasureTokenLength(Loc: spellLoc, SM, LangOpts);
910 if (tokLen == 0)
911 return false;
912
913 SourceLocation afterLoc = loc.getLocWithOffset(Offset: tokLen);
914 SourceLocation expansionLoc;
915 if (!SM.isAtEndOfImmediateMacroExpansion(Loc: afterLoc, MacroEnd: &expansionLoc))
916 return false;
917
918 if (expansionLoc.isFileID()) {
919 // No other macro expansions.
920 if (MacroEnd)
921 *MacroEnd = expansionLoc;
922 return true;
923 }
924
925 return isAtEndOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroEnd);
926}
927
928static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
929 const SourceManager &SM,
930 const LangOptions &LangOpts) {
931 SourceLocation Begin = Range.getBegin();
932 SourceLocation End = Range.getEnd();
933 assert(Begin.isFileID() && End.isFileID());
934 if (Range.isTokenRange()) {
935 End = Lexer::getLocForEndOfToken(Loc: End, Offset: 0, SM,LangOpts);
936 if (End.isInvalid())
937 return {};
938 }
939
940 // Break down the source locations.
941 auto [FID, BeginOffs] = SM.getDecomposedLoc(Loc: Begin);
942 if (FID.isInvalid())
943 return {};
944
945 unsigned EndOffs;
946 if (!SM.isInFileID(Loc: End, FID, RelativeOffset: &EndOffs) ||
947 BeginOffs > EndOffs)
948 return {};
949
950 return CharSourceRange::getCharRange(B: Begin, E: End);
951}
952
953// Assumes that `Loc` is in an expansion.
954static bool isInExpansionTokenRange(const SourceLocation Loc,
955 const SourceManager &SM) {
956 return SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Loc))
957 .getExpansion()
958 .isExpansionTokenRange();
959}
960
961CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
962 const SourceManager &SM,
963 const LangOptions &LangOpts) {
964 SourceLocation Begin = Range.getBegin();
965 SourceLocation End = Range.getEnd();
966 if (Begin.isInvalid() || End.isInvalid())
967 return {};
968
969 if (Begin.isFileID() && End.isFileID())
970 return makeRangeFromFileLocs(Range, SM, LangOpts);
971
972 if (Begin.isMacroID() && End.isFileID()) {
973 if (!isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &Begin))
974 return {};
975 Range.setBegin(Begin);
976 return makeRangeFromFileLocs(Range, SM, LangOpts);
977 }
978
979 if (Begin.isFileID() && End.isMacroID()) {
980 if (Range.isTokenRange()) {
981 if (!isAtEndOfMacroExpansion(loc: End, SM, LangOpts, MacroEnd: &End))
982 return {};
983 // Use the *original* end, not the expanded one in `End`.
984 Range.setTokenRange(isInExpansionTokenRange(Loc: Range.getEnd(), SM));
985 } else if (!isAtStartOfMacroExpansion(loc: End, SM, LangOpts, MacroBegin: &End))
986 return {};
987 Range.setEnd(End);
988 return makeRangeFromFileLocs(Range, SM, LangOpts);
989 }
990
991 assert(Begin.isMacroID() && End.isMacroID());
992 SourceLocation MacroBegin, MacroEnd;
993 if (isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &MacroBegin) &&
994 ((Range.isTokenRange() && isAtEndOfMacroExpansion(loc: End, SM, LangOpts,
995 MacroEnd: &MacroEnd)) ||
996 (Range.isCharRange() && isAtStartOfMacroExpansion(loc: End, SM, LangOpts,
997 MacroBegin: &MacroEnd)))) {
998 Range.setBegin(MacroBegin);
999 Range.setEnd(MacroEnd);
1000 // Use the *original* `End`, not the expanded one in `MacroEnd`.
1001 if (Range.isTokenRange())
1002 Range.setTokenRange(isInExpansionTokenRange(Loc: End, SM));
1003 return makeRangeFromFileLocs(Range, SM, LangOpts);
1004 }
1005
1006 bool Invalid = false;
1007 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Begin),
1008 Invalid: &Invalid);
1009 if (Invalid)
1010 return {};
1011
1012 if (BeginEntry.getExpansion().isMacroArgExpansion()) {
1013 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: End),
1014 Invalid: &Invalid);
1015 if (Invalid)
1016 return {};
1017
1018 if (EndEntry.getExpansion().isMacroArgExpansion() &&
1019 BeginEntry.getExpansion().getExpansionLocStart() ==
1020 EndEntry.getExpansion().getExpansionLocStart()) {
1021 Range.setBegin(SM.getImmediateSpellingLoc(Loc: Begin));
1022 Range.setEnd(SM.getImmediateSpellingLoc(Loc: End));
1023 return makeFileCharRange(Range, SM, LangOpts);
1024 }
1025 }
1026
1027 return {};
1028}
1029
1030StringRef Lexer::getSourceText(CharSourceRange Range,
1031 const SourceManager &SM,
1032 const LangOptions &LangOpts,
1033 bool *Invalid) {
1034 Range = makeFileCharRange(Range, SM, LangOpts);
1035 if (Range.isInvalid()) {
1036 if (Invalid) *Invalid = true;
1037 return {};
1038 }
1039
1040 // Break down the source location.
1041 FileIDAndOffset beginInfo = SM.getDecomposedLoc(Loc: Range.getBegin());
1042 if (beginInfo.first.isInvalid()) {
1043 if (Invalid) *Invalid = true;
1044 return {};
1045 }
1046
1047 unsigned EndOffs;
1048 if (!SM.isInFileID(Loc: Range.getEnd(), FID: beginInfo.first, RelativeOffset: &EndOffs) ||
1049 beginInfo.second > EndOffs) {
1050 if (Invalid) *Invalid = true;
1051 return {};
1052 }
1053
1054 // Try to the load the file buffer.
1055 bool invalidTemp = false;
1056 StringRef file = SM.getBufferData(FID: beginInfo.first, Invalid: &invalidTemp);
1057 if (invalidTemp) {
1058 if (Invalid) *Invalid = true;
1059 return {};
1060 }
1061
1062 if (Invalid) *Invalid = false;
1063 return file.substr(Start: beginInfo.second, N: EndOffs - beginInfo.second);
1064}
1065
1066StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1067 const SourceManager &SM,
1068 const LangOptions &LangOpts) {
1069 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1070
1071 // Find the location of the immediate macro expansion.
1072 while (true) {
1073 FileID FID = SM.getFileID(SpellingLoc: Loc);
1074 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
1075 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
1076 Loc = Expansion.getExpansionLocStart();
1077 if (!Expansion.isMacroArgExpansion())
1078 break;
1079
1080 // For macro arguments we need to check that the argument did not come
1081 // from an inner macro, e.g: "MAC1( MAC2(foo) )"
1082
1083 // Loc points to the argument id of the macro definition, move to the
1084 // macro expansion.
1085 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1086 SourceLocation SpellLoc = Expansion.getSpellingLoc();
1087 if (SpellLoc.isFileID())
1088 break; // No inner macro.
1089
1090 // If spelling location resides in the same FileID as macro expansion
1091 // location, it means there is no inner macro.
1092 FileID MacroFID = SM.getFileID(SpellingLoc: Loc);
1093 if (SM.isInFileID(Loc: SpellLoc, FID: MacroFID))
1094 break;
1095
1096 // Argument came from inner macro.
1097 Loc = SpellLoc;
1098 }
1099
1100 // Find the spelling location of the start of the non-argument expansion
1101 // range. This is where the macro name was spelled in order to begin
1102 // expanding this macro.
1103 Loc = SM.getSpellingLoc(Loc);
1104
1105 // Dig out the buffer where the macro name was spelled and the extents of the
1106 // name so that we can render it into the expansion note.
1107 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1108 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1109 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1110 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1111}
1112
1113StringRef Lexer::getImmediateMacroNameForDiagnostics(
1114 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
1115 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1116 // Walk past macro argument expansions.
1117 while (SM.isMacroArgExpansion(Loc))
1118 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1119
1120 // If the macro's spelling isn't FileID or from scratch space, then it's
1121 // actually a token paste or stringization (or similar) and not a macro at
1122 // all.
1123 SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
1124 if (!SpellLoc.isFileID() || SM.isWrittenInScratchSpace(Loc: SpellLoc))
1125 return {};
1126
1127 // Find the spelling location of the start of the non-argument expansion
1128 // range. This is where the macro name was spelled in order to begin
1129 // expanding this macro.
1130 Loc = SM.getSpellingLoc(Loc: SM.getImmediateExpansionRange(Loc).getBegin());
1131
1132 // Dig out the buffer where the macro name was spelled and the extents of the
1133 // name so that we can render it into the expansion note.
1134 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1135 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1136 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1137 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1138}
1139
1140bool Lexer::isAsciiIdentifierContinueChar(char c, const LangOptions &LangOpts) {
1141 return isAsciiIdentifierContinue(c, AllowDollar: LangOpts.DollarIdents);
1142}
1143
1144bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1145 assert(isVerticalWhitespace(Str[0]));
1146 if (Str - 1 < BufferStart)
1147 return false;
1148
1149 if ((Str[0] == '\n' && Str[-1] == '\r') ||
1150 (Str[0] == '\r' && Str[-1] == '\n')) {
1151 if (Str - 2 < BufferStart)
1152 return false;
1153 --Str;
1154 }
1155 --Str;
1156
1157 // Rewind to first non-space character:
1158 while (Str > BufferStart && isHorizontalWhitespace(c: *Str))
1159 --Str;
1160
1161 return *Str == '\\';
1162}
1163
1164StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1165 const SourceManager &SM) {
1166 if (Loc.isInvalid() || Loc.isMacroID())
1167 return {};
1168 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1169 if (LocInfo.first.isInvalid())
1170 return {};
1171 bool Invalid = false;
1172 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
1173 if (Invalid)
1174 return {};
1175 const char *Line = findBeginningOfLine(Buffer, Offset: LocInfo.second);
1176 if (!Line)
1177 return {};
1178 StringRef Rest = Buffer.substr(Start: Line - Buffer.data());
1179 size_t NumWhitespaceChars = Rest.find_first_not_of(Chars: " \t");
1180 return NumWhitespaceChars == StringRef::npos
1181 ? ""
1182 : Rest.take_front(N: NumWhitespaceChars);
1183}
1184
1185//===----------------------------------------------------------------------===//
1186// Diagnostics forwarding code.
1187//===----------------------------------------------------------------------===//
1188
1189/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1190/// lexer buffer was all expanded at a single point, perform the mapping.
1191/// This is currently only used for _Pragma implementation, so it is the slow
1192/// path of the hot getSourceLocation method. Do not allow it to be inlined.
1193static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1194 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1195static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1196 SourceLocation FileLoc,
1197 unsigned CharNo, unsigned TokLen) {
1198 assert(FileLoc.isMacroID() && "Must be a macro expansion");
1199
1200 // Otherwise, we're lexing "mapped tokens". This is used for things like
1201 // _Pragma handling. Combine the expansion location of FileLoc with the
1202 // spelling location.
1203 SourceManager &SM = PP.getSourceManager();
1204
1205 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
1206 // characters come from spelling(FileLoc)+Offset.
1207 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc: FileLoc);
1208 SpellingLoc = SpellingLoc.getLocWithOffset(Offset: CharNo);
1209
1210 // Figure out the expansion loc range, which is the range covered by the
1211 // original _Pragma(...) sequence.
1212 CharSourceRange II = SM.getImmediateExpansionRange(Loc: FileLoc);
1213
1214 return SM.createExpansionLoc(SpellingLoc, ExpansionLocStart: II.getBegin(), ExpansionLocEnd: II.getEnd(), Length: TokLen);
1215}
1216
1217/// getSourceLocation - Return a source location identifier for the specified
1218/// offset in the current file.
1219SourceLocation Lexer::getSourceLocation(const char *Loc,
1220 unsigned TokLen) const {
1221 assert(Loc >= BufferStart && Loc <= BufferEnd &&
1222 "Location out of range for this buffer!");
1223
1224 // In the normal case, we're just lexing from a simple file buffer, return
1225 // the file id from FileLoc with the offset specified.
1226 unsigned CharNo = Loc-BufferStart;
1227 if (FileLoc.isFileID())
1228 return FileLoc.getLocWithOffset(Offset: CharNo);
1229
1230 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1231 // tokens are lexed from where the _Pragma was defined.
1232 assert(PP && "This doesn't work on raw lexers");
1233 return GetMappedTokenLoc(PP&: *PP, FileLoc, CharNo, TokLen);
1234}
1235
1236/// Diag - Forwarding function for diagnostics. This translate a source
1237/// position in the current buffer into a SourceLocation object for rendering.
1238DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1239 return PP->Diag(Loc: getSourceLocation(Loc), DiagID);
1240}
1241
1242//===----------------------------------------------------------------------===//
1243// Trigraph and Escaped Newline Handling Code.
1244//===----------------------------------------------------------------------===//
1245
1246/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1247/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1248static char GetTrigraphCharForLetter(char Letter) {
1249 switch (Letter) {
1250 default: return 0;
1251 case '=': return '#';
1252 case ')': return ']';
1253 case '(': return '[';
1254 case '!': return '|';
1255 case '\'': return '^';
1256 case '>': return '}';
1257 case '/': return '\\';
1258 case '<': return '{';
1259 case '-': return '~';
1260 }
1261}
1262
1263/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1264/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
1265/// return the result character. Finally, emit a warning about trigraph use
1266/// whether trigraphs are enabled or not.
1267static char DecodeTrigraphChar(const char *CP, Lexer *L, bool Trigraphs) {
1268 char Res = GetTrigraphCharForLetter(Letter: *CP);
1269 if (!Res)
1270 return Res;
1271
1272 if (!Trigraphs) {
1273 if (L && !L->isLexingRawMode())
1274 L->Diag(Loc: CP-2, DiagID: diag::trigraph_ignored);
1275 return 0;
1276 }
1277
1278 if (L && !L->isLexingRawMode())
1279 L->Diag(Loc: CP-2, DiagID: diag::trigraph_converted) << StringRef(&Res, 1);
1280 return Res;
1281}
1282
1283/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1284/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1285/// trigraph equivalent on entry to this function.
1286unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1287 unsigned Size = 0;
1288 while (isWhitespace(c: Ptr[Size])) {
1289 ++Size;
1290
1291 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
1292 continue;
1293
1294 // If this is a \r\n or \n\r, skip the other half.
1295 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
1296 Ptr[Size-1] != Ptr[Size])
1297 ++Size;
1298
1299 return Size;
1300 }
1301
1302 // Not an escaped newline, must be a \t or something else.
1303 return 0;
1304}
1305
1306/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1307/// them), skip over them and return the first non-escaped-newline found,
1308/// otherwise return P.
1309const char *Lexer::SkipEscapedNewLines(const char *P) {
1310 while (true) {
1311 const char *AfterEscape;
1312 if (*P == '\\') {
1313 AfterEscape = P+1;
1314 } else if (*P == '?') {
1315 // If not a trigraph for escape, bail out.
1316 if (P[1] != '?' || P[2] != '/')
1317 return P;
1318 // FIXME: Take LangOpts into account; the language might not
1319 // support trigraphs.
1320 AfterEscape = P+3;
1321 } else {
1322 return P;
1323 }
1324
1325 unsigned NewLineSize = Lexer::getEscapedNewLineSize(Ptr: AfterEscape);
1326 if (NewLineSize == 0) return P;
1327 P = AfterEscape+NewLineSize;
1328 }
1329}
1330
1331std::optional<Token> Lexer::findNextToken(SourceLocation Loc,
1332 const SourceManager &SM,
1333 const LangOptions &LangOpts,
1334 bool IncludeComments) {
1335 if (Loc.isMacroID()) {
1336 if (!Lexer::isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
1337 return std::nullopt;
1338 }
1339 Loc = Lexer::getLocForEndOfToken(Loc, Offset: 0, SM, LangOpts);
1340
1341 // Break down the source location.
1342 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1343
1344 // Try to load the file buffer.
1345 bool InvalidTemp = false;
1346 StringRef File = SM.getBufferData(FID: LocInfo.first, Invalid: &InvalidTemp);
1347 if (InvalidTemp)
1348 return std::nullopt;
1349
1350 const char *TokenBegin = File.data() + LocInfo.second;
1351
1352 // Lex from the start of the given location.
1353 Lexer lexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts, File.begin(),
1354 TokenBegin, File.end());
1355 lexer.SetCommentRetentionState(IncludeComments);
1356 // Find the token.
1357 Token Tok;
1358 lexer.LexFromRawLexer(Result&: Tok);
1359 return Tok;
1360}
1361
1362std::optional<Token> Lexer::findPreviousToken(SourceLocation Loc,
1363 const SourceManager &SM,
1364 const LangOptions &LangOpts,
1365 bool IncludeComments) {
1366 const auto StartOfFile = SM.getLocForStartOfFile(FID: SM.getFileID(SpellingLoc: Loc));
1367 while (Loc != StartOfFile) {
1368 Loc = Loc.getLocWithOffset(Offset: -1);
1369 if (Loc.isInvalid())
1370 return std::nullopt;
1371
1372 Loc = GetBeginningOfToken(Loc, SM, LangOpts);
1373 Token Tok;
1374 if (getRawToken(Loc, Result&: Tok, SM, LangOpts))
1375 continue; // Not a token, go to prev location.
1376 if (!Tok.is(K: tok::comment) || IncludeComments) {
1377 return Tok;
1378 }
1379 }
1380 return std::nullopt;
1381}
1382
1383/// Checks that the given token is the first token that occurs after the
1384/// given location (this excludes comments and whitespace). Returns the location
1385/// immediately after the specified token. If the token is not found or the
1386/// location is inside a macro, the returned source location will be invalid.
1387SourceLocation Lexer::findLocationAfterToken(
1388 SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
1389 const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
1390 std::optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
1391 if (!Tok || Tok->isNot(K: TKind))
1392 return {};
1393 SourceLocation TokenLoc = Tok->getLocation();
1394
1395 // Calculate how much whitespace needs to be skipped if any.
1396 unsigned NumWhitespaceChars = 0;
1397 if (SkipTrailingWhitespaceAndNewLine) {
1398 const char *TokenEnd = SM.getCharacterData(SL: TokenLoc) + Tok->getLength();
1399 unsigned char C = *TokenEnd;
1400 while (isHorizontalWhitespace(c: C)) {
1401 C = *(++TokenEnd);
1402 NumWhitespaceChars++;
1403 }
1404
1405 // Skip \r, \n, \r\n, or \n\r
1406 if (C == '\n' || C == '\r') {
1407 char PrevC = C;
1408 C = *(++TokenEnd);
1409 NumWhitespaceChars++;
1410 if ((C == '\n' || C == '\r') && C != PrevC)
1411 NumWhitespaceChars++;
1412 }
1413 }
1414
1415 return TokenLoc.getLocWithOffset(Offset: Tok->getLength() + NumWhitespaceChars);
1416}
1417
1418/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1419/// get its size, and return it. This is tricky in several cases:
1420/// 1. If currently at the start of a trigraph, we warn about the trigraph,
1421/// then either return the trigraph (skipping 3 chars) or the '?',
1422/// depending on whether trigraphs are enabled or not.
1423/// 2. If this is an escaped newline (potentially with whitespace between
1424/// the backslash and newline), implicitly skip the newline and return
1425/// the char after it.
1426///
1427/// This handles the slow/uncommon case of the getCharAndSize method. Here we
1428/// know that we can accumulate into Size, and that we have already incremented
1429/// Ptr by Size bytes.
1430///
1431/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1432/// be updated to match.
1433Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
1434 unsigned Size = 0;
1435 // If we have a slash, look for an escaped newline.
1436 if (Ptr[0] == '\\') {
1437 ++Size;
1438 ++Ptr;
1439Slash:
1440 // Common case, backslash-char where the char is not whitespace.
1441 if (!isWhitespace(c: Ptr[0]))
1442 return {.Char: '\\', .Size: Size};
1443
1444 // See if we have optional whitespace characters between the slash and
1445 // newline.
1446 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1447 // Remember that this token needs to be cleaned.
1448 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1449
1450 // Warn if there was whitespace between the backslash and newline.
1451 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1452 Diag(Loc: Ptr, DiagID: diag::backslash_newline_space);
1453
1454 // Found backslash<whitespace><newline>. Parse the char after it.
1455 Size += EscapedNewLineSize;
1456 Ptr += EscapedNewLineSize;
1457
1458 // Use slow version to accumulate a correct size field.
1459 auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
1460 CharAndSize.Size += Size;
1461 return CharAndSize;
1462 }
1463
1464 // Otherwise, this is not an escaped newline, just return the slash.
1465 return {.Char: '\\', .Size: Size};
1466 }
1467
1468 // If this is a trigraph, process it.
1469 if (Ptr[0] == '?' && Ptr[1] == '?') {
1470 // If this is actually a legal trigraph (not something like "??x"), emit
1471 // a trigraph warning. If so, and if trigraphs are enabled, return it.
1472 if (char C = DecodeTrigraphChar(CP: Ptr + 2, L: Tok ? this : nullptr,
1473 Trigraphs: LangOpts.Trigraphs)) {
1474 // Remember that this token needs to be cleaned.
1475 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1476
1477 Ptr += 3;
1478 Size += 3;
1479 if (C == '\\') goto Slash;
1480 return {.Char: C, .Size: Size};
1481 }
1482 }
1483
1484 // If this is neither, return a single character.
1485 return {.Char: *Ptr, .Size: Size + 1u};
1486}
1487
1488/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1489/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
1490/// and that we have already incremented Ptr by Size bytes.
1491///
1492/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1493/// be updated to match.
1494Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
1495 const LangOptions &LangOpts) {
1496
1497 unsigned Size = 0;
1498 // If we have a slash, look for an escaped newline.
1499 if (Ptr[0] == '\\') {
1500 ++Size;
1501 ++Ptr;
1502Slash:
1503 // Common case, backslash-char where the char is not whitespace.
1504 if (!isWhitespace(c: Ptr[0]))
1505 return {.Char: '\\', .Size: Size};
1506
1507 // See if we have optional whitespace characters followed by a newline.
1508 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1509 // Found backslash<whitespace><newline>. Parse the char after it.
1510 Size += EscapedNewLineSize;
1511 Ptr += EscapedNewLineSize;
1512
1513 // Use slow version to accumulate a correct size field.
1514 auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
1515 CharAndSize.Size += Size;
1516 return CharAndSize;
1517 }
1518
1519 // Otherwise, this is not an escaped newline, just return the slash.
1520 return {.Char: '\\', .Size: Size};
1521 }
1522
1523 // If this is a trigraph, process it.
1524 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1525 // If this is actually a legal trigraph (not something like "??x"), return
1526 // it.
1527 if (char C = GetTrigraphCharForLetter(Letter: Ptr[2])) {
1528 Ptr += 3;
1529 Size += 3;
1530 if (C == '\\') goto Slash;
1531 return {.Char: C, .Size: Size};
1532 }
1533 }
1534
1535 // If this is neither, return a single character.
1536 return {.Char: *Ptr, .Size: Size + 1u};
1537}
1538
1539//===----------------------------------------------------------------------===//
1540// Helper methods for lexing.
1541//===----------------------------------------------------------------------===//
1542
1543/// Routine that indiscriminately sets the offset into the source file.
1544void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1545 BufferPtr = BufferStart + Offset;
1546 if (BufferPtr > BufferEnd)
1547 BufferPtr = BufferEnd;
1548 // FIXME: What exactly does the StartOfLine bit mean? There are two
1549 // possible meanings for the "start" of the line: the first token on the
1550 // unexpanded line, or the first token on the expanded line.
1551 IsAtStartOfLine = StartOfLine;
1552 IsAtPhysicalStartOfLine = StartOfLine;
1553}
1554
1555static bool isUnicodeWhitespace(uint32_t Codepoint) {
1556 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
1557 UnicodeWhitespaceCharRanges);
1558 return UnicodeWhitespaceChars.contains(C: Codepoint);
1559}
1560
1561static llvm::SmallString<5> codepointAsHexString(uint32_t C) {
1562 llvm::SmallString<5> CharBuf;
1563 llvm::raw_svector_ostream CharOS(CharBuf);
1564 llvm::write_hex(S&: CharOS, N: C, Style: llvm::HexPrintStyle::Upper, Width: 4);
1565 return CharBuf;
1566}
1567
1568// To mitigate https://github.com/llvm/llvm-project/issues/54732,
1569// we allow "Mathematical Notation Characters" in identifiers.
1570// This is a proposed profile that extends the XID_Start/XID_continue
1571// with mathematical symbols, superscipts and subscripts digits
1572// found in some production software.
1573// https://www.unicode.org/L2/L2022/22230-math-profile.pdf
1574static bool isMathematicalExtensionID(uint32_t C, const LangOptions &LangOpts,
1575 bool IsStart, bool &IsExtension) {
1576 static const llvm::sys::UnicodeCharSet MathStartChars(
1577 MathematicalNotationProfileIDStartRanges);
1578 static const llvm::sys::UnicodeCharSet MathContinueChars(
1579 MathematicalNotationProfileIDContinueRanges);
1580 if (MathStartChars.contains(C) ||
1581 (!IsStart && MathContinueChars.contains(C))) {
1582 IsExtension = true;
1583 return true;
1584 }
1585 return false;
1586}
1587
1588static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts,
1589 bool &IsExtension) {
1590 if (LangOpts.AsmPreprocessor) {
1591 return false;
1592 } else if (LangOpts.DollarIdents && '$' == C) {
1593 return true;
1594 } else if (LangOpts.CPlusPlus || LangOpts.C23) {
1595 // A non-leading codepoint must have the XID_Continue property.
1596 // XIDContinueRanges doesn't contains characters also in XIDStartRanges,
1597 // so we need to check both tables.
1598 // '_' doesn't have the XID_Continue property but is allowed in C and C++.
1599 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1600 static const llvm::sys::UnicodeCharSet XIDContinueChars(XIDContinueRanges);
1601 if (C == '_' || XIDStartChars.contains(C) || XIDContinueChars.contains(C))
1602 return true;
1603 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/false,
1604 IsExtension);
1605 } else if (LangOpts.C11) {
1606 static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
1607 C11AllowedIDCharRanges);
1608 return C11AllowedIDChars.contains(C);
1609 } else {
1610 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1611 C99AllowedIDCharRanges);
1612 return C99AllowedIDChars.contains(C);
1613 }
1614}
1615
1616static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts,
1617 bool &IsExtension) {
1618 assert(C > 0x7F && "isAllowedInitiallyIDChar called with an ASCII codepoint");
1619 IsExtension = false;
1620 if (LangOpts.AsmPreprocessor) {
1621 return false;
1622 }
1623 if (LangOpts.CPlusPlus || LangOpts.C23) {
1624 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1625 if (XIDStartChars.contains(C))
1626 return true;
1627 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/true,
1628 IsExtension);
1629 }
1630 if (!isAllowedIDChar(C, LangOpts, IsExtension))
1631 return false;
1632 if (LangOpts.C11) {
1633 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
1634 C11DisallowedInitialIDCharRanges);
1635 return !C11DisallowedInitialIDChars.contains(C);
1636 }
1637 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1638 C99DisallowedInitialIDCharRanges);
1639 return !C99DisallowedInitialIDChars.contains(C);
1640}
1641
1642static void diagnoseExtensionInIdentifier(DiagnosticsEngine &Diags, uint32_t C,
1643 CharSourceRange Range) {
1644
1645 static const llvm::sys::UnicodeCharSet MathStartChars(
1646 MathematicalNotationProfileIDStartRanges);
1647 static const llvm::sys::UnicodeCharSet MathContinueChars(
1648 MathematicalNotationProfileIDContinueRanges);
1649
1650 (void)MathStartChars;
1651 (void)MathContinueChars;
1652 assert((MathStartChars.contains(C) || MathContinueChars.contains(C)) &&
1653 "Unexpected mathematical notation codepoint");
1654 Diags.Report(Loc: Range.getBegin(), DiagID: diag::ext_mathematical_notation)
1655 << codepointAsHexString(C) << Range;
1656}
1657
1658static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1659 const char *End) {
1660 return CharSourceRange::getCharRange(B: L.getSourceLocation(Loc: Begin),
1661 E: L.getSourceLocation(Loc: End));
1662}
1663
1664static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
1665 CharSourceRange Range, bool IsFirst) {
1666 // Check C99 compatibility.
1667 if (!Diags.isIgnored(DiagID: diag::warn_c99_compat_unicode_id, Loc: Range.getBegin())) {
1668 enum {
1669 CannotAppearInIdentifier = 0,
1670 CannotStartIdentifier
1671 };
1672
1673 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1674 C99AllowedIDCharRanges);
1675 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1676 C99DisallowedInitialIDCharRanges);
1677 if (!C99AllowedIDChars.contains(C)) {
1678 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1679 << Range
1680 << CannotAppearInIdentifier;
1681 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) {
1682 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1683 << Range
1684 << CannotStartIdentifier;
1685 }
1686 }
1687}
1688
1689/// After encountering UTF-8 character C and interpreting it as an identifier
1690/// character, check whether it's a homoglyph for a common non-identifier
1691/// source character that is unlikely to be an intentional identifier
1692/// character and warn if so.
1693static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
1694 CharSourceRange Range) {
1695 // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
1696 struct HomoglyphPair {
1697 uint32_t Character;
1698 char LooksLike;
1699 bool operator<(HomoglyphPair R) const { return Character < R.Character; }
1700 };
1701 static constexpr HomoglyphPair SortedHomoglyphs[] = {
1702 {.Character: U'\u00ad', .LooksLike: 0}, // SOFT HYPHEN
1703 {.Character: U'\u01c3', .LooksLike: '!'}, // LATIN LETTER RETROFLEX CLICK
1704 {.Character: U'\u037e', .LooksLike: ';'}, // GREEK QUESTION MARK
1705 {.Character: U'\u200b', .LooksLike: 0}, // ZERO WIDTH SPACE
1706 {.Character: U'\u200c', .LooksLike: 0}, // ZERO WIDTH NON-JOINER
1707 {.Character: U'\u200d', .LooksLike: 0}, // ZERO WIDTH JOINER
1708 {.Character: U'\u2060', .LooksLike: 0}, // WORD JOINER
1709 {.Character: U'\u2061', .LooksLike: 0}, // FUNCTION APPLICATION
1710 {.Character: U'\u2062', .LooksLike: 0}, // INVISIBLE TIMES
1711 {.Character: U'\u2063', .LooksLike: 0}, // INVISIBLE SEPARATOR
1712 {.Character: U'\u2064', .LooksLike: 0}, // INVISIBLE PLUS
1713 {.Character: U'\u2212', .LooksLike: '-'}, // MINUS SIGN
1714 {.Character: U'\u2215', .LooksLike: '/'}, // DIVISION SLASH
1715 {.Character: U'\u2216', .LooksLike: '\\'}, // SET MINUS
1716 {.Character: U'\u2217', .LooksLike: '*'}, // ASTERISK OPERATOR
1717 {.Character: U'\u2223', .LooksLike: '|'}, // DIVIDES
1718 {.Character: U'\u2227', .LooksLike: '^'}, // LOGICAL AND
1719 {.Character: U'\u2236', .LooksLike: ':'}, // RATIO
1720 {.Character: U'\u223c', .LooksLike: '~'}, // TILDE OPERATOR
1721 {.Character: U'\ua789', .LooksLike: ':'}, // MODIFIER LETTER COLON
1722 {.Character: U'\ufeff', .LooksLike: 0}, // ZERO WIDTH NO-BREAK SPACE
1723 {.Character: U'\uff01', .LooksLike: '!'}, // FULLWIDTH EXCLAMATION MARK
1724 {.Character: U'\uff03', .LooksLike: '#'}, // FULLWIDTH NUMBER SIGN
1725 {.Character: U'\uff04', .LooksLike: '$'}, // FULLWIDTH DOLLAR SIGN
1726 {.Character: U'\uff05', .LooksLike: '%'}, // FULLWIDTH PERCENT SIGN
1727 {.Character: U'\uff06', .LooksLike: '&'}, // FULLWIDTH AMPERSAND
1728 {.Character: U'\uff08', .LooksLike: '('}, // FULLWIDTH LEFT PARENTHESIS
1729 {.Character: U'\uff09', .LooksLike: ')'}, // FULLWIDTH RIGHT PARENTHESIS
1730 {.Character: U'\uff0a', .LooksLike: '*'}, // FULLWIDTH ASTERISK
1731 {.Character: U'\uff0b', .LooksLike: '+'}, // FULLWIDTH ASTERISK
1732 {.Character: U'\uff0c', .LooksLike: ','}, // FULLWIDTH COMMA
1733 {.Character: U'\uff0d', .LooksLike: '-'}, // FULLWIDTH HYPHEN-MINUS
1734 {.Character: U'\uff0e', .LooksLike: '.'}, // FULLWIDTH FULL STOP
1735 {.Character: U'\uff0f', .LooksLike: '/'}, // FULLWIDTH SOLIDUS
1736 {.Character: U'\uff1a', .LooksLike: ':'}, // FULLWIDTH COLON
1737 {.Character: U'\uff1b', .LooksLike: ';'}, // FULLWIDTH SEMICOLON
1738 {.Character: U'\uff1c', .LooksLike: '<'}, // FULLWIDTH LESS-THAN SIGN
1739 {.Character: U'\uff1d', .LooksLike: '='}, // FULLWIDTH EQUALS SIGN
1740 {.Character: U'\uff1e', .LooksLike: '>'}, // FULLWIDTH GREATER-THAN SIGN
1741 {.Character: U'\uff1f', .LooksLike: '?'}, // FULLWIDTH QUESTION MARK
1742 {.Character: U'\uff20', .LooksLike: '@'}, // FULLWIDTH COMMERCIAL AT
1743 {.Character: U'\uff3b', .LooksLike: '['}, // FULLWIDTH LEFT SQUARE BRACKET
1744 {.Character: U'\uff3c', .LooksLike: '\\'}, // FULLWIDTH REVERSE SOLIDUS
1745 {.Character: U'\uff3d', .LooksLike: ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
1746 {.Character: U'\uff3e', .LooksLike: '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
1747 {.Character: U'\uff5b', .LooksLike: '{'}, // FULLWIDTH LEFT CURLY BRACKET
1748 {.Character: U'\uff5c', .LooksLike: '|'}, // FULLWIDTH VERTICAL LINE
1749 {.Character: U'\uff5d', .LooksLike: '}'}, // FULLWIDTH RIGHT CURLY BRACKET
1750 {.Character: U'\uff5e', .LooksLike: '~'}, // FULLWIDTH TILDE
1751 {.Character: 0, .LooksLike: 0}
1752 };
1753 auto Homoglyph =
1754 std::lower_bound(first: std::begin(arr: SortedHomoglyphs),
1755 last: std::end(arr: SortedHomoglyphs) - 1, val: HomoglyphPair{.Character: C, .LooksLike: '\0'});
1756 if (Homoglyph->Character == C) {
1757 if (Homoglyph->LooksLike) {
1758 const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
1759 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_homoglyph)
1760 << Range << codepointAsHexString(C) << LooksLikeStr;
1761 } else {
1762 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_zero_width)
1763 << Range << codepointAsHexString(C);
1764 }
1765 }
1766}
1767
1768static void diagnoseInvalidUnicodeCodepointInIdentifier(
1769 DiagnosticsEngine &Diags, const LangOptions &LangOpts, uint32_t CodePoint,
1770 CharSourceRange Range, bool IsFirst) {
1771 if (isASCII(c: CodePoint))
1772 return;
1773
1774 bool IsExtension;
1775 bool IsIDStart = isAllowedInitiallyIDChar(C: CodePoint, LangOpts, IsExtension);
1776 bool IsIDContinue =
1777 IsIDStart || isAllowedIDChar(C: CodePoint, LangOpts, IsExtension);
1778
1779 if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
1780 return;
1781
1782 bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
1783
1784 if (!IsFirst || InvalidOnlyAtStart) {
1785 Diags.Report(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed_identifier)
1786 << Range << codepointAsHexString(C: CodePoint) << int(InvalidOnlyAtStart)
1787 << FixItHint::CreateRemoval(RemoveRange: Range);
1788 } else {
1789 Diags.Report(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed)
1790 << Range << codepointAsHexString(C: CodePoint)
1791 << FixItHint::CreateRemoval(RemoveRange: Range);
1792 }
1793}
1794
1795bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1796 Token &Result) {
1797 const char *UCNPtr = CurPtr + Size;
1798 uint32_t CodePoint = tryReadUCN(StartPtr&: UCNPtr, SlashLoc: CurPtr, /*Token=*/Result: nullptr);
1799 if (CodePoint == 0) {
1800 return false;
1801 }
1802 bool IsExtension = false;
1803 if (!isAllowedIDChar(C: CodePoint, LangOpts, IsExtension)) {
1804 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1805 return false;
1806 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1807 !PP->isPreprocessedOutput())
1808 diagnoseInvalidUnicodeCodepointInIdentifier(
1809 Diags&: PP->getDiagnostics(), LangOpts, CodePoint,
1810 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1811 /*IsFirst=*/false);
1812
1813 // We got a unicode codepoint that is neither a space nor a
1814 // a valid identifier part.
1815 // Carry on as if the codepoint was valid for recovery purposes.
1816 } else if (!isLexingRawMode()) {
1817 if (IsExtension)
1818 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C: CodePoint,
1819 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr));
1820
1821 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1822 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1823 /*IsFirst=*/false);
1824 }
1825
1826 Result.setFlag(Token::HasUCN);
1827 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') ||
1828 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
1829 CurPtr = UCNPtr;
1830 else
1831 while (CurPtr != UCNPtr)
1832 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
1833 return true;
1834}
1835
1836bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
1837 llvm::UTF32 CodePoint;
1838
1839 // If a UTF-8 codepoint appears immediately after an escaped new line,
1840 // CurPtr may point to the splicing \ on the preceding line,
1841 // so we need to skip it.
1842 unsigned FirstCodeUnitSize;
1843 getCharAndSize(Ptr: CurPtr, Size&: FirstCodeUnitSize);
1844 const char *CharStart = CurPtr + FirstCodeUnitSize - 1;
1845 const char *UnicodePtr = CharStart;
1846
1847 llvm::ConversionResult ConvResult = llvm::convertUTF8Sequence(
1848 source: (const llvm::UTF8 **)&UnicodePtr, sourceEnd: (const llvm::UTF8 *)BufferEnd,
1849 target: &CodePoint, flags: llvm::strictConversion);
1850 if (ConvResult != llvm::conversionOK)
1851 return false;
1852
1853 bool IsExtension = false;
1854 if (!isAllowedIDChar(C: static_cast<uint32_t>(CodePoint), LangOpts,
1855 IsExtension)) {
1856 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1857 return false;
1858
1859 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1860 !PP->isPreprocessedOutput())
1861 diagnoseInvalidUnicodeCodepointInIdentifier(
1862 Diags&: PP->getDiagnostics(), LangOpts, CodePoint,
1863 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr), /*IsFirst=*/false);
1864 // We got a unicode codepoint that is neither a space nor a
1865 // a valid identifier part. Carry on as if the codepoint was
1866 // valid for recovery purposes.
1867 } else if (!isLexingRawMode()) {
1868 if (IsExtension)
1869 diagnoseExtensionInIdentifier(
1870 Diags&: PP->getDiagnostics(), C: CodePoint,
1871 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1872 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1873 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr),
1874 /*IsFirst=*/false);
1875 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C: CodePoint,
1876 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1877 }
1878
1879 // Once we sucessfully parsed some UTF-8,
1880 // calling ConsumeChar ensures the NeedsCleaning flag is set on the token
1881 // being lexed, and that warnings about trailing spaces are emitted.
1882 ConsumeChar(Ptr: CurPtr, Size: FirstCodeUnitSize, Tok&: Result);
1883 CurPtr = UnicodePtr;
1884 return true;
1885}
1886
1887bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
1888 const char *CurPtr) {
1889 bool IsExtension = false;
1890 if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
1891 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1892 !PP->isPreprocessedOutput()) {
1893 if (IsExtension)
1894 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C,
1895 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1896 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C,
1897 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1898 /*IsFirst=*/true);
1899 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C,
1900 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1901 }
1902
1903 MIOpt.ReadToken();
1904 return LexIdentifierContinue(Result, CurPtr);
1905 }
1906
1907 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1908 !PP->isPreprocessedOutput() && !isASCII(c: *BufferPtr) &&
1909 !isUnicodeWhitespace(Codepoint: C)) {
1910 // Non-ASCII characters tend to creep into source code unintentionally.
1911 // Instead of letting the parser complain about the unknown token,
1912 // just drop the character.
1913 // Note that we can /only/ do this when the non-ASCII character is actually
1914 // spelled as Unicode, not written as a UCN. The standard requires that
1915 // we not throw away any possible preprocessor tokens, but there's a
1916 // loophole in the mapping of Unicode characters to basic character set
1917 // characters that allows us to map these particular characters to, say,
1918 // whitespace.
1919 diagnoseInvalidUnicodeCodepointInIdentifier(
1920 Diags&: PP->getDiagnostics(), LangOpts, CodePoint: C,
1921 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr), /*IsStart*/ IsFirst: true);
1922 BufferPtr = CurPtr;
1923 return false;
1924 }
1925
1926 // Otherwise, we have an explicit UCN or a character that's unlikely to show
1927 // up by accident.
1928 MIOpt.ReadToken();
1929 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
1930 return true;
1931}
1932
1933static const char *
1934fastParseASCIIIdentifier(const char *CurPtr,
1935 [[maybe_unused]] const char *BufferEnd) {
1936#ifdef __SSE4_2__
1937 alignas(16) static constexpr char AsciiIdentifierRange[16] = {
1938 '_', '_', 'A', 'Z', 'a', 'z', '0', '9',
1939 };
1940 constexpr ssize_t BytesPerRegister = 16;
1941
1942 __m128i AsciiIdentifierRangeV =
1943 _mm_load_si128((const __m128i *)AsciiIdentifierRange);
1944
1945 while (LLVM_LIKELY(BufferEnd - CurPtr >= BytesPerRegister)) {
1946 __m128i Cv = _mm_loadu_si128((const __m128i *)(CurPtr));
1947
1948 int Consumed = _mm_cmpistri(AsciiIdentifierRangeV, Cv,
1949 _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES |
1950 _SIDD_UBYTE_OPS | _SIDD_NEGATIVE_POLARITY);
1951 CurPtr += Consumed;
1952 if (Consumed == BytesPerRegister)
1953 continue;
1954 return CurPtr;
1955 }
1956#endif
1957
1958 unsigned char C = *CurPtr;
1959 while (isAsciiIdentifierContinue(c: C))
1960 C = *++CurPtr;
1961 return CurPtr;
1962}
1963
1964bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
1965 // Match [_A-Za-z0-9]*, we have already matched an identifier start.
1966
1967 while (true) {
1968
1969 CurPtr = fastParseASCIIIdentifier(CurPtr, BufferEnd);
1970
1971 unsigned Size;
1972 // Slow path: handle trigraph, unicode codepoints, UCNs.
1973 unsigned char C = getCharAndSize(Ptr: CurPtr, Size);
1974 if (isAsciiIdentifierContinue(c: C)) {
1975 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
1976 continue;
1977 }
1978 if (C == '$') {
1979 // If we hit a $ and they are not supported in identifiers, we are done.
1980 if (!LangOpts.DollarIdents)
1981 break;
1982 // Otherwise, emit a diagnostic and continue.
1983 if (!isLexingRawMode())
1984 Diag(Loc: CurPtr, DiagID: diag::ext_dollar_in_identifier);
1985 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
1986 continue;
1987 }
1988 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
1989 continue;
1990 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
1991 continue;
1992 // Neither an expected Unicode codepoint nor a UCN.
1993 break;
1994 }
1995
1996 const char *IdStart = BufferPtr;
1997 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::raw_identifier);
1998 Result.setRawIdentifierData(IdStart);
1999
2000 // If we are in raw mode, return this identifier raw. There is no need to
2001 // look up identifier information or attempt to macro expand it.
2002 if (LexingRawMode)
2003 return true;
2004
2005 // Fill in Result.IdentifierInfo and update the token kind,
2006 // looking up the identifier in the identifier table.
2007 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
2008 // Note that we have to call PP->LookUpIdentifierInfo() even for code
2009 // completion, it writes IdentifierInfo into Result, and callers rely on it.
2010
2011 // If the completion point is at the end of an identifier, we want to treat
2012 // the identifier as incomplete even if it resolves to a macro or a keyword.
2013 // This allows e.g. 'class^' to complete to 'classifier'.
2014 if (isCodeCompletionPoint(CurPtr)) {
2015 // Return the code-completion token.
2016 Result.setKind(tok::code_completion);
2017 // Skip the code-completion char and all immediate identifier characters.
2018 // This ensures we get consistent behavior when completing at any point in
2019 // an identifier (i.e. at the start, in the middle, at the end). Note that
2020 // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
2021 // simpler.
2022 assert(*CurPtr == 0 && "Completion character must be 0");
2023 ++CurPtr;
2024 // Note that code completion token is not added as a separate character
2025 // when the completion point is at the end of the buffer. Therefore, we need
2026 // to check if the buffer has ended.
2027 if (CurPtr < BufferEnd) {
2028 while (isAsciiIdentifierContinue(c: *CurPtr))
2029 ++CurPtr;
2030 }
2031 BufferPtr = CurPtr;
2032 return true;
2033 }
2034
2035 // Finally, now that we know we have an identifier, pass this off to the
2036 // preprocessor, which may macro expand it or something.
2037 if (II->isHandleIdentifierCase())
2038 return PP->HandleIdentifier(Identifier&: Result);
2039
2040 return true;
2041}
2042
2043/// isHexaLiteral - Return true if Start points to a hex constant.
2044/// in microsoft mode (where this is supposed to be several different tokens).
2045bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
2046 auto CharAndSize1 = Lexer::getCharAndSizeNoWarn(Ptr: Start, LangOpts);
2047 char C1 = CharAndSize1.Char;
2048 if (C1 != '0')
2049 return false;
2050
2051 auto CharAndSize2 =
2052 Lexer::getCharAndSizeNoWarn(Ptr: Start + CharAndSize1.Size, LangOpts);
2053 char C2 = CharAndSize2.Char;
2054 return (C2 == 'x' || C2 == 'X');
2055}
2056
2057/// LexNumericConstant - Lex the remainder of a integer or floating point
2058/// constant. From[-1] is the first character lexed. Return the end of the
2059/// constant.
2060bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
2061 unsigned Size;
2062 char C = getCharAndSize(Ptr: CurPtr, Size);
2063 char PrevCh = 0;
2064 while (isPreprocessingNumberBody(c: C)) {
2065 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2066 PrevCh = C;
2067 if (LangOpts.HLSL && C == '.' && (*CurPtr == 'x' || *CurPtr == 'r')) {
2068 CurPtr -= Size;
2069 break;
2070 }
2071 C = getCharAndSize(Ptr: CurPtr, Size);
2072 }
2073
2074 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
2075 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
2076 // If we are in Microsoft mode, don't continue if the constant is hex.
2077 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
2078 if (!LangOpts.MicrosoftExt || !isHexaLiteral(Start: BufferPtr, LangOpts))
2079 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2080 }
2081
2082 // If we have a hex FP constant, continue.
2083 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
2084 // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
2085 // not-quite-conforming extension. Only do so if this looks like it's
2086 // actually meant to be a hexfloat, and not if it has a ud-suffix.
2087 bool IsHexFloat = true;
2088 if (!LangOpts.C99) {
2089 if (!isHexaLiteral(Start: BufferPtr, LangOpts))
2090 IsHexFloat = false;
2091 else if (!LangOpts.CPlusPlus17 &&
2092 std::find(first: BufferPtr, last: CurPtr, val: '_') != CurPtr)
2093 IsHexFloat = false;
2094 }
2095 if (IsHexFloat)
2096 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2097 }
2098
2099 // If we have a digit separator, continue.
2100 if (C == '\'' && (LangOpts.CPlusPlus14 || LangOpts.C23)) {
2101 auto [Next, NextSize] = getCharAndSizeNoWarn(Ptr: CurPtr + Size, LangOpts);
2102 if (isAsciiIdentifierContinue(c: Next)) {
2103 if (!isLexingRawMode())
2104 Diag(Loc: CurPtr, DiagID: LangOpts.CPlusPlus
2105 ? diag::warn_cxx11_compat_digit_separator
2106 : diag::warn_c23_compat_digit_separator);
2107 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2108 CurPtr = ConsumeChar(Ptr: CurPtr, Size: NextSize, Tok&: Result);
2109 return LexNumericConstant(Result, CurPtr);
2110 }
2111 }
2112
2113 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
2114 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2115 return LexNumericConstant(Result, CurPtr);
2116 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2117 return LexNumericConstant(Result, CurPtr);
2118
2119 // Update the location of token as well as BufferPtr.
2120 const char *TokStart = BufferPtr;
2121 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::numeric_constant);
2122 Result.setLiteralData(TokStart);
2123 return true;
2124}
2125
2126/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
2127/// in C++11, or warn on a ud-suffix in C++98.
2128const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
2129 bool IsStringLiteral) {
2130 assert(LangOpts.CPlusPlus);
2131
2132 // Maximally munch an identifier.
2133 unsigned Size;
2134 char C = getCharAndSize(Ptr: CurPtr, Size);
2135 bool Consumed = false;
2136
2137 if (!isAsciiIdentifierStart(c: C)) {
2138 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2139 Consumed = true;
2140 else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2141 Consumed = true;
2142 else
2143 return CurPtr;
2144 }
2145
2146 if (!LangOpts.CPlusPlus11) {
2147 if (!isLexingRawMode())
2148 Diag(Loc: CurPtr,
2149 DiagID: C == '_' ? diag::warn_cxx11_compat_user_defined_literal
2150 : diag::warn_cxx11_compat_reserved_user_defined_literal)
2151 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2152 return CurPtr;
2153 }
2154
2155 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
2156 // that does not start with an underscore is ill-formed. As a conforming
2157 // extension, we treat all such suffixes as if they had whitespace before
2158 // them. We assume a suffix beginning with a UCN or UTF-8 character is more
2159 // likely to be a ud-suffix than a macro, however, and accept that.
2160 if (!Consumed) {
2161 bool IsUDSuffix = false;
2162 if (C == '_')
2163 IsUDSuffix = true;
2164 else if (IsStringLiteral && LangOpts.CPlusPlus14) {
2165 // In C++1y, we need to look ahead a few characters to see if this is a
2166 // valid suffix for a string literal or a numeric literal (this could be
2167 // the 'operator""if' defining a numeric literal operator).
2168 const unsigned MaxStandardSuffixLength = 3;
2169 char Buffer[MaxStandardSuffixLength] = { C };
2170 unsigned Consumed = Size;
2171 unsigned Chars = 1;
2172 while (true) {
2173 auto [Next, NextSize] =
2174 getCharAndSizeNoWarn(Ptr: CurPtr + Consumed, LangOpts);
2175 if (!isAsciiIdentifierContinue(c: Next)) {
2176 // End of suffix. Check whether this is on the allowed list.
2177 const StringRef CompleteSuffix(Buffer, Chars);
2178 IsUDSuffix =
2179 StringLiteralParser::isValidUDSuffix(LangOpts, Suffix: CompleteSuffix);
2180 break;
2181 }
2182
2183 if (Chars == MaxStandardSuffixLength)
2184 // Too long: can't be a standard suffix.
2185 break;
2186
2187 Buffer[Chars++] = Next;
2188 Consumed += NextSize;
2189 }
2190 }
2191
2192 if (!IsUDSuffix) {
2193 if (!isLexingRawMode())
2194 Diag(Loc: CurPtr, DiagID: LangOpts.MSVCCompat
2195 ? diag::ext_ms_reserved_user_defined_literal
2196 : diag::ext_reserved_user_defined_literal)
2197 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2198 return CurPtr;
2199 }
2200
2201 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2202 }
2203
2204 Result.setFlag(Token::HasUDSuffix);
2205 while (true) {
2206 C = getCharAndSize(Ptr: CurPtr, Size);
2207 if (isAsciiIdentifierContinue(c: C)) {
2208 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2209 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
2210 } else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result)) {
2211 } else
2212 break;
2213 }
2214
2215 return CurPtr;
2216}
2217
2218/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
2219/// either " or L" or u8" or u" or U".
2220bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
2221 tok::TokenKind Kind) {
2222 const char *AfterQuote = CurPtr;
2223 // Does this string contain the \0 character?
2224 const char *NulCharacter = nullptr;
2225
2226 if (!isLexingRawMode() &&
2227 (Kind == tok::utf8_string_literal ||
2228 Kind == tok::utf16_string_literal ||
2229 Kind == tok::utf32_string_literal))
2230 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
2231 : diag::warn_c99_compat_unicode_literal);
2232
2233 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2234 while (C != '"') {
2235 // Skip escaped characters. Escaped newlines will already be processed by
2236 // getAndAdvanceChar.
2237 if (C == '\\')
2238 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2239
2240 if (C == '\n' || C == '\r' || // Newline.
2241 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2242 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2243 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 1;
2244 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2245 return true;
2246 }
2247
2248 if (C == 0) {
2249 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2250 if (ParsingFilename)
2251 codeCompleteIncludedFile(PathStart: AfterQuote, CompletionPoint: CurPtr - 1, /*IsAngled=*/false);
2252 else
2253 PP->CodeCompleteNaturalLanguage();
2254 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2255 cutOffLexing();
2256 return true;
2257 }
2258
2259 NulCharacter = CurPtr-1;
2260 }
2261 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2262 }
2263
2264 // If we are in C++11, lex the optional ud-suffix.
2265 if (LangOpts.CPlusPlus)
2266 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2267
2268 // If a nul character existed in the string, warn about it.
2269 if (NulCharacter && !isLexingRawMode())
2270 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2271
2272 // Update the location of the token as well as the BufferPtr instance var.
2273 const char *TokStart = BufferPtr;
2274 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2275 Result.setLiteralData(TokStart);
2276 return true;
2277}
2278
2279/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
2280/// having lexed R", LR", u8R", uR", or UR".
2281bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2282 tok::TokenKind Kind) {
2283 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
2284 // Between the initial and final double quote characters of the raw string,
2285 // any transformations performed in phases 1 and 2 (trigraphs,
2286 // universal-character-names, and line splicing) are reverted.
2287
2288 if (!isLexingRawMode())
2289 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_raw_string_literal);
2290
2291 unsigned PrefixLen = 0;
2292
2293 while (PrefixLen != 16 && isRawStringDelimBody(c: CurPtr[PrefixLen])) {
2294 if (!isLexingRawMode() &&
2295 llvm::is_contained(Set: {'$', '@', '`'}, Element: CurPtr[PrefixLen])) {
2296 const char *Pos = &CurPtr[PrefixLen];
2297 Diag(Loc: Pos, DiagID: LangOpts.CPlusPlus26
2298 ? diag::warn_cxx26_compat_raw_string_literal_character_set
2299 : diag::ext_cxx26_raw_string_literal_character_set)
2300 << StringRef(Pos, 1);
2301 }
2302 ++PrefixLen;
2303 }
2304
2305 // If the last character was not a '(', then we didn't lex a valid delimiter.
2306 if (CurPtr[PrefixLen] != '(') {
2307 if (!isLexingRawMode()) {
2308 const char *PrefixEnd = &CurPtr[PrefixLen];
2309 if (PrefixLen == 16) {
2310 Diag(Loc: PrefixEnd, DiagID: diag::err_raw_delim_too_long);
2311 } else if (*PrefixEnd == '\n') {
2312 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_newline_raw_delim);
2313 } else {
2314 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_char_raw_delim)
2315 << StringRef(PrefixEnd, 1);
2316 }
2317 }
2318
2319 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2320 // it's possible the '"' was intended to be part of the raw string, but
2321 // there's not much we can do about that.
2322 while (true) {
2323 char C = *CurPtr++;
2324
2325 if (C == '"')
2326 break;
2327 if (C == 0 && CurPtr-1 == BufferEnd) {
2328 --CurPtr;
2329 break;
2330 }
2331 }
2332
2333 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2334 return true;
2335 }
2336
2337 // Save prefix and move CurPtr past it
2338 const char *Prefix = CurPtr;
2339 CurPtr += PrefixLen + 1; // skip over prefix and '('
2340
2341 while (true) {
2342 char C = *CurPtr++;
2343
2344 if (C == ')') {
2345 // Check for prefix match and closing quote.
2346 if (strncmp(s1: CurPtr, s2: Prefix, n: PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
2347 CurPtr += PrefixLen + 1; // skip over prefix and '"'
2348 break;
2349 }
2350 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
2351 if (!isLexingRawMode())
2352 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_raw_string)
2353 << StringRef(Prefix, PrefixLen);
2354 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2355 return true;
2356 }
2357 }
2358
2359 // If we are in C++11, lex the optional ud-suffix.
2360 if (LangOpts.CPlusPlus)
2361 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2362
2363 // Update the location of token as well as BufferPtr.
2364 const char *TokStart = BufferPtr;
2365 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2366 Result.setLiteralData(TokStart);
2367 return true;
2368}
2369
2370/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2371/// after having lexed the '<' character. This is used for #include filenames.
2372bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2373 // Does this string contain the \0 character?
2374 const char *NulCharacter = nullptr;
2375 const char *AfterLessPos = CurPtr;
2376 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2377 while (C != '>') {
2378 // Skip escaped characters. Escaped newlines will already be processed by
2379 // getAndAdvanceChar.
2380 if (C == '\\')
2381 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2382
2383 if (isVerticalWhitespace(c: C) || // Newline.
2384 (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
2385 // If the filename is unterminated, then it must just be a lone <
2386 // character. Return this as such.
2387 FormTokenWithChars(Result, TokEnd: AfterLessPos, Kind: tok::less);
2388 return true;
2389 }
2390
2391 if (C == 0) {
2392 if (isCodeCompletionPoint(CurPtr: CurPtr - 1)) {
2393 codeCompleteIncludedFile(PathStart: AfterLessPos, CompletionPoint: CurPtr - 1, /*IsAngled=*/true);
2394 cutOffLexing();
2395 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2396 return true;
2397 }
2398 NulCharacter = CurPtr-1;
2399 }
2400 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2401 }
2402
2403 // If a nul character existed in the string, warn about it.
2404 if (NulCharacter && !isLexingRawMode())
2405 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2406
2407 // Update the location of token as well as BufferPtr.
2408 const char *TokStart = BufferPtr;
2409 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::header_name);
2410 Result.setLiteralData(TokStart);
2411 return true;
2412}
2413
2414void Lexer::codeCompleteIncludedFile(const char *PathStart,
2415 const char *CompletionPoint,
2416 bool IsAngled) {
2417 // Completion only applies to the filename, after the last slash.
2418 StringRef PartialPath(PathStart, CompletionPoint - PathStart);
2419 llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
2420 auto Slash = PartialPath.find_last_of(Chars: SlashChars);
2421 StringRef Dir =
2422 (Slash == StringRef::npos) ? "" : PartialPath.take_front(N: Slash);
2423 const char *StartOfFilename =
2424 (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
2425 // Code completion filter range is the filename only, up to completion point.
2426 PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
2427 Name: StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
2428 // We should replace the characters up to the closing quote or closest slash,
2429 // if any.
2430 while (CompletionPoint < BufferEnd) {
2431 char Next = *(CompletionPoint + 1);
2432 if (Next == 0 || Next == '\r' || Next == '\n')
2433 break;
2434 ++CompletionPoint;
2435 if (Next == (IsAngled ? '>' : '"'))
2436 break;
2437 if (SlashChars.contains(C: Next))
2438 break;
2439 }
2440
2441 PP->setCodeCompletionTokenRange(
2442 Start: FileLoc.getLocWithOffset(Offset: StartOfFilename - BufferStart),
2443 End: FileLoc.getLocWithOffset(Offset: CompletionPoint - BufferStart));
2444 PP->CodeCompleteIncludedFile(Dir, IsAngled);
2445}
2446
2447/// LexCharConstant - Lex the remainder of a character constant, after having
2448/// lexed either ' or L' or u8' or u' or U'.
2449bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2450 tok::TokenKind Kind) {
2451 // Does this character contain the \0 character?
2452 const char *NulCharacter = nullptr;
2453
2454 if (!isLexingRawMode()) {
2455 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
2456 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2457 ? diag::warn_cxx98_compat_unicode_literal
2458 : diag::warn_c99_compat_unicode_literal);
2459 else if (Kind == tok::utf8_char_constant)
2460 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2461 ? diag::warn_cxx14_compat_u8_character_literal
2462 : diag::warn_c17_compat_u8_character_literal);
2463 }
2464
2465 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2466 if (C == '\'') {
2467 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2468 Diag(Loc: BufferPtr, DiagID: diag::ext_empty_character);
2469 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2470 return true;
2471 }
2472
2473 while (C != '\'') {
2474 // Skip escaped characters.
2475 if (C == '\\')
2476 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2477
2478 if (C == '\n' || C == '\r' || // Newline.
2479 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2480 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2481 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 0;
2482 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2483 return true;
2484 }
2485
2486 if (C == 0) {
2487 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2488 PP->CodeCompleteNaturalLanguage();
2489 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2490 cutOffLexing();
2491 return true;
2492 }
2493
2494 NulCharacter = CurPtr-1;
2495 }
2496 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2497 }
2498
2499 // If we are in C++11, lex the optional ud-suffix.
2500 if (LangOpts.CPlusPlus)
2501 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: false);
2502
2503 // If a nul character existed in the character, warn about it.
2504 if (NulCharacter && !isLexingRawMode())
2505 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 0;
2506
2507 // Update the location of token as well as BufferPtr.
2508 const char *TokStart = BufferPtr;
2509 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2510 Result.setLiteralData(TokStart);
2511 return true;
2512}
2513
2514/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2515/// Update BufferPtr to point to the next non-whitespace character and return.
2516///
2517/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2518bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
2519 bool &TokAtPhysicalStartOfLine) {
2520 // Whitespace - Skip it, then return the token after the whitespace.
2521 bool SawNewline = isVerticalWhitespace(c: CurPtr[-1]);
2522
2523 unsigned char Char = *CurPtr;
2524
2525 const char *lastNewLine = nullptr;
2526 auto setLastNewLine = [&](const char *Ptr) {
2527 lastNewLine = Ptr;
2528 if (!NewLinePtr)
2529 NewLinePtr = Ptr;
2530 };
2531 if (SawNewline)
2532 setLastNewLine(CurPtr - 1);
2533
2534 // Skip consecutive spaces efficiently.
2535 while (true) {
2536 // Skip horizontal whitespace very aggressively.
2537 while (isHorizontalWhitespace(c: Char))
2538 Char = *++CurPtr;
2539
2540 // Otherwise if we have something other than whitespace, we're done.
2541 if (!isVerticalWhitespace(c: Char))
2542 break;
2543
2544 if (ParsingPreprocessorDirective) {
2545 // End of preprocessor directive line, let LexTokenInternal handle this.
2546 BufferPtr = CurPtr;
2547 return false;
2548 }
2549
2550 // OK, but handle newline.
2551 if (*CurPtr == '\n')
2552 setLastNewLine(CurPtr);
2553 SawNewline = true;
2554 Char = *++CurPtr;
2555 }
2556
2557 // If the client wants us to return whitespace, return it now.
2558 if (isKeepWhitespaceMode()) {
2559 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2560 if (SawNewline) {
2561 IsAtStartOfLine = true;
2562 IsAtPhysicalStartOfLine = true;
2563 }
2564 // FIXME: The next token will not have LeadingSpace set.
2565 return true;
2566 }
2567
2568 // If this isn't immediately after a newline, there is leading space.
2569 char PrevChar = CurPtr[-1];
2570 bool HasLeadingSpace = !isVerticalWhitespace(c: PrevChar);
2571
2572 Result.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace);
2573 if (SawNewline) {
2574 Result.setFlag(Token::StartOfLine);
2575 TokAtPhysicalStartOfLine = true;
2576
2577 if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) {
2578 if (auto *Handler = PP->getEmptylineHandler())
2579 Handler->HandleEmptyline(Range: SourceRange(getSourceLocation(Loc: NewLinePtr + 1),
2580 getSourceLocation(Loc: lastNewLine)));
2581 }
2582 }
2583
2584 BufferPtr = CurPtr;
2585 return false;
2586}
2587
2588/// We have just read the // characters from input. Skip until we find the
2589/// newline character that terminates the comment. Then update BufferPtr and
2590/// return.
2591///
2592/// If we're in KeepCommentMode or any CommentHandler has inserted
2593/// some tokens, this will store the first token and return true.
2594bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
2595 bool &TokAtPhysicalStartOfLine) {
2596 // If Line comments aren't explicitly enabled for this language, emit an
2597 // extension warning.
2598 if (!LineComment) {
2599 if (!isLexingRawMode()) // There's no PP in raw mode, so can't emit diags.
2600 Diag(Loc: BufferPtr, DiagID: diag::ext_line_comment);
2601
2602 // Mark them enabled so we only emit one warning for this translation
2603 // unit.
2604 LineComment = true;
2605 }
2606
2607 // Scan over the body of the comment. The common case, when scanning, is that
2608 // the comment contains normal ascii characters with nothing interesting in
2609 // them. As such, optimize for this case with the inner loop.
2610 //
2611 // This loop terminates with CurPtr pointing at the newline (or end of buffer)
2612 // character that ends the line comment.
2613
2614 // C++23 [lex.phases] p1
2615 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2616 // diagnostic only once per entire ill-formed subsequence to avoid
2617 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2618 bool UnicodeDecodingAlreadyDiagnosed = false;
2619
2620 char C;
2621 while (true) {
2622 C = *CurPtr;
2623 // Skip over characters in the fast loop.
2624 while (isASCII(c: C) && C != 0 && // Potentially EOF.
2625 C != '\n' && C != '\r') { // Newline or DOS-style newline.
2626 C = *++CurPtr;
2627 UnicodeDecodingAlreadyDiagnosed = false;
2628 }
2629
2630 if (!isASCII(c: C)) {
2631 unsigned Length = llvm::getUTF8SequenceSize(
2632 source: (const llvm::UTF8 *)CurPtr, sourceEnd: (const llvm::UTF8 *)BufferEnd);
2633 if (Length == 0) {
2634 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
2635 Diag(Loc: CurPtr, DiagID: diag::warn_invalid_utf8_in_comment);
2636 UnicodeDecodingAlreadyDiagnosed = true;
2637 ++CurPtr;
2638 } else {
2639 UnicodeDecodingAlreadyDiagnosed = false;
2640 CurPtr += Length;
2641 }
2642 continue;
2643 }
2644
2645 const char *NextLine = CurPtr;
2646 if (C != 0) {
2647 // We found a newline, see if it's escaped.
2648 const char *EscapePtr = CurPtr-1;
2649 bool HasSpace = false;
2650 while (isHorizontalWhitespace(c: *EscapePtr)) { // Skip whitespace.
2651 --EscapePtr;
2652 HasSpace = true;
2653 }
2654
2655 if (*EscapePtr == '\\')
2656 // Escaped newline.
2657 CurPtr = EscapePtr;
2658 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
2659 EscapePtr[-2] == '?' && LangOpts.Trigraphs)
2660 // Trigraph-escaped newline.
2661 CurPtr = EscapePtr-2;
2662 else
2663 break; // This is a newline, we're done.
2664
2665 // If there was space between the backslash and newline, warn about it.
2666 if (HasSpace && !isLexingRawMode())
2667 Diag(Loc: EscapePtr, DiagID: diag::backslash_newline_space);
2668 }
2669
2670 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
2671 // properly decode the character. Read it in raw mode to avoid emitting
2672 // diagnostics about things like trigraphs. If we see an escaped newline,
2673 // we'll handle it below.
2674 const char *OldPtr = CurPtr;
2675 bool OldRawMode = isLexingRawMode();
2676 LexingRawMode = true;
2677 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2678 LexingRawMode = OldRawMode;
2679
2680 // If we only read only one character, then no special handling is needed.
2681 // We're done and can skip forward to the newline.
2682 if (C != 0 && CurPtr == OldPtr+1) {
2683 CurPtr = NextLine;
2684 break;
2685 }
2686
2687 // If we read multiple characters, and one of those characters was a \r or
2688 // \n, then we had an escaped newline within the comment. Emit diagnostic
2689 // unless the next line is also a // comment.
2690 if (CurPtr != OldPtr + 1 && C != '/' &&
2691 (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) {
2692 for (; OldPtr != CurPtr; ++OldPtr)
2693 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
2694 // Okay, we found a // comment that ends in a newline, if the next
2695 // line is also a // comment, but has spaces, don't emit a diagnostic.
2696 if (isWhitespace(c: C)) {
2697 const char *ForwardPtr = CurPtr;
2698 while (isWhitespace(c: *ForwardPtr)) // Skip whitespace.
2699 ++ForwardPtr;
2700 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
2701 break;
2702 }
2703
2704 if (!isLexingRawMode())
2705 Diag(Loc: OldPtr-1, DiagID: diag::ext_multi_line_line_comment);
2706 break;
2707 }
2708 }
2709
2710 if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
2711 --CurPtr;
2712 break;
2713 }
2714
2715 if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2716 PP->CodeCompleteNaturalLanguage();
2717 cutOffLexing();
2718 return false;
2719 }
2720 }
2721
2722 // Found but did not consume the newline. Notify comment handlers about the
2723 // comment unless we're in a #if 0 block.
2724 if (PP && !isLexingRawMode() &&
2725 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
2726 getSourceLocation(Loc: CurPtr)))) {
2727 BufferPtr = CurPtr;
2728 return true; // A token has to be returned.
2729 }
2730
2731 // If we are returning comments as tokens, return this comment as a token.
2732 if (inKeepCommentMode())
2733 return SaveLineComment(Result, CurPtr);
2734
2735 // If we are inside a preprocessor directive and we see the end of line,
2736 // return immediately, so that the lexer can return this as an EOD token.
2737 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
2738 BufferPtr = CurPtr;
2739 return false;
2740 }
2741
2742 // Otherwise, eat the \n character. We don't care if this is a \n\r or
2743 // \r\n sequence. This is an efficiency hack (because we know the \n can't
2744 // contribute to another token), it isn't needed for correctness. Note that
2745 // this is ok even in KeepWhitespaceMode, because we would have returned the
2746 // comment above in that mode.
2747 NewLinePtr = CurPtr++;
2748
2749 // The next returned token is at the start of the line.
2750 Result.setFlag(Token::StartOfLine);
2751 TokAtPhysicalStartOfLine = true;
2752 // No leading whitespace seen so far.
2753 Result.clearFlag(Flag: Token::LeadingSpace);
2754 BufferPtr = CurPtr;
2755 return false;
2756}
2757
2758/// If in save-comment mode, package up this Line comment in an appropriate
2759/// way and return it.
2760bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2761 // If we're not in a preprocessor directive, just return the // comment
2762 // directly.
2763 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
2764
2765 if (!ParsingPreprocessorDirective || LexingRawMode)
2766 return true;
2767
2768 // If this Line-style comment is in a macro definition, transmogrify it into
2769 // a C-style block comment.
2770 bool Invalid = false;
2771 std::string Spelling = PP->getSpelling(Tok: Result, Invalid: &Invalid);
2772 if (Invalid)
2773 return true;
2774
2775 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
2776 Spelling[1] = '*'; // Change prefix to "/*".
2777 Spelling += "*/"; // add suffix.
2778
2779 Result.setKind(tok::comment);
2780 PP->CreateString(Str: Spelling, Tok&: Result,
2781 ExpansionLocStart: Result.getLocation(), ExpansionLocEnd: Result.getLocation());
2782 return true;
2783}
2784
2785/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2786/// character (either \\n or \\r) is part of an escaped newline sequence. Issue
2787/// a diagnostic if so. We know that the newline is inside of a block comment.
2788static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, Lexer *L,
2789 bool Trigraphs) {
2790 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
2791
2792 // Position of the first trigraph in the ending sequence.
2793 const char *TrigraphPos = nullptr;
2794 // Position of the first whitespace after a '\' in the ending sequence.
2795 const char *SpacePos = nullptr;
2796
2797 while (true) {
2798 // Back up off the newline.
2799 --CurPtr;
2800
2801 // If this is a two-character newline sequence, skip the other character.
2802 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
2803 // \n\n or \r\r -> not escaped newline.
2804 if (CurPtr[0] == CurPtr[1])
2805 return false;
2806 // \n\r or \r\n -> skip the newline.
2807 --CurPtr;
2808 }
2809
2810 // If we have horizontal whitespace, skip over it. We allow whitespace
2811 // between the slash and newline.
2812 while (isHorizontalWhitespace(c: *CurPtr) || *CurPtr == 0) {
2813 SpacePos = CurPtr;
2814 --CurPtr;
2815 }
2816
2817 // If we have a slash, this is an escaped newline.
2818 if (*CurPtr == '\\') {
2819 --CurPtr;
2820 } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') {
2821 // This is a trigraph encoding of a slash.
2822 TrigraphPos = CurPtr - 2;
2823 CurPtr -= 3;
2824 } else {
2825 return false;
2826 }
2827
2828 // If the character preceding the escaped newline is a '*', then after line
2829 // splicing we have a '*/' ending the comment.
2830 if (*CurPtr == '*')
2831 break;
2832
2833 if (*CurPtr != '\n' && *CurPtr != '\r')
2834 return false;
2835 }
2836
2837 if (TrigraphPos) {
2838 // If no trigraphs are enabled, warn that we ignored this trigraph and
2839 // ignore this * character.
2840 if (!Trigraphs) {
2841 if (!L->isLexingRawMode())
2842 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ignored_block_comment);
2843 return false;
2844 }
2845 if (!L->isLexingRawMode())
2846 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ends_block_comment);
2847 }
2848
2849 // Warn about having an escaped newline between the */ characters.
2850 if (!L->isLexingRawMode())
2851 L->Diag(Loc: CurPtr + 1, DiagID: diag::escaped_newline_block_comment_end);
2852
2853 // If there was space between the backslash and newline, warn about it.
2854 if (SpacePos && !L->isLexingRawMode())
2855 L->Diag(Loc: SpacePos, DiagID: diag::backslash_newline_space);
2856
2857 return true;
2858}
2859
2860#ifdef __SSE2__
2861#include <emmintrin.h>
2862#elif __ALTIVEC__
2863#include <altivec.h>
2864#undef bool
2865#endif
2866
2867/// We have just read from input the / and * characters that started a comment.
2868/// Read until we find the * and / characters that terminate the comment.
2869/// Note that we don't bother decoding trigraphs or escaped newlines in block
2870/// comments, because they cannot cause the comment to end. The only thing
2871/// that can happen is the comment could end with an escaped newline between
2872/// the terminating * and /.
2873///
2874/// If we're in KeepCommentMode or any CommentHandler has inserted
2875/// some tokens, this will store the first token and return true.
2876bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
2877 bool &TokAtPhysicalStartOfLine) {
2878 // Scan one character past where we should, looking for a '/' character. Once
2879 // we find it, check to see if it was preceded by a *. This common
2880 // optimization helps people who like to put a lot of * characters in their
2881 // comments.
2882
2883 // The first character we get with newlines and trigraphs skipped to handle
2884 // the degenerate /*/ case below correctly if the * has an escaped newline
2885 // after it.
2886 unsigned CharSize;
2887 unsigned char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
2888 CurPtr += CharSize;
2889 if (C == 0 && CurPtr == BufferEnd+1) {
2890 if (!isLexingRawMode())
2891 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
2892 --CurPtr;
2893
2894 // KeepWhitespaceMode should return this broken comment as a token. Since
2895 // it isn't a well formed comment, just return it as an 'unknown' token.
2896 if (isKeepWhitespaceMode()) {
2897 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2898 return true;
2899 }
2900
2901 BufferPtr = CurPtr;
2902 return false;
2903 }
2904
2905 // Check to see if the first character after the '/*' is another /. If so,
2906 // then this slash does not end the block comment, it is part of it.
2907 if (C == '/')
2908 C = *CurPtr++;
2909
2910 // C++23 [lex.phases] p1
2911 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2912 // diagnostic only once per entire ill-formed subsequence to avoid
2913 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2914 bool UnicodeDecodingAlreadyDiagnosed = false;
2915
2916 while (true) {
2917 // Skip over all non-interesting characters until we find end of buffer or a
2918 // (probably ending) '/' character.
2919 if (CurPtr + 24 < BufferEnd &&
2920 // If there is a code-completion point avoid the fast scan because it
2921 // doesn't check for '\0'.
2922 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
2923 // While not aligned to a 16-byte boundary.
2924 while (C != '/' && (intptr_t)CurPtr % 16 != 0) {
2925 if (!isASCII(c: C))
2926 goto MultiByteUTF8;
2927 C = *CurPtr++;
2928 }
2929 if (C == '/') goto FoundSlash;
2930
2931#ifdef __SSE2__
2932 __m128i Slashes = _mm_set1_epi8(b: '/');
2933 while (CurPtr + 16 < BufferEnd) {
2934 int Mask = _mm_movemask_epi8(a: *(const __m128i *)CurPtr);
2935 if (LLVM_UNLIKELY(Mask != 0)) {
2936 goto MultiByteUTF8;
2937 }
2938 // look for slashes
2939 int cmp = _mm_movemask_epi8(a: _mm_cmpeq_epi8(a: *(const __m128i*)CurPtr,
2940 b: Slashes));
2941 if (cmp != 0) {
2942 // Adjust the pointer to point directly after the first slash. It's
2943 // not necessary to set C here, it will be overwritten at the end of
2944 // the outer loop.
2945 CurPtr += llvm::countr_zero<unsigned>(Val: cmp) + 1;
2946 goto FoundSlash;
2947 }
2948 CurPtr += 16;
2949 }
2950#elif __ALTIVEC__
2951 __vector unsigned char LongUTF = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
2952 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
2953 0x80, 0x80, 0x80, 0x80};
2954 __vector unsigned char Slashes = {
2955 '/', '/', '/', '/', '/', '/', '/', '/',
2956 '/', '/', '/', '/', '/', '/', '/', '/'
2957 };
2958 while (CurPtr + 16 < BufferEnd) {
2959 if (LLVM_UNLIKELY(
2960 vec_any_ge(*(const __vector unsigned char *)CurPtr, LongUTF)))
2961 goto MultiByteUTF8;
2962 if (vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) {
2963 break;
2964 }
2965 CurPtr += 16;
2966 }
2967
2968#else
2969 while (CurPtr + 16 < BufferEnd) {
2970 bool HasNonASCII = false;
2971 for (unsigned I = 0; I < 16; ++I)
2972 HasNonASCII |= !isASCII(CurPtr[I]);
2973
2974 if (LLVM_UNLIKELY(HasNonASCII))
2975 goto MultiByteUTF8;
2976
2977 bool HasSlash = false;
2978 for (unsigned I = 0; I < 16; ++I)
2979 HasSlash |= CurPtr[I] == '/';
2980 if (HasSlash)
2981 break;
2982 CurPtr += 16;
2983 }
2984#endif
2985
2986 // It has to be one of the bytes scanned, increment to it and read one.
2987 C = *CurPtr++;
2988 }
2989
2990 // Loop to scan the remainder, warning on invalid UTF-8
2991 // if the corresponding warning is enabled, emitting a diagnostic only once
2992 // per sequence that cannot be decoded.
2993 while (C != '/' && C != '\0') {
2994 if (isASCII(c: C)) {
2995 UnicodeDecodingAlreadyDiagnosed = false;
2996 C = *CurPtr++;
2997 continue;
2998 }
2999 MultiByteUTF8:
3000 // CurPtr is 1 code unit past C, so to decode
3001 // the codepoint, we need to read from the previous position.
3002 unsigned Length = llvm::getUTF8SequenceSize(
3003 source: (const llvm::UTF8 *)CurPtr - 1, sourceEnd: (const llvm::UTF8 *)BufferEnd);
3004 if (Length == 0) {
3005 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
3006 Diag(Loc: CurPtr - 1, DiagID: diag::warn_invalid_utf8_in_comment);
3007 UnicodeDecodingAlreadyDiagnosed = true;
3008 } else {
3009 UnicodeDecodingAlreadyDiagnosed = false;
3010 CurPtr += Length - 1;
3011 }
3012 C = *CurPtr++;
3013 }
3014
3015 if (C == '/') {
3016 FoundSlash:
3017 if (CurPtr[-2] == '*') // We found the final */. We're done!
3018 break;
3019
3020 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
3021 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr: CurPtr - 2, L: this,
3022 Trigraphs: LangOpts.Trigraphs)) {
3023 // We found the final */, though it had an escaped newline between the
3024 // * and /. We're done!
3025 break;
3026 }
3027 }
3028 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
3029 // If this is a /* inside of the comment, emit a warning. Don't do this
3030 // if this is a /*/, which will end the comment. This misses cases with
3031 // embedded escaped newlines, but oh well.
3032 if (!isLexingRawMode())
3033 Diag(Loc: CurPtr-1, DiagID: diag::warn_nested_block_comment);
3034 }
3035 } else if (C == 0 && CurPtr == BufferEnd+1) {
3036 if (!isLexingRawMode())
3037 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
3038 // Note: the user probably forgot a */. We could continue immediately
3039 // after the /*, but this would involve lexing a lot of what really is the
3040 // comment, which surely would confuse the parser.
3041 --CurPtr;
3042
3043 // KeepWhitespaceMode should return this broken comment as a token. Since
3044 // it isn't a well formed comment, just return it as an 'unknown' token.
3045 if (isKeepWhitespaceMode()) {
3046 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3047 return true;
3048 }
3049
3050 BufferPtr = CurPtr;
3051 return false;
3052 } else if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3053 PP->CodeCompleteNaturalLanguage();
3054 cutOffLexing();
3055 return false;
3056 }
3057
3058 C = *CurPtr++;
3059 }
3060
3061 // Notify comment handlers about the comment unless we're in a #if 0 block.
3062 if (PP && !isLexingRawMode() &&
3063 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
3064 getSourceLocation(Loc: CurPtr)))) {
3065 BufferPtr = CurPtr;
3066 return true; // A token has to be returned.
3067 }
3068
3069 // If we are returning comments as tokens, return this comment as a token.
3070 if (inKeepCommentMode()) {
3071 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
3072 return true;
3073 }
3074
3075 // It is common for the tokens immediately after a /**/ comment to be
3076 // whitespace. Instead of going through the big switch, handle it
3077 // efficiently now. This is safe even in KeepWhitespaceMode because we would
3078 // have already returned above with the comment as a token.
3079 if (isHorizontalWhitespace(c: *CurPtr)) {
3080 SkipWhitespace(Result, CurPtr: CurPtr+1, TokAtPhysicalStartOfLine);
3081 return false;
3082 }
3083
3084 // Otherwise, just return so that the next character will be lexed as a token.
3085 BufferPtr = CurPtr;
3086 Result.setFlag(Token::LeadingSpace);
3087 return false;
3088}
3089
3090//===----------------------------------------------------------------------===//
3091// Primary Lexing Entry Points
3092//===----------------------------------------------------------------------===//
3093
3094/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
3095/// uninterpreted string. This switches the lexer out of directive mode.
3096void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
3097 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
3098 "Must be in a preprocessing directive!");
3099 Token Tmp;
3100 Tmp.startToken();
3101
3102 // CurPtr - Cache BufferPtr in an automatic variable.
3103 const char *CurPtr = BufferPtr;
3104 while (true) {
3105 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Tmp);
3106 switch (Char) {
3107 default:
3108 if (Result)
3109 Result->push_back(Elt: Char);
3110 break;
3111 case 0: // Null.
3112 // Found end of file?
3113 if (CurPtr-1 != BufferEnd) {
3114 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3115 PP->CodeCompleteNaturalLanguage();
3116 cutOffLexing();
3117 return;
3118 }
3119
3120 // Nope, normal character, continue.
3121 if (Result)
3122 Result->push_back(Elt: Char);
3123 break;
3124 }
3125 // FALL THROUGH.
3126 [[fallthrough]];
3127 case '\r':
3128 case '\n':
3129 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
3130 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
3131 BufferPtr = CurPtr-1;
3132
3133 // Next, lex the character, which should handle the EOD transition.
3134 Lex(Result&: Tmp);
3135 if (Tmp.is(K: tok::code_completion)) {
3136 if (PP)
3137 PP->CodeCompleteNaturalLanguage();
3138 Lex(Result&: Tmp);
3139 }
3140 assert(Tmp.is(tok::eod) && "Unexpected token!");
3141
3142 // Finally, we're done;
3143 return;
3144 }
3145 }
3146}
3147
3148/// LexEndOfFile - CurPtr points to the end of this file. Handle this
3149/// condition, reporting diagnostics and handling other edge cases as required.
3150/// This returns true if Result contains a token, false if PP.Lex should be
3151/// called again.
3152bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
3153 // If we hit the end of the file while parsing a preprocessor directive,
3154 // end the preprocessor directive first. The next token returned will
3155 // then be the end of file.
3156 if (ParsingPreprocessorDirective) {
3157 // Done parsing the "line".
3158 ParsingPreprocessorDirective = false;
3159 // Update the location of token as well as BufferPtr.
3160 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::eod);
3161
3162 // Restore comment saving mode, in case it was disabled for directive.
3163 if (PP)
3164 resetExtendedTokenMode();
3165 return true; // Have a token.
3166 }
3167
3168 // If we are in raw mode, return this event as an EOF token. Let the caller
3169 // that put us in raw mode handle the event.
3170 if (isLexingRawMode()) {
3171 Result.startToken();
3172 BufferPtr = BufferEnd;
3173 FormTokenWithChars(Result, TokEnd: BufferEnd, Kind: tok::eof);
3174 return true;
3175 }
3176
3177 if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) {
3178 PP->setRecordedPreambleConditionalStack(ConditionalStack);
3179 // If the preamble cuts off the end of a header guard, consider it guarded.
3180 // The guard is valid for the preamble content itself, and for tools the
3181 // most useful answer is "yes, this file has a header guard".
3182 if (!ConditionalStack.empty())
3183 MIOpt.ExitTopLevelConditional();
3184 ConditionalStack.clear();
3185 }
3186
3187 // Issue diagnostics for unterminated #if and missing newline.
3188
3189 // If we are in a #if directive, emit an error.
3190 while (!ConditionalStack.empty()) {
3191 if (PP->getCodeCompletionFileLoc() != FileLoc)
3192 PP->Diag(Loc: ConditionalStack.back().IfLoc,
3193 DiagID: diag::err_pp_unterminated_conditional);
3194 ConditionalStack.pop_back();
3195 }
3196
3197 // Before C++11 and C2y, a file not ending with a newline was UB. Both
3198 // standards changed this behavior (as a DR or equivalent), but we still have
3199 // an opt-in diagnostic to warn about it.
3200 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
3201 Diag(Loc: BufferEnd, DiagID: diag::warn_no_newline_eof)
3202 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: BufferEnd), Code: "\n");
3203
3204 BufferPtr = CurPtr;
3205
3206 // Finally, let the preprocessor handle this.
3207 return PP->HandleEndOfFile(Result, isEndOfMacro: isPragmaLexer());
3208}
3209
3210/// peekNextPPToken - Return std::nullopt if there are no more tokens in the
3211/// buffer controlled by this lexer, otherwise return the next unexpanded
3212/// token.
3213std::optional<Token> Lexer::peekNextPPToken() {
3214 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
3215
3216 if (isDependencyDirectivesLexer()) {
3217 if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size())
3218 return std::nullopt;
3219 Token Result;
3220 (void)convertDependencyDirectiveToken(
3221 DDTok: DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result);
3222 return Result;
3223 }
3224
3225 // Switch to 'skipping' mode. This will ensure that we can lex a token
3226 // without emitting diagnostics, disables macro expansion, and will cause EOF
3227 // to return an EOF token instead of popping the include stack.
3228 LexingRawMode = true;
3229
3230 // Save state that can be changed while lexing so that we can restore it.
3231 const char *TmpBufferPtr = BufferPtr;
3232 bool inPPDirectiveMode = ParsingPreprocessorDirective;
3233 bool atStartOfLine = IsAtStartOfLine;
3234 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3235 bool leadingSpace = HasLeadingSpace;
3236
3237 Token Tok;
3238 Lex(Result&: Tok);
3239
3240 // Restore state that may have changed.
3241 BufferPtr = TmpBufferPtr;
3242 ParsingPreprocessorDirective = inPPDirectiveMode;
3243 HasLeadingSpace = leadingSpace;
3244 IsAtStartOfLine = atStartOfLine;
3245 IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
3246 // Restore the lexer back to non-skipping mode.
3247 LexingRawMode = false;
3248
3249 if (Tok.is(K: tok::eof))
3250 return std::nullopt;
3251 return Tok;
3252}
3253
3254/// Find the end of a version control conflict marker.
3255static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
3256 ConflictMarkerKind CMK) {
3257 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
3258 size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
3259 auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(Start: TermLen);
3260 size_t Pos = RestOfBuffer.find(Str: Terminator);
3261 while (Pos != StringRef::npos) {
3262 // Must occur at start of line.
3263 if (Pos == 0 ||
3264 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) {
3265 RestOfBuffer = RestOfBuffer.substr(Start: Pos+TermLen);
3266 Pos = RestOfBuffer.find(Str: Terminator);
3267 continue;
3268 }
3269 return RestOfBuffer.data()+Pos;
3270 }
3271 return nullptr;
3272}
3273
3274/// IsStartOfConflictMarker - If the specified pointer is the start of a version
3275/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
3276/// and recover nicely. This returns true if it is a conflict marker and false
3277/// if not.
3278bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
3279 // Only a conflict marker if it starts at the beginning of a line.
3280 if (CurPtr != BufferStart &&
3281 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3282 return false;
3283
3284 // Check to see if we have <<<<<<< or >>>>.
3285 if (!StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: "<<<<<<<") &&
3286 !StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: ">>>> "))
3287 return false;
3288
3289 // If we have a situation where we don't care about conflict markers, ignore
3290 // it.
3291 if (CurrentConflictMarkerState || isLexingRawMode())
3292 return false;
3293
3294 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
3295
3296 // Check to see if there is an ending marker somewhere in the buffer at the
3297 // start of a line to terminate this conflict marker.
3298 if (FindConflictEnd(CurPtr, BufferEnd, CMK: Kind)) {
3299 // We found a match. We are really in a conflict marker.
3300 // Diagnose this, and ignore to the end of line.
3301 Diag(Loc: CurPtr, DiagID: diag::err_conflict_marker);
3302 CurrentConflictMarkerState = Kind;
3303
3304 // Skip ahead to the end of line. We know this exists because the
3305 // end-of-conflict marker starts with \r or \n.
3306 while (*CurPtr != '\r' && *CurPtr != '\n') {
3307 assert(CurPtr != BufferEnd && "Didn't find end of line");
3308 ++CurPtr;
3309 }
3310 BufferPtr = CurPtr;
3311 return true;
3312 }
3313
3314 // No end of conflict marker found.
3315 return false;
3316}
3317
3318/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
3319/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
3320/// is the end of a conflict marker. Handle it by ignoring up until the end of
3321/// the line. This returns true if it is a conflict marker and false if not.
3322bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
3323 // Only a conflict marker if it starts at the beginning of a line.
3324 if (CurPtr != BufferStart &&
3325 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3326 return false;
3327
3328 // If we have a situation where we don't care about conflict markers, ignore
3329 // it.
3330 if (!CurrentConflictMarkerState || isLexingRawMode())
3331 return false;
3332
3333 // Check to see if we have the marker (4 characters in a row).
3334 for (unsigned i = 1; i != 4; ++i)
3335 if (CurPtr[i] != CurPtr[0])
3336 return false;
3337
3338 // If we do have it, search for the end of the conflict marker. This could
3339 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
3340 // be the end of conflict marker.
3341 if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
3342 CMK: CurrentConflictMarkerState)) {
3343 CurPtr = End;
3344
3345 // Skip ahead to the end of line.
3346 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
3347 ++CurPtr;
3348
3349 BufferPtr = CurPtr;
3350
3351 // No longer in the conflict marker.
3352 CurrentConflictMarkerState = CMK_None;
3353 return true;
3354 }
3355
3356 return false;
3357}
3358
3359static const char *findPlaceholderEnd(const char *CurPtr,
3360 const char *BufferEnd) {
3361 if (CurPtr == BufferEnd)
3362 return nullptr;
3363 BufferEnd -= 1; // Scan until the second last character.
3364 for (; CurPtr != BufferEnd; ++CurPtr) {
3365 if (CurPtr[0] == '#' && CurPtr[1] == '>')
3366 return CurPtr + 2;
3367 }
3368 return nullptr;
3369}
3370
3371bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
3372 assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!");
3373 if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
3374 return false;
3375 const char *End = findPlaceholderEnd(CurPtr: CurPtr + 1, BufferEnd);
3376 if (!End)
3377 return false;
3378 const char *Start = CurPtr - 1;
3379 if (!LangOpts.AllowEditorPlaceholders)
3380 Diag(Loc: Start, DiagID: diag::err_placeholder_in_source);
3381 Result.startToken();
3382 FormTokenWithChars(Result, TokEnd: End, Kind: tok::raw_identifier);
3383 Result.setRawIdentifierData(Start);
3384 PP->LookUpIdentifierInfo(Identifier&: Result);
3385 Result.setFlag(Token::IsEditorPlaceholder);
3386 BufferPtr = End;
3387 return true;
3388}
3389
3390bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
3391 if (PP && PP->isCodeCompletionEnabled()) {
3392 SourceLocation Loc = FileLoc.getLocWithOffset(Offset: CurPtr-BufferStart);
3393 return Loc == PP->getCodeCompletionLoc();
3394 }
3395
3396 return false;
3397}
3398
3399void Lexer::DiagnoseDelimitedOrNamedEscapeSequence(SourceLocation Loc,
3400 bool Named,
3401 const LangOptions &Opts,
3402 DiagnosticsEngine &Diags) {
3403 unsigned DiagId;
3404 if (Opts.CPlusPlus23)
3405 DiagId = diag::warn_cxx23_delimited_escape_sequence;
3406 else if (Opts.C2y && !Named)
3407 DiagId = diag::warn_c2y_delimited_escape_sequence;
3408 else
3409 DiagId = diag::ext_delimited_escape_sequence;
3410
3411 // The trailing arguments are only used by the extension warning; either this
3412 // is a C2y extension or a C++23 extension, unless it's a named escape
3413 // sequence in C, then it's a Clang extension.
3414 unsigned Ext;
3415 if (!Opts.CPlusPlus)
3416 Ext = Named ? 2 /* Clang extension */ : 1 /* C2y extension */;
3417 else
3418 Ext = 0; // C++23 extension
3419
3420 Diags.Report(Loc, DiagID: DiagId) << Named << Ext;
3421}
3422
3423std::optional<uint32_t> Lexer::tryReadNumericUCN(const char *&StartPtr,
3424 const char *SlashLoc,
3425 Token *Result) {
3426 unsigned CharSize;
3427 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3428 assert((Kind == 'u' || Kind == 'U') && "expected a UCN");
3429
3430 unsigned NumHexDigits;
3431 if (Kind == 'u')
3432 NumHexDigits = 4;
3433 else if (Kind == 'U')
3434 NumHexDigits = 8;
3435
3436 bool Delimited = false;
3437 bool FoundEndDelimiter = false;
3438 unsigned Count = 0;
3439 bool Diagnose = Result && !isLexingRawMode();
3440
3441 if (!LangOpts.CPlusPlus && !LangOpts.C99) {
3442 if (Diagnose)
3443 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_not_valid_in_c89);
3444 return std::nullopt;
3445 }
3446
3447 const char *CurPtr = StartPtr + CharSize;
3448 const char *KindLoc = &CurPtr[-1];
3449
3450 uint32_t CodePoint = 0;
3451 while (Count != NumHexDigits || Delimited) {
3452 char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3453 if (!Delimited && Count == 0 && C == '{') {
3454 Delimited = true;
3455 CurPtr += CharSize;
3456 continue;
3457 }
3458
3459 if (Delimited && C == '}') {
3460 CurPtr += CharSize;
3461 FoundEndDelimiter = true;
3462 break;
3463 }
3464
3465 unsigned Value = llvm::hexDigitValue(C);
3466 if (Value == std::numeric_limits<unsigned>::max()) {
3467 if (!Delimited)
3468 break;
3469 if (Diagnose)
3470 Diag(Loc: SlashLoc, DiagID: diag::warn_delimited_ucn_incomplete)
3471 << StringRef(KindLoc, 1);
3472 return std::nullopt;
3473 }
3474
3475 if (CodePoint & 0xF000'0000) {
3476 if (Diagnose)
3477 Diag(Loc: KindLoc, DiagID: diag::err_escape_too_large) << 0;
3478 return std::nullopt;
3479 }
3480
3481 CodePoint <<= 4;
3482 CodePoint |= Value;
3483 CurPtr += CharSize;
3484 Count++;
3485 }
3486
3487 if (Count == 0) {
3488 if (Diagnose)
3489 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3490 : diag::warn_ucn_escape_no_digits)
3491 << StringRef(KindLoc, 1);
3492 return std::nullopt;
3493 }
3494
3495 if (Delimited && Kind == 'U') {
3496 if (Diagnose)
3497 Diag(Loc: SlashLoc, DiagID: diag::err_hex_escape_no_digits) << StringRef(KindLoc, 1);
3498 return std::nullopt;
3499 }
3500
3501 if (!Delimited && Count != NumHexDigits) {
3502 if (Diagnose) {
3503 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3504 // If the user wrote \U1234, suggest a fixit to \u.
3505 if (Count == 4 && NumHexDigits == 8) {
3506 CharSourceRange URange = makeCharRange(L&: *this, Begin: KindLoc, End: KindLoc + 1);
3507 Diag(Loc: KindLoc, DiagID: diag::note_ucn_four_not_eight)
3508 << FixItHint::CreateReplacement(RemoveRange: URange, Code: "u");
3509 }
3510 }
3511 return std::nullopt;
3512 }
3513
3514 if (Delimited && PP)
3515 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: false,
3516 Opts: PP->getLangOpts(),
3517 Diags&: PP->getDiagnostics());
3518
3519 if (Result) {
3520 Result->setFlag(Token::HasUCN);
3521 // If the UCN contains either a trigraph or a line splicing,
3522 // we need to call getAndAdvanceChar again to set the appropriate flags
3523 // on Result.
3524 if (CurPtr - StartPtr == (ptrdiff_t)(Count + 1 + (Delimited ? 2 : 0)))
3525 StartPtr = CurPtr;
3526 else
3527 while (StartPtr != CurPtr)
3528 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3529 } else {
3530 StartPtr = CurPtr;
3531 }
3532 return CodePoint;
3533}
3534
3535std::optional<uint32_t> Lexer::tryReadNamedUCN(const char *&StartPtr,
3536 const char *SlashLoc,
3537 Token *Result) {
3538 unsigned CharSize;
3539 bool Diagnose = Result && !isLexingRawMode();
3540
3541 char C = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3542 assert(C == 'N' && "expected \\N{...}");
3543
3544 const char *CurPtr = StartPtr + CharSize;
3545 const char *KindLoc = &CurPtr[-1];
3546
3547 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3548 if (C != '{') {
3549 if (Diagnose)
3550 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3551 return std::nullopt;
3552 }
3553 CurPtr += CharSize;
3554 const char *StartName = CurPtr;
3555 bool FoundEndDelimiter = false;
3556 llvm::SmallVector<char, 30> Buffer;
3557 while (C) {
3558 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3559 CurPtr += CharSize;
3560 if (C == '}') {
3561 FoundEndDelimiter = true;
3562 break;
3563 }
3564
3565 if (isVerticalWhitespace(c: C))
3566 break;
3567 Buffer.push_back(Elt: C);
3568 }
3569
3570 if (!FoundEndDelimiter || Buffer.empty()) {
3571 if (Diagnose)
3572 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3573 : diag::warn_delimited_ucn_incomplete)
3574 << StringRef(KindLoc, 1);
3575 return std::nullopt;
3576 }
3577
3578 StringRef Name(Buffer.data(), Buffer.size());
3579 std::optional<char32_t> Match =
3580 llvm::sys::unicode::nameToCodepointStrict(Name);
3581 std::optional<llvm::sys::unicode::LooseMatchingResult> LooseMatch;
3582 if (!Match) {
3583 LooseMatch = llvm::sys::unicode::nameToCodepointLooseMatching(Name);
3584 if (Diagnose) {
3585 Diag(Loc: StartName, DiagID: diag::err_invalid_ucn_name)
3586 << StringRef(Buffer.data(), Buffer.size())
3587 << makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize);
3588 if (LooseMatch) {
3589 Diag(Loc: StartName, DiagID: diag::note_invalid_ucn_name_loose_matching)
3590 << FixItHint::CreateReplacement(
3591 RemoveRange: makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize),
3592 Code: LooseMatch->Name);
3593 }
3594 }
3595 // We do not offer misspelled character names suggestions here
3596 // as the set of what would be a valid suggestion depends on context,
3597 // and we should not make invalid suggestions.
3598 }
3599
3600 if (Diagnose && Match)
3601 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: true,
3602 Opts: PP->getLangOpts(),
3603 Diags&: PP->getDiagnostics());
3604
3605 // If no diagnostic has been emitted yet, likely because we are doing a
3606 // tentative lexing, we do not want to recover here to make sure the token
3607 // will not be incorrectly considered valid. This function will be called
3608 // again and a diagnostic emitted then.
3609 if (LooseMatch && Diagnose)
3610 Match = LooseMatch->CodePoint;
3611
3612 if (Result) {
3613 Result->setFlag(Token::HasUCN);
3614 // If the UCN contains either a trigraph or a line splicing,
3615 // we need to call getAndAdvanceChar again to set the appropriate flags
3616 // on Result.
3617 if (CurPtr - StartPtr == (ptrdiff_t)(Buffer.size() + 3))
3618 StartPtr = CurPtr;
3619 else
3620 while (StartPtr != CurPtr)
3621 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3622 } else {
3623 StartPtr = CurPtr;
3624 }
3625 return Match ? std::optional<uint32_t>(*Match) : std::nullopt;
3626}
3627
3628uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3629 Token *Result) {
3630
3631 unsigned CharSize;
3632 std::optional<uint32_t> CodePointOpt;
3633 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3634 if (Kind == 'u' || Kind == 'U')
3635 CodePointOpt = tryReadNumericUCN(StartPtr, SlashLoc, Result);
3636 else if (Kind == 'N')
3637 CodePointOpt = tryReadNamedUCN(StartPtr, SlashLoc, Result);
3638
3639 if (!CodePointOpt)
3640 return 0;
3641
3642 uint32_t CodePoint = *CodePointOpt;
3643
3644 // Don't apply C family restrictions to UCNs in assembly mode
3645 if (LangOpts.AsmPreprocessor)
3646 return CodePoint;
3647
3648 // C23 6.4.3p2: A universal character name shall not designate a code point
3649 // where the hexadecimal value is:
3650 // - in the range D800 through DFFF inclusive; or
3651 // - greater than 10FFFF.
3652 // A universal-character-name outside the c-char-sequence of a character
3653 // constant, or the s-char-sequence of a string-literal shall not designate
3654 // a control character or a character in the basic character set.
3655
3656 // C++11 [lex.charset]p2: If the hexadecimal value for a
3657 // universal-character-name corresponds to a surrogate code point (in the
3658 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
3659 // if the hexadecimal value for a universal-character-name outside the
3660 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or
3661 // string literal corresponds to a control character (in either of the
3662 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
3663 // basic source character set, the program is ill-formed.
3664 if (CodePoint < 0xA0) {
3665 // We don't use isLexingRawMode() here because we need to warn about bad
3666 // UCNs even when skipping preprocessing tokens in a #if block.
3667 if (Result && PP) {
3668 if (CodePoint < 0x20 || CodePoint >= 0x7F)
3669 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_control_character);
3670 else {
3671 char C = static_cast<char>(CodePoint);
3672 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
3673 }
3674 }
3675
3676 return 0;
3677 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) {
3678 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
3679 // We don't use isLexingRawMode() here because we need to diagnose bad
3680 // UCNs even when skipping preprocessing tokens in a #if block.
3681 if (Result && PP) {
3682 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11)
3683 Diag(Loc: BufferPtr, DiagID: diag::warn_ucn_escape_surrogate);
3684 else
3685 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_invalid);
3686 }
3687 return 0;
3688 }
3689
3690 return CodePoint;
3691}
3692
3693bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3694 const char *CurPtr) {
3695 if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
3696 isUnicodeWhitespace(Codepoint: C)) {
3697 Diag(Loc: BufferPtr, DiagID: diag::ext_unicode_whitespace)
3698 << makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr);
3699
3700 Result.setFlag(Token::LeadingSpace);
3701 return true;
3702 }
3703 return false;
3704}
3705
3706void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3707 IsAtStartOfLine = Result.isAtStartOfLine();
3708 HasLeadingSpace = Result.hasLeadingSpace();
3709 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
3710 // Note that this doesn't affect IsAtPhysicalStartOfLine.
3711}
3712
3713bool Lexer::Lex(Token &Result) {
3714 assert(!isDependencyDirectivesLexer());
3715
3716 // Start a new token.
3717 Result.startToken();
3718
3719 // Set up misc whitespace flags for LexTokenInternal.
3720 if (IsAtStartOfLine) {
3721 Result.setFlag(Token::StartOfLine);
3722 IsAtStartOfLine = false;
3723 }
3724
3725 if (HasLeadingSpace) {
3726 Result.setFlag(Token::LeadingSpace);
3727 HasLeadingSpace = false;
3728 }
3729
3730 if (HasLeadingEmptyMacro) {
3731 Result.setFlag(Token::LeadingEmptyMacro);
3732 HasLeadingEmptyMacro = false;
3733 }
3734
3735 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3736 IsAtPhysicalStartOfLine = false;
3737 bool isRawLex = isLexingRawMode();
3738 (void) isRawLex;
3739 bool returnedToken = LexTokenInternal(Result, TokAtPhysicalStartOfLine: atPhysicalStartOfLine);
3740 // (After the LexTokenInternal call, the lexer might be destroyed.)
3741 assert((returnedToken || !isRawLex) && "Raw lex must succeed");
3742 return returnedToken;
3743}
3744
3745/// LexTokenInternal - This implements a simple C family lexer. It is an
3746/// extremely performance critical piece of code. This assumes that the buffer
3747/// has a null character at the end of the file. This returns a preprocessing
3748/// token, not a normal token, as such, it is an internal interface. It assumes
3749/// that the Flags of result have been cleared before calling this.
3750bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
3751LexStart:
3752 assert(!Result.needsCleaning() && "Result needs cleaning");
3753 assert(!Result.hasPtrData() && "Result has not been reset");
3754
3755 // CurPtr - Cache BufferPtr in an automatic variable.
3756 const char *CurPtr = BufferPtr;
3757
3758 // Small amounts of horizontal whitespace is very common between tokens.
3759 if (isHorizontalWhitespace(c: *CurPtr)) {
3760 do {
3761 ++CurPtr;
3762 } while (isHorizontalWhitespace(c: *CurPtr));
3763
3764 // If we are keeping whitespace and other tokens, just return what we just
3765 // skipped. The next lexer invocation will return the token after the
3766 // whitespace.
3767 if (isKeepWhitespaceMode()) {
3768 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3769 // FIXME: The next token will not have LeadingSpace set.
3770 return true;
3771 }
3772
3773 BufferPtr = CurPtr;
3774 Result.setFlag(Token::LeadingSpace);
3775 }
3776
3777 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
3778
3779 // Read a character, advancing over it.
3780 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3781 tok::TokenKind Kind;
3782
3783 if (!isVerticalWhitespace(c: Char))
3784 NewLinePtr = nullptr;
3785
3786 switch (Char) {
3787 case 0: // Null.
3788 // Found end of file?
3789 if (CurPtr-1 == BufferEnd)
3790 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3791
3792 // Check if we are performing code completion.
3793 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3794 // Return the code-completion token.
3795 Result.startToken();
3796 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::code_completion);
3797 return true;
3798 }
3799
3800 if (!isLexingRawMode())
3801 Diag(Loc: CurPtr-1, DiagID: diag::null_in_file);
3802 Result.setFlag(Token::LeadingSpace);
3803 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3804 return true; // KeepWhitespaceMode
3805
3806 // We know the lexer hasn't changed, so just try again with this lexer.
3807 // (We manually eliminate the tail call to avoid recursion.)
3808 goto LexNextToken;
3809
3810 case 26: // DOS & CP/M EOF: "^Z".
3811 // If we're in Microsoft extensions mode, treat this as end of file.
3812 if (LangOpts.MicrosoftExt) {
3813 if (!isLexingRawMode())
3814 Diag(Loc: CurPtr-1, DiagID: diag::ext_ctrl_z_eof_microsoft);
3815 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3816 }
3817
3818 // If Microsoft extensions are disabled, this is just random garbage.
3819 Kind = tok::unknown;
3820 break;
3821
3822 case '\r':
3823 if (CurPtr[0] == '\n')
3824 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3825 [[fallthrough]];
3826 case '\n':
3827 // If we are inside a preprocessor directive and we see the end of line,
3828 // we know we are done with the directive, so return an EOD token.
3829 if (ParsingPreprocessorDirective) {
3830 // Done parsing the "line".
3831 ParsingPreprocessorDirective = false;
3832
3833 // Restore comment saving mode, in case it was disabled for directive.
3834 if (PP)
3835 resetExtendedTokenMode();
3836
3837 // Since we consumed a newline, we are back at the start of a line.
3838 IsAtStartOfLine = true;
3839 IsAtPhysicalStartOfLine = true;
3840 NewLinePtr = CurPtr - 1;
3841
3842 Kind = tok::eod;
3843 break;
3844 }
3845
3846 // No leading whitespace seen so far.
3847 Result.clearFlag(Flag: Token::LeadingSpace);
3848
3849 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3850 return true; // KeepWhitespaceMode
3851
3852 // We only saw whitespace, so just try again with this lexer.
3853 // (We manually eliminate the tail call to avoid recursion.)
3854 goto LexNextToken;
3855 case ' ':
3856 case '\t':
3857 case '\f':
3858 case '\v':
3859 SkipHorizontalWhitespace:
3860 Result.setFlag(Token::LeadingSpace);
3861 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3862 return true; // KeepWhitespaceMode
3863
3864 SkipIgnoredUnits:
3865 CurPtr = BufferPtr;
3866
3867 // If the next token is obviously a // or /* */ comment, skip it efficiently
3868 // too (without going through the big switch stmt).
3869 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
3870 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
3871 if (SkipLineComment(Result, CurPtr: CurPtr+2, TokAtPhysicalStartOfLine))
3872 return true; // There is a token to return.
3873 goto SkipIgnoredUnits;
3874 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
3875 if (SkipBlockComment(Result, CurPtr: CurPtr+2, TokAtPhysicalStartOfLine))
3876 return true; // There is a token to return.
3877 goto SkipIgnoredUnits;
3878 } else if (isHorizontalWhitespace(c: *CurPtr)) {
3879 goto SkipHorizontalWhitespace;
3880 }
3881 // We only saw whitespace, so just try again with this lexer.
3882 // (We manually eliminate the tail call to avoid recursion.)
3883 goto LexNextToken;
3884
3885 // C99 6.4.4.1: Integer Constants.
3886 // C99 6.4.4.2: Floating Constants.
3887 case '0': case '1': case '2': case '3': case '4':
3888 case '5': case '6': case '7': case '8': case '9':
3889 // Notify MIOpt that we read a non-whitespace/non-comment token.
3890 MIOpt.ReadToken();
3891 return LexNumericConstant(Result, CurPtr);
3892
3893 // Identifier (e.g., uber), or
3894 // UTF-8 (C23/C++17) or UTF-16 (C11/C++11) character literal, or
3895 // UTF-8 or UTF-16 string literal (C11/C++11).
3896 case 'u':
3897 // Notify MIOpt that we read a non-whitespace/non-comment token.
3898 MIOpt.ReadToken();
3899
3900 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
3901 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3902
3903 // UTF-16 string literal
3904 if (Char == '"')
3905 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3906 Kind: tok::utf16_string_literal);
3907
3908 // UTF-16 character constant
3909 if (Char == '\'')
3910 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3911 Kind: tok::utf16_char_constant);
3912
3913 // UTF-16 raw string literal
3914 if (Char == 'R' && LangOpts.RawStringLiterals &&
3915 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
3916 return LexRawStringLiteral(Result,
3917 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3918 Size: SizeTmp2, Tok&: Result),
3919 Kind: tok::utf16_string_literal);
3920
3921 if (Char == '8') {
3922 char Char2 = getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2);
3923
3924 // UTF-8 string literal
3925 if (Char2 == '"')
3926 return LexStringLiteral(Result,
3927 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3928 Size: SizeTmp2, Tok&: Result),
3929 Kind: tok::utf8_string_literal);
3930 if (Char2 == '\'' && (LangOpts.CPlusPlus17 || LangOpts.C23))
3931 return LexCharConstant(
3932 Result, CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3933 Size: SizeTmp2, Tok&: Result),
3934 Kind: tok::utf8_char_constant);
3935
3936 if (Char2 == 'R' && LangOpts.RawStringLiterals) {
3937 unsigned SizeTmp3;
3938 char Char3 = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
3939 // UTF-8 raw string literal
3940 if (Char3 == '"') {
3941 return LexRawStringLiteral(Result,
3942 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3943 Size: SizeTmp2, Tok&: Result),
3944 Size: SizeTmp3, Tok&: Result),
3945 Kind: tok::utf8_string_literal);
3946 }
3947 }
3948 }
3949 }
3950
3951 // treat u like the start of an identifier.
3952 return LexIdentifierContinue(Result, CurPtr);
3953
3954 case 'U': // Identifier (e.g. Uber) or C11/C++11 UTF-32 string literal
3955 // Notify MIOpt that we read a non-whitespace/non-comment token.
3956 MIOpt.ReadToken();
3957
3958 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
3959 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3960
3961 // UTF-32 string literal
3962 if (Char == '"')
3963 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3964 Kind: tok::utf32_string_literal);
3965
3966 // UTF-32 character constant
3967 if (Char == '\'')
3968 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3969 Kind: tok::utf32_char_constant);
3970
3971 // UTF-32 raw string literal
3972 if (Char == 'R' && LangOpts.RawStringLiterals &&
3973 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
3974 return LexRawStringLiteral(Result,
3975 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3976 Size: SizeTmp2, Tok&: Result),
3977 Kind: tok::utf32_string_literal);
3978 }
3979
3980 // treat U like the start of an identifier.
3981 return LexIdentifierContinue(Result, CurPtr);
3982
3983 case 'R': // Identifier or C++0x raw string literal
3984 // Notify MIOpt that we read a non-whitespace/non-comment token.
3985 MIOpt.ReadToken();
3986
3987 if (LangOpts.RawStringLiterals) {
3988 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3989
3990 if (Char == '"')
3991 return LexRawStringLiteral(Result,
3992 CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3993 Kind: tok::string_literal);
3994 }
3995
3996 // treat R like the start of an identifier.
3997 return LexIdentifierContinue(Result, CurPtr);
3998
3999 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
4000 // Notify MIOpt that we read a non-whitespace/non-comment token.
4001 MIOpt.ReadToken();
4002 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4003
4004 // Wide string literal.
4005 if (Char == '"')
4006 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4007 Kind: tok::wide_string_literal);
4008
4009 // Wide raw string literal.
4010 if (LangOpts.RawStringLiterals && Char == 'R' &&
4011 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4012 return LexRawStringLiteral(Result,
4013 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4014 Size: SizeTmp2, Tok&: Result),
4015 Kind: tok::wide_string_literal);
4016
4017 // Wide character constant.
4018 if (Char == '\'')
4019 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4020 Kind: tok::wide_char_constant);
4021 // FALL THROUGH, treating L like the start of an identifier.
4022 [[fallthrough]];
4023
4024 // C99 6.4.2: Identifiers.
4025 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
4026 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
4027 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
4028 case 'V': case 'W': case 'X': case 'Y': case 'Z':
4029 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
4030 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
4031 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
4032 case 'v': case 'w': case 'x': case 'y': case 'z':
4033 case '_': {
4034 // Notify MIOpt that we read a non-whitespace/non-comment token.
4035 MIOpt.ReadToken();
4036
4037 // LexIdentifierContinue may trigger HandleEndOfFile which would
4038 // normally destroy this Lexer. However, the Preprocessor now defers
4039 // lexer destruction until the stack of Lexer unwinds (LexLevel == 0),
4040 // so it's safe to access member variables after this call returns.
4041 bool returnedToken = LexIdentifierContinue(Result, CurPtr);
4042
4043 if (returnedToken && !LexingRawMode && !Is_PragmaLexer &&
4044 !ParsingPreprocessorDirective && LangOpts.CPlusPlusModules &&
4045 Result.isModuleContextualKeyword() &&
4046 PP->HandleModuleContextualKeyword(Result, TokAtPhysicalStartOfLine))
4047 goto HandleDirective;
4048 return returnedToken;
4049 }
4050 case '$': // $ in identifiers.
4051 if (LangOpts.DollarIdents) {
4052 if (!isLexingRawMode())
4053 Diag(Loc: CurPtr-1, DiagID: diag::ext_dollar_in_identifier);
4054 // Notify MIOpt that we read a non-whitespace/non-comment token.
4055 MIOpt.ReadToken();
4056 return LexIdentifierContinue(Result, CurPtr);
4057 }
4058
4059 Kind = tok::unknown;
4060 break;
4061
4062 // C99 6.4.4: Character Constants.
4063 case '\'':
4064 // Notify MIOpt that we read a non-whitespace/non-comment token.
4065 MIOpt.ReadToken();
4066 return LexCharConstant(Result, CurPtr, Kind: tok::char_constant);
4067
4068 // C99 6.4.5: String Literals.
4069 case '"':
4070 // Notify MIOpt that we read a non-whitespace/non-comment token.
4071 MIOpt.ReadToken();
4072 return LexStringLiteral(Result, CurPtr,
4073 Kind: ParsingFilename ? tok::header_name
4074 : tok::string_literal);
4075
4076 // C99 6.4.6: Punctuators.
4077 case '?':
4078 Kind = tok::question;
4079 break;
4080 case '[':
4081 Kind = tok::l_square;
4082 break;
4083 case ']':
4084 Kind = tok::r_square;
4085 break;
4086 case '(':
4087 Kind = tok::l_paren;
4088 break;
4089 case ')':
4090 Kind = tok::r_paren;
4091 break;
4092 case '{':
4093 Kind = tok::l_brace;
4094 break;
4095 case '}':
4096 Kind = tok::r_brace;
4097 break;
4098 case '.':
4099 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4100 if (Char >= '0' && Char <= '9') {
4101 // Notify MIOpt that we read a non-whitespace/non-comment token.
4102 MIOpt.ReadToken();
4103
4104 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result));
4105 } else if (LangOpts.CPlusPlus && Char == '*') {
4106 Kind = tok::periodstar;
4107 CurPtr += SizeTmp;
4108 } else if (Char == '.' &&
4109 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '.') {
4110 Kind = tok::ellipsis;
4111 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4112 Size: SizeTmp2, Tok&: Result);
4113 } else {
4114 Kind = tok::period;
4115 }
4116 break;
4117 case '&':
4118 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4119 if (Char == '&') {
4120 Kind = tok::ampamp;
4121 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4122 } else if (Char == '=') {
4123 Kind = tok::ampequal;
4124 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4125 } else {
4126 Kind = tok::amp;
4127 }
4128 break;
4129 case '*':
4130 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4131 Kind = tok::starequal;
4132 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4133 } else {
4134 Kind = tok::star;
4135 }
4136 break;
4137 case '+':
4138 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4139 if (Char == '+') {
4140 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4141 Kind = tok::plusplus;
4142 } else if (Char == '=') {
4143 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4144 Kind = tok::plusequal;
4145 } else {
4146 Kind = tok::plus;
4147 }
4148 break;
4149 case '-':
4150 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4151 if (Char == '-') { // --
4152 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4153 Kind = tok::minusminus;
4154 } else if (Char == '>' && LangOpts.CPlusPlus &&
4155 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '*') { // C++ ->*
4156 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4157 Size: SizeTmp2, Tok&: Result);
4158 Kind = tok::arrowstar;
4159 } else if (Char == '>') { // ->
4160 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4161 Kind = tok::arrow;
4162 } else if (Char == '=') { // -=
4163 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4164 Kind = tok::minusequal;
4165 } else {
4166 Kind = tok::minus;
4167 }
4168 break;
4169 case '~':
4170 Kind = tok::tilde;
4171 break;
4172 case '!':
4173 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4174 Kind = tok::exclaimequal;
4175 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4176 } else {
4177 Kind = tok::exclaim;
4178 }
4179 break;
4180 case '/':
4181 // 6.4.9: Comments
4182 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4183 if (Char == '/') { // Line comment.
4184 // Even if Line comments are disabled (e.g. in C89 mode), we generally
4185 // want to lex this as a comment. There is one problem with this though,
4186 // that in one particular corner case, this can change the behavior of the
4187 // resultant program. For example, In "foo //**/ bar", C89 would lex
4188 // this as "foo / bar" and languages with Line comments would lex it as
4189 // "foo". Check to see if the character after the second slash is a '*'.
4190 // If so, we will lex that as a "/" instead of the start of a comment.
4191 // However, we never do this if we are just preprocessing.
4192 bool TreatAsComment =
4193 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP);
4194 if (!TreatAsComment)
4195 if (!(PP && PP->isPreprocessedOutput()))
4196 TreatAsComment = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) != '*';
4197
4198 if (TreatAsComment) {
4199 if (SkipLineComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4200 TokAtPhysicalStartOfLine))
4201 return true; // There is a token to return.
4202
4203 // It is common for the tokens immediately after a // comment to be
4204 // whitespace (indentation for the next line). Instead of going through
4205 // the big switch, handle it efficiently now.
4206 goto SkipIgnoredUnits;
4207 }
4208 }
4209
4210 if (Char == '*') { // /**/ comment.
4211 if (SkipBlockComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4212 TokAtPhysicalStartOfLine))
4213 return true; // There is a token to return.
4214
4215 // We only saw whitespace, so just try again with this lexer.
4216 // (We manually eliminate the tail call to avoid recursion.)
4217 goto LexNextToken;
4218 }
4219
4220 if (Char == '=') {
4221 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4222 Kind = tok::slashequal;
4223 } else {
4224 Kind = tok::slash;
4225 }
4226 break;
4227 case '%':
4228 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4229 if (Char == '=') {
4230 Kind = tok::percentequal;
4231 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4232 } else if (LangOpts.Digraphs && Char == '>') {
4233 Kind = tok::r_brace; // '%>' -> '}'
4234 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4235 } else if (LangOpts.Digraphs && Char == ':') {
4236 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4237 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4238 if (Char == '%' && getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == ':') {
4239 Kind = tok::hashhash; // '%:%:' -> '##'
4240 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4241 Size: SizeTmp2, Tok&: Result);
4242 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
4243 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4244 if (!isLexingRawMode())
4245 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4246 Kind = tok::hashat;
4247 } else { // '%:' -> '#'
4248 // We parsed a # character. If this occurs at the start of the line,
4249 // it's actually the start of a preprocessing directive. Callback to
4250 // the preprocessor to handle it.
4251 // TODO: -fpreprocessed mode??
4252 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) {
4253 // We parsed a # character and it's the start of a preprocessing
4254 // directive.
4255 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::hash);
4256 goto HandleDirective;
4257 }
4258
4259 Kind = tok::hash;
4260 }
4261 } else {
4262 Kind = tok::percent;
4263 }
4264 break;
4265 case '<':
4266 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4267 if (ParsingFilename) {
4268 return LexAngledStringLiteral(Result, CurPtr);
4269 } else if (Char == '<') {
4270 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4271 if (After == '=') {
4272 Kind = tok::lesslessequal;
4273 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4274 Size: SizeTmp2, Tok&: Result);
4275 } else if (After == '<' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4276 // If this is actually a '<<<<<<<' version control conflict marker,
4277 // recognize it as such and recover nicely.
4278 goto LexNextToken;
4279 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4280 // If this is '<<<<' and we're in a Perforce-style conflict marker,
4281 // ignore it.
4282 goto LexNextToken;
4283 } else if (LangOpts.CUDA && After == '<') {
4284 Kind = tok::lesslessless;
4285 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4286 Size: SizeTmp2, Tok&: Result);
4287 } else {
4288 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4289 Kind = tok::lessless;
4290 }
4291 } else if (Char == '=') {
4292 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4293 if (After == '>') {
4294 if (LangOpts.CPlusPlus20) {
4295 if (!isLexingRawMode())
4296 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx17_compat_spaceship);
4297 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4298 Size: SizeTmp2, Tok&: Result);
4299 Kind = tok::spaceship;
4300 break;
4301 }
4302 // Suggest adding a space between the '<=' and the '>' to avoid a
4303 // change in semantics if this turns up in C++ <=17 mode.
4304 if (LangOpts.CPlusPlus && !isLexingRawMode()) {
4305 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx20_compat_spaceship)
4306 << FixItHint::CreateInsertion(
4307 InsertionLoc: getSourceLocation(Loc: CurPtr + SizeTmp, TokLen: SizeTmp2), Code: " ");
4308 }
4309 }
4310 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4311 Kind = tok::lessequal;
4312 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '['
4313 if (LangOpts.CPlusPlus11 &&
4314 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == ':') {
4315 // C++0x [lex.pptoken]p3:
4316 // Otherwise, if the next three characters are <:: and the subsequent
4317 // character is neither : nor >, the < is treated as a preprocessor
4318 // token by itself and not as the first character of the alternative
4319 // token <:.
4320 unsigned SizeTmp3;
4321 char After = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4322 if (After != ':' && After != '>') {
4323 Kind = tok::less;
4324 if (!isLexingRawMode())
4325 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_less_colon_colon);
4326 break;
4327 }
4328 }
4329
4330 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4331 Kind = tok::l_square;
4332 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{'
4333 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4334 Kind = tok::l_brace;
4335 } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 &&
4336 lexEditorPlaceholder(Result, CurPtr)) {
4337 return true;
4338 } else {
4339 Kind = tok::less;
4340 }
4341 break;
4342 case '>':
4343 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4344 if (Char == '=') {
4345 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4346 Kind = tok::greaterequal;
4347 } else if (Char == '>') {
4348 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4349 if (After == '=') {
4350 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4351 Size: SizeTmp2, Tok&: Result);
4352 Kind = tok::greatergreaterequal;
4353 } else if (After == '>' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4354 // If this is actually a '>>>>' conflict marker, recognize it as such
4355 // and recover nicely.
4356 goto LexNextToken;
4357 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4358 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
4359 goto LexNextToken;
4360 } else if (LangOpts.CUDA && After == '>') {
4361 Kind = tok::greatergreatergreater;
4362 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4363 Size: SizeTmp2, Tok&: Result);
4364 } else {
4365 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4366 Kind = tok::greatergreater;
4367 }
4368 } else {
4369 Kind = tok::greater;
4370 }
4371 break;
4372 case '^':
4373 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4374 if (Char == '=') {
4375 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4376 Kind = tok::caretequal;
4377 } else {
4378 if (LangOpts.OpenCL && Char == '^')
4379 Diag(Loc: CurPtr, DiagID: diag::err_opencl_logical_exclusive_or);
4380 Kind = tok::caret;
4381 }
4382 break;
4383 case '|':
4384 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4385 if (Char == '=') {
4386 Kind = tok::pipeequal;
4387 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4388 } else if (Char == '|') {
4389 // If this is '|||||||' and we're in a conflict marker, ignore it.
4390 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4391 goto LexNextToken;
4392 Kind = tok::pipepipe;
4393 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4394 } else {
4395 Kind = tok::pipe;
4396 }
4397 break;
4398 case ':':
4399 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4400 if (LangOpts.Digraphs && Char == '>') {
4401 Kind = tok::r_square; // ':>' -> ']'
4402 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4403 } else if (Char == ':') {
4404 Kind = tok::coloncolon;
4405 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4406 } else {
4407 Kind = tok::colon;
4408 }
4409 break;
4410 case ';':
4411 Kind = tok::semi;
4412 break;
4413 case '=':
4414 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4415 if (Char == '=') {
4416 // If this is '====' and we're in a conflict marker, ignore it.
4417 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4418 goto LexNextToken;
4419
4420 Kind = tok::equalequal;
4421 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4422 } else {
4423 Kind = tok::equal;
4424 }
4425 break;
4426 case ',':
4427 Kind = tok::comma;
4428 break;
4429 case '#':
4430 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4431 if (Char == '#') {
4432 Kind = tok::hashhash;
4433 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4434 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize
4435 Kind = tok::hashat;
4436 if (!isLexingRawMode())
4437 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4438 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4439 } else {
4440 // We parsed a # character. If this occurs at the start of the line,
4441 // it's actually the start of a preprocessing directive. Callback to
4442 // the preprocessor to handle it.
4443 // TODO: -fpreprocessed mode??
4444 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) {
4445 // We parsed a # character and it's the start of a preprocessing
4446 // directive.
4447 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::hash);
4448 goto HandleDirective;
4449 }
4450
4451 Kind = tok::hash;
4452 }
4453 break;
4454
4455 case '@':
4456 // Objective C support.
4457 if (CurPtr[-1] == '@' && LangOpts.ObjC)
4458 Kind = tok::at;
4459 else
4460 Kind = tok::unknown;
4461 break;
4462
4463 // UCNs (C99 6.4.3, C++11 [lex.charset]p2)
4464 case '\\':
4465 if (!LangOpts.AsmPreprocessor) {
4466 if (uint32_t CodePoint = tryReadUCN(StartPtr&: CurPtr, SlashLoc: BufferPtr, Result: &Result)) {
4467 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4468 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4469 return true; // KeepWhitespaceMode
4470
4471 // We only saw whitespace, so just try again with this lexer.
4472 // (We manually eliminate the tail call to avoid recursion.)
4473 goto LexNextToken;
4474 }
4475
4476 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4477 }
4478 }
4479
4480 Kind = tok::unknown;
4481 break;
4482
4483 default: {
4484 if (isASCII(c: Char)) {
4485 Kind = tok::unknown;
4486 break;
4487 }
4488
4489 llvm::UTF32 CodePoint;
4490
4491 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
4492 // an escaped newline.
4493 --CurPtr;
4494 llvm::ConversionResult Status =
4495 llvm::convertUTF8Sequence(source: (const llvm::UTF8 **)&CurPtr,
4496 sourceEnd: (const llvm::UTF8 *)BufferEnd,
4497 target: &CodePoint,
4498 flags: llvm::strictConversion);
4499 if (Status == llvm::conversionOK) {
4500 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4501 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4502 return true; // KeepWhitespaceMode
4503
4504 // We only saw whitespace, so just try again with this lexer.
4505 // (We manually eliminate the tail call to avoid recursion.)
4506 goto LexNextToken;
4507 }
4508 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4509 }
4510
4511 if (isLexingRawMode() || ParsingPreprocessorDirective ||
4512 PP->isPreprocessedOutput()) {
4513 ++CurPtr;
4514 Kind = tok::unknown;
4515 break;
4516 }
4517
4518 // Non-ASCII characters tend to creep into source code unintentionally.
4519 // Instead of letting the parser complain about the unknown token,
4520 // just diagnose the invalid UTF-8, then drop the character.
4521 Diag(Loc: CurPtr, DiagID: diag::err_invalid_utf8);
4522
4523 BufferPtr = CurPtr+1;
4524 // We're pretending the character didn't exist, so just try again with
4525 // this lexer.
4526 // (We manually eliminate the tail call to avoid recursion.)
4527 goto LexNextToken;
4528 }
4529 }
4530
4531 // Notify MIOpt that we read a non-whitespace/non-comment token.
4532 MIOpt.ReadToken();
4533
4534 // Update the location of token as well as BufferPtr.
4535 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
4536 return true;
4537
4538HandleDirective:
4539 PP->HandleDirective(Result);
4540
4541 if (PP->hadModuleLoaderFatalFailure())
4542 // With a fatal failure in the module loader, we abort parsing.
4543 return true;
4544
4545 // We parsed the directive; lex a token with the new state.
4546 return false;
4547
4548LexNextToken:
4549 Result.clearFlag(Flag: Token::NeedsCleaning);
4550 goto LexStart;
4551}
4552
4553const char *Lexer::convertDependencyDirectiveToken(
4554 const dependency_directives_scan::Token &DDTok, Token &Result) {
4555 const char *TokPtr = BufferStart + DDTok.Offset;
4556 Result.startToken();
4557 Result.setLocation(getSourceLocation(Loc: TokPtr));
4558 Result.setKind(DDTok.Kind);
4559 Result.setFlag((Token::TokenFlags)DDTok.Flags);
4560 Result.setLength(DDTok.Length);
4561 if (Result.is(K: tok::raw_identifier))
4562 Result.setRawIdentifierData(TokPtr);
4563 else if (Result.isLiteral())
4564 Result.setLiteralData(TokPtr);
4565 BufferPtr = TokPtr + DDTok.Length;
4566 return TokPtr;
4567}
4568
4569bool Lexer::LexDependencyDirectiveToken(Token &Result) {
4570 assert(isDependencyDirectivesLexer());
4571
4572 using namespace dependency_directives_scan;
4573
4574 if (BufferPtr == BufferEnd)
4575 return LexEndOfFile(Result, CurPtr: BufferPtr);
4576
4577 while (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) {
4578 if (DepDirectives.front().Kind == pp_eof)
4579 return LexEndOfFile(Result, CurPtr: BufferEnd);
4580 if (DepDirectives.front().Kind == tokens_present_before_eof)
4581 MIOpt.ReadToken();
4582 NextDepDirectiveTokenIndex = 0;
4583 DepDirectives = DepDirectives.drop_front();
4584 }
4585
4586 const dependency_directives_scan::Token &DDTok =
4587 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex++];
4588 if (NextDepDirectiveTokenIndex > 1 || DDTok.Kind != tok::hash) {
4589 // Read something other than a preprocessor directive hash.
4590 MIOpt.ReadToken();
4591 }
4592
4593 if (ParsingFilename && DDTok.is(K: tok::less)) {
4594 BufferPtr = BufferStart + DDTok.Offset;
4595 LexAngledStringLiteral(Result, CurPtr: BufferPtr + 1);
4596 if (Result.isNot(K: tok::header_name))
4597 return true;
4598 // Advance the index of lexed tokens.
4599 while (true) {
4600 const dependency_directives_scan::Token &NextTok =
4601 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex];
4602 if (BufferStart + NextTok.Offset >= BufferPtr)
4603 break;
4604 ++NextDepDirectiveTokenIndex;
4605 }
4606 return true;
4607 }
4608
4609 const char *TokPtr = convertDependencyDirectiveToken(DDTok, Result);
4610
4611 if (Result.is(K: tok::hash) && Result.isAtStartOfLine()) {
4612 PP->HandleDirective(Result);
4613 if (PP->hadModuleLoaderFatalFailure())
4614 // With a fatal failure in the module loader, we abort parsing.
4615 return true;
4616 return false;
4617 }
4618 if (Result.is(K: tok::raw_identifier)) {
4619 Result.setRawIdentifierData(TokPtr);
4620 if (!isLexingRawMode()) {
4621 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
4622 if (LangOpts.CPlusPlusModules && Result.isModuleContextualKeyword() &&
4623 PP->HandleModuleContextualKeyword(Result, TokAtPhysicalStartOfLine: Result.isAtStartOfLine())) {
4624 PP->HandleDirective(Result);
4625 return false;
4626 }
4627 if (II->isHandleIdentifierCase())
4628 return PP->HandleIdentifier(Identifier&: Result);
4629 }
4630 return true;
4631 }
4632 if (Result.isLiteral())
4633 return true;
4634 if (Result.is(K: tok::colon)) {
4635 // Convert consecutive colons to 'tok::coloncolon'.
4636 if (*BufferPtr == ':') {
4637 assert(DepDirectives.front().Tokens[NextDepDirectiveTokenIndex].is(
4638 tok::colon));
4639 ++NextDepDirectiveTokenIndex;
4640 Result.setKind(tok::coloncolon);
4641 }
4642 return true;
4643 }
4644 if (Result.is(K: tok::eod))
4645 ParsingPreprocessorDirective = false;
4646
4647 return true;
4648}
4649
4650bool Lexer::LexDependencyDirectiveTokenWhileSkipping(Token &Result) {
4651 assert(isDependencyDirectivesLexer());
4652
4653 using namespace dependency_directives_scan;
4654
4655 bool Stop = false;
4656 unsigned NestedIfs = 0;
4657 do {
4658 DepDirectives = DepDirectives.drop_front();
4659 switch (DepDirectives.front().Kind) {
4660 case pp_none:
4661 llvm_unreachable("unexpected 'pp_none'");
4662 case pp_include:
4663 case pp___include_macros:
4664 case pp_define:
4665 case pp_undef:
4666 case pp_import:
4667 case pp_pragma_import:
4668 case pp_pragma_once:
4669 case pp_pragma_push_macro:
4670 case pp_pragma_pop_macro:
4671 case pp_pragma_include_alias:
4672 case pp_pragma_system_header:
4673 case pp_include_next:
4674 case decl_at_import:
4675 case cxx_module_decl:
4676 case cxx_import_decl:
4677 case cxx_export_module_decl:
4678 case cxx_export_import_decl:
4679 case tokens_present_before_eof:
4680 break;
4681 case pp_if:
4682 case pp_ifdef:
4683 case pp_ifndef:
4684 ++NestedIfs;
4685 break;
4686 case pp_elif:
4687 case pp_elifdef:
4688 case pp_elifndef:
4689 case pp_else:
4690 if (!NestedIfs) {
4691 Stop = true;
4692 }
4693 break;
4694 case pp_endif:
4695 if (!NestedIfs) {
4696 Stop = true;
4697 } else {
4698 --NestedIfs;
4699 }
4700 break;
4701 case pp_eof:
4702 NextDepDirectiveTokenIndex = 0;
4703 return LexEndOfFile(Result, CurPtr: BufferEnd);
4704 }
4705 } while (!Stop);
4706
4707 const dependency_directives_scan::Token &DDTok =
4708 DepDirectives.front().Tokens.front();
4709 assert(DDTok.is(tok::hash));
4710 NextDepDirectiveTokenIndex = 1;
4711
4712 convertDependencyDirectiveToken(DDTok, Result);
4713 return false;
4714}
4715