1//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Lexer and Token interfaces.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Lex/Lexer.h"
14#include "UnicodeCharSets.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/Diagnostic.h"
17#include "clang/Basic/IdentifierTable.h"
18#include "clang/Basic/LLVM.h"
19#include "clang/Basic/LangOptions.h"
20#include "clang/Basic/SourceLocation.h"
21#include "clang/Basic/SourceManager.h"
22#include "clang/Basic/TokenKinds.h"
23#include "clang/Lex/LexDiagnostic.h"
24#include "clang/Lex/LiteralSupport.h"
25#include "clang/Lex/MultipleIncludeOpt.h"
26#include "clang/Lex/Preprocessor.h"
27#include "clang/Lex/PreprocessorOptions.h"
28#include "clang/Lex/Token.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/StringSwitch.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/ConvertUTF.h"
35#include "llvm/Support/MemoryBufferRef.h"
36#include "llvm/Support/NativeFormatting.h"
37#include "llvm/Support/SaveAndRestore.h"
38#include "llvm/Support/Unicode.h"
39#include "llvm/Support/UnicodeCharRanges.h"
40#include <algorithm>
41#include <cassert>
42#include <cstddef>
43#include <cstdint>
44#include <cstring>
45#include <limits>
46#include <optional>
47#include <string>
48
49#ifdef __SSE4_2__
50#include <nmmintrin.h>
51#endif
52
53using namespace clang;
54
55//===----------------------------------------------------------------------===//
56// Token Class Implementation
57//===----------------------------------------------------------------------===//
58
59/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
60bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
61 if (isAnnotation())
62 return false;
63 if (const IdentifierInfo *II = getIdentifierInfo())
64 return II->getObjCKeywordID() == objcKey;
65 return false;
66}
67
68/// getObjCKeywordID - Return the ObjC keyword kind.
69tok::ObjCKeywordKind Token::getObjCKeywordID() const {
70 if (isAnnotation())
71 return tok::objc_not_keyword;
72 const IdentifierInfo *specId = getIdentifierInfo();
73 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
74}
75
76bool Token::isModuleContextualKeyword(bool AllowExport) const {
77 if (AllowExport && is(K: tok::kw_export))
78 return true;
79 if (isOneOf(Ks: tok::kw_import, Ks: tok::kw_module))
80 return true;
81 if (isNot(K: tok::identifier))
82 return false;
83 const auto *II = getIdentifierInfo();
84 return II->isImportKeyword() || II->isModuleKeyword();
85}
86
87/// Determine whether the token kind starts a simple-type-specifier.
88bool Token::isSimpleTypeSpecifier(const LangOptions &LangOpts) const {
89 switch (getKind()) {
90 case tok::annot_typename:
91 case tok::annot_decltype:
92 case tok::annot_pack_indexing_type:
93 return true;
94
95 case tok::kw_short:
96 case tok::kw_long:
97 case tok::kw___int64:
98 case tok::kw___int128:
99 case tok::kw_signed:
100 case tok::kw_unsigned:
101 case tok::kw_void:
102 case tok::kw_char:
103 case tok::kw_int:
104 case tok::kw_half:
105 case tok::kw_float:
106 case tok::kw_double:
107 case tok::kw___bf16:
108 case tok::kw__Float16:
109 case tok::kw___float128:
110 case tok::kw___ibm128:
111 case tok::kw_wchar_t:
112 case tok::kw_bool:
113 case tok::kw__Bool:
114 case tok::kw__Accum:
115 case tok::kw__Fract:
116 case tok::kw__Sat:
117#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
118#include "clang/Basic/Traits.inc"
119 case tok::kw___auto_type:
120 case tok::kw_char16_t:
121 case tok::kw_char32_t:
122 case tok::kw_typeof:
123 case tok::kw_decltype:
124 case tok::kw_char8_t:
125 return getIdentifierInfo()->isKeyword(LangOpts);
126
127 default:
128 return false;
129 }
130}
131
132//===----------------------------------------------------------------------===//
133// Lexer Class Implementation
134//===----------------------------------------------------------------------===//
135
136void Lexer::anchor() {}
137
138void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
139 const char *BufEnd) {
140 BufferStart = BufStart;
141 BufferPtr = BufPtr;
142 BufferEnd = BufEnd;
143
144 assert(BufEnd[0] == 0 &&
145 "We assume that the input buffer has a null character at the end"
146 " to simplify lexing!");
147
148 // Check whether we have a BOM in the beginning of the buffer. If yes - act
149 // accordingly. Right now we support only UTF-8 with and without BOM, so, just
150 // skip the UTF-8 BOM if it's present.
151 if (BufferStart == BufferPtr) {
152 // Determine the size of the BOM.
153 StringRef Buf(BufferStart, BufferEnd - BufferStart);
154 size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
155 .StartsWith(S: "\xEF\xBB\xBF", Value: 3) // UTF-8 BOM
156 .Default(Value: 0);
157
158 // Skip the BOM.
159 BufferPtr += BOMLength;
160 }
161
162 Is_PragmaLexer = false;
163 CurrentConflictMarkerState = CMK_None;
164
165 // Start of the file is a start of line.
166 IsAtStartOfLine = true;
167 IsAtPhysicalStartOfLine = true;
168
169 HasLeadingSpace = false;
170 HasLeadingEmptyMacro = false;
171
172 // We are not after parsing a #.
173 ParsingPreprocessorDirective = false;
174
175 // We are not after parsing #include.
176 ParsingFilename = false;
177
178 // We are not in raw mode. Raw mode disables diagnostics and interpretation
179 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
180 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
181 // or otherwise skipping over tokens.
182 LexingRawMode = false;
183
184 // Default to not keeping comments.
185 ExtendedTokenMode = 0;
186
187 NewLinePtr = nullptr;
188}
189
190/// Lexer constructor - Create a new lexer object for the specified buffer
191/// with the specified preprocessor managing the lexing process. This lexer
192/// assumes that the associated file buffer and Preprocessor objects will
193/// outlive it, so it doesn't take ownership of either of them.
194Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
195 Preprocessor &PP, bool IsFirstIncludeOfFile)
196 : PreprocessorLexer(&PP, FID),
197 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
198 LangOpts(PP.getLangOpts()), LineComment(LangOpts.LineComment),
199 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
200 InitLexer(BufStart: InputFile.getBufferStart(), BufPtr: InputFile.getBufferStart(),
201 BufEnd: InputFile.getBufferEnd());
202
203 resetExtendedTokenMode();
204}
205
206/// Lexer constructor - Create a new raw lexer object. This object is only
207/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
208/// range will outlive it, so it doesn't take ownership of it.
209Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
210 const char *BufStart, const char *BufPtr, const char *BufEnd,
211 bool IsFirstIncludeOfFile)
212 : FileLoc(fileloc), LangOpts(langOpts), LineComment(LangOpts.LineComment),
213 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
214 InitLexer(BufStart, BufPtr, BufEnd);
215
216 // We *are* in raw mode.
217 LexingRawMode = true;
218}
219
220/// Lexer constructor - Create a new raw lexer object. This object is only
221/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
222/// range will outlive it, so it doesn't take ownership of it.
223Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
224 const SourceManager &SM, const LangOptions &langOpts,
225 bool IsFirstIncludeOfFile)
226 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
227 FromFile.getBufferStart(), FromFile.getBufferEnd(),
228 IsFirstIncludeOfFile) {}
229
230void Lexer::resetExtendedTokenMode() {
231 assert(PP && "Cannot reset token mode without a preprocessor");
232 if (LangOpts.TraditionalCPP)
233 SetKeepWhitespaceMode(true);
234 else
235 SetCommentRetentionState(PP->getCommentRetentionState());
236}
237
238/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
239/// _Pragma expansion. This has a variety of magic semantics that this method
240/// sets up.
241///
242/// On entrance to this routine, TokStartLoc is a macro location which has a
243/// spelling loc that indicates the bytes to be lexed for the token and an
244/// expansion location that indicates where all lexed tokens should be
245/// "expanded from".
246///
247/// TODO: It would really be nice to make _Pragma just be a wrapper around a
248/// normal lexer that remaps tokens as they fly by. This would require making
249/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
250/// interface that could handle this stuff. This would pull GetMappedTokenLoc
251/// out of the critical path of the lexer!
252///
253std::unique_ptr<Lexer> Lexer::Create_PragmaLexer(
254 SourceLocation SpellingLoc, SourceLocation ExpansionLocStart,
255 SourceLocation ExpansionLocEnd, unsigned TokLen, Preprocessor &PP) {
256 SourceManager &SM = PP.getSourceManager();
257
258 // Create the lexer as if we were going to lex the file normally.
259 FileID SpellingFID = SM.getFileID(SpellingLoc);
260 llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(FID: SpellingFID);
261 auto L = std::make_unique<Lexer>(args&: SpellingFID, args&: InputFile, args&: PP);
262
263 // Now that the lexer is created, change the start/end locations so that we
264 // just lex the subsection of the file that we want. This is lexing from a
265 // scratch buffer.
266 const char *StrData = SM.getCharacterData(SL: SpellingLoc);
267
268 L->BufferPtr = StrData;
269 L->BufferEnd = StrData+TokLen;
270 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
271
272 // Set the SourceLocation with the remapping information. This ensures that
273 // GetMappedTokenLoc will remap the tokens as they are lexed.
274 L->FileLoc = SM.createExpansionLoc(SpellingLoc: SM.getLocForStartOfFile(FID: SpellingFID),
275 ExpansionLocStart,
276 ExpansionLocEnd, Length: TokLen);
277
278 // Ensure that the lexer thinks it is inside a directive, so that end \n will
279 // return an EOD token.
280 L->ParsingPreprocessorDirective = true;
281
282 // This lexer really is for _Pragma.
283 L->Is_PragmaLexer = true;
284 return L;
285}
286
287void Lexer::seek(unsigned Offset, bool IsAtStartOfLine) {
288 this->IsAtPhysicalStartOfLine = IsAtStartOfLine;
289 this->IsAtStartOfLine = IsAtStartOfLine;
290 assert((BufferStart + Offset) <= BufferEnd);
291 BufferPtr = BufferStart + Offset;
292}
293
294template <typename T> static void StringifyImpl(T &Str, char Quote) {
295 typename T::size_type i = 0, e = Str.size();
296 while (i < e) {
297 if (Str[i] == '\\' || Str[i] == Quote) {
298 Str.insert(Str.begin() + i, '\\');
299 i += 2;
300 ++e;
301 } else if (Str[i] == '\n' || Str[i] == '\r') {
302 // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
303 if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') &&
304 Str[i] != Str[i + 1]) {
305 Str[i] = '\\';
306 Str[i + 1] = 'n';
307 } else {
308 // Replace '\n' and '\r' to '\\' followed by 'n'.
309 Str[i] = '\\';
310 Str.insert(Str.begin() + i + 1, 'n');
311 ++e;
312 }
313 i += 2;
314 } else
315 ++i;
316 }
317}
318
319std::string Lexer::Stringify(StringRef Str, bool Charify) {
320 std::string Result = std::string(Str);
321 char Quote = Charify ? '\'' : '"';
322 StringifyImpl(Str&: Result, Quote);
323 return Result;
324}
325
326void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, Quote: '"'); }
327
328//===----------------------------------------------------------------------===//
329// Token Spelling
330//===----------------------------------------------------------------------===//
331
332/// Slow case of getSpelling. Extract the characters comprising the
333/// spelling of this token from the provided input buffer.
334static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
335 const LangOptions &LangOpts, char *Spelling) {
336 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
337
338 size_t Length = 0;
339 const char *BufEnd = BufPtr + Tok.getLength();
340
341 if (tok::isStringLiteral(K: Tok.getKind())) {
342 // Munch the encoding-prefix and opening double-quote.
343 while (BufPtr < BufEnd) {
344 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
345 Spelling[Length++] = CharAndSize.Char;
346 BufPtr += CharAndSize.Size;
347
348 if (Spelling[Length - 1] == '"')
349 break;
350 }
351
352 // Raw string literals need special handling; trigraph expansion and line
353 // splicing do not occur within their d-char-sequence nor within their
354 // r-char-sequence.
355 if (Length >= 2 &&
356 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') {
357 // Search backwards from the end of the token to find the matching closing
358 // quote.
359 const char *RawEnd = BufEnd;
360 do --RawEnd; while (*RawEnd != '"');
361 size_t RawLength = RawEnd - BufPtr + 1;
362
363 // Everything between the quotes is included verbatim in the spelling.
364 memcpy(dest: Spelling + Length, src: BufPtr, n: RawLength);
365 Length += RawLength;
366 BufPtr += RawLength;
367
368 // The rest of the token is lexed normally.
369 }
370 }
371
372 while (BufPtr < BufEnd) {
373 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
374 Spelling[Length++] = CharAndSize.Char;
375 BufPtr += CharAndSize.Size;
376 }
377
378 assert(Length < Tok.getLength() &&
379 "NeedsCleaning flag set on token that didn't need cleaning!");
380 return Length;
381}
382
383/// getSpelling() - Return the 'spelling' of this token. The spelling of a
384/// token are the characters used to represent the token in the source file
385/// after trigraph expansion and escaped-newline folding. In particular, this
386/// wants to get the true, uncanonicalized, spelling of things like digraphs
387/// UCNs, etc.
388StringRef Lexer::getSpelling(SourceLocation loc,
389 SmallVectorImpl<char> &buffer,
390 const SourceManager &SM,
391 const LangOptions &options,
392 bool *invalid) {
393 // Break down the source location.
394 FileIDAndOffset locInfo = SM.getDecomposedLoc(Loc: loc);
395
396 // Try to the load the file buffer.
397 bool invalidTemp = false;
398 StringRef file = SM.getBufferData(FID: locInfo.first, Invalid: &invalidTemp);
399 if (invalidTemp) {
400 if (invalid) *invalid = true;
401 return {};
402 }
403
404 const char *tokenBegin = file.data() + locInfo.second;
405
406 // Lex from the start of the given location.
407 Lexer lexer(SM.getLocForStartOfFile(FID: locInfo.first), options,
408 file.begin(), tokenBegin, file.end());
409 Token token;
410 lexer.LexFromRawLexer(Result&: token);
411
412 unsigned length = token.getLength();
413
414 // Common case: no need for cleaning.
415 if (!token.needsCleaning())
416 return StringRef(tokenBegin, length);
417
418 // Hard case, we need to relex the characters into the string.
419 buffer.resize(N: length);
420 buffer.resize(N: getSpellingSlow(Tok: token, BufPtr: tokenBegin, LangOpts: options, Spelling: buffer.data()));
421 return StringRef(buffer.data(), buffer.size());
422}
423
424/// getSpelling() - Return the 'spelling' of this token. The spelling of a
425/// token are the characters used to represent the token in the source file
426/// after trigraph expansion and escaped-newline folding. In particular, this
427/// wants to get the true, uncanonicalized, spelling of things like digraphs
428/// UCNs, etc.
429std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
430 const LangOptions &LangOpts, bool *Invalid) {
431 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
432
433 bool CharDataInvalid = false;
434 const char *TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(),
435 Invalid: &CharDataInvalid);
436 if (Invalid)
437 *Invalid = CharDataInvalid;
438 if (CharDataInvalid)
439 return {};
440
441 // If this token contains nothing interesting, return it directly.
442 if (!Tok.needsCleaning())
443 return std::string(TokStart, TokStart + Tok.getLength());
444
445 std::string Result;
446 Result.resize(n: Tok.getLength());
447 Result.resize(n: getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: &*Result.begin()));
448 return Result;
449}
450
451/// getSpelling - This method is used to get the spelling of a token into a
452/// preallocated buffer, instead of as an std::string. The caller is required
453/// to allocate enough space for the token, which is guaranteed to be at least
454/// Tok.getLength() bytes long. The actual length of the token is returned.
455///
456/// Note that this method may do two possible things: it may either fill in
457/// the buffer specified with characters, or it may *change the input pointer*
458/// to point to a constant buffer with the data already in it (avoiding a
459/// copy). The caller is not allowed to modify the returned buffer pointer
460/// if an internal buffer is returned.
461unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
462 const SourceManager &SourceMgr,
463 const LangOptions &LangOpts, bool *Invalid) {
464 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
465
466 const char *TokStart = nullptr;
467 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
468 if (Tok.is(K: tok::raw_identifier))
469 TokStart = Tok.getRawIdentifier().data();
470 else if (!Tok.hasUCN()) {
471 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
472 // Just return the string from the identifier table, which is very quick.
473 Buffer = II->getNameStart();
474 return II->getLength();
475 }
476 }
477
478 // NOTE: this can be checked even after testing for an IdentifierInfo.
479 if (Tok.isLiteral())
480 TokStart = Tok.getLiteralData();
481
482 if (!TokStart) {
483 // Compute the start of the token in the input lexer buffer.
484 bool CharDataInvalid = false;
485 TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(), Invalid: &CharDataInvalid);
486 if (Invalid)
487 *Invalid = CharDataInvalid;
488 if (CharDataInvalid) {
489 Buffer = "";
490 return 0;
491 }
492 }
493
494 // If this token contains nothing interesting, return it directly.
495 if (!Tok.needsCleaning()) {
496 Buffer = TokStart;
497 return Tok.getLength();
498 }
499
500 // Otherwise, hard case, relex the characters into the string.
501 return getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: const_cast<char*>(Buffer));
502}
503
504/// MeasureTokenLength - Relex the token at the specified location and return
505/// its length in bytes in the input file. If the token needs cleaning (e.g.
506/// includes a trigraph or an escaped newline) then this count includes bytes
507/// that are part of that.
508unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
509 const SourceManager &SM,
510 const LangOptions &LangOpts) {
511 Token TheTok;
512 if (getRawToken(Loc, Result&: TheTok, SM, LangOpts))
513 return 0;
514 return TheTok.getLength();
515}
516
517SourceLocation Lexer::findEndOfIdentifierContinuation(
518 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
519 Loc = SM.getExpansionLoc(Loc);
520 const FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
521 bool Invalid = false;
522 const StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
523 if (Invalid)
524 return Loc;
525
526 const char *StrData = Buffer.data() + LocInfo.second;
527 if (StrData >= Buffer.end())
528 return Loc;
529
530 // Use the lexer continuation rules directly, without requiring identifier
531 // start at Loc.
532 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
533 Buffer.begin(), StrData, Buffer.end());
534 Token Tok;
535 Tok.startToken();
536 TheLexer.LexIdentifierContinue(Result&: Tok, CurPtr: StrData);
537 return Loc.getLocWithOffset(Offset: Tok.getLength());
538}
539
540/// Relex the token at the specified location.
541/// \returns true if there was a failure, false on success.
542bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
543 const SourceManager &SM,
544 const LangOptions &LangOpts,
545 bool IgnoreWhiteSpace) {
546 // TODO: this could be special cased for common tokens like identifiers, ')',
547 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
548 // all obviously single-char tokens. This could use
549 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
550 // something.
551
552 // If this comes from a macro expansion, we really do want the macro name, not
553 // the token this macro expanded to.
554 Loc = SM.getExpansionLoc(Loc);
555 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
556 bool Invalid = false;
557 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
558 if (Invalid)
559 return true;
560
561 const char *StrData = Buffer.data()+LocInfo.second;
562
563 if (!IgnoreWhiteSpace && isWhitespace(c: SkipEscapedNewLines(P: StrData)[0]))
564 return true;
565
566 // Create a lexer starting at the beginning of this token.
567 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
568 Buffer.begin(), StrData, Buffer.end());
569 TheLexer.SetCommentRetentionState(true);
570 TheLexer.LexFromRawLexer(Result);
571 return false;
572}
573
574/// Returns the pointer that points to the beginning of line that contains
575/// the given offset, or null if the offset if invalid.
576static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
577 const char *BufStart = Buffer.data();
578 if (Offset >= Buffer.size())
579 return nullptr;
580
581 const char *LexStart = BufStart + Offset;
582 for (; LexStart != BufStart; --LexStart) {
583 if (isVerticalWhitespace(c: LexStart[0]) &&
584 !Lexer::isNewLineEscaped(BufferStart: BufStart, Str: LexStart)) {
585 // LexStart should point at first character of logical line.
586 ++LexStart;
587 break;
588 }
589 }
590 return LexStart;
591}
592
593static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
594 const SourceManager &SM,
595 const LangOptions &LangOpts) {
596 assert(Loc.isFileID());
597 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
598 if (LocInfo.first.isInvalid())
599 return Loc;
600
601 bool Invalid = false;
602 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
603 if (Invalid)
604 return Loc;
605
606 // Back up from the current location until we hit the beginning of a line
607 // (or the buffer). We'll relex from that point.
608 const char *StrData = Buffer.data() + LocInfo.second;
609 const char *LexStart = findBeginningOfLine(Buffer, Offset: LocInfo.second);
610 if (!LexStart || LexStart == StrData)
611 return Loc;
612
613 // Create a lexer starting at the beginning of this token.
614 SourceLocation LexerStartLoc = Loc.getLocWithOffset(Offset: -LocInfo.second);
615 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
616 Buffer.end());
617 TheLexer.SetCommentRetentionState(true);
618
619 // Lex tokens until we find the token that contains the source location.
620 Token TheTok;
621 do {
622 TheLexer.LexFromRawLexer(Result&: TheTok);
623
624 if (TheLexer.getBufferLocation() > StrData) {
625 // Lexing this token has taken the lexer past the source location we're
626 // looking for. If the current token encompasses our source location,
627 // return the beginning of that token.
628 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
629 return TheTok.getLocation();
630
631 // We ended up skipping over the source location entirely, which means
632 // that it points into whitespace. We're done here.
633 break;
634 }
635 } while (TheTok.getKind() != tok::eof);
636
637 // We've passed our source location; just return the original source location.
638 return Loc;
639}
640
641SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
642 const SourceManager &SM,
643 const LangOptions &LangOpts) {
644 if (Loc.isFileID())
645 return getBeginningOfFileToken(Loc, SM, LangOpts);
646
647 if (!SM.isMacroArgExpansion(Loc))
648 return Loc;
649
650 SourceLocation FileLoc = SM.getSpellingLoc(Loc);
651 SourceLocation BeginFileLoc = getBeginningOfFileToken(Loc: FileLoc, SM, LangOpts);
652 FileIDAndOffset FileLocInfo = SM.getDecomposedLoc(Loc: FileLoc);
653 FileIDAndOffset BeginFileLocInfo = SM.getDecomposedLoc(Loc: BeginFileLoc);
654 assert(FileLocInfo.first == BeginFileLocInfo.first &&
655 FileLocInfo.second >= BeginFileLocInfo.second);
656 return Loc.getLocWithOffset(Offset: BeginFileLocInfo.second - FileLocInfo.second);
657}
658
659namespace {
660
661enum PreambleDirectiveKind {
662 PDK_Skipped,
663 PDK_Unknown
664};
665
666} // namespace
667
668PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
669 const LangOptions &LangOpts,
670 unsigned MaxLines) {
671 // Create a lexer starting at the beginning of the file. Note that we use a
672 // "fake" file source location at offset 1 so that the lexer will track our
673 // position within the file.
674 const SourceLocation::UIntTy StartOffset = 1;
675 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(Encoding: StartOffset);
676 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
677 Buffer.end());
678 TheLexer.SetCommentRetentionState(true);
679
680 bool InPreprocessorDirective = false;
681 Token TheTok;
682 SourceLocation ActiveCommentLoc;
683
684 unsigned MaxLineOffset = 0;
685 if (MaxLines) {
686 const char *CurPtr = Buffer.begin();
687 unsigned CurLine = 0;
688 while (CurPtr != Buffer.end()) {
689 char ch = *CurPtr++;
690 if (ch == '\n') {
691 ++CurLine;
692 if (CurLine == MaxLines)
693 break;
694 }
695 }
696 if (CurPtr != Buffer.end())
697 MaxLineOffset = CurPtr - Buffer.begin();
698 }
699
700 do {
701 TheLexer.LexFromRawLexer(Result&: TheTok);
702
703 if (InPreprocessorDirective) {
704 // If we've hit the end of the file, we're done.
705 if (TheTok.getKind() == tok::eof) {
706 break;
707 }
708
709 // If we haven't hit the end of the preprocessor directive, skip this
710 // token.
711 if (!TheTok.isAtStartOfLine())
712 continue;
713
714 // We've passed the end of the preprocessor directive, and will look
715 // at this token again below.
716 InPreprocessorDirective = false;
717 }
718
719 // Keep track of the # of lines in the preamble.
720 if (TheTok.isAtStartOfLine()) {
721 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
722
723 // If we were asked to limit the number of lines in the preamble,
724 // and we're about to exceed that limit, we're done.
725 if (MaxLineOffset && TokOffset >= MaxLineOffset)
726 break;
727 }
728
729 // Comments are okay; skip over them.
730 if (TheTok.getKind() == tok::comment) {
731 if (ActiveCommentLoc.isInvalid())
732 ActiveCommentLoc = TheTok.getLocation();
733 continue;
734 }
735
736 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
737 // This is the start of a preprocessor directive.
738 Token HashTok = TheTok;
739 InPreprocessorDirective = true;
740 ActiveCommentLoc = SourceLocation();
741
742 // Figure out which directive this is. Since we're lexing raw tokens,
743 // we don't have an identifier table available. Instead, just look at
744 // the raw identifier to recognize and categorize preprocessor directives.
745 TheLexer.LexFromRawLexer(Result&: TheTok);
746 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
747 StringRef Keyword = TheTok.getRawIdentifier();
748 PreambleDirectiveKind PDK
749 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
750 .Case(S: "include", Value: PDK_Skipped)
751 .Case(S: "__include_macros", Value: PDK_Skipped)
752 .Case(S: "define", Value: PDK_Skipped)
753 .Case(S: "undef", Value: PDK_Skipped)
754 .Case(S: "line", Value: PDK_Skipped)
755 .Case(S: "error", Value: PDK_Skipped)
756 .Case(S: "pragma", Value: PDK_Skipped)
757 .Case(S: "import", Value: PDK_Skipped)
758 .Case(S: "include_next", Value: PDK_Skipped)
759 .Case(S: "warning", Value: PDK_Skipped)
760 .Case(S: "ident", Value: PDK_Skipped)
761 .Case(S: "sccs", Value: PDK_Skipped)
762 .Case(S: "assert", Value: PDK_Skipped)
763 .Case(S: "unassert", Value: PDK_Skipped)
764 .Case(S: "if", Value: PDK_Skipped)
765 .Case(S: "ifdef", Value: PDK_Skipped)
766 .Case(S: "ifndef", Value: PDK_Skipped)
767 .Case(S: "elif", Value: PDK_Skipped)
768 .Case(S: "elifdef", Value: PDK_Skipped)
769 .Case(S: "elifndef", Value: PDK_Skipped)
770 .Case(S: "else", Value: PDK_Skipped)
771 .Case(S: "endif", Value: PDK_Skipped)
772 .Default(Value: PDK_Unknown);
773
774 switch (PDK) {
775 case PDK_Skipped:
776 continue;
777
778 case PDK_Unknown:
779 // We don't know what this directive is; stop at the '#'.
780 break;
781 }
782 }
783
784 // We only end up here if we didn't recognize the preprocessor
785 // directive or it was one that can't occur in the preamble at this
786 // point. Roll back the current token to the location of the '#'.
787 TheTok = HashTok;
788 } else if (TheTok.isAtStartOfLine() &&
789 TheTok.getKind() == tok::raw_identifier &&
790 TheTok.getRawIdentifier() == "module" &&
791 LangOpts.CPlusPlusModules) {
792 // The initial global module fragment introducer "module;" is part of
793 // the preamble, which runs up to the module declaration "module foo;".
794 Token ModuleTok = TheTok;
795 do {
796 TheLexer.LexFromRawLexer(Result&: TheTok);
797 } while (TheTok.getKind() == tok::comment);
798 if (TheTok.getKind() != tok::semi) {
799 // Not global module fragment, roll back.
800 TheTok = ModuleTok;
801 break;
802 }
803 continue;
804 }
805
806 // We hit a token that we don't recognize as being in the
807 // "preprocessing only" part of the file, so we're no longer in
808 // the preamble.
809 break;
810 } while (true);
811
812 SourceLocation End;
813 if (ActiveCommentLoc.isValid())
814 End = ActiveCommentLoc; // don't truncate a decl comment.
815 else
816 End = TheTok.getLocation();
817
818 return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
819 TheTok.isAtStartOfLine());
820}
821
822unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
823 const SourceManager &SM,
824 const LangOptions &LangOpts) {
825 // Figure out how many physical characters away the specified expansion
826 // character is. This needs to take into consideration newlines and
827 // trigraphs.
828 bool Invalid = false;
829 const char *TokPtr = SM.getCharacterData(SL: TokStart, Invalid: &Invalid);
830
831 // If they request the first char of the token, we're trivially done.
832 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(C: *TokPtr)))
833 return 0;
834
835 unsigned PhysOffset = 0;
836
837 // The usual case is that tokens don't contain anything interesting. Skip
838 // over the uninteresting characters. If a token only consists of simple
839 // chars, this method is extremely fast.
840 while (Lexer::isObviouslySimpleCharacter(C: *TokPtr)) {
841 if (CharNo == 0)
842 return PhysOffset;
843 ++TokPtr;
844 --CharNo;
845 ++PhysOffset;
846 }
847
848 // If we have a character that may be a trigraph or escaped newline, use a
849 // lexer to parse it correctly.
850 for (; CharNo; --CharNo) {
851 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: TokPtr, LangOpts);
852 TokPtr += CharAndSize.Size;
853 PhysOffset += CharAndSize.Size;
854 }
855
856 // Final detail: if we end up on an escaped newline, we want to return the
857 // location of the actual byte of the token. For example foo\<newline>bar
858 // advanced by 3 should return the location of b, not of \\. One compounding
859 // detail of this is that the escape may be made by a trigraph.
860 if (!Lexer::isObviouslySimpleCharacter(C: *TokPtr))
861 PhysOffset += Lexer::SkipEscapedNewLines(P: TokPtr)-TokPtr;
862
863 return PhysOffset;
864}
865
866/// Computes the source location just past the end of the
867/// token at this source location.
868///
869/// This routine can be used to produce a source location that
870/// points just past the end of the token referenced by \p Loc, and
871/// is generally used when a diagnostic needs to point just after a
872/// token where it expected something different that it received. If
873/// the returned source location would not be meaningful (e.g., if
874/// it points into a macro), this routine returns an invalid
875/// source location.
876///
877/// \param Offset an offset from the end of the token, where the source
878/// location should refer to. The default offset (0) produces a source
879/// location pointing just past the end of the token; an offset of 1 produces
880/// a source location pointing to the last character in the token, etc.
881SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
882 const SourceManager &SM,
883 const LangOptions &LangOpts) {
884 if (Loc.isInvalid())
885 return {};
886
887 if (Loc.isMacroID()) {
888 // Token split (for example, splitting '>>' into two '>' tokens) is
889 // represented in SourceManager as an ExpansionInfo (see
890 // createForTokenSplit), so these locations are MacroIDs even when no user
891 // macro is involved. For split expansions, the expansion end is already
892 // the correct insertion point.
893 const FileID LocFileID = SM.getFileID(SpellingLoc: Loc);
894 if (Offset > 0 || !isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
895 return {}; // Points inside the macro expansion.
896 if (!SM.getSLocEntry(FID: LocFileID).getExpansion().isExpansionTokenRange())
897 return Loc;
898 }
899
900 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
901 if (Len > Offset)
902 Len = Len - Offset;
903 else
904 return Loc;
905
906 return Loc.getLocWithOffset(Offset: Len);
907}
908
909/// Returns true if the given MacroID location points at the first
910/// token of the macro expansion.
911bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
912 const SourceManager &SM,
913 const LangOptions &LangOpts,
914 SourceLocation *MacroBegin) {
915 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
916
917 SourceLocation expansionLoc;
918 if (!SM.isAtStartOfImmediateMacroExpansion(Loc: loc, MacroBegin: &expansionLoc))
919 return false;
920
921 if (expansionLoc.isFileID()) {
922 // No other macro expansions, this is the first.
923 if (MacroBegin)
924 *MacroBegin = expansionLoc;
925 return true;
926 }
927
928 return isAtStartOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroBegin);
929}
930
931/// Returns true if the given MacroID location points at the last
932/// token of the macro expansion.
933bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
934 const SourceManager &SM,
935 const LangOptions &LangOpts,
936 SourceLocation *MacroEnd) {
937 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
938
939 SourceLocation spellLoc = SM.getSpellingLoc(Loc: loc);
940 unsigned tokLen = MeasureTokenLength(Loc: spellLoc, SM, LangOpts);
941 if (tokLen == 0)
942 return false;
943
944 SourceLocation afterLoc = loc.getLocWithOffset(Offset: tokLen);
945 SourceLocation expansionLoc;
946 FileID FID = SM.getFileID(SpellingLoc: loc);
947
948 if (SM.isInFileID(Loc: afterLoc, FID)) {
949 if (!SM.isAtEndOfImmediateMacroExpansion(Loc: afterLoc, MacroEnd: &expansionLoc))
950 return false;
951 } else {
952 // During error recovery, a zero-length synthetic token might be inserted
953 // past the end of the FileID, e.g. inserting ")" when a macro-arg
954 // containing a comma should be guarded by parentheses. In this case,
955 // afterLoc reaches the `NextLocalOffset` boundary, any operations on
956 // afterLoc will be invalid!
957 const SrcMgr::SLocEntry &Entry = SM.getSLocEntry(FID);
958 assert(Entry.isExpansion() && "Should be in an expansion");
959 expansionLoc = Entry.getExpansion().getExpansionLocEnd();
960 }
961
962 if (expansionLoc.isFileID()) {
963 // No other macro expansions.
964 if (MacroEnd)
965 *MacroEnd = expansionLoc;
966 return true;
967 }
968
969 return isAtEndOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroEnd);
970}
971
972static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
973 const SourceManager &SM,
974 const LangOptions &LangOpts) {
975 SourceLocation Begin = Range.getBegin();
976 SourceLocation End = Range.getEnd();
977 assert(Begin.isFileID() && End.isFileID());
978 if (Range.isTokenRange()) {
979 End = Lexer::getLocForEndOfToken(Loc: End, Offset: 0, SM,LangOpts);
980 if (End.isInvalid())
981 return {};
982 }
983
984 // Break down the source locations.
985 auto [FID, BeginOffs] = SM.getDecomposedLoc(Loc: Begin);
986 if (FID.isInvalid())
987 return {};
988
989 unsigned EndOffs;
990 if (!SM.isInFileID(Loc: End, FID, RelativeOffset: &EndOffs) ||
991 BeginOffs > EndOffs)
992 return {};
993
994 return CharSourceRange::getCharRange(B: Begin, E: End);
995}
996
997// Assumes that `Loc` is in an expansion.
998static bool isInExpansionTokenRange(const SourceLocation Loc,
999 const SourceManager &SM) {
1000 return SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Loc))
1001 .getExpansion()
1002 .isExpansionTokenRange();
1003}
1004
1005CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
1006 const SourceManager &SM,
1007 const LangOptions &LangOpts) {
1008 SourceLocation Begin = Range.getBegin();
1009 SourceLocation End = Range.getEnd();
1010 if (Begin.isInvalid() || End.isInvalid())
1011 return {};
1012
1013 if (Begin.isFileID() && End.isFileID())
1014 return makeRangeFromFileLocs(Range, SM, LangOpts);
1015
1016 if (Begin.isMacroID() && End.isFileID()) {
1017 if (!isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &Begin))
1018 return {};
1019 Range.setBegin(Begin);
1020 return makeRangeFromFileLocs(Range, SM, LangOpts);
1021 }
1022
1023 if (Begin.isFileID() && End.isMacroID()) {
1024 if (Range.isTokenRange()) {
1025 if (!isAtEndOfMacroExpansion(loc: End, SM, LangOpts, MacroEnd: &End))
1026 return {};
1027 // Use the *original* end, not the expanded one in `End`.
1028 Range.setTokenRange(isInExpansionTokenRange(Loc: Range.getEnd(), SM));
1029 } else if (!isAtStartOfMacroExpansion(loc: End, SM, LangOpts, MacroBegin: &End))
1030 return {};
1031 Range.setEnd(End);
1032 return makeRangeFromFileLocs(Range, SM, LangOpts);
1033 }
1034
1035 assert(Begin.isMacroID() && End.isMacroID());
1036 SourceLocation MacroBegin, MacroEnd;
1037 if (isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &MacroBegin) &&
1038 ((Range.isTokenRange() && isAtEndOfMacroExpansion(loc: End, SM, LangOpts,
1039 MacroEnd: &MacroEnd)) ||
1040 (Range.isCharRange() && isAtStartOfMacroExpansion(loc: End, SM, LangOpts,
1041 MacroBegin: &MacroEnd)))) {
1042 Range.setBegin(MacroBegin);
1043 Range.setEnd(MacroEnd);
1044 // Use the *original* `End`, not the expanded one in `MacroEnd`.
1045 if (Range.isTokenRange())
1046 Range.setTokenRange(isInExpansionTokenRange(Loc: End, SM));
1047 return makeRangeFromFileLocs(Range, SM, LangOpts);
1048 }
1049
1050 bool Invalid = false;
1051 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Begin),
1052 Invalid: &Invalid);
1053 if (Invalid)
1054 return {};
1055
1056 if (BeginEntry.getExpansion().isMacroArgExpansion()) {
1057 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: End),
1058 Invalid: &Invalid);
1059 if (Invalid)
1060 return {};
1061
1062 if (EndEntry.getExpansion().isMacroArgExpansion() &&
1063 BeginEntry.getExpansion().getExpansionLocStart() ==
1064 EndEntry.getExpansion().getExpansionLocStart()) {
1065 Range.setBegin(SM.getImmediateSpellingLoc(Loc: Begin));
1066 Range.setEnd(SM.getImmediateSpellingLoc(Loc: End));
1067 return makeFileCharRange(Range, SM, LangOpts);
1068 }
1069 }
1070
1071 return {};
1072}
1073
1074StringRef Lexer::getSourceText(CharSourceRange Range,
1075 const SourceManager &SM,
1076 const LangOptions &LangOpts,
1077 bool *Invalid) {
1078 Range = makeFileCharRange(Range, SM, LangOpts);
1079 if (Range.isInvalid()) {
1080 if (Invalid) *Invalid = true;
1081 return {};
1082 }
1083
1084 // Break down the source location.
1085 FileIDAndOffset beginInfo = SM.getDecomposedLoc(Loc: Range.getBegin());
1086 if (beginInfo.first.isInvalid()) {
1087 if (Invalid) *Invalid = true;
1088 return {};
1089 }
1090
1091 unsigned EndOffs;
1092 if (!SM.isInFileID(Loc: Range.getEnd(), FID: beginInfo.first, RelativeOffset: &EndOffs) ||
1093 beginInfo.second > EndOffs) {
1094 if (Invalid) *Invalid = true;
1095 return {};
1096 }
1097
1098 // Try to the load the file buffer.
1099 bool invalidTemp = false;
1100 StringRef file = SM.getBufferData(FID: beginInfo.first, Invalid: &invalidTemp);
1101 if (invalidTemp) {
1102 if (Invalid) *Invalid = true;
1103 return {};
1104 }
1105
1106 if (Invalid) *Invalid = false;
1107 return file.substr(Start: beginInfo.second, N: EndOffs - beginInfo.second);
1108}
1109
1110StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1111 const SourceManager &SM,
1112 const LangOptions &LangOpts) {
1113 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1114
1115 // Find the location of the immediate macro expansion.
1116 while (true) {
1117 FileID FID = SM.getFileID(SpellingLoc: Loc);
1118 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
1119 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
1120 Loc = Expansion.getExpansionLocStart();
1121 if (!Expansion.isMacroArgExpansion())
1122 break;
1123
1124 // For macro arguments we need to check that the argument did not come
1125 // from an inner macro, e.g: "MAC1( MAC2(foo) )"
1126
1127 // Loc points to the argument id of the macro definition, move to the
1128 // macro expansion.
1129 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1130 SourceLocation SpellLoc = Expansion.getSpellingLoc();
1131 if (SpellLoc.isFileID())
1132 break; // No inner macro.
1133
1134 // If spelling location resides in the same FileID as macro expansion
1135 // location, it means there is no inner macro.
1136 FileID MacroFID = SM.getFileID(SpellingLoc: Loc);
1137 if (SM.isInFileID(Loc: SpellLoc, FID: MacroFID))
1138 break;
1139
1140 // Argument came from inner macro.
1141 Loc = SpellLoc;
1142 }
1143
1144 // Find the spelling location of the start of the non-argument expansion
1145 // range. This is where the macro name was spelled in order to begin
1146 // expanding this macro.
1147 Loc = SM.getSpellingLoc(Loc);
1148
1149 // Dig out the buffer where the macro name was spelled and the extents of the
1150 // name so that we can render it into the expansion note.
1151 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1152 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1153 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1154 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1155}
1156
1157StringRef Lexer::getImmediateMacroNameForDiagnostics(
1158 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
1159 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1160 // Walk past macro argument expansions.
1161 while (SM.isMacroArgExpansion(Loc))
1162 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1163
1164 // If the macro's spelling isn't FileID or from scratch space, then it's
1165 // actually a token paste or stringization (or similar) and not a macro at
1166 // all.
1167 SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
1168 if (!SpellLoc.isFileID() || SM.isWrittenInScratchSpace(Loc: SpellLoc))
1169 return {};
1170
1171 // Find the spelling location of the start of the non-argument expansion
1172 // range. This is where the macro name was spelled in order to begin
1173 // expanding this macro.
1174 Loc = SM.getSpellingLoc(Loc: SM.getImmediateExpansionRange(Loc).getBegin());
1175
1176 // Dig out the buffer where the macro name was spelled and the extents of the
1177 // name so that we can render it into the expansion note.
1178 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1179 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1180 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1181 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1182}
1183
1184bool Lexer::isAsciiIdentifierContinueChar(char c, const LangOptions &LangOpts) {
1185 return isAsciiIdentifierContinue(c, AllowDollar: LangOpts.DollarIdents);
1186}
1187
1188bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1189 assert(isVerticalWhitespace(Str[0]));
1190 if (Str - 1 < BufferStart)
1191 return false;
1192
1193 if ((Str[0] == '\n' && Str[-1] == '\r') ||
1194 (Str[0] == '\r' && Str[-1] == '\n')) {
1195 if (Str - 2 < BufferStart)
1196 return false;
1197 --Str;
1198 }
1199 --Str;
1200
1201 // Rewind to first non-space character:
1202 while (Str > BufferStart && isHorizontalWhitespace(c: *Str))
1203 --Str;
1204
1205 return *Str == '\\';
1206}
1207
1208StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1209 const SourceManager &SM) {
1210 if (Loc.isInvalid() || Loc.isMacroID())
1211 return {};
1212 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1213 if (LocInfo.first.isInvalid())
1214 return {};
1215 bool Invalid = false;
1216 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
1217 if (Invalid)
1218 return {};
1219 const char *Line = findBeginningOfLine(Buffer, Offset: LocInfo.second);
1220 if (!Line)
1221 return {};
1222 StringRef Rest = Buffer.substr(Start: Line - Buffer.data());
1223 size_t NumWhitespaceChars = Rest.find_first_not_of(Chars: " \t");
1224 return NumWhitespaceChars == StringRef::npos
1225 ? ""
1226 : Rest.take_front(N: NumWhitespaceChars);
1227}
1228
1229//===----------------------------------------------------------------------===//
1230// Diagnostics forwarding code.
1231//===----------------------------------------------------------------------===//
1232
1233/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1234/// lexer buffer was all expanded at a single point, perform the mapping.
1235/// This is currently only used for _Pragma implementation, so it is the slow
1236/// path of the hot getSourceLocation method. Do not allow it to be inlined.
1237static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1238 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1239static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1240 SourceLocation FileLoc,
1241 unsigned CharNo, unsigned TokLen) {
1242 assert(FileLoc.isMacroID() && "Must be a macro expansion");
1243
1244 // Otherwise, we're lexing "mapped tokens". This is used for things like
1245 // _Pragma handling. Combine the expansion location of FileLoc with the
1246 // spelling location.
1247 SourceManager &SM = PP.getSourceManager();
1248
1249 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
1250 // characters come from spelling(FileLoc)+Offset.
1251 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc: FileLoc);
1252 SpellingLoc = SpellingLoc.getLocWithOffset(Offset: CharNo);
1253
1254 // Figure out the expansion loc range, which is the range covered by the
1255 // original _Pragma(...) sequence.
1256 CharSourceRange II = SM.getImmediateExpansionRange(Loc: FileLoc);
1257
1258 return SM.createExpansionLoc(SpellingLoc, ExpansionLocStart: II.getBegin(), ExpansionLocEnd: II.getEnd(), Length: TokLen);
1259}
1260
1261/// getSourceLocation - Return a source location identifier for the specified
1262/// offset in the current file.
1263SourceLocation Lexer::getSourceLocation(const char *Loc,
1264 unsigned TokLen) const {
1265 assert(Loc >= BufferStart && Loc <= BufferEnd &&
1266 "Location out of range for this buffer!");
1267
1268 // In the normal case, we're just lexing from a simple file buffer, return
1269 // the file id from FileLoc with the offset specified.
1270 unsigned CharNo = Loc-BufferStart;
1271 if (FileLoc.isFileID())
1272 return FileLoc.getLocWithOffset(Offset: CharNo);
1273
1274 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1275 // tokens are lexed from where the _Pragma was defined.
1276 assert(PP && "This doesn't work on raw lexers");
1277 return GetMappedTokenLoc(PP&: *PP, FileLoc, CharNo, TokLen);
1278}
1279
1280/// Diag - Forwarding function for diagnostics. This translate a source
1281/// position in the current buffer into a SourceLocation object for rendering.
1282DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1283 return PP->Diag(Loc: getSourceLocation(Loc), DiagID);
1284}
1285
1286//===----------------------------------------------------------------------===//
1287// Trigraph and Escaped Newline Handling Code.
1288//===----------------------------------------------------------------------===//
1289
1290/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1291/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1292static char GetTrigraphCharForLetter(char Letter) {
1293 switch (Letter) {
1294 default: return 0;
1295 case '=': return '#';
1296 case ')': return ']';
1297 case '(': return '[';
1298 case '!': return '|';
1299 case '\'': return '^';
1300 case '>': return '}';
1301 case '/': return '\\';
1302 case '<': return '{';
1303 case '-': return '~';
1304 }
1305}
1306
1307/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1308/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
1309/// return the result character. Finally, emit a warning about trigraph use
1310/// whether trigraphs are enabled or not.
1311static char DecodeTrigraphChar(const char *CP, Lexer *L, bool Trigraphs) {
1312 char Res = GetTrigraphCharForLetter(Letter: *CP);
1313 if (!Res)
1314 return Res;
1315
1316 if (!Trigraphs) {
1317 if (L && !L->isLexingRawMode())
1318 L->Diag(Loc: CP-2, DiagID: diag::trigraph_ignored);
1319 return 0;
1320 }
1321
1322 if (L && !L->isLexingRawMode())
1323 L->Diag(Loc: CP-2, DiagID: diag::trigraph_converted) << StringRef(&Res, 1);
1324 return Res;
1325}
1326
1327/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1328/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1329/// trigraph equivalent on entry to this function.
1330unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1331 unsigned Size = 0;
1332 while (isWhitespace(c: Ptr[Size])) {
1333 ++Size;
1334
1335 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
1336 continue;
1337
1338 // If this is a \r\n or \n\r, skip the other half.
1339 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
1340 Ptr[Size-1] != Ptr[Size])
1341 ++Size;
1342
1343 return Size;
1344 }
1345
1346 // Not an escaped newline, must be a \t or something else.
1347 return 0;
1348}
1349
1350/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1351/// them), skip over them and return the first non-escaped-newline found,
1352/// otherwise return P.
1353const char *Lexer::SkipEscapedNewLines(const char *P) {
1354 while (true) {
1355 const char *AfterEscape;
1356 if (*P == '\\') {
1357 AfterEscape = P+1;
1358 } else if (*P == '?') {
1359 // If not a trigraph for escape, bail out.
1360 if (P[1] != '?' || P[2] != '/')
1361 return P;
1362 // FIXME: Take LangOpts into account; the language might not
1363 // support trigraphs.
1364 AfterEscape = P+3;
1365 } else {
1366 return P;
1367 }
1368
1369 unsigned NewLineSize = Lexer::getEscapedNewLineSize(Ptr: AfterEscape);
1370 if (NewLineSize == 0) return P;
1371 P = AfterEscape+NewLineSize;
1372 }
1373}
1374
1375std::optional<Token> Lexer::findNextToken(SourceLocation Loc,
1376 const SourceManager &SM,
1377 const LangOptions &LangOpts,
1378 bool IncludeComments) {
1379 if (Loc.isMacroID()) {
1380 if (!Lexer::isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
1381 return std::nullopt;
1382 }
1383 Loc = Lexer::getLocForEndOfToken(Loc, Offset: 0, SM, LangOpts);
1384
1385 // Break down the source location.
1386 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1387
1388 // Try to load the file buffer.
1389 bool InvalidTemp = false;
1390 StringRef File = SM.getBufferData(FID: LocInfo.first, Invalid: &InvalidTemp);
1391 if (InvalidTemp)
1392 return std::nullopt;
1393
1394 const char *TokenBegin = File.data() + LocInfo.second;
1395
1396 // Lex from the start of the given location.
1397 Lexer lexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts, File.begin(),
1398 TokenBegin, File.end());
1399 lexer.SetCommentRetentionState(IncludeComments);
1400 // Find the token.
1401 Token Tok;
1402 lexer.LexFromRawLexer(Result&: Tok);
1403 return Tok;
1404}
1405
1406std::optional<Token> Lexer::findPreviousToken(SourceLocation Loc,
1407 const SourceManager &SM,
1408 const LangOptions &LangOpts,
1409 bool IncludeComments) {
1410 const auto StartOfFile = SM.getLocForStartOfFile(FID: SM.getFileID(SpellingLoc: Loc));
1411 while (Loc != StartOfFile) {
1412 Loc = Loc.getLocWithOffset(Offset: -1);
1413 if (Loc.isInvalid())
1414 return std::nullopt;
1415
1416 Loc = GetBeginningOfToken(Loc, SM, LangOpts);
1417 Token Tok;
1418 if (getRawToken(Loc, Result&: Tok, SM, LangOpts))
1419 continue; // Not a token, go to prev location.
1420 if (!Tok.is(K: tok::comment) || IncludeComments) {
1421 return Tok;
1422 }
1423 }
1424 return std::nullopt;
1425}
1426
1427/// Checks that the given token is the first token that occurs after the
1428/// given location (this excludes comments and whitespace). Returns the location
1429/// immediately after the specified token. If the token is not found or the
1430/// location is inside a macro, the returned source location will be invalid.
1431SourceLocation Lexer::findLocationAfterToken(
1432 SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
1433 const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
1434 std::optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
1435 if (!Tok || Tok->isNot(K: TKind))
1436 return {};
1437 SourceLocation TokenLoc = Tok->getLocation();
1438
1439 // Calculate how much whitespace needs to be skipped if any.
1440 unsigned NumWhitespaceChars = 0;
1441 if (SkipTrailingWhitespaceAndNewLine) {
1442 const char *TokenEnd = SM.getCharacterData(SL: TokenLoc) + Tok->getLength();
1443 unsigned char C = *TokenEnd;
1444 while (isHorizontalWhitespace(c: C)) {
1445 C = *(++TokenEnd);
1446 NumWhitespaceChars++;
1447 }
1448
1449 // Skip \r, \n, \r\n, or \n\r
1450 if (C == '\n' || C == '\r') {
1451 char PrevC = C;
1452 C = *(++TokenEnd);
1453 NumWhitespaceChars++;
1454 if ((C == '\n' || C == '\r') && C != PrevC)
1455 NumWhitespaceChars++;
1456 }
1457 }
1458
1459 return TokenLoc.getLocWithOffset(Offset: Tok->getLength() + NumWhitespaceChars);
1460}
1461
1462/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1463/// get its size, and return it. This is tricky in several cases:
1464/// 1. If currently at the start of a trigraph, we warn about the trigraph,
1465/// then either return the trigraph (skipping 3 chars) or the '?',
1466/// depending on whether trigraphs are enabled or not.
1467/// 2. If this is an escaped newline (potentially with whitespace between
1468/// the backslash and newline), implicitly skip the newline and return
1469/// the char after it.
1470///
1471/// This handles the slow/uncommon case of the getCharAndSize method. Here we
1472/// know that we can accumulate into Size, and that we have already incremented
1473/// Ptr by Size bytes.
1474///
1475/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1476/// be updated to match.
1477Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
1478 unsigned Size = 0;
1479 // If we have a slash, look for an escaped newline.
1480 if (Ptr[0] == '\\') {
1481 ++Size;
1482 ++Ptr;
1483Slash:
1484 // Common case, backslash-char where the char is not whitespace.
1485 if (!isWhitespace(c: Ptr[0]))
1486 return {.Char: '\\', .Size: Size};
1487
1488 // See if we have optional whitespace characters between the slash and
1489 // newline.
1490 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1491 // Remember that this token needs to be cleaned.
1492 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1493
1494 // Warn if there was whitespace between the backslash and newline.
1495 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1496 Diag(Loc: Ptr, DiagID: diag::backslash_newline_space);
1497
1498 // Found backslash<whitespace><newline>. Parse the char after it.
1499 Size += EscapedNewLineSize;
1500 Ptr += EscapedNewLineSize;
1501
1502 // Use slow version to accumulate a correct size field.
1503 auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
1504 CharAndSize.Size += Size;
1505 return CharAndSize;
1506 }
1507
1508 // Otherwise, this is not an escaped newline, just return the slash.
1509 return {.Char: '\\', .Size: Size};
1510 }
1511
1512 // If this is a trigraph, process it.
1513 if (Ptr[0] == '?' && Ptr[1] == '?') {
1514 // If this is actually a legal trigraph (not something like "??x"), emit
1515 // a trigraph warning. If so, and if trigraphs are enabled, return it.
1516 if (char C = DecodeTrigraphChar(CP: Ptr + 2, L: Tok ? this : nullptr,
1517 Trigraphs: LangOpts.Trigraphs)) {
1518 // Remember that this token needs to be cleaned.
1519 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1520
1521 Ptr += 3;
1522 Size += 3;
1523 if (C == '\\') goto Slash;
1524 return {.Char: C, .Size: Size};
1525 }
1526 }
1527
1528 // If this is neither, return a single character.
1529 return {.Char: *Ptr, .Size: Size + 1u};
1530}
1531
1532/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1533/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
1534/// and that we have already incremented Ptr by Size bytes.
1535///
1536/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1537/// be updated to match.
1538Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
1539 const LangOptions &LangOpts) {
1540
1541 unsigned Size = 0;
1542 // If we have a slash, look for an escaped newline.
1543 if (Ptr[0] == '\\') {
1544 ++Size;
1545 ++Ptr;
1546Slash:
1547 // Common case, backslash-char where the char is not whitespace.
1548 if (!isWhitespace(c: Ptr[0]))
1549 return {.Char: '\\', .Size: Size};
1550
1551 // See if we have optional whitespace characters followed by a newline.
1552 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1553 // Found backslash<whitespace><newline>. Parse the char after it.
1554 Size += EscapedNewLineSize;
1555 Ptr += EscapedNewLineSize;
1556
1557 // Use slow version to accumulate a correct size field.
1558 auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
1559 CharAndSize.Size += Size;
1560 return CharAndSize;
1561 }
1562
1563 // Otherwise, this is not an escaped newline, just return the slash.
1564 return {.Char: '\\', .Size: Size};
1565 }
1566
1567 // If this is a trigraph, process it.
1568 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1569 // If this is actually a legal trigraph (not something like "??x"), return
1570 // it.
1571 if (char C = GetTrigraphCharForLetter(Letter: Ptr[2])) {
1572 Ptr += 3;
1573 Size += 3;
1574 if (C == '\\') goto Slash;
1575 return {.Char: C, .Size: Size};
1576 }
1577 }
1578
1579 // If this is neither, return a single character.
1580 return {.Char: *Ptr, .Size: Size + 1u};
1581}
1582
1583//===----------------------------------------------------------------------===//
1584// Helper methods for lexing.
1585//===----------------------------------------------------------------------===//
1586
1587/// Routine that indiscriminately sets the offset into the source file.
1588void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1589 BufferPtr = BufferStart + Offset;
1590 if (BufferPtr > BufferEnd)
1591 BufferPtr = BufferEnd;
1592 // FIXME: What exactly does the StartOfLine bit mean? There are two
1593 // possible meanings for the "start" of the line: the first token on the
1594 // unexpanded line, or the first token on the expanded line.
1595 IsAtStartOfLine = StartOfLine;
1596 IsAtPhysicalStartOfLine = StartOfLine;
1597}
1598
1599static bool isUnicodeWhitespace(uint32_t Codepoint) {
1600 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
1601 UnicodeWhitespaceCharRanges);
1602 return UnicodeWhitespaceChars.contains(C: Codepoint);
1603}
1604
1605static llvm::SmallString<5> codepointAsHexString(uint32_t C) {
1606 llvm::SmallString<5> CharBuf;
1607 llvm::raw_svector_ostream CharOS(CharBuf);
1608 llvm::write_hex(S&: CharOS, N: C, Style: llvm::HexPrintStyle::Upper, Width: 4);
1609 return CharBuf;
1610}
1611
1612// To mitigate https://github.com/llvm/llvm-project/issues/54732,
1613// we allow "Mathematical Notation Characters" in identifiers.
1614// This is a proposed profile that extends the XID_Start/XID_continue
1615// with mathematical symbols, superscipts and subscripts digits
1616// found in some production software.
1617// https://www.unicode.org/L2/L2022/22230-math-profile.pdf
1618static bool isMathematicalExtensionID(uint32_t C, const LangOptions &LangOpts,
1619 bool IsStart, bool &IsExtension) {
1620 static const llvm::sys::UnicodeCharSet MathStartChars(
1621 MathematicalNotationProfileIDStartRanges);
1622 static const llvm::sys::UnicodeCharSet MathContinueChars(
1623 MathematicalNotationProfileIDContinueRanges);
1624 if (MathStartChars.contains(C) ||
1625 (!IsStart && MathContinueChars.contains(C))) {
1626 IsExtension = true;
1627 return true;
1628 }
1629 return false;
1630}
1631
1632static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts,
1633 bool &IsExtension) {
1634 if (LangOpts.AsmPreprocessor) {
1635 return false;
1636 } else if (LangOpts.DollarIdents && '$' == C) {
1637 return true;
1638 } else if (LangOpts.CPlusPlus || LangOpts.C23) {
1639 // A non-leading codepoint must have the XID_Continue property.
1640 // XIDContinueRanges doesn't contains characters also in XIDStartRanges,
1641 // so we need to check both tables.
1642 // '_' doesn't have the XID_Continue property but is allowed in C and C++.
1643 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1644 static const llvm::sys::UnicodeCharSet XIDContinueChars(XIDContinueRanges);
1645 if (C == '_' || XIDStartChars.contains(C) || XIDContinueChars.contains(C))
1646 return true;
1647 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/false,
1648 IsExtension);
1649 } else if (LangOpts.C11) {
1650 static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
1651 C11AllowedIDCharRanges);
1652 return C11AllowedIDChars.contains(C);
1653 } else {
1654 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1655 C99AllowedIDCharRanges);
1656 return C99AllowedIDChars.contains(C);
1657 }
1658}
1659
1660static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts,
1661 bool &IsExtension) {
1662 assert(C > 0x7F && "isAllowedInitiallyIDChar called with an ASCII codepoint");
1663 IsExtension = false;
1664 if (LangOpts.AsmPreprocessor) {
1665 return false;
1666 }
1667 if (LangOpts.CPlusPlus || LangOpts.C23) {
1668 static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1669 if (XIDStartChars.contains(C))
1670 return true;
1671 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/true,
1672 IsExtension);
1673 }
1674 if (!isAllowedIDChar(C, LangOpts, IsExtension))
1675 return false;
1676 if (LangOpts.C11) {
1677 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
1678 C11DisallowedInitialIDCharRanges);
1679 return !C11DisallowedInitialIDChars.contains(C);
1680 }
1681 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1682 C99DisallowedInitialIDCharRanges);
1683 return !C99DisallowedInitialIDChars.contains(C);
1684}
1685
1686static void diagnoseExtensionInIdentifier(DiagnosticsEngine &Diags, uint32_t C,
1687 CharSourceRange Range) {
1688
1689 static const llvm::sys::UnicodeCharSet MathStartChars(
1690 MathematicalNotationProfileIDStartRanges);
1691 static const llvm::sys::UnicodeCharSet MathContinueChars(
1692 MathematicalNotationProfileIDContinueRanges);
1693
1694 (void)MathStartChars;
1695 (void)MathContinueChars;
1696 assert((MathStartChars.contains(C) || MathContinueChars.contains(C)) &&
1697 "Unexpected mathematical notation codepoint");
1698 Diags.Report(Loc: Range.getBegin(), DiagID: diag::ext_mathematical_notation)
1699 << codepointAsHexString(C) << Range;
1700}
1701
1702static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1703 const char *End) {
1704 return CharSourceRange::getCharRange(B: L.getSourceLocation(Loc: Begin),
1705 E: L.getSourceLocation(Loc: End));
1706}
1707
1708static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
1709 CharSourceRange Range, bool IsFirst) {
1710 // Check C99 compatibility.
1711 if (!Diags.isIgnored(DiagID: diag::warn_c99_compat_unicode_id, Loc: Range.getBegin())) {
1712 enum {
1713 CannotAppearInIdentifier = 0,
1714 CannotStartIdentifier
1715 };
1716
1717 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1718 C99AllowedIDCharRanges);
1719 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1720 C99DisallowedInitialIDCharRanges);
1721 if (!C99AllowedIDChars.contains(C)) {
1722 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1723 << Range
1724 << CannotAppearInIdentifier;
1725 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) {
1726 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1727 << Range
1728 << CannotStartIdentifier;
1729 }
1730 }
1731}
1732
1733/// After encountering UTF-8 character C and interpreting it as an identifier
1734/// character, check whether it's a homoglyph for a common non-identifier
1735/// source character that is unlikely to be an intentional identifier
1736/// character and warn if so.
1737static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
1738 CharSourceRange Range) {
1739 // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
1740 struct HomoglyphPair {
1741 uint32_t Character;
1742 char LooksLike;
1743 bool operator<(HomoglyphPair R) const { return Character < R.Character; }
1744 };
1745 static constexpr HomoglyphPair SortedHomoglyphs[] = {
1746 {.Character: U'\u00ad', .LooksLike: 0}, // SOFT HYPHEN
1747 {.Character: U'\u01c3', .LooksLike: '!'}, // LATIN LETTER RETROFLEX CLICK
1748 {.Character: U'\u037e', .LooksLike: ';'}, // GREEK QUESTION MARK
1749 {.Character: U'\u200b', .LooksLike: 0}, // ZERO WIDTH SPACE
1750 {.Character: U'\u200c', .LooksLike: 0}, // ZERO WIDTH NON-JOINER
1751 {.Character: U'\u200d', .LooksLike: 0}, // ZERO WIDTH JOINER
1752 {.Character: U'\u2060', .LooksLike: 0}, // WORD JOINER
1753 {.Character: U'\u2061', .LooksLike: 0}, // FUNCTION APPLICATION
1754 {.Character: U'\u2062', .LooksLike: 0}, // INVISIBLE TIMES
1755 {.Character: U'\u2063', .LooksLike: 0}, // INVISIBLE SEPARATOR
1756 {.Character: U'\u2064', .LooksLike: 0}, // INVISIBLE PLUS
1757 {.Character: U'\u2212', .LooksLike: '-'}, // MINUS SIGN
1758 {.Character: U'\u2215', .LooksLike: '/'}, // DIVISION SLASH
1759 {.Character: U'\u2216', .LooksLike: '\\'}, // SET MINUS
1760 {.Character: U'\u2217', .LooksLike: '*'}, // ASTERISK OPERATOR
1761 {.Character: U'\u2223', .LooksLike: '|'}, // DIVIDES
1762 {.Character: U'\u2227', .LooksLike: '^'}, // LOGICAL AND
1763 {.Character: U'\u2236', .LooksLike: ':'}, // RATIO
1764 {.Character: U'\u223c', .LooksLike: '~'}, // TILDE OPERATOR
1765 {.Character: U'\ua789', .LooksLike: ':'}, // MODIFIER LETTER COLON
1766 {.Character: U'\ufeff', .LooksLike: 0}, // ZERO WIDTH NO-BREAK SPACE
1767 {.Character: U'\uff01', .LooksLike: '!'}, // FULLWIDTH EXCLAMATION MARK
1768 {.Character: U'\uff03', .LooksLike: '#'}, // FULLWIDTH NUMBER SIGN
1769 {.Character: U'\uff04', .LooksLike: '$'}, // FULLWIDTH DOLLAR SIGN
1770 {.Character: U'\uff05', .LooksLike: '%'}, // FULLWIDTH PERCENT SIGN
1771 {.Character: U'\uff06', .LooksLike: '&'}, // FULLWIDTH AMPERSAND
1772 {.Character: U'\uff08', .LooksLike: '('}, // FULLWIDTH LEFT PARENTHESIS
1773 {.Character: U'\uff09', .LooksLike: ')'}, // FULLWIDTH RIGHT PARENTHESIS
1774 {.Character: U'\uff0a', .LooksLike: '*'}, // FULLWIDTH ASTERISK
1775 {.Character: U'\uff0b', .LooksLike: '+'}, // FULLWIDTH ASTERISK
1776 {.Character: U'\uff0c', .LooksLike: ','}, // FULLWIDTH COMMA
1777 {.Character: U'\uff0d', .LooksLike: '-'}, // FULLWIDTH HYPHEN-MINUS
1778 {.Character: U'\uff0e', .LooksLike: '.'}, // FULLWIDTH FULL STOP
1779 {.Character: U'\uff0f', .LooksLike: '/'}, // FULLWIDTH SOLIDUS
1780 {.Character: U'\uff1a', .LooksLike: ':'}, // FULLWIDTH COLON
1781 {.Character: U'\uff1b', .LooksLike: ';'}, // FULLWIDTH SEMICOLON
1782 {.Character: U'\uff1c', .LooksLike: '<'}, // FULLWIDTH LESS-THAN SIGN
1783 {.Character: U'\uff1d', .LooksLike: '='}, // FULLWIDTH EQUALS SIGN
1784 {.Character: U'\uff1e', .LooksLike: '>'}, // FULLWIDTH GREATER-THAN SIGN
1785 {.Character: U'\uff1f', .LooksLike: '?'}, // FULLWIDTH QUESTION MARK
1786 {.Character: U'\uff20', .LooksLike: '@'}, // FULLWIDTH COMMERCIAL AT
1787 {.Character: U'\uff3b', .LooksLike: '['}, // FULLWIDTH LEFT SQUARE BRACKET
1788 {.Character: U'\uff3c', .LooksLike: '\\'}, // FULLWIDTH REVERSE SOLIDUS
1789 {.Character: U'\uff3d', .LooksLike: ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
1790 {.Character: U'\uff3e', .LooksLike: '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
1791 {.Character: U'\uff5b', .LooksLike: '{'}, // FULLWIDTH LEFT CURLY BRACKET
1792 {.Character: U'\uff5c', .LooksLike: '|'}, // FULLWIDTH VERTICAL LINE
1793 {.Character: U'\uff5d', .LooksLike: '}'}, // FULLWIDTH RIGHT CURLY BRACKET
1794 {.Character: U'\uff5e', .LooksLike: '~'}, // FULLWIDTH TILDE
1795 {.Character: 0, .LooksLike: 0}
1796 };
1797 auto Homoglyph =
1798 std::lower_bound(first: std::begin(arr: SortedHomoglyphs),
1799 last: std::end(arr: SortedHomoglyphs) - 1, val: HomoglyphPair{.Character: C, .LooksLike: '\0'});
1800 if (Homoglyph->Character == C) {
1801 if (Homoglyph->LooksLike) {
1802 const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
1803 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_homoglyph)
1804 << Range << codepointAsHexString(C) << LooksLikeStr;
1805 } else {
1806 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_zero_width)
1807 << Range << codepointAsHexString(C);
1808 }
1809 }
1810}
1811
1812static bool CheckCodepointValidInIdentifier(const Preprocessor *PP,
1813 const LangOptions &LangOpts,
1814 uint32_t CodePoint,
1815 CharSourceRange Range, bool IsFirst,
1816 bool Diagnose) {
1817 if (isASCII(c: CodePoint))
1818 return true;
1819
1820 bool IsExtension;
1821 bool IsIDStart = isAllowedInitiallyIDChar(C: CodePoint, LangOpts, IsExtension);
1822 bool IsIDContinue =
1823 IsIDStart || isAllowedIDChar(C: CodePoint, LangOpts, IsExtension);
1824
1825 if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
1826 return true;
1827
1828 if (!Diagnose)
1829 return false;
1830
1831 bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
1832
1833 if (!IsFirst || InvalidOnlyAtStart) {
1834 PP->Diag(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed_identifier)
1835 << Range << codepointAsHexString(C: CodePoint) << int(InvalidOnlyAtStart)
1836 << FixItHint::CreateRemoval(RemoveRange: Range);
1837 } else {
1838 PP->Diag(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed)
1839 << Range << codepointAsHexString(C: CodePoint)
1840 << FixItHint::CreateRemoval(RemoveRange: Range);
1841 }
1842 return false;
1843}
1844
1845bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1846 Token &Result) {
1847 const char *UCNPtr = CurPtr + Size;
1848 uint32_t CodePoint = tryReadUCN(StartPtr&: UCNPtr, SlashLoc: CurPtr, /*Token=*/Result: nullptr);
1849 if (CodePoint == 0) {
1850 return false;
1851 }
1852 bool IsExtension = false;
1853 if (!isAllowedIDChar(C: CodePoint, LangOpts, IsExtension)) {
1854 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1855 return false;
1856
1857 bool DiagnoseAndContinue = !isLexingRawMode() &&
1858 !ParsingPreprocessorDirective &&
1859 !PP->isPreprocessedOutput();
1860 if (!CheckCodepointValidInIdentifier(
1861 PP, LangOpts, CodePoint, Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1862 /*IsFirst=*/false, Diagnose: DiagnoseAndContinue) &&
1863 !DiagnoseAndContinue)
1864 return false;
1865 // We got a unicode codepoint that is neither a space nor a
1866 // a valid identifier part.
1867 // Carry on as if the codepoint was valid for recovery purposes.
1868 } else if (!isLexingRawMode()) {
1869 if (IsExtension)
1870 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C: CodePoint,
1871 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr));
1872
1873 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1874 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1875 /*IsFirst=*/false);
1876 }
1877
1878 Result.setFlag(Token::HasUCN);
1879 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') ||
1880 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
1881 CurPtr = UCNPtr;
1882 else
1883 while (CurPtr != UCNPtr)
1884 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
1885 return true;
1886}
1887
1888bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
1889 llvm::UTF32 CodePoint;
1890
1891 // If a UTF-8 codepoint appears immediately after an escaped new line,
1892 // CurPtr may point to the splicing \ on the preceding line,
1893 // so we need to skip it.
1894 unsigned FirstCodeUnitSize;
1895 getCharAndSize(Ptr: CurPtr, Size&: FirstCodeUnitSize);
1896 const char *CharStart = CurPtr + FirstCodeUnitSize - 1;
1897 const char *UnicodePtr = CharStart;
1898
1899 llvm::ConversionResult ConvResult = llvm::convertUTF8Sequence(
1900 source: (const llvm::UTF8 **)&UnicodePtr, sourceEnd: (const llvm::UTF8 *)BufferEnd,
1901 target: &CodePoint, flags: llvm::strictConversion);
1902 if (ConvResult != llvm::conversionOK)
1903 return false;
1904
1905 bool IsExtension = false;
1906 if (!isAllowedIDChar(C: static_cast<uint32_t>(CodePoint), LangOpts,
1907 IsExtension)) {
1908 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1909 return false;
1910
1911 bool DiagnoseAndContinue = !isLexingRawMode() &&
1912 !ParsingPreprocessorDirective &&
1913 !PP->isPreprocessedOutput();
1914
1915 if (!CheckCodepointValidInIdentifier(
1916 PP, LangOpts, CodePoint,
1917 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr), /*IsFirst=*/false,
1918 Diagnose: DiagnoseAndContinue) &&
1919 !DiagnoseAndContinue)
1920 return false;
1921 // We got a unicode codepoint that is neither a space nor a
1922 // a valid identifier part. Carry on as if the codepoint was
1923 // valid for recovery purposes.
1924 } else if (!isLexingRawMode()) {
1925 if (IsExtension)
1926 diagnoseExtensionInIdentifier(
1927 Diags&: PP->getDiagnostics(), C: CodePoint,
1928 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1929 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1930 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr),
1931 /*IsFirst=*/false);
1932 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C: CodePoint,
1933 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1934 }
1935
1936 // Once we sucessfully parsed some UTF-8,
1937 // calling ConsumeChar ensures the NeedsCleaning flag is set on the token
1938 // being lexed, and that warnings about trailing spaces are emitted.
1939 ConsumeChar(Ptr: CurPtr, Size: FirstCodeUnitSize, Tok&: Result);
1940 CurPtr = UnicodePtr;
1941 return true;
1942}
1943
1944bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
1945 const char *CurPtr) {
1946 bool IsExtension = false;
1947 if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
1948 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1949 !PP->isPreprocessedOutput()) {
1950 if (IsExtension)
1951 diagnoseExtensionInIdentifier(Diags&: PP->getDiagnostics(), C,
1952 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1953 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C,
1954 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1955 /*IsFirst=*/true);
1956 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C,
1957 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1958 }
1959
1960 MIOpt.ReadToken();
1961 return LexIdentifierContinue(Result, CurPtr);
1962 }
1963
1964 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1965 !PP->isPreprocessedOutput() && !isASCII(c: *BufferPtr) &&
1966 !isUnicodeWhitespace(Codepoint: C)) {
1967 // Non-ASCII characters tend to creep into source code unintentionally.
1968 // Instead of letting the parser complain about the unknown token,
1969 // just drop the character.
1970 // Note that we can /only/ do this when the non-ASCII character is actually
1971 // spelled as Unicode, not written as a UCN. The standard requires that
1972 // we not throw away any possible preprocessor tokens, but there's a
1973 // loophole in the mapping of Unicode characters to basic character set
1974 // characters that allows us to map these particular characters to, say,
1975 // whitespace.
1976 CheckCodepointValidInIdentifier(PP, LangOpts, CodePoint: C,
1977 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1978 /*IsStart=*/IsFirst: true, /*Diagnose=*/true);
1979 BufferPtr = CurPtr;
1980 return false;
1981 }
1982
1983 // Otherwise, we have an explicit UCN or a character that's unlikely to show
1984 // up by accident.
1985 MIOpt.ReadToken();
1986 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
1987 return true;
1988}
1989
1990static const char *
1991fastParseASCIIIdentifier(const char *CurPtr,
1992 [[maybe_unused]] const char *BufferEnd) {
1993#ifdef __SSE4_2__
1994 alignas(16) static constexpr char AsciiIdentifierRange[16] = {
1995 '_', '_', 'A', 'Z', 'a', 'z', '0', '9',
1996 };
1997 constexpr ssize_t BytesPerRegister = 16;
1998
1999 __m128i AsciiIdentifierRangeV =
2000 _mm_load_si128((const __m128i *)AsciiIdentifierRange);
2001
2002 while (LLVM_LIKELY(BufferEnd - CurPtr >= BytesPerRegister)) {
2003 __m128i Cv = _mm_loadu_si128((const __m128i *)(CurPtr));
2004
2005 int Consumed = _mm_cmpistri(AsciiIdentifierRangeV, Cv,
2006 _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES |
2007 _SIDD_UBYTE_OPS | _SIDD_NEGATIVE_POLARITY);
2008 CurPtr += Consumed;
2009 if (Consumed == BytesPerRegister)
2010 continue;
2011 return CurPtr;
2012 }
2013#endif
2014
2015 unsigned char C = *CurPtr;
2016 while (isAsciiIdentifierContinue(c: C))
2017 C = *++CurPtr;
2018 return CurPtr;
2019}
2020
2021bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
2022 // Match [_A-Za-z0-9]*, we have already matched an identifier start.
2023
2024 while (true) {
2025
2026 CurPtr = fastParseASCIIIdentifier(CurPtr, BufferEnd);
2027
2028 unsigned Size;
2029 // Slow path: handle trigraph, unicode codepoints, UCNs.
2030 unsigned char C = getCharAndSize(Ptr: CurPtr, Size);
2031 if (isAsciiIdentifierContinue(c: C)) {
2032 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2033 continue;
2034 }
2035 if (C == '$') {
2036 // If we hit a $ and they are not supported in identifiers, we are done.
2037 if (!LangOpts.DollarIdents)
2038 break;
2039 // Otherwise, emit a diagnostic and continue.
2040 if (!isLexingRawMode())
2041 Diag(Loc: CurPtr, DiagID: diag::ext_dollar_in_identifier);
2042 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2043 continue;
2044 }
2045 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2046 continue;
2047 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2048 continue;
2049 // Neither an expected Unicode codepoint nor a UCN.
2050 break;
2051 }
2052
2053 const char *IdStart = BufferPtr;
2054 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::raw_identifier);
2055 Result.setRawIdentifierData(IdStart);
2056
2057 // If we are in raw mode, return this identifier raw. There is no need to
2058 // look up identifier information or attempt to macro expand it.
2059 if (LexingRawMode)
2060 return true;
2061
2062 // Fill in Result.IdentifierInfo and update the token kind,
2063 // looking up the identifier in the identifier table.
2064 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
2065 // Note that we have to call PP->LookUpIdentifierInfo() even for code
2066 // completion, it writes IdentifierInfo into Result, and callers rely on it.
2067
2068 // If the completion point is at the end of an identifier, we want to treat
2069 // the identifier as incomplete even if it resolves to a macro or a keyword.
2070 // This allows e.g. 'class^' to complete to 'classifier'.
2071 if (isCodeCompletionPoint(CurPtr)) {
2072 // Return the code-completion token.
2073 Result.setKind(tok::code_completion);
2074 // Skip the code-completion char and all immediate identifier characters.
2075 // This ensures we get consistent behavior when completing at any point in
2076 // an identifier (i.e. at the start, in the middle, at the end). Note that
2077 // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
2078 // simpler.
2079 assert(*CurPtr == 0 && "Completion character must be 0");
2080 ++CurPtr;
2081 // Note that code completion token is not added as a separate character
2082 // when the completion point is at the end of the buffer. Therefore, we need
2083 // to check if the buffer has ended.
2084 if (CurPtr < BufferEnd) {
2085 while (isAsciiIdentifierContinue(c: *CurPtr))
2086 ++CurPtr;
2087 }
2088 BufferPtr = CurPtr;
2089 return true;
2090 }
2091
2092 // Finally, now that we know we have an identifier, pass this off to the
2093 // preprocessor, which may macro expand it or something.
2094 if (II->isHandleIdentifierCase() || II->isModuleKeyword() ||
2095 II->isImportKeyword() || II->getTokenID() == tok::kw_export)
2096 return PP->HandleIdentifier(Identifier&: Result);
2097
2098 return true;
2099}
2100
2101/// isHexaLiteral - Return true if Start points to a hex constant.
2102/// in microsoft mode (where this is supposed to be several different tokens).
2103bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
2104 auto CharAndSize1 = Lexer::getCharAndSizeNoWarn(Ptr: Start, LangOpts);
2105 char C1 = CharAndSize1.Char;
2106 if (C1 != '0')
2107 return false;
2108
2109 auto CharAndSize2 =
2110 Lexer::getCharAndSizeNoWarn(Ptr: Start + CharAndSize1.Size, LangOpts);
2111 char C2 = CharAndSize2.Char;
2112 return (C2 == 'x' || C2 == 'X');
2113}
2114
2115/// LexNumericConstant - Lex the remainder of a integer or floating point
2116/// constant. From[-1] is the first character lexed. Return the end of the
2117/// constant.
2118bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
2119 unsigned Size;
2120 char C = getCharAndSize(Ptr: CurPtr, Size);
2121 char PrevCh = 0;
2122 while (isPreprocessingNumberBody(c: C)) {
2123 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2124 PrevCh = C;
2125 if (LangOpts.HLSL && C == '.' && (*CurPtr == 'x' || *CurPtr == 'r')) {
2126 CurPtr -= Size;
2127 break;
2128 }
2129 C = getCharAndSize(Ptr: CurPtr, Size);
2130 }
2131
2132 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
2133 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
2134 // If we are in Microsoft mode, don't continue if the constant is hex.
2135 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
2136 if (!LangOpts.MicrosoftExt || !isHexaLiteral(Start: BufferPtr, LangOpts))
2137 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2138 }
2139
2140 // If we have a hex FP constant, continue.
2141 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
2142 // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
2143 // not-quite-conforming extension. Only do so if this looks like it's
2144 // actually meant to be a hexfloat, and not if it has a ud-suffix.
2145 bool IsHexFloat = true;
2146 if (!LangOpts.C99) {
2147 if (!isHexaLiteral(Start: BufferPtr, LangOpts))
2148 IsHexFloat = false;
2149 else if (!LangOpts.CPlusPlus17 &&
2150 std::find(first: BufferPtr, last: CurPtr, val: '_') != CurPtr)
2151 IsHexFloat = false;
2152 }
2153 if (IsHexFloat)
2154 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2155 }
2156
2157 // If we have a digit separator, continue.
2158 if (C == '\'' && LangOpts.AllowLiteralDigitSeparator) {
2159 auto [Next, NextSize] = getCharAndSizeNoWarn(Ptr: CurPtr + Size, LangOpts);
2160 // A digit or non-digit.
2161 if (isAsciiIdentifierContinue(c: Next)) {
2162 if (!isLexingRawMode())
2163 Diag(Loc: CurPtr, DiagID: LangOpts.CPlusPlus
2164 ? diag::warn_cxx11_compat_digit_separator
2165 : diag::warn_c23_compat_digit_separator);
2166 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2167 CurPtr = ConsumeChar(Ptr: CurPtr, Size: NextSize, Tok&: Result);
2168 return LexNumericConstant(Result, CurPtr);
2169 }
2170 }
2171
2172 if (C == '$' && LangOpts.DollarIdents) {
2173 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2174 return LexNumericConstant(Result, CurPtr);
2175 }
2176
2177 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
2178 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2179 return LexNumericConstant(Result, CurPtr);
2180 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2181 return LexNumericConstant(Result, CurPtr);
2182
2183 // Update the location of token as well as BufferPtr.
2184 const char *TokStart = BufferPtr;
2185 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::numeric_constant);
2186 Result.setLiteralData(TokStart);
2187 return true;
2188}
2189
2190/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
2191/// in C++11, or warn on a ud-suffix in C++98.
2192const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
2193 bool IsStringLiteral) {
2194 assert(LangOpts.CPlusPlus);
2195
2196 // Maximally munch an identifier.
2197 unsigned Size;
2198 char C = getCharAndSize(Ptr: CurPtr, Size);
2199 bool Consumed = false;
2200
2201 if (!isAsciiIdentifierStart(c: C, AllowDollar: LangOpts.DollarIdents)) {
2202 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2203 Consumed = true;
2204 else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2205 Consumed = true;
2206 else
2207 return CurPtr;
2208 }
2209
2210 if (!LangOpts.CPlusPlus11) {
2211 if (!isLexingRawMode())
2212 Diag(Loc: CurPtr,
2213 DiagID: C == '_' ? diag::warn_cxx11_compat_user_defined_literal
2214 : diag::warn_cxx11_compat_reserved_user_defined_literal)
2215 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2216 return CurPtr;
2217 }
2218
2219 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
2220 // that does not start with an underscore is ill-formed. As a conforming
2221 // extension, we treat all such suffixes as if they had whitespace before
2222 // them. We assume a suffix beginning with a UCN or UTF-8 character is more
2223 // likely to be a ud-suffix than a macro, however, and accept that.
2224 if (!Consumed) {
2225 bool IsUDSuffix = false;
2226 if (C == '_')
2227 IsUDSuffix = true;
2228 else if (IsStringLiteral && LangOpts.CPlusPlus14) {
2229 // In C++1y, we need to look ahead a few characters to see if this is a
2230 // valid suffix for a string literal or a numeric literal (this could be
2231 // the 'operator""if' defining a numeric literal operator).
2232 const unsigned MaxStandardSuffixLength = 3;
2233 char Buffer[MaxStandardSuffixLength] = { C };
2234 unsigned Consumed = Size;
2235 unsigned Chars = 1;
2236 while (true) {
2237 auto [Next, NextSize] =
2238 getCharAndSizeNoWarn(Ptr: CurPtr + Consumed, LangOpts);
2239 if (!isAsciiIdentifierContinue(c: Next, AllowDollar: LangOpts.DollarIdents)) {
2240 // End of suffix. Check whether this is on the allowed list.
2241 const StringRef CompleteSuffix(Buffer, Chars);
2242 IsUDSuffix =
2243 StringLiteralParser::isValidUDSuffix(LangOpts, Suffix: CompleteSuffix);
2244 break;
2245 }
2246
2247 if (Chars == MaxStandardSuffixLength)
2248 // Too long: can't be a standard suffix.
2249 break;
2250
2251 Buffer[Chars++] = Next;
2252 Consumed += NextSize;
2253 }
2254 }
2255
2256 if (!IsUDSuffix) {
2257 if (!isLexingRawMode())
2258 Diag(Loc: CurPtr, DiagID: LangOpts.MSVCCompat
2259 ? diag::ext_ms_reserved_user_defined_literal
2260 : diag::ext_reserved_user_defined_literal)
2261 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2262 return CurPtr;
2263 }
2264
2265 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2266 }
2267
2268 Result.setFlag(Token::HasUDSuffix);
2269 while (true) {
2270 C = getCharAndSize(Ptr: CurPtr, Size);
2271 if (isAsciiIdentifierContinue(c: C, AllowDollar: LangOpts.DollarIdents)) {
2272 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2273 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
2274 } else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result)) {
2275 } else
2276 break;
2277 }
2278
2279 return CurPtr;
2280}
2281
2282/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
2283/// either " or L" or u8" or u" or U".
2284bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
2285 tok::TokenKind Kind) {
2286 const char *AfterQuote = CurPtr;
2287 // Does this string contain the \0 character?
2288 const char *NulCharacter = nullptr;
2289
2290 if (!isLexingRawMode() &&
2291 (Kind == tok::utf8_string_literal ||
2292 Kind == tok::utf16_string_literal ||
2293 Kind == tok::utf32_string_literal))
2294 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
2295 : diag::warn_c99_compat_unicode_literal);
2296
2297 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2298 while (C != '"') {
2299 // Skip escaped characters. Escaped newlines will already be processed by
2300 // getAndAdvanceChar.
2301 if (C == '\\') {
2302 const char *SavedCurPtr = CurPtr;
2303 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2304
2305 // lex.header
2306 //
2307 // header-name:
2308 // ...
2309 // " q-char-sequence "
2310 // ...
2311 // q-char-sequence:
2312 // q-char q-char-sequence[opt]
2313 // q-char:
2314 // any member of the translation character set except new-line and
2315 // U+0022 quotation mark
2316 //
2317 // The implementation-defined semantics cannot be taken as causing '\' to
2318 // "escape" the following " because there is no provision for " in a
2319 // q-char-sequence.
2320 if (ParsingFilename && C == '"')
2321 CurPtr = SavedCurPtr;
2322 }
2323
2324 if (C == '\n' || C == '\r' || // Newline.
2325 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2326 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2327 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 1;
2328 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2329 return true;
2330 }
2331
2332 if (C == 0) {
2333 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2334 if (ParsingFilename)
2335 codeCompleteIncludedFile(PathStart: AfterQuote, CompletionPoint: CurPtr - 1, /*IsAngled=*/false);
2336 else
2337 PP->CodeCompleteNaturalLanguage();
2338 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2339 cutOffLexing();
2340 return true;
2341 }
2342
2343 NulCharacter = CurPtr-1;
2344 }
2345 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2346 }
2347
2348 // If we are in C++11, lex the optional ud-suffix.
2349 if (LangOpts.CPlusPlus)
2350 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2351
2352 // If a nul character existed in the string, warn about it.
2353 if (NulCharacter && !isLexingRawMode())
2354 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2355
2356 // Update the location of the token as well as the BufferPtr instance var.
2357 const char *TokStart = BufferPtr;
2358 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2359 Result.setLiteralData(TokStart);
2360 return true;
2361}
2362
2363/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
2364/// having lexed R", LR", u8R", uR", or UR".
2365bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2366 tok::TokenKind Kind) {
2367 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
2368 // Between the initial and final double quote characters of the raw string,
2369 // any transformations performed in phases 1 and 2 (trigraphs,
2370 // universal-character-names, and line splicing) are reverted.
2371
2372 if (!isLexingRawMode())
2373 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_raw_string_literal);
2374
2375 unsigned PrefixLen = 0;
2376
2377 while (PrefixLen != 16 && isRawStringDelimBody(c: CurPtr[PrefixLen])) {
2378 if (!isLexingRawMode() &&
2379 llvm::is_contained(Set: {'$', '@', '`'}, Element: CurPtr[PrefixLen])) {
2380 const char *Pos = &CurPtr[PrefixLen];
2381 Diag(Loc: Pos, DiagID: LangOpts.CPlusPlus26
2382 ? diag::warn_cxx26_compat_raw_string_literal_character_set
2383 : diag::ext_cxx26_raw_string_literal_character_set)
2384 << StringRef(Pos, 1);
2385 }
2386 ++PrefixLen;
2387 }
2388
2389 // If the last character was not a '(', then we didn't lex a valid delimiter.
2390 if (CurPtr[PrefixLen] != '(') {
2391 if (!isLexingRawMode()) {
2392 const char *PrefixEnd = &CurPtr[PrefixLen];
2393 if (PrefixLen == 16) {
2394 Diag(Loc: PrefixEnd, DiagID: diag::err_raw_delim_too_long);
2395 } else if (*PrefixEnd == '\n') {
2396 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_newline_raw_delim);
2397 } else {
2398 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_char_raw_delim)
2399 << StringRef(PrefixEnd, 1);
2400 }
2401 }
2402
2403 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2404 // it's possible the '"' was intended to be part of the raw string, but
2405 // there's not much we can do about that.
2406 while (true) {
2407 char C = *CurPtr++;
2408
2409 if (C == '"')
2410 break;
2411 if (C == 0 && CurPtr-1 == BufferEnd) {
2412 --CurPtr;
2413 break;
2414 }
2415 }
2416
2417 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2418 return true;
2419 }
2420
2421 // Save prefix and move CurPtr past it
2422 const char *Prefix = CurPtr;
2423 CurPtr += PrefixLen + 1; // skip over prefix and '('
2424
2425 while (true) {
2426 char C = *CurPtr++;
2427
2428 if (C == ')') {
2429 // Check for prefix match and closing quote.
2430 if (strncmp(s1: CurPtr, s2: Prefix, n: PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
2431 CurPtr += PrefixLen + 1; // skip over prefix and '"'
2432 break;
2433 }
2434 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
2435 if (!isLexingRawMode())
2436 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_raw_string)
2437 << StringRef(Prefix, PrefixLen);
2438 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2439 return true;
2440 }
2441 }
2442
2443 // If we are in C++11, lex the optional ud-suffix.
2444 if (LangOpts.CPlusPlus)
2445 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2446
2447 // Update the location of token as well as BufferPtr.
2448 const char *TokStart = BufferPtr;
2449 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2450 Result.setLiteralData(TokStart);
2451 return true;
2452}
2453
2454/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2455/// after having lexed the '<' character. This is used for #include filenames.
2456bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2457 // Does this string contain the \0 character?
2458 const char *NulCharacter = nullptr;
2459 const char *AfterLessPos = CurPtr;
2460 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2461 while (C != '>') {
2462 // Skip escaped characters. Escaped newlines will already be processed by
2463 // getAndAdvanceChar.
2464 if (C == '\\')
2465 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2466
2467 if (isVerticalWhitespace(c: C) || // Newline.
2468 (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
2469 // If the filename is unterminated, then it must just be a lone <
2470 // character. Return this as such.
2471 FormTokenWithChars(Result, TokEnd: AfterLessPos, Kind: tok::less);
2472 return true;
2473 }
2474
2475 if (C == 0) {
2476 if (isCodeCompletionPoint(CurPtr: CurPtr - 1)) {
2477 codeCompleteIncludedFile(PathStart: AfterLessPos, CompletionPoint: CurPtr - 1, /*IsAngled=*/true);
2478 cutOffLexing();
2479 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2480 return true;
2481 }
2482 NulCharacter = CurPtr-1;
2483 }
2484 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2485 }
2486
2487 // If a nul character existed in the string, warn about it.
2488 if (NulCharacter && !isLexingRawMode())
2489 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2490
2491 // Update the location of token as well as BufferPtr.
2492 const char *TokStart = BufferPtr;
2493 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::header_name);
2494 Result.setLiteralData(TokStart);
2495 return true;
2496}
2497
2498void Lexer::codeCompleteIncludedFile(const char *PathStart,
2499 const char *CompletionPoint,
2500 bool IsAngled) {
2501 // Completion only applies to the filename, after the last slash.
2502 StringRef PartialPath(PathStart, CompletionPoint - PathStart);
2503 llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
2504 auto Slash = PartialPath.find_last_of(Chars: SlashChars);
2505 StringRef Dir =
2506 (Slash == StringRef::npos) ? "" : PartialPath.take_front(N: Slash);
2507 const char *StartOfFilename =
2508 (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
2509 // Code completion filter range is the filename only, up to completion point.
2510 PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
2511 Name: StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
2512 // We should replace the characters up to the closing quote or closest slash,
2513 // if any.
2514 while (CompletionPoint < BufferEnd) {
2515 char Next = *(CompletionPoint + 1);
2516 if (Next == 0 || Next == '\r' || Next == '\n')
2517 break;
2518 ++CompletionPoint;
2519 if (Next == (IsAngled ? '>' : '"'))
2520 break;
2521 if (SlashChars.contains(C: Next))
2522 break;
2523 }
2524
2525 PP->setCodeCompletionTokenRange(
2526 Start: FileLoc.getLocWithOffset(Offset: StartOfFilename - BufferStart),
2527 End: FileLoc.getLocWithOffset(Offset: CompletionPoint - BufferStart));
2528 PP->CodeCompleteIncludedFile(Dir, IsAngled);
2529}
2530
2531/// LexCharConstant - Lex the remainder of a character constant, after having
2532/// lexed either ' or L' or u8' or u' or U'.
2533bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2534 tok::TokenKind Kind) {
2535 // Does this character contain the \0 character?
2536 const char *NulCharacter = nullptr;
2537
2538 if (!isLexingRawMode()) {
2539 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
2540 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2541 ? diag::warn_cxx98_compat_unicode_literal
2542 : diag::warn_c99_compat_unicode_literal);
2543 else if (Kind == tok::utf8_char_constant)
2544 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2545 ? diag::warn_cxx14_compat_u8_character_literal
2546 : diag::warn_c17_compat_u8_character_literal);
2547 }
2548
2549 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2550 if (C == '\'') {
2551 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2552 Diag(Loc: BufferPtr, DiagID: diag::ext_empty_character);
2553 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2554 return true;
2555 }
2556
2557 while (C != '\'') {
2558 // Skip escaped characters.
2559 if (C == '\\')
2560 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2561
2562 if (C == '\n' || C == '\r' || // Newline.
2563 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2564 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2565 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 0;
2566 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2567 return true;
2568 }
2569
2570 if (C == 0) {
2571 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2572 PP->CodeCompleteNaturalLanguage();
2573 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2574 cutOffLexing();
2575 return true;
2576 }
2577
2578 NulCharacter = CurPtr-1;
2579 }
2580 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2581 }
2582
2583 // If we are in C++11, lex the optional ud-suffix.
2584 if (LangOpts.CPlusPlus)
2585 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: false);
2586
2587 // If a nul character existed in the character, warn about it.
2588 if (NulCharacter && !isLexingRawMode())
2589 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 0;
2590
2591 // Update the location of token as well as BufferPtr.
2592 const char *TokStart = BufferPtr;
2593 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2594 Result.setLiteralData(TokStart);
2595 return true;
2596}
2597
2598/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2599/// Update BufferPtr to point to the next non-whitespace character and return.
2600///
2601/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2602bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
2603 // Whitespace - Skip it, then return the token after the whitespace.
2604 bool SawNewline = isVerticalWhitespace(c: CurPtr[-1]);
2605
2606 unsigned char Char = *CurPtr;
2607
2608 const char *lastNewLine = nullptr;
2609 auto setLastNewLine = [&](const char *Ptr) {
2610 lastNewLine = Ptr;
2611 if (!NewLinePtr)
2612 NewLinePtr = Ptr;
2613 };
2614 if (SawNewline)
2615 setLastNewLine(CurPtr - 1);
2616
2617 // Skip consecutive spaces efficiently.
2618 while (true) {
2619 // Skip horizontal whitespace, especially space, very aggressively.
2620 while (Char == ' ' || isHorizontalWhitespace(c: Char))
2621 Char = *++CurPtr;
2622
2623 // Otherwise if we have something other than whitespace, we're done.
2624 if (!isVerticalWhitespace(c: Char))
2625 break;
2626
2627 if (ParsingPreprocessorDirective) {
2628 // End of preprocessor directive line, let LexTokenInternal handle this.
2629 BufferPtr = CurPtr;
2630 return false;
2631 }
2632
2633 // OK, but handle newline.
2634 if (*CurPtr == '\n')
2635 setLastNewLine(CurPtr);
2636 SawNewline = true;
2637 Char = *++CurPtr;
2638 }
2639
2640 // If the client wants us to return whitespace, return it now.
2641 if (isKeepWhitespaceMode()) {
2642 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2643 if (SawNewline) {
2644 IsAtStartOfLine = true;
2645 IsAtPhysicalStartOfLine = true;
2646 }
2647 // FIXME: The next token will not have LeadingSpace set.
2648 return true;
2649 }
2650
2651 // If this isn't immediately after a newline, there is leading space.
2652 char PrevChar = CurPtr[-1];
2653 bool HasLeadingSpace = !isVerticalWhitespace(c: PrevChar);
2654
2655 Result.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace);
2656 if (SawNewline) {
2657 Result.setFlag(Token::StartOfLine);
2658 Result.setFlag(Token::PhysicalStartOfLine);
2659
2660 if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) {
2661 if (auto *Handler = PP->getEmptylineHandler())
2662 Handler->HandleEmptyline(Range: SourceRange(getSourceLocation(Loc: NewLinePtr + 1),
2663 getSourceLocation(Loc: lastNewLine)));
2664 }
2665 }
2666
2667 BufferPtr = CurPtr;
2668 return false;
2669}
2670
2671/// We have just read the // characters from input. Skip until we find the
2672/// newline character that terminates the comment. Then update BufferPtr and
2673/// return.
2674///
2675/// If we're in KeepCommentMode or any CommentHandler has inserted
2676/// some tokens, this will store the first token and return true.
2677bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) {
2678 // If Line comments aren't explicitly enabled for this language, emit an
2679 // extension warning.
2680 if (!LineComment) {
2681 if (!isLexingRawMode()) // There's no PP in raw mode, so can't emit diags.
2682 Diag(Loc: BufferPtr, DiagID: diag::ext_line_comment);
2683
2684 // Mark them enabled so we only emit one warning for this translation
2685 // unit.
2686 LineComment = true;
2687 }
2688
2689 // Scan over the body of the comment. The common case, when scanning, is that
2690 // the comment contains normal ascii characters with nothing interesting in
2691 // them. As such, optimize for this case with the inner loop.
2692 //
2693 // This loop terminates with CurPtr pointing at the newline (or end of buffer)
2694 // character that ends the line comment.
2695
2696 // C++23 [lex.phases] p1
2697 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2698 // diagnostic only once per entire ill-formed subsequence to avoid
2699 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2700 bool UnicodeDecodingAlreadyDiagnosed = false;
2701
2702 char C;
2703 while (true) {
2704 C = *CurPtr;
2705 // Skip over characters in the fast loop.
2706 while (isASCII(c: C) && C != 0 && // Potentially EOF.
2707 C != '\n' && C != '\r') { // Newline or DOS-style newline.
2708 C = *++CurPtr;
2709 UnicodeDecodingAlreadyDiagnosed = false;
2710 }
2711
2712 if (!isASCII(c: C)) {
2713 unsigned Length = llvm::getUTF8SequenceSize(
2714 source: (const llvm::UTF8 *)CurPtr, sourceEnd: (const llvm::UTF8 *)BufferEnd);
2715 if (Length == 0) {
2716 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
2717 Diag(Loc: CurPtr, DiagID: diag::warn_invalid_utf8_in_comment);
2718 UnicodeDecodingAlreadyDiagnosed = true;
2719 ++CurPtr;
2720 } else {
2721 UnicodeDecodingAlreadyDiagnosed = false;
2722 CurPtr += Length;
2723 }
2724 continue;
2725 }
2726
2727 const char *NextLine = CurPtr;
2728 if (C != 0) {
2729 // We found a newline, see if it's escaped.
2730 const char *EscapePtr = CurPtr-1;
2731 bool HasSpace = false;
2732 while (isHorizontalWhitespace(c: *EscapePtr)) { // Skip whitespace.
2733 --EscapePtr;
2734 HasSpace = true;
2735 }
2736
2737 if (*EscapePtr == '\\')
2738 // Escaped newline.
2739 CurPtr = EscapePtr;
2740 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
2741 EscapePtr[-2] == '?' && LangOpts.Trigraphs)
2742 // Trigraph-escaped newline.
2743 CurPtr = EscapePtr-2;
2744 else
2745 break; // This is a newline, we're done.
2746
2747 // If there was space between the backslash and newline, warn about it.
2748 if (HasSpace && !isLexingRawMode())
2749 Diag(Loc: EscapePtr, DiagID: diag::backslash_newline_space);
2750 }
2751
2752 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
2753 // properly decode the character. Read it in raw mode to avoid emitting
2754 // diagnostics about things like trigraphs. If we see an escaped newline,
2755 // we'll handle it below.
2756 const char *OldPtr = CurPtr;
2757 bool OldRawMode = isLexingRawMode();
2758 LexingRawMode = true;
2759 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2760 LexingRawMode = OldRawMode;
2761
2762 // If we only read only one character, then no special handling is needed.
2763 // We're done and can skip forward to the newline.
2764 if (C != 0 && CurPtr == OldPtr+1) {
2765 CurPtr = NextLine;
2766 break;
2767 }
2768
2769 // If we read multiple characters, and one of those characters was a \r or
2770 // \n, then we had an escaped newline within the comment. Emit diagnostic
2771 // unless the next line is also a // comment.
2772 if (CurPtr != OldPtr + 1 && C != '/' &&
2773 (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) {
2774 for (; OldPtr != CurPtr; ++OldPtr)
2775 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
2776 // Okay, we found a // comment that ends in a newline, if the next
2777 // line is also a // comment, but has spaces, don't emit a diagnostic.
2778 if (isWhitespace(c: C)) {
2779 const char *ForwardPtr = CurPtr;
2780 while (isWhitespace(c: *ForwardPtr)) // Skip whitespace.
2781 ++ForwardPtr;
2782 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
2783 break;
2784 }
2785
2786 if (!isLexingRawMode())
2787 Diag(Loc: OldPtr-1, DiagID: diag::ext_multi_line_line_comment);
2788 break;
2789 }
2790 }
2791
2792 if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
2793 --CurPtr;
2794 break;
2795 }
2796
2797 if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2798 PP->CodeCompleteNaturalLanguage();
2799 cutOffLexing();
2800 return false;
2801 }
2802 }
2803
2804 // Found but did not consume the newline. Notify comment handlers about the
2805 // comment unless we're in a #if 0 block.
2806 if (PP && !isLexingRawMode() &&
2807 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
2808 getSourceLocation(Loc: CurPtr)))) {
2809 BufferPtr = CurPtr;
2810 return true; // A token has to be returned.
2811 }
2812
2813 // If we are returning comments as tokens, return this comment as a token.
2814 if (inKeepCommentMode())
2815 return SaveLineComment(Result, CurPtr);
2816
2817 // If we are inside a preprocessor directive and we see the end of line,
2818 // return immediately, so that the lexer can return this as an EOD token.
2819 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
2820 BufferPtr = CurPtr;
2821 return false;
2822 }
2823
2824 // Otherwise, eat the \n character. We don't care if this is a \n\r or
2825 // \r\n sequence. This is an efficiency hack (because we know the \n can't
2826 // contribute to another token), it isn't needed for correctness. Note that
2827 // this is ok even in KeepWhitespaceMode, because we would have returned the
2828 // comment above in that mode.
2829 NewLinePtr = CurPtr++;
2830
2831 // The next returned token is at the start of the line.
2832 Result.setFlag(Token::StartOfLine);
2833 Result.setFlag(Token::PhysicalStartOfLine);
2834 // No leading whitespace seen so far.
2835 Result.clearFlag(Flag: Token::LeadingSpace);
2836 BufferPtr = CurPtr;
2837 return false;
2838}
2839
2840/// If in save-comment mode, package up this Line comment in an appropriate
2841/// way and return it.
2842bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2843 // If we're not in a preprocessor directive, just return the // comment
2844 // directly.
2845 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
2846
2847 if (!ParsingPreprocessorDirective || LexingRawMode)
2848 return true;
2849
2850 // If this Line-style comment is in a macro definition, transmogrify it into
2851 // a C-style block comment.
2852 bool Invalid = false;
2853 std::string Spelling = PP->getSpelling(Tok: Result, Invalid: &Invalid);
2854 if (Invalid)
2855 return true;
2856
2857 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
2858 Spelling[1] = '*'; // Change prefix to "/*".
2859 Spelling += "*/"; // add suffix.
2860
2861 Result.setKind(tok::comment);
2862 PP->CreateString(Str: Spelling, Tok&: Result,
2863 ExpansionLocStart: Result.getLocation(), ExpansionLocEnd: Result.getLocation());
2864 return true;
2865}
2866
2867/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2868/// character (either \\n or \\r) is part of an escaped newline sequence. Issue
2869/// a diagnostic if so. We know that the newline is inside of a block comment.
2870static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, Lexer *L,
2871 bool Trigraphs) {
2872 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
2873
2874 // Position of the first trigraph in the ending sequence.
2875 const char *TrigraphPos = nullptr;
2876 // Position of the first whitespace after a '\' in the ending sequence.
2877 const char *SpacePos = nullptr;
2878
2879 while (true) {
2880 // Back up off the newline.
2881 --CurPtr;
2882
2883 // If this is a two-character newline sequence, skip the other character.
2884 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
2885 // \n\n or \r\r -> not escaped newline.
2886 if (CurPtr[0] == CurPtr[1])
2887 return false;
2888 // \n\r or \r\n -> skip the newline.
2889 --CurPtr;
2890 }
2891
2892 // If we have horizontal whitespace, skip over it. We allow whitespace
2893 // between the slash and newline.
2894 while (isHorizontalWhitespace(c: *CurPtr) || *CurPtr == 0) {
2895 SpacePos = CurPtr;
2896 --CurPtr;
2897 }
2898
2899 // If we have a slash, this is an escaped newline.
2900 if (*CurPtr == '\\') {
2901 --CurPtr;
2902 } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') {
2903 // This is a trigraph encoding of a slash.
2904 TrigraphPos = CurPtr - 2;
2905 CurPtr -= 3;
2906 } else {
2907 return false;
2908 }
2909
2910 // If the character preceding the escaped newline is a '*', then after line
2911 // splicing we have a '*/' ending the comment.
2912 if (*CurPtr == '*')
2913 break;
2914
2915 if (*CurPtr != '\n' && *CurPtr != '\r')
2916 return false;
2917 }
2918
2919 if (TrigraphPos) {
2920 // If no trigraphs are enabled, warn that we ignored this trigraph and
2921 // ignore this * character.
2922 if (!Trigraphs) {
2923 if (!L->isLexingRawMode())
2924 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ignored_block_comment);
2925 return false;
2926 }
2927 if (!L->isLexingRawMode())
2928 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ends_block_comment);
2929 }
2930
2931 // Warn about having an escaped newline between the */ characters.
2932 if (!L->isLexingRawMode())
2933 L->Diag(Loc: CurPtr + 1, DiagID: diag::escaped_newline_block_comment_end);
2934
2935 // If there was space between the backslash and newline, warn about it.
2936 if (SpacePos && !L->isLexingRawMode())
2937 L->Diag(Loc: SpacePos, DiagID: diag::backslash_newline_space);
2938
2939 return true;
2940}
2941
2942#ifdef __SSE2__
2943#include <emmintrin.h>
2944#elif __ALTIVEC__
2945#include <altivec.h>
2946#undef bool
2947#endif
2948
2949/// We have just read from input the / and * characters that started a comment.
2950/// Read until we find the * and / characters that terminate the comment.
2951/// Note that we don't bother decoding trigraphs or escaped newlines in block
2952/// comments, because they cannot cause the comment to end. The only thing
2953/// that can happen is the comment could end with an escaped newline between
2954/// the terminating * and /.
2955///
2956/// If we're in KeepCommentMode or any CommentHandler has inserted
2957/// some tokens, this will store the first token and return true.
2958bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
2959 // Scan one character past where we should, looking for a '/' character. Once
2960 // we find it, check to see if it was preceded by a *. This common
2961 // optimization helps people who like to put a lot of * characters in their
2962 // comments.
2963
2964 // The first character we get with newlines and trigraphs skipped to handle
2965 // the degenerate /*/ case below correctly if the * has an escaped newline
2966 // after it.
2967 unsigned CharSize;
2968 unsigned char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
2969 CurPtr += CharSize;
2970 if (C == 0 && CurPtr == BufferEnd+1) {
2971 if (!isLexingRawMode())
2972 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
2973 --CurPtr;
2974
2975 // KeepWhitespaceMode should return this broken comment as a token. Since
2976 // it isn't a well formed comment, just return it as an 'unknown' token.
2977 if (isKeepWhitespaceMode()) {
2978 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2979 return true;
2980 }
2981
2982 BufferPtr = CurPtr;
2983 return false;
2984 }
2985
2986 // Check to see if the first character after the '/*' is another /. If so,
2987 // then this slash does not end the block comment, it is part of it.
2988 if (C == '/')
2989 C = *CurPtr++;
2990
2991 // C++23 [lex.phases] p1
2992 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2993 // diagnostic only once per entire ill-formed subsequence to avoid
2994 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2995 bool UnicodeDecodingAlreadyDiagnosed = false;
2996
2997 while (true) {
2998 // Skip over all non-interesting characters until we find end of buffer or a
2999 // (probably ending) '/' character.
3000 if (CurPtr + 24 < BufferEnd &&
3001 // If there is a code-completion point avoid the fast scan because it
3002 // doesn't check for '\0'.
3003 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
3004 // While not aligned to a 16-byte boundary.
3005 while (C != '/' && (intptr_t)CurPtr % 16 != 0) {
3006 if (!isASCII(c: C))
3007 goto MultiByteUTF8;
3008 C = *CurPtr++;
3009 }
3010 if (C == '/') goto FoundSlash;
3011
3012#ifdef __SSE2__
3013 __m128i Slashes = _mm_set1_epi8(b: '/');
3014 while (CurPtr + 16 < BufferEnd) {
3015 int Mask = _mm_movemask_epi8(a: *(const __m128i *)CurPtr);
3016 if (LLVM_UNLIKELY(Mask != 0)) {
3017 goto MultiByteUTF8;
3018 }
3019 // look for slashes
3020 int cmp = _mm_movemask_epi8(a: _mm_cmpeq_epi8(a: *(const __m128i*)CurPtr,
3021 b: Slashes));
3022 if (cmp != 0) {
3023 // Adjust the pointer to point directly after the first slash. It's
3024 // not necessary to set C here, it will be overwritten at the end of
3025 // the outer loop.
3026 CurPtr += llvm::countr_zero<unsigned>(Val: cmp) + 1;
3027 goto FoundSlash;
3028 }
3029 CurPtr += 16;
3030 }
3031#elif __ALTIVEC__
3032 __vector unsigned char LongUTF = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
3033 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
3034 0x80, 0x80, 0x80, 0x80};
3035 __vector unsigned char Slashes = {
3036 '/', '/', '/', '/', '/', '/', '/', '/',
3037 '/', '/', '/', '/', '/', '/', '/', '/'
3038 };
3039 while (CurPtr + 16 < BufferEnd) {
3040 if (LLVM_UNLIKELY(
3041 vec_any_ge(*(const __vector unsigned char *)CurPtr, LongUTF)))
3042 goto MultiByteUTF8;
3043 if (vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) {
3044 break;
3045 }
3046 CurPtr += 16;
3047 }
3048
3049#else
3050 while (CurPtr + 16 < BufferEnd) {
3051 bool HasNonASCII = false;
3052 for (unsigned I = 0; I < 16; ++I)
3053 HasNonASCII |= !isASCII(CurPtr[I]);
3054
3055 if (LLVM_UNLIKELY(HasNonASCII))
3056 goto MultiByteUTF8;
3057
3058 bool HasSlash = false;
3059 for (unsigned I = 0; I < 16; ++I)
3060 HasSlash |= CurPtr[I] == '/';
3061 if (HasSlash)
3062 break;
3063 CurPtr += 16;
3064 }
3065#endif
3066
3067 // It has to be one of the bytes scanned, increment to it and read one.
3068 C = *CurPtr++;
3069 }
3070
3071 // Loop to scan the remainder, warning on invalid UTF-8
3072 // if the corresponding warning is enabled, emitting a diagnostic only once
3073 // per sequence that cannot be decoded.
3074 while (C != '/' && C != '\0') {
3075 if (isASCII(c: C)) {
3076 UnicodeDecodingAlreadyDiagnosed = false;
3077 C = *CurPtr++;
3078 continue;
3079 }
3080 MultiByteUTF8:
3081 // CurPtr is 1 code unit past C, so to decode
3082 // the codepoint, we need to read from the previous position.
3083 unsigned Length = llvm::getUTF8SequenceSize(
3084 source: (const llvm::UTF8 *)CurPtr - 1, sourceEnd: (const llvm::UTF8 *)BufferEnd);
3085 if (Length == 0) {
3086 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
3087 Diag(Loc: CurPtr - 1, DiagID: diag::warn_invalid_utf8_in_comment);
3088 UnicodeDecodingAlreadyDiagnosed = true;
3089 } else {
3090 UnicodeDecodingAlreadyDiagnosed = false;
3091 CurPtr += Length - 1;
3092 }
3093 C = *CurPtr++;
3094 }
3095
3096 if (C == '/') {
3097 FoundSlash:
3098 if (CurPtr[-2] == '*') // We found the final */. We're done!
3099 break;
3100
3101 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
3102 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr: CurPtr - 2, L: this,
3103 Trigraphs: LangOpts.Trigraphs)) {
3104 // We found the final */, though it had an escaped newline between the
3105 // * and /. We're done!
3106 break;
3107 }
3108 }
3109 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
3110 // If this is a /* inside of the comment, emit a warning. Don't do this
3111 // if this is a /*/, which will end the comment. This misses cases with
3112 // embedded escaped newlines, but oh well.
3113 if (!isLexingRawMode())
3114 Diag(Loc: CurPtr-1, DiagID: diag::warn_nested_block_comment);
3115 }
3116 } else if (C == 0 && CurPtr == BufferEnd+1) {
3117 if (!isLexingRawMode())
3118 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
3119 // Note: the user probably forgot a */. We could continue immediately
3120 // after the /*, but this would involve lexing a lot of what really is the
3121 // comment, which surely would confuse the parser.
3122 --CurPtr;
3123
3124 // KeepWhitespaceMode should return this broken comment as a token. Since
3125 // it isn't a well formed comment, just return it as an 'unknown' token.
3126 if (isKeepWhitespaceMode()) {
3127 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3128 return true;
3129 }
3130
3131 BufferPtr = CurPtr;
3132 return false;
3133 } else if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3134 PP->CodeCompleteNaturalLanguage();
3135 cutOffLexing();
3136 return false;
3137 }
3138
3139 C = *CurPtr++;
3140 }
3141
3142 // Notify comment handlers about the comment unless we're in a #if 0 block.
3143 if (PP && !isLexingRawMode() &&
3144 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
3145 getSourceLocation(Loc: CurPtr)))) {
3146 BufferPtr = CurPtr;
3147 return true; // A token has to be returned.
3148 }
3149
3150 // If we are returning comments as tokens, return this comment as a token.
3151 if (inKeepCommentMode()) {
3152 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
3153 return true;
3154 }
3155
3156 // It is common for the tokens immediately after a /**/ comment to be
3157 // whitespace. Instead of going through the big switch, handle it
3158 // efficiently now. This is safe even in KeepWhitespaceMode because we would
3159 // have already returned above with the comment as a token.
3160 if (isHorizontalWhitespace(c: *CurPtr)) {
3161 SkipWhitespace(Result, CurPtr: CurPtr + 1);
3162 return false;
3163 }
3164
3165 // Otherwise, just return so that the next character will be lexed as a token.
3166 BufferPtr = CurPtr;
3167 Result.setFlag(Token::LeadingSpace);
3168 return false;
3169}
3170
3171//===----------------------------------------------------------------------===//
3172// Primary Lexing Entry Points
3173//===----------------------------------------------------------------------===//
3174
3175/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
3176/// uninterpreted string. This switches the lexer out of directive mode.
3177void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
3178 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
3179 "Must be in a preprocessing directive!");
3180 Token Tmp;
3181 Tmp.startToken();
3182
3183 // CurPtr - Cache BufferPtr in an automatic variable.
3184 const char *CurPtr = BufferPtr;
3185 while (true) {
3186 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Tmp);
3187 switch (Char) {
3188 default:
3189 if (Result)
3190 Result->push_back(Elt: Char);
3191 break;
3192 case 0: // Null.
3193 // Found end of file?
3194 if (CurPtr-1 != BufferEnd) {
3195 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3196 PP->CodeCompleteNaturalLanguage();
3197 cutOffLexing();
3198 return;
3199 }
3200
3201 // Nope, normal character, continue.
3202 if (Result)
3203 Result->push_back(Elt: Char);
3204 break;
3205 }
3206 // FALL THROUGH.
3207 [[fallthrough]];
3208 case '\r':
3209 case '\n':
3210 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
3211 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
3212 BufferPtr = CurPtr-1;
3213
3214 // Next, lex the character, which should handle the EOD transition.
3215 Lex(Result&: Tmp);
3216 if (Tmp.is(K: tok::code_completion)) {
3217 if (PP)
3218 PP->CodeCompleteNaturalLanguage();
3219 Lex(Result&: Tmp);
3220 }
3221 assert(Tmp.is(tok::eod) && "Unexpected token!");
3222
3223 // Finally, we're done;
3224 return;
3225 }
3226 }
3227}
3228
3229/// LexEndOfFile - CurPtr points to the end of this file. Handle this
3230/// condition, reporting diagnostics and handling other edge cases as required.
3231/// This returns true if Result contains a token, false if PP.Lex should be
3232/// called again.
3233bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
3234 // If we hit the end of the file while parsing a preprocessor directive,
3235 // end the preprocessor directive first. The next token returned will
3236 // then be the end of file.
3237 if (ParsingPreprocessorDirective) {
3238 // Done parsing the "line".
3239 ParsingPreprocessorDirective = false;
3240 // Update the location of token as well as BufferPtr.
3241 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::eod);
3242
3243 // Restore comment saving mode, in case it was disabled for directive.
3244 if (PP)
3245 resetExtendedTokenMode();
3246 return true; // Have a token.
3247 }
3248
3249 // If we are in raw mode, return this event as an EOF token. Let the caller
3250 // that put us in raw mode handle the event.
3251 if (isLexingRawMode()) {
3252 Result.startToken();
3253 BufferPtr = BufferEnd;
3254 FormTokenWithChars(Result, TokEnd: BufferEnd, Kind: tok::eof);
3255 return true;
3256 }
3257
3258 if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) {
3259 PP->setRecordedPreambleConditionalStack(ConditionalStack);
3260 // If the preamble cuts off the end of a header guard, consider it guarded.
3261 // The guard is valid for the preamble content itself, and for tools the
3262 // most useful answer is "yes, this file has a header guard".
3263 if (!ConditionalStack.empty())
3264 MIOpt.ExitTopLevelConditional();
3265 ConditionalStack.clear();
3266 }
3267
3268 // Issue diagnostics for unterminated #if and missing newline.
3269
3270 // If we are in a #if directive, emit an error.
3271 while (!ConditionalStack.empty()) {
3272 if (PP->getCodeCompletionFileLoc() != FileLoc)
3273 PP->Diag(Loc: ConditionalStack.back().IfLoc,
3274 DiagID: diag::err_pp_unterminated_conditional);
3275 ConditionalStack.pop_back();
3276 }
3277
3278 // Before C++11 and C2y, a file not ending with a newline was UB. Both
3279 // standards changed this behavior (as a DR or equivalent), but we still have
3280 // an opt-in diagnostic to warn about it.
3281 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
3282 Diag(Loc: BufferEnd, DiagID: diag::warn_no_newline_eof)
3283 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: BufferEnd), Code: "\n");
3284
3285 BufferPtr = CurPtr;
3286
3287 // Finally, let the preprocessor handle this.
3288 return PP->HandleEndOfFile(Result, isEndOfMacro: isPragmaLexer());
3289}
3290
3291/// peekNextPPToken - Return std::nullopt if there are no more tokens in the
3292/// buffer controlled by this lexer, otherwise return the next unexpanded
3293/// token.
3294std::optional<Token> Lexer::peekNextPPToken() {
3295 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
3296
3297 if (isDependencyDirectivesLexer()) {
3298 if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size())
3299 return std::nullopt;
3300 Token Result;
3301 (void)convertDependencyDirectiveToken(
3302 DDTok: DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result);
3303 return Result;
3304 }
3305
3306 // Switch to 'skipping' mode. This will ensure that we can lex a token
3307 // without emitting diagnostics, disables macro expansion, and will cause EOF
3308 // to return an EOF token instead of popping the include stack.
3309 LexingRawMode = true;
3310
3311 // Save state that can be changed while lexing so that we can restore it.
3312 const char *TmpBufferPtr = BufferPtr;
3313 bool inPPDirectiveMode = ParsingPreprocessorDirective;
3314 bool atStartOfLine = IsAtStartOfLine;
3315 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3316 bool leadingSpace = HasLeadingSpace;
3317 MultipleIncludeOpt MIOptState = MIOpt;
3318
3319 Token Tok;
3320 Lex(Result&: Tok);
3321
3322 // Restore state that may have changed.
3323 BufferPtr = TmpBufferPtr;
3324 ParsingPreprocessorDirective = inPPDirectiveMode;
3325 HasLeadingSpace = leadingSpace;
3326 IsAtStartOfLine = atStartOfLine;
3327 IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
3328 MIOpt = MIOptState;
3329 // Restore the lexer back to non-skipping mode.
3330 LexingRawMode = false;
3331
3332 if (Tok.is(K: tok::eof))
3333 return std::nullopt;
3334 return Tok;
3335}
3336
3337/// Find the end of a version control conflict marker.
3338static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
3339 ConflictMarkerKind CMK) {
3340 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
3341 size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
3342 auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(Start: TermLen);
3343 size_t Pos = RestOfBuffer.find(Str: Terminator);
3344 while (Pos != StringRef::npos) {
3345 // Must occur at start of line.
3346 if (Pos == 0 ||
3347 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) {
3348 RestOfBuffer = RestOfBuffer.substr(Start: Pos+TermLen);
3349 Pos = RestOfBuffer.find(Str: Terminator);
3350 continue;
3351 }
3352 return RestOfBuffer.data()+Pos;
3353 }
3354 return nullptr;
3355}
3356
3357/// IsStartOfConflictMarker - If the specified pointer is the start of a version
3358/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
3359/// and recover nicely. This returns true if it is a conflict marker and false
3360/// if not.
3361bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
3362 // Only a conflict marker if it starts at the beginning of a line.
3363 if (CurPtr != BufferStart &&
3364 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3365 return false;
3366
3367 // Check to see if we have <<<<<<< or >>>>.
3368 if (!StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: "<<<<<<<") &&
3369 !StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: ">>>> "))
3370 return false;
3371
3372 // If we have a situation where we don't care about conflict markers, ignore
3373 // it.
3374 if (CurrentConflictMarkerState || isLexingRawMode())
3375 return false;
3376
3377 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
3378
3379 // Check to see if there is an ending marker somewhere in the buffer at the
3380 // start of a line to terminate this conflict marker.
3381 if (FindConflictEnd(CurPtr, BufferEnd, CMK: Kind)) {
3382 // We found a match. We are really in a conflict marker.
3383 // Diagnose this, and ignore to the end of line.
3384 Diag(Loc: CurPtr, DiagID: diag::err_conflict_marker);
3385 CurrentConflictMarkerState = Kind;
3386
3387 // Skip ahead to the end of line. We know this exists because the
3388 // end-of-conflict marker starts with \r or \n.
3389 while (*CurPtr != '\r' && *CurPtr != '\n') {
3390 assert(CurPtr != BufferEnd && "Didn't find end of line");
3391 ++CurPtr;
3392 }
3393 BufferPtr = CurPtr;
3394 return true;
3395 }
3396
3397 // No end of conflict marker found.
3398 return false;
3399}
3400
3401/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
3402/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
3403/// is the end of a conflict marker. Handle it by ignoring up until the end of
3404/// the line. This returns true if it is a conflict marker and false if not.
3405bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
3406 // Only a conflict marker if it starts at the beginning of a line.
3407 if (CurPtr != BufferStart &&
3408 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3409 return false;
3410
3411 // If we have a situation where we don't care about conflict markers, ignore
3412 // it.
3413 if (!CurrentConflictMarkerState || isLexingRawMode())
3414 return false;
3415
3416 // Check to see if we have the marker (4 characters in a row).
3417 for (unsigned i = 1; i != 4; ++i)
3418 if (CurPtr[i] != CurPtr[0])
3419 return false;
3420
3421 // If we do have it, search for the end of the conflict marker. This could
3422 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
3423 // be the end of conflict marker.
3424 if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
3425 CMK: CurrentConflictMarkerState)) {
3426 CurPtr = End;
3427
3428 // Skip ahead to the end of line.
3429 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
3430 ++CurPtr;
3431
3432 BufferPtr = CurPtr;
3433
3434 // No longer in the conflict marker.
3435 CurrentConflictMarkerState = CMK_None;
3436 return true;
3437 }
3438
3439 return false;
3440}
3441
3442static const char *findPlaceholderEnd(const char *CurPtr,
3443 const char *BufferEnd) {
3444 if (CurPtr == BufferEnd)
3445 return nullptr;
3446 BufferEnd -= 1; // Scan until the second last character.
3447 for (; CurPtr != BufferEnd; ++CurPtr) {
3448 if (CurPtr[0] == '#' && CurPtr[1] == '>')
3449 return CurPtr + 2;
3450 }
3451 return nullptr;
3452}
3453
3454bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
3455 assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!");
3456 if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
3457 return false;
3458 const char *End = findPlaceholderEnd(CurPtr: CurPtr + 1, BufferEnd);
3459 if (!End)
3460 return false;
3461 const char *Start = CurPtr - 1;
3462 if (!LangOpts.AllowEditorPlaceholders)
3463 Diag(Loc: Start, DiagID: diag::err_placeholder_in_source);
3464 Result.startToken();
3465 FormTokenWithChars(Result, TokEnd: End, Kind: tok::raw_identifier);
3466 Result.setRawIdentifierData(Start);
3467 PP->LookUpIdentifierInfo(Identifier&: Result);
3468 Result.setFlag(Token::IsEditorPlaceholder);
3469 BufferPtr = End;
3470 return true;
3471}
3472
3473bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
3474 if (PP && PP->isCodeCompletionEnabled()) {
3475 SourceLocation Loc = FileLoc.getLocWithOffset(Offset: CurPtr-BufferStart);
3476 return Loc == PP->getCodeCompletionLoc();
3477 }
3478
3479 return false;
3480}
3481
3482void Lexer::DiagnoseDelimitedOrNamedEscapeSequence(SourceLocation Loc,
3483 bool Named,
3484 const LangOptions &Opts,
3485 DiagnosticsEngine &Diags) {
3486 unsigned DiagId;
3487 if (Opts.CPlusPlus23)
3488 DiagId = diag::warn_cxx23_delimited_escape_sequence;
3489 else if (Opts.C2y && !Named)
3490 DiagId = diag::warn_c2y_delimited_escape_sequence;
3491 else
3492 DiagId = diag::ext_delimited_escape_sequence;
3493
3494 // The trailing arguments are only used by the extension warning; either this
3495 // is a C2y extension or a C++23 extension, unless it's a named escape
3496 // sequence in C, then it's a Clang extension.
3497 unsigned Ext;
3498 if (!Opts.CPlusPlus)
3499 Ext = Named ? 2 /* Clang extension */ : 1 /* C2y extension */;
3500 else
3501 Ext = 0; // C++23 extension
3502
3503 Diags.Report(Loc, DiagID: DiagId) << Named << Ext;
3504}
3505
3506std::optional<uint32_t> Lexer::tryReadNumericUCN(const char *&StartPtr,
3507 const char *SlashLoc,
3508 Token *Result) {
3509 unsigned CharSize;
3510 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3511 assert((Kind == 'u' || Kind == 'U') && "expected a UCN");
3512
3513 unsigned NumHexDigits;
3514 if (Kind == 'u')
3515 NumHexDigits = 4;
3516 else if (Kind == 'U')
3517 NumHexDigits = 8;
3518
3519 bool Delimited = false;
3520 bool FoundEndDelimiter = false;
3521 unsigned Count = 0;
3522 bool Diagnose = Result && !isLexingRawMode();
3523
3524 if (!LangOpts.CPlusPlus && !LangOpts.C99) {
3525 if (Diagnose)
3526 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_not_valid_in_c89);
3527 return std::nullopt;
3528 }
3529
3530 const char *CurPtr = StartPtr + CharSize;
3531 const char *KindLoc = &CurPtr[-1];
3532
3533 uint32_t CodePoint = 0;
3534 while (Count != NumHexDigits || Delimited) {
3535 char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3536 if (!Delimited && Count == 0 && C == '{') {
3537 Delimited = true;
3538 CurPtr += CharSize;
3539 continue;
3540 }
3541
3542 if (Delimited && C == '}') {
3543 CurPtr += CharSize;
3544 FoundEndDelimiter = true;
3545 break;
3546 }
3547
3548 unsigned Value = llvm::hexDigitValue(C);
3549 if (Value == std::numeric_limits<unsigned>::max()) {
3550 if (!Delimited)
3551 break;
3552 if (Diagnose)
3553 Diag(Loc: SlashLoc, DiagID: diag::warn_delimited_ucn_incomplete)
3554 << StringRef(KindLoc, 1);
3555 return std::nullopt;
3556 }
3557
3558 if (CodePoint & 0xF000'0000) {
3559 if (Diagnose)
3560 Diag(Loc: KindLoc, DiagID: diag::err_escape_too_large) << 0;
3561 return std::nullopt;
3562 }
3563
3564 CodePoint <<= 4;
3565 CodePoint |= Value;
3566 CurPtr += CharSize;
3567 Count++;
3568 }
3569
3570 if (Count == 0) {
3571 if (Diagnose)
3572 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3573 : diag::warn_ucn_escape_no_digits)
3574 << StringRef(KindLoc, 1);
3575 return std::nullopt;
3576 }
3577
3578 if (Delimited && Kind == 'U') {
3579 if (Diagnose)
3580 Diag(Loc: SlashLoc, DiagID: diag::err_hex_escape_no_digits) << StringRef(KindLoc, 1);
3581 return std::nullopt;
3582 }
3583
3584 if (!Delimited && Count != NumHexDigits) {
3585 if (Diagnose) {
3586 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3587 // If the user wrote \U1234, suggest a fixit to \u.
3588 if (Count == 4 && NumHexDigits == 8) {
3589 CharSourceRange URange = makeCharRange(L&: *this, Begin: KindLoc, End: KindLoc + 1);
3590 Diag(Loc: KindLoc, DiagID: diag::note_ucn_four_not_eight)
3591 << FixItHint::CreateReplacement(RemoveRange: URange, Code: "u");
3592 }
3593 }
3594 return std::nullopt;
3595 }
3596
3597 if (Delimited && PP)
3598 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: false,
3599 Opts: PP->getLangOpts(),
3600 Diags&: PP->getDiagnostics());
3601
3602 if (Result) {
3603 Result->setFlag(Token::HasUCN);
3604 // If the UCN contains either a trigraph or a line splicing,
3605 // we need to call getAndAdvanceChar again to set the appropriate flags
3606 // on Result.
3607 if (CurPtr - StartPtr == (ptrdiff_t)(Count + 1 + (Delimited ? 2 : 0)))
3608 StartPtr = CurPtr;
3609 else
3610 while (StartPtr != CurPtr)
3611 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3612 } else {
3613 StartPtr = CurPtr;
3614 }
3615 return CodePoint;
3616}
3617
3618std::optional<uint32_t> Lexer::tryReadNamedUCN(const char *&StartPtr,
3619 const char *SlashLoc,
3620 Token *Result) {
3621 unsigned CharSize;
3622 bool Diagnose = Result && !isLexingRawMode();
3623
3624 char C = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3625 assert(C == 'N' && "expected \\N{...}");
3626
3627 const char *CurPtr = StartPtr + CharSize;
3628 const char *KindLoc = &CurPtr[-1];
3629
3630 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3631 if (C != '{') {
3632 if (Diagnose)
3633 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3634 return std::nullopt;
3635 }
3636 CurPtr += CharSize;
3637 const char *StartName = CurPtr;
3638 bool FoundEndDelimiter = false;
3639 llvm::SmallVector<char, 30> Buffer;
3640 while (C) {
3641 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3642 CurPtr += CharSize;
3643 if (C == '}') {
3644 FoundEndDelimiter = true;
3645 break;
3646 }
3647
3648 if (isVerticalWhitespace(c: C))
3649 break;
3650 Buffer.push_back(Elt: C);
3651 }
3652
3653 if (!FoundEndDelimiter || Buffer.empty()) {
3654 if (Diagnose)
3655 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3656 : diag::warn_delimited_ucn_incomplete)
3657 << StringRef(KindLoc, 1);
3658 return std::nullopt;
3659 }
3660
3661 StringRef Name(Buffer.data(), Buffer.size());
3662 std::optional<char32_t> Match =
3663 llvm::sys::unicode::nameToCodepointStrict(Name);
3664 std::optional<llvm::sys::unicode::LooseMatchingResult> LooseMatch;
3665 if (!Match) {
3666 LooseMatch = llvm::sys::unicode::nameToCodepointLooseMatching(Name);
3667 if (Diagnose) {
3668 Diag(Loc: StartName, DiagID: diag::err_invalid_ucn_name)
3669 << StringRef(Buffer.data(), Buffer.size())
3670 << makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize);
3671 if (LooseMatch) {
3672 Diag(Loc: StartName, DiagID: diag::note_invalid_ucn_name_loose_matching)
3673 << FixItHint::CreateReplacement(
3674 RemoveRange: makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize),
3675 Code: LooseMatch->Name);
3676 }
3677 }
3678 // We do not offer misspelled character names suggestions here
3679 // as the set of what would be a valid suggestion depends on context,
3680 // and we should not make invalid suggestions.
3681 }
3682
3683 if (Diagnose && Match)
3684 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: true,
3685 Opts: PP->getLangOpts(),
3686 Diags&: PP->getDiagnostics());
3687
3688 // If no diagnostic has been emitted yet, likely because we are doing a
3689 // tentative lexing, we do not want to recover here to make sure the token
3690 // will not be incorrectly considered valid. This function will be called
3691 // again and a diagnostic emitted then.
3692 if (LooseMatch && Diagnose)
3693 Match = LooseMatch->CodePoint;
3694
3695 if (Result) {
3696 Result->setFlag(Token::HasUCN);
3697 // If the UCN contains either a trigraph or a line splicing,
3698 // we need to call getAndAdvanceChar again to set the appropriate flags
3699 // on Result.
3700 if (CurPtr - StartPtr == (ptrdiff_t)(Buffer.size() + 3))
3701 StartPtr = CurPtr;
3702 else
3703 while (StartPtr != CurPtr)
3704 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3705 } else {
3706 StartPtr = CurPtr;
3707 }
3708 return Match ? std::optional<uint32_t>(*Match) : std::nullopt;
3709}
3710
3711uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3712 Token *Result) {
3713
3714 unsigned CharSize;
3715 std::optional<uint32_t> CodePointOpt;
3716 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3717 if (Kind == 'u' || Kind == 'U')
3718 CodePointOpt = tryReadNumericUCN(StartPtr, SlashLoc, Result);
3719 else if (Kind == 'N')
3720 CodePointOpt = tryReadNamedUCN(StartPtr, SlashLoc, Result);
3721
3722 if (!CodePointOpt)
3723 return 0;
3724
3725 uint32_t CodePoint = *CodePointOpt;
3726
3727 // Don't apply C family restrictions to UCNs in assembly mode
3728 if (LangOpts.AsmPreprocessor)
3729 return CodePoint;
3730
3731 // C23 6.4.3p2: A universal character name shall not designate a code point
3732 // where the hexadecimal value is:
3733 // - in the range D800 through DFFF inclusive; or
3734 // - greater than 10FFFF.
3735 // A universal-character-name outside the c-char-sequence of a character
3736 // constant, or the s-char-sequence of a string-literal shall not designate
3737 // a control character or a character in the basic character set.
3738
3739 // C++11 [lex.charset]p2: If the hexadecimal value for a
3740 // universal-character-name corresponds to a surrogate code point (in the
3741 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
3742 // if the hexadecimal value for a universal-character-name outside the
3743 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or
3744 // string literal corresponds to a control character (in either of the
3745 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
3746 // basic source character set, the program is ill-formed.
3747 if (CodePoint < 0xA0) {
3748 // We don't use isLexingRawMode() here because we need to warn about bad
3749 // UCNs even when skipping preprocessing tokens in a #if block.
3750 if (Result && PP) {
3751 if (CodePoint < 0x20 || CodePoint >= 0x7F)
3752 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_control_character);
3753 else {
3754 char C = static_cast<char>(CodePoint);
3755 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
3756 }
3757 }
3758
3759 return 0;
3760 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) {
3761 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
3762 // We don't use isLexingRawMode() here because we need to diagnose bad
3763 // UCNs even when skipping preprocessing tokens in a #if block.
3764 if (Result && PP) {
3765 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11)
3766 Diag(Loc: BufferPtr, DiagID: diag::warn_ucn_escape_surrogate);
3767 else
3768 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_invalid);
3769 }
3770 return 0;
3771 }
3772
3773 return CodePoint;
3774}
3775
3776bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3777 const char *CurPtr) {
3778 if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
3779 isUnicodeWhitespace(Codepoint: C)) {
3780 Diag(Loc: BufferPtr, DiagID: diag::ext_unicode_whitespace)
3781 << makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr);
3782
3783 Result.setFlag(Token::LeadingSpace);
3784 return true;
3785 }
3786 return false;
3787}
3788
3789void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3790 IsAtStartOfLine = Result.isAtStartOfLine();
3791 HasLeadingSpace = Result.hasLeadingSpace();
3792 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
3793 // Note that this doesn't affect IsAtPhysicalStartOfLine.
3794}
3795
3796bool Lexer::Lex(Token &Result) {
3797 assert(!isDependencyDirectivesLexer());
3798
3799 // Start a new token.
3800 Result.startToken();
3801
3802 // Set up misc whitespace flags for LexTokenInternal.
3803 if (IsAtStartOfLine) {
3804 Result.setFlag(Token::StartOfLine);
3805 IsAtStartOfLine = false;
3806 }
3807
3808 if (IsAtPhysicalStartOfLine) {
3809 Result.setFlag(Token::PhysicalStartOfLine);
3810 IsAtPhysicalStartOfLine = false;
3811 }
3812
3813 if (HasLeadingSpace) {
3814 Result.setFlag(Token::LeadingSpace);
3815 HasLeadingSpace = false;
3816 }
3817
3818 if (HasLeadingEmptyMacro) {
3819 Result.setFlag(Token::LeadingEmptyMacro);
3820 HasLeadingEmptyMacro = false;
3821 }
3822
3823 bool isRawLex = isLexingRawMode();
3824 (void) isRawLex;
3825 bool returnedToken = LexTokenInternal(Result);
3826 // (After the LexTokenInternal call, the lexer might be destroyed.)
3827 assert((returnedToken || !isRawLex) && "Raw lex must succeed");
3828 return returnedToken;
3829}
3830
3831/// LexTokenInternal - This implements a simple C family lexer. It is an
3832/// extremely performance critical piece of code. This assumes that the buffer
3833/// has a null character at the end of the file. This returns a preprocessing
3834/// token, not a normal token, as such, it is an internal interface. It assumes
3835/// that the Flags of result have been cleared before calling this.
3836bool Lexer::LexTokenInternal(Token &Result) {
3837LexStart:
3838 assert(!Result.needsCleaning() && "Result needs cleaning");
3839 assert(!Result.hasPtrData() && "Result has not been reset");
3840
3841 // CurPtr - Cache BufferPtr in an automatic variable.
3842 const char *CurPtr = BufferPtr;
3843
3844 // Small amounts of horizontal whitespace is very common between tokens.
3845 // Check for space character separately to skip the expensive
3846 // isHorizontalWhitespace() check
3847 if (*CurPtr == ' ' || isHorizontalWhitespace(c: *CurPtr)) {
3848 do {
3849 ++CurPtr;
3850 } while (*CurPtr == ' ' || isHorizontalWhitespace(c: *CurPtr));
3851
3852 // If we are keeping whitespace and other tokens, just return what we just
3853 // skipped. The next lexer invocation will return the token after the
3854 // whitespace.
3855 if (isKeepWhitespaceMode()) {
3856 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3857 // FIXME: The next token will not have LeadingSpace set.
3858 return true;
3859 }
3860
3861 BufferPtr = CurPtr;
3862 Result.setFlag(Token::LeadingSpace);
3863 }
3864
3865 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
3866
3867 // Read a character, advancing over it.
3868 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3869 tok::TokenKind Kind;
3870
3871 if (!isVerticalWhitespace(c: Char))
3872 NewLinePtr = nullptr;
3873
3874 switch (Char) {
3875 case 0: // Null.
3876 // Found end of file?
3877 if (CurPtr-1 == BufferEnd)
3878 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3879
3880 // Check if we are performing code completion.
3881 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3882 // Return the code-completion token.
3883 Result.startToken();
3884 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::code_completion);
3885 return true;
3886 }
3887
3888 if (!isLexingRawMode())
3889 Diag(Loc: CurPtr-1, DiagID: diag::null_in_file);
3890 Result.setFlag(Token::LeadingSpace);
3891 if (SkipWhitespace(Result, CurPtr))
3892 return true; // KeepWhitespaceMode
3893
3894 // We know the lexer hasn't changed, so just try again with this lexer.
3895 // (We manually eliminate the tail call to avoid recursion.)
3896 goto LexNextToken;
3897
3898 case 26: // DOS & CP/M EOF: "^Z".
3899 // If we're in Microsoft extensions mode, treat this as end of file.
3900 if (LangOpts.MicrosoftExt) {
3901 if (!isLexingRawMode())
3902 Diag(Loc: CurPtr-1, DiagID: diag::ext_ctrl_z_eof_microsoft);
3903 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3904 }
3905
3906 // If Microsoft extensions are disabled, this is just random garbage.
3907 Kind = tok::unknown;
3908 break;
3909
3910 case '\r':
3911 if (CurPtr[0] == '\n')
3912 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3913 [[fallthrough]];
3914 case '\n':
3915 // If we are inside a preprocessor directive and we see the end of line,
3916 // we know we are done with the directive, so return an EOD token.
3917 if (ParsingPreprocessorDirective) {
3918 // Done parsing the "line".
3919 ParsingPreprocessorDirective = false;
3920
3921 // Restore comment saving mode, in case it was disabled for directive.
3922 if (PP)
3923 resetExtendedTokenMode();
3924
3925 // Since we consumed a newline, we are back at the start of a line.
3926 IsAtStartOfLine = true;
3927 IsAtPhysicalStartOfLine = true;
3928 NewLinePtr = CurPtr - 1;
3929
3930 Kind = tok::eod;
3931 break;
3932 }
3933
3934 // No leading whitespace seen so far.
3935 Result.clearFlag(Flag: Token::LeadingSpace);
3936
3937 if (SkipWhitespace(Result, CurPtr))
3938 return true; // KeepWhitespaceMode
3939
3940 // We only saw whitespace, so just try again with this lexer.
3941 // (We manually eliminate the tail call to avoid recursion.)
3942 goto LexNextToken;
3943 case ' ':
3944 case '\t':
3945 case '\f':
3946 case '\v':
3947 SkipHorizontalWhitespace:
3948 Result.setFlag(Token::LeadingSpace);
3949 if (SkipWhitespace(Result, CurPtr))
3950 return true; // KeepWhitespaceMode
3951
3952 SkipIgnoredUnits:
3953 CurPtr = BufferPtr;
3954
3955 // If the next token is obviously a // or /* */ comment, skip it efficiently
3956 // too (without going through the big switch stmt).
3957 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
3958 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
3959 if (SkipLineComment(Result, CurPtr: CurPtr + 2))
3960 return true; // There is a token to return.
3961 goto SkipIgnoredUnits;
3962 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
3963 if (SkipBlockComment(Result, CurPtr: CurPtr + 2))
3964 return true; // There is a token to return.
3965 goto SkipIgnoredUnits;
3966 } else if (isHorizontalWhitespace(c: *CurPtr)) {
3967 goto SkipHorizontalWhitespace;
3968 }
3969 // We only saw whitespace, so just try again with this lexer.
3970 // (We manually eliminate the tail call to avoid recursion.)
3971 goto LexNextToken;
3972
3973 // C99 6.4.4.1: Integer Constants.
3974 // C99 6.4.4.2: Floating Constants.
3975 case '0': case '1': case '2': case '3': case '4':
3976 case '5': case '6': case '7': case '8': case '9':
3977 // Notify MIOpt that we read a non-whitespace/non-comment token.
3978 MIOpt.ReadToken();
3979 return LexNumericConstant(Result, CurPtr);
3980
3981 // Identifier (e.g., uber), or
3982 // UTF-8 (C23/C++17) or UTF-16 (C11/C++11) character literal, or
3983 // UTF-8 or UTF-16 string literal (C11/C++11).
3984 case 'u':
3985 // Notify MIOpt that we read a non-whitespace/non-comment token.
3986 MIOpt.ReadToken();
3987
3988 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
3989 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
3990
3991 // UTF-16 string literal
3992 if (Char == '"')
3993 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3994 Kind: tok::utf16_string_literal);
3995
3996 // UTF-16 character constant
3997 if (Char == '\'')
3998 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
3999 Kind: tok::utf16_char_constant);
4000
4001 // UTF-16 raw string literal
4002 if (Char == 'R' && LangOpts.RawStringLiterals &&
4003 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4004 return LexRawStringLiteral(Result,
4005 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4006 Size: SizeTmp2, Tok&: Result),
4007 Kind: tok::utf16_string_literal);
4008
4009 if (Char == '8') {
4010 char Char2 = getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2);
4011
4012 // UTF-8 string literal
4013 if (Char2 == '"')
4014 return LexStringLiteral(Result,
4015 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4016 Size: SizeTmp2, Tok&: Result),
4017 Kind: tok::utf8_string_literal);
4018 if (Char2 == '\'' && (LangOpts.CPlusPlus17 || LangOpts.C23))
4019 return LexCharConstant(
4020 Result, CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4021 Size: SizeTmp2, Tok&: Result),
4022 Kind: tok::utf8_char_constant);
4023
4024 if (Char2 == 'R' && LangOpts.RawStringLiterals) {
4025 unsigned SizeTmp3;
4026 char Char3 = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4027 // UTF-8 raw string literal
4028 if (Char3 == '"') {
4029 return LexRawStringLiteral(Result,
4030 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4031 Size: SizeTmp2, Tok&: Result),
4032 Size: SizeTmp3, Tok&: Result),
4033 Kind: tok::utf8_string_literal);
4034 }
4035 }
4036 }
4037 }
4038
4039 // treat u like the start of an identifier.
4040 return LexIdentifierContinue(Result, CurPtr);
4041
4042 case 'U': // Identifier (e.g. Uber) or C11/C++11 UTF-32 string literal
4043 // Notify MIOpt that we read a non-whitespace/non-comment token.
4044 MIOpt.ReadToken();
4045
4046 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
4047 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4048
4049 // UTF-32 string literal
4050 if (Char == '"')
4051 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4052 Kind: tok::utf32_string_literal);
4053
4054 // UTF-32 character constant
4055 if (Char == '\'')
4056 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4057 Kind: tok::utf32_char_constant);
4058
4059 // UTF-32 raw string literal
4060 if (Char == 'R' && LangOpts.RawStringLiterals &&
4061 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4062 return LexRawStringLiteral(Result,
4063 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4064 Size: SizeTmp2, Tok&: Result),
4065 Kind: tok::utf32_string_literal);
4066 }
4067
4068 // treat U like the start of an identifier.
4069 return LexIdentifierContinue(Result, CurPtr);
4070
4071 case 'R': // Identifier or C++0x raw string literal
4072 // Notify MIOpt that we read a non-whitespace/non-comment token.
4073 MIOpt.ReadToken();
4074
4075 if (LangOpts.RawStringLiterals) {
4076 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4077
4078 if (Char == '"')
4079 return LexRawStringLiteral(Result,
4080 CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4081 Kind: tok::string_literal);
4082 }
4083
4084 // treat R like the start of an identifier.
4085 return LexIdentifierContinue(Result, CurPtr);
4086
4087 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
4088 // Notify MIOpt that we read a non-whitespace/non-comment token.
4089 MIOpt.ReadToken();
4090 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4091
4092 // Wide string literal.
4093 if (Char == '"')
4094 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4095 Kind: tok::wide_string_literal);
4096
4097 // Wide raw string literal.
4098 if (LangOpts.RawStringLiterals && Char == 'R' &&
4099 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4100 return LexRawStringLiteral(Result,
4101 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4102 Size: SizeTmp2, Tok&: Result),
4103 Kind: tok::wide_string_literal);
4104
4105 // Wide character constant.
4106 if (Char == '\'')
4107 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4108 Kind: tok::wide_char_constant);
4109 // FALL THROUGH, treating L like the start of an identifier.
4110 [[fallthrough]];
4111
4112 // C99 6.4.2: Identifiers.
4113 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
4114 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
4115 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
4116 case 'V': case 'W': case 'X': case 'Y': case 'Z':
4117 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
4118 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
4119 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
4120 case 'v': case 'w': case 'x': case 'y': case 'z':
4121 case '_':
4122 // Notify MIOpt that we read a non-whitespace/non-comment token.
4123 MIOpt.ReadToken();
4124 return LexIdentifierContinue(Result, CurPtr);
4125 case '$': // $ in identifiers.
4126 if (LangOpts.DollarIdents) {
4127 if (!isLexingRawMode())
4128 Diag(Loc: CurPtr-1, DiagID: diag::ext_dollar_in_identifier);
4129 // Notify MIOpt that we read a non-whitespace/non-comment token.
4130 MIOpt.ReadToken();
4131 return LexIdentifierContinue(Result, CurPtr);
4132 }
4133
4134 Kind = tok::unknown;
4135 break;
4136
4137 // C99 6.4.4: Character Constants.
4138 case '\'':
4139 // Notify MIOpt that we read a non-whitespace/non-comment token.
4140 MIOpt.ReadToken();
4141 return LexCharConstant(Result, CurPtr, Kind: tok::char_constant);
4142
4143 // C99 6.4.5: String Literals.
4144 case '"':
4145 // Notify MIOpt that we read a non-whitespace/non-comment token.
4146 MIOpt.ReadToken();
4147 return LexStringLiteral(Result, CurPtr,
4148 Kind: ParsingFilename ? tok::header_name
4149 : tok::string_literal);
4150
4151 // C99 6.4.6: Punctuators.
4152 case '?':
4153 Kind = tok::question;
4154 break;
4155 case '[':
4156 Kind = tok::l_square;
4157 break;
4158 case ']':
4159 Kind = tok::r_square;
4160 break;
4161 case '(':
4162 Kind = tok::l_paren;
4163 break;
4164 case ')':
4165 Kind = tok::r_paren;
4166 break;
4167 case '{':
4168 Kind = tok::l_brace;
4169 break;
4170 case '}':
4171 Kind = tok::r_brace;
4172 break;
4173 case '.':
4174 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4175 if (Char >= '0' && Char <= '9') {
4176 // Notify MIOpt that we read a non-whitespace/non-comment token.
4177 MIOpt.ReadToken();
4178
4179 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result));
4180 } else if (LangOpts.CPlusPlus && Char == '*') {
4181 Kind = tok::periodstar;
4182 CurPtr += SizeTmp;
4183 } else if (Char == '.' &&
4184 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '.') {
4185 Kind = tok::ellipsis;
4186 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4187 Size: SizeTmp2, Tok&: Result);
4188 } else {
4189 Kind = tok::period;
4190 }
4191 break;
4192 case '&':
4193 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4194 if (Char == '&') {
4195 Kind = tok::ampamp;
4196 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4197 } else if (Char == '=') {
4198 Kind = tok::ampequal;
4199 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4200 } else {
4201 Kind = tok::amp;
4202 }
4203 break;
4204 case '*':
4205 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4206 Kind = tok::starequal;
4207 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4208 } else {
4209 Kind = tok::star;
4210 }
4211 break;
4212 case '+':
4213 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4214 if (Char == '+') {
4215 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4216 Kind = tok::plusplus;
4217 } else if (Char == '=') {
4218 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4219 Kind = tok::plusequal;
4220 } else {
4221 Kind = tok::plus;
4222 }
4223 break;
4224 case '-':
4225 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4226 if (Char == '-') { // --
4227 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4228 Kind = tok::minusminus;
4229 } else if (Char == '>' && LangOpts.CPlusPlus &&
4230 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '*') { // C++ ->*
4231 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4232 Size: SizeTmp2, Tok&: Result);
4233 Kind = tok::arrowstar;
4234 } else if (Char == '>') { // ->
4235 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4236 Kind = tok::arrow;
4237 } else if (Char == '=') { // -=
4238 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4239 Kind = tok::minusequal;
4240 } else {
4241 Kind = tok::minus;
4242 }
4243 break;
4244 case '~':
4245 Kind = tok::tilde;
4246 break;
4247 case '!':
4248 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4249 Kind = tok::exclaimequal;
4250 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4251 } else {
4252 Kind = tok::exclaim;
4253 }
4254 break;
4255 case '/':
4256 // 6.4.9: Comments
4257 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4258 if (Char == '/') { // Line comment.
4259 // Even if Line comments are disabled (e.g. in C89 mode), we generally
4260 // want to lex this as a comment. There is one problem with this though,
4261 // that in one particular corner case, this can change the behavior of the
4262 // resultant program. For example, In "foo //**/ bar", C89 would lex
4263 // this as "foo / bar" and languages with Line comments would lex it as
4264 // "foo". Check to see if the character after the second slash is a '*'.
4265 // If so, we will lex that as a "/" instead of the start of a comment.
4266 // However, we never do this if we are just preprocessing.
4267 bool TreatAsComment =
4268 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP);
4269 if (!TreatAsComment)
4270 if (!(PP && PP->isPreprocessedOutput()))
4271 TreatAsComment = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) != '*';
4272
4273 if (TreatAsComment) {
4274 if (SkipLineComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result)))
4275 return true; // There is a token to return.
4276
4277 // It is common for the tokens immediately after a // comment to be
4278 // whitespace (indentation for the next line). Instead of going through
4279 // the big switch, handle it efficiently now.
4280 goto SkipIgnoredUnits;
4281 }
4282 }
4283
4284 if (Char == '*') { // /**/ comment.
4285 if (SkipBlockComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result)))
4286 return true; // There is a token to return.
4287
4288 // We only saw whitespace, so just try again with this lexer.
4289 // (We manually eliminate the tail call to avoid recursion.)
4290 goto LexNextToken;
4291 }
4292
4293 if (Char == '=') {
4294 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4295 Kind = tok::slashequal;
4296 } else {
4297 Kind = tok::slash;
4298 }
4299 break;
4300 case '%':
4301 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4302 if (Char == '=') {
4303 Kind = tok::percentequal;
4304 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4305 } else if (LangOpts.Digraphs && Char == '>') {
4306 Kind = tok::r_brace; // '%>' -> '}'
4307 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4308 } else if (LangOpts.Digraphs && Char == ':') {
4309 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4310 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4311 if (Char == '%' && getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == ':') {
4312 Kind = tok::hashhash; // '%:%:' -> '##'
4313 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4314 Size: SizeTmp2, Tok&: Result);
4315 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
4316 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4317 if (!isLexingRawMode())
4318 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4319 Kind = tok::hashat;
4320 } else { // '%:' -> '#'
4321 // We parsed a # character. If this occurs at the start of the line,
4322 // it's actually the start of a preprocessing directive. Callback to
4323 // the preprocessor to handle it.
4324 // TODO: -fpreprocessed mode??
4325 if (Result.isAtPhysicalStartOfLine() && !LexingRawMode &&
4326 !Is_PragmaLexer)
4327 goto HandleDirective;
4328
4329 Kind = tok::hash;
4330 }
4331 } else {
4332 Kind = tok::percent;
4333 }
4334 break;
4335 case '<':
4336 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4337 if (ParsingFilename) {
4338 return LexAngledStringLiteral(Result, CurPtr);
4339 } else if (Char == '<') {
4340 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4341 if (After == '=') {
4342 Kind = tok::lesslessequal;
4343 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4344 Size: SizeTmp2, Tok&: Result);
4345 } else if (After == '<' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4346 // If this is actually a '<<<<<<<' version control conflict marker,
4347 // recognize it as such and recover nicely.
4348 goto LexNextToken;
4349 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4350 // If this is '<<<<' and we're in a Perforce-style conflict marker,
4351 // ignore it.
4352 goto LexNextToken;
4353 } else if (LangOpts.CUDA && After == '<') {
4354 Kind = tok::lesslessless;
4355 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4356 Size: SizeTmp2, Tok&: Result);
4357 } else {
4358 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4359 Kind = tok::lessless;
4360 }
4361 } else if (Char == '=') {
4362 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4363 if (After == '>') {
4364 if (LangOpts.CPlusPlus20) {
4365 if (!isLexingRawMode())
4366 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx17_compat_spaceship);
4367 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4368 Size: SizeTmp2, Tok&: Result);
4369 Kind = tok::spaceship;
4370 break;
4371 }
4372 // Suggest adding a space between the '<=' and the '>' to avoid a
4373 // change in semantics if this turns up in C++ <=17 mode.
4374 if (LangOpts.CPlusPlus && !isLexingRawMode()) {
4375 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx20_compat_spaceship)
4376 << FixItHint::CreateInsertion(
4377 InsertionLoc: getSourceLocation(Loc: CurPtr + SizeTmp, TokLen: SizeTmp2), Code: " ");
4378 }
4379 }
4380 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4381 Kind = tok::lessequal;
4382 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '['
4383 if (LangOpts.CPlusPlus11 &&
4384 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == ':') {
4385 // C++0x [lex.pptoken]p3:
4386 // Otherwise, if the next three characters are <:: and the subsequent
4387 // character is neither : nor >, the < is treated as a preprocessor
4388 // token by itself and not as the first character of the alternative
4389 // token <:.
4390 unsigned SizeTmp3;
4391 char After = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4392 if (After != ':' && After != '>') {
4393 Kind = tok::less;
4394 if (!isLexingRawMode())
4395 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_less_colon_colon);
4396 break;
4397 }
4398 }
4399
4400 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4401 Kind = tok::l_square;
4402 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{'
4403 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4404 Kind = tok::l_brace;
4405 } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 &&
4406 lexEditorPlaceholder(Result, CurPtr)) {
4407 return true;
4408 } else {
4409 Kind = tok::less;
4410 }
4411 break;
4412 case '>':
4413 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4414 if (Char == '=') {
4415 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4416 Kind = tok::greaterequal;
4417 } else if (Char == '>') {
4418 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4419 if (After == '=') {
4420 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4421 Size: SizeTmp2, Tok&: Result);
4422 Kind = tok::greatergreaterequal;
4423 } else if (After == '>' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4424 // If this is actually a '>>>>' conflict marker, recognize it as such
4425 // and recover nicely.
4426 goto LexNextToken;
4427 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4428 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
4429 goto LexNextToken;
4430 } else if (LangOpts.CUDA && After == '>') {
4431 Kind = tok::greatergreatergreater;
4432 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4433 Size: SizeTmp2, Tok&: Result);
4434 } else {
4435 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4436 Kind = tok::greatergreater;
4437 }
4438 } else {
4439 Kind = tok::greater;
4440 }
4441 break;
4442 case '^':
4443 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4444 if (Char == '=') {
4445 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4446 Kind = tok::caretequal;
4447 } else if (LangOpts.Reflection && Char == '^') {
4448 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4449 Kind = tok::caretcaret;
4450 } else {
4451 if (LangOpts.OpenCL && Char == '^')
4452 Diag(Loc: CurPtr, DiagID: diag::err_opencl_logical_exclusive_or);
4453 Kind = tok::caret;
4454 }
4455 break;
4456 case '|':
4457 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4458 if (Char == '=') {
4459 Kind = tok::pipeequal;
4460 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4461 } else if (Char == '|') {
4462 // If this is '|||||||' and we're in a conflict marker, ignore it.
4463 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4464 goto LexNextToken;
4465 Kind = tok::pipepipe;
4466 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4467 } else {
4468 Kind = tok::pipe;
4469 }
4470 break;
4471 case ':':
4472 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4473 if (LangOpts.Digraphs && Char == '>') {
4474 Kind = tok::r_square; // ':>' -> ']'
4475 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4476 } else if (Char == ':') {
4477 Kind = tok::coloncolon;
4478 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4479 } else {
4480 Kind = tok::colon;
4481 }
4482 break;
4483 case ';':
4484 Kind = tok::semi;
4485 break;
4486 case '=':
4487 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4488 if (Char == '=') {
4489 // If this is '====' and we're in a conflict marker, ignore it.
4490 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4491 goto LexNextToken;
4492
4493 Kind = tok::equalequal;
4494 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4495 } else {
4496 Kind = tok::equal;
4497 }
4498 break;
4499 case ',':
4500 Kind = tok::comma;
4501 break;
4502 case '#':
4503 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4504 if (Char == '#') {
4505 Kind = tok::hashhash;
4506 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4507 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize
4508 Kind = tok::hashat;
4509 if (!isLexingRawMode())
4510 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4511 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4512 } else {
4513 // We parsed a # character. If this occurs at the start of the line,
4514 // it's actually the start of a preprocessing directive. Callback to
4515 // the preprocessor to handle it.
4516 // TODO: -fpreprocessed mode??
4517 if (Result.isAtPhysicalStartOfLine() && !LexingRawMode && !Is_PragmaLexer)
4518 goto HandleDirective;
4519
4520 Kind = tok::hash;
4521 }
4522 break;
4523
4524 case '@':
4525 // Objective C support.
4526 if (CurPtr[-1] == '@' && LangOpts.ObjC) {
4527 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::at);
4528 if (PP && Result.isAtPhysicalStartOfLine() && !LexingRawMode &&
4529 !Is_PragmaLexer) {
4530 Token NextPPTok;
4531 NextPPTok.startToken();
4532 {
4533 llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective(
4534 this->ParsingPreprocessorDirective, true);
4535 auto NextTokOr = peekNextPPToken();
4536 if (NextTokOr.has_value()) {
4537 NextPPTok = *NextTokOr;
4538 }
4539 }
4540 if (NextPPTok.is(K: tok::raw_identifier) &&
4541 NextPPTok.getRawIdentifier() == "import") {
4542 PP->HandleDirective(Result);
4543 return false;
4544 }
4545 }
4546 return true;
4547 } else
4548 Kind = tok::unknown;
4549 break;
4550
4551 // UCNs (C99 6.4.3, C++11 [lex.charset]p2)
4552 case '\\':
4553 if (!LangOpts.AsmPreprocessor) {
4554 if (uint32_t CodePoint = tryReadUCN(StartPtr&: CurPtr, SlashLoc: BufferPtr, Result: &Result)) {
4555 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4556 if (SkipWhitespace(Result, CurPtr))
4557 return true; // KeepWhitespaceMode
4558
4559 // We only saw whitespace, so just try again with this lexer.
4560 // (We manually eliminate the tail call to avoid recursion.)
4561 goto LexNextToken;
4562 }
4563
4564 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4565 }
4566 }
4567
4568 Kind = tok::unknown;
4569 break;
4570
4571 default: {
4572 if (isASCII(c: Char)) {
4573 Kind = tok::unknown;
4574 break;
4575 }
4576
4577 llvm::UTF32 CodePoint;
4578
4579 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
4580 // an escaped newline.
4581 --CurPtr;
4582 llvm::ConversionResult Status =
4583 llvm::convertUTF8Sequence(source: (const llvm::UTF8 **)&CurPtr,
4584 sourceEnd: (const llvm::UTF8 *)BufferEnd,
4585 target: &CodePoint,
4586 flags: llvm::strictConversion);
4587 if (Status == llvm::conversionOK) {
4588 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4589 if (SkipWhitespace(Result, CurPtr))
4590 return true; // KeepWhitespaceMode
4591
4592 // We only saw whitespace, so just try again with this lexer.
4593 // (We manually eliminate the tail call to avoid recursion.)
4594 goto LexNextToken;
4595 }
4596 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4597 }
4598
4599 if (isLexingRawMode() || ParsingPreprocessorDirective ||
4600 PP->isPreprocessedOutput()) {
4601 ++CurPtr;
4602 Kind = tok::unknown;
4603 break;
4604 }
4605
4606 // Non-ASCII characters tend to creep into source code unintentionally.
4607 // Instead of letting the parser complain about the unknown token,
4608 // just diagnose the invalid UTF-8, then drop the character.
4609 Diag(Loc: CurPtr, DiagID: diag::err_invalid_utf8);
4610
4611 BufferPtr = CurPtr+1;
4612 // We're pretending the character didn't exist, so just try again with
4613 // this lexer.
4614 // (We manually eliminate the tail call to avoid recursion.)
4615 goto LexNextToken;
4616 }
4617 }
4618
4619 // Notify MIOpt that we read a non-whitespace/non-comment token.
4620 MIOpt.ReadToken();
4621
4622 // Update the location of token as well as BufferPtr.
4623 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
4624 return true;
4625
4626HandleDirective:
4627
4628 // We parsed a # character and it's the start of a preprocessing directive.
4629 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::hash);
4630 PP->HandleDirective(Result);
4631
4632 if (PP->hadModuleLoaderFatalFailure())
4633 // With a fatal failure in the module loader, we abort parsing.
4634 return true;
4635
4636 // We parsed the directive; lex a token with the new state.
4637 return false;
4638
4639LexNextToken:
4640 Result.clearFlag(Flag: Token::NeedsCleaning);
4641 goto LexStart;
4642}
4643
4644const char *Lexer::convertDependencyDirectiveToken(
4645 const dependency_directives_scan::Token &DDTok, Token &Result) {
4646 const char *TokPtr = BufferStart + DDTok.Offset;
4647 Result.startToken();
4648 Result.setLocation(getSourceLocation(Loc: TokPtr));
4649 Result.setKind(DDTok.Kind);
4650 Result.setFlag((Token::TokenFlags)DDTok.Flags);
4651 Result.setLength(DDTok.Length);
4652 if (Result.is(K: tok::raw_identifier))
4653 Result.setRawIdentifierData(TokPtr);
4654 else if (Result.isLiteral())
4655 Result.setLiteralData(TokPtr);
4656 BufferPtr = TokPtr + DDTok.Length;
4657 return TokPtr;
4658}
4659
4660bool Lexer::LexDependencyDirectiveToken(Token &Result) {
4661 assert(isDependencyDirectivesLexer());
4662
4663 using namespace dependency_directives_scan;
4664
4665 if (BufferPtr == BufferEnd)
4666 return LexEndOfFile(Result, CurPtr: BufferPtr);
4667
4668 while (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) {
4669 if (DepDirectives.front().Kind == pp_eof)
4670 return LexEndOfFile(Result, CurPtr: BufferEnd);
4671 if (DepDirectives.front().Kind == tokens_present_before_eof)
4672 MIOpt.ReadToken();
4673 NextDepDirectiveTokenIndex = 0;
4674 DepDirectives = DepDirectives.drop_front();
4675 }
4676
4677 const dependency_directives_scan::Token &DDTok =
4678 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex++];
4679 if (NextDepDirectiveTokenIndex > 1 || DDTok.Kind != tok::hash) {
4680 // Read something other than a preprocessor directive hash.
4681 MIOpt.ReadToken();
4682 }
4683
4684 if (ParsingFilename && DDTok.is(K: tok::less)) {
4685 BufferPtr = BufferStart + DDTok.Offset;
4686 LexAngledStringLiteral(Result, CurPtr: BufferPtr + 1);
4687 if (Result.isNot(K: tok::header_name))
4688 return true;
4689 // Advance the index of lexed tokens.
4690 while (true) {
4691 const dependency_directives_scan::Token &NextTok =
4692 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex];
4693 if (BufferStart + NextTok.Offset >= BufferPtr)
4694 break;
4695 ++NextDepDirectiveTokenIndex;
4696 }
4697 return true;
4698 }
4699
4700 const char *TokPtr = convertDependencyDirectiveToken(DDTok, Result);
4701
4702 if (Result.is(K: tok::hash) && Result.isAtStartOfLine()) {
4703 PP->HandleDirective(Result);
4704 if (PP->hadModuleLoaderFatalFailure())
4705 // With a fatal failure in the module loader, we abort parsing.
4706 return true;
4707 return false;
4708 }
4709 if (Result.is(K: tok::at) && Result.isAtStartOfLine()) {
4710 auto NextTok = peekNextPPToken();
4711 if (NextTok && NextTok->is(K: tok::raw_identifier) &&
4712 NextTok->getRawIdentifier() == "import") {
4713 PP->HandleDirective(Result);
4714 if (PP->hadModuleLoaderFatalFailure())
4715 return true;
4716 return false;
4717 }
4718 }
4719 if (Result.is(K: tok::raw_identifier)) {
4720 Result.setRawIdentifierData(TokPtr);
4721 if (!isLexingRawMode()) {
4722 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
4723 if (LangOpts.CPlusPlusModules && Result.isModuleContextualKeyword() &&
4724 PP->HandleModuleContextualKeyword(Result)) {
4725 PP->HandleDirective(Result);
4726 return false;
4727 }
4728 if (II->isHandleIdentifierCase())
4729 return PP->HandleIdentifier(Identifier&: Result);
4730 }
4731 return true;
4732 }
4733 if (Result.isLiteral())
4734 return true;
4735 if (Result.is(K: tok::colon)) {
4736 // Convert consecutive colons to 'tok::coloncolon'.
4737 if (*BufferPtr == ':') {
4738 assert(DepDirectives.front().Tokens[NextDepDirectiveTokenIndex].is(
4739 tok::colon));
4740 ++NextDepDirectiveTokenIndex;
4741 Result.setKind(tok::coloncolon);
4742 }
4743 return true;
4744 }
4745 if (Result.is(K: tok::eod))
4746 ParsingPreprocessorDirective = false;
4747
4748 return true;
4749}
4750
4751bool Lexer::LexDependencyDirectiveTokenWhileSkipping(Token &Result) {
4752 assert(isDependencyDirectivesLexer());
4753
4754 using namespace dependency_directives_scan;
4755
4756 bool Stop = false;
4757 unsigned NestedIfs = 0;
4758 do {
4759 DepDirectives = DepDirectives.drop_front();
4760 switch (DepDirectives.front().Kind) {
4761 case pp_none:
4762 llvm_unreachable("unexpected 'pp_none'");
4763 case pp_include:
4764 case pp___include_macros:
4765 case pp_define:
4766 case pp_undef:
4767 case pp_import:
4768 case pp_pragma_import:
4769 case pp_pragma_once:
4770 case pp_pragma_push_macro:
4771 case pp_pragma_pop_macro:
4772 case pp_pragma_include_alias:
4773 case pp_pragma_system_header:
4774 case pp_include_next:
4775 case decl_at_import:
4776 case cxx_module_decl:
4777 case cxx_import_decl:
4778 case cxx_export_module_decl:
4779 case cxx_export_import_decl:
4780 case tokens_present_before_eof:
4781 break;
4782 case pp_if:
4783 case pp_ifdef:
4784 case pp_ifndef:
4785 ++NestedIfs;
4786 break;
4787 case pp_elif:
4788 case pp_elifdef:
4789 case pp_elifndef:
4790 case pp_else:
4791 if (!NestedIfs) {
4792 Stop = true;
4793 }
4794 break;
4795 case pp_endif:
4796 if (!NestedIfs) {
4797 Stop = true;
4798 } else {
4799 --NestedIfs;
4800 }
4801 break;
4802 case pp_eof:
4803 NextDepDirectiveTokenIndex = 0;
4804 return LexEndOfFile(Result, CurPtr: BufferEnd);
4805 }
4806 } while (!Stop);
4807
4808 const dependency_directives_scan::Token &DDTok =
4809 DepDirectives.front().Tokens.front();
4810 assert(DDTok.is(tok::hash));
4811 NextDepDirectiveTokenIndex = 1;
4812
4813 convertDependencyDirectiveToken(DDTok, Result);
4814 return false;
4815}
4816