1//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Lexer and Token interfaces.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Lex/Lexer.h"
14#include "UnicodeCharSets.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/Diagnostic.h"
17#include "clang/Basic/IdentifierTable.h"
18#include "clang/Basic/LLVM.h"
19#include "clang/Basic/LangOptions.h"
20#include "clang/Basic/SourceLocation.h"
21#include "clang/Basic/SourceManager.h"
22#include "clang/Basic/TokenKinds.h"
23#include "clang/Lex/LexDiagnostic.h"
24#include "clang/Lex/LiteralSupport.h"
25#include "clang/Lex/MultipleIncludeOpt.h"
26#include "clang/Lex/Preprocessor.h"
27#include "clang/Lex/PreprocessorOptions.h"
28#include "clang/Lex/Token.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/StringSwitch.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/ConvertUTF.h"
35#include "llvm/Support/MemoryBufferRef.h"
36#include "llvm/Support/NativeFormatting.h"
37#include "llvm/Support/SaveAndRestore.h"
38#include "llvm/Support/Unicode.h"
39#include "llvm/Support/UnicodeCharRanges.h"
40
41#include <algorithm>
42#include <cassert>
43#include <cstddef>
44#include <cstdint>
45#include <cstring>
46#include <limits>
47#include <optional>
48#include <string>
49
50#if LLVM_IS_X86
51#include <nmmintrin.h>
52#endif
53
54using namespace clang;
55
56//===----------------------------------------------------------------------===//
57// Token Class Implementation
58//===----------------------------------------------------------------------===//
59
60/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
61bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
62 if (isAnnotation())
63 return false;
64 if (const IdentifierInfo *II = getIdentifierInfo())
65 return II->getObjCKeywordID() == objcKey;
66 return false;
67}
68
69/// getObjCKeywordID - Return the ObjC keyword kind.
70tok::ObjCKeywordKind Token::getObjCKeywordID() const {
71 if (isAnnotation())
72 return tok::objc_not_keyword;
73 const IdentifierInfo *specId = getIdentifierInfo();
74 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
75}
76
77bool Token::isModuleContextualKeyword(bool AllowExport) const {
78 if (AllowExport && is(K: tok::kw_export))
79 return true;
80 if (isOneOf(Ks: tok::kw_import, Ks: tok::kw_module))
81 return true;
82 if (isNot(K: tok::identifier))
83 return false;
84 const auto *II = getIdentifierInfo();
85 return II->isImportKeyword() || II->isModuleKeyword();
86}
87
88/// Determine whether the token kind starts a simple-type-specifier.
89bool Token::isSimpleTypeSpecifier(const LangOptions &LangOpts) const {
90 switch (getKind()) {
91 case tok::annot_typename:
92 case tok::annot_decltype:
93 case tok::annot_pack_indexing_type:
94 return true;
95
96 case tok::kw_short:
97 case tok::kw_long:
98 case tok::kw___int64:
99 case tok::kw___int128:
100 case tok::kw_signed:
101 case tok::kw_unsigned:
102 case tok::kw_void:
103 case tok::kw_char:
104 case tok::kw_int:
105 case tok::kw_half:
106 case tok::kw_float:
107 case tok::kw_double:
108 case tok::kw___bf16:
109 case tok::kw__Float16:
110 case tok::kw___float128:
111 case tok::kw___ibm128:
112 case tok::kw_wchar_t:
113 case tok::kw_bool:
114 case tok::kw__Bool:
115 case tok::kw__Accum:
116 case tok::kw__Fract:
117 case tok::kw__Sat:
118#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
119#include "clang/Basic/BuiltinTraits.inc"
120 case tok::kw___auto_type:
121 case tok::kw_char16_t:
122 case tok::kw_char32_t:
123 case tok::kw_typeof:
124 case tok::kw_decltype:
125 case tok::kw_char8_t:
126 return getIdentifierInfo()->isKeyword(LangOpts);
127
128 default:
129 return false;
130 }
131}
132
133//===----------------------------------------------------------------------===//
134// Lexer Class Implementation
135//===----------------------------------------------------------------------===//
136
137void Lexer::anchor() {}
138
139void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
140 const char *BufEnd) {
141 BufferStart = BufStart;
142 BufferPtr = BufPtr;
143 BufferEnd = BufEnd;
144
145 assert(BufEnd[0] == 0 &&
146 "We assume that the input buffer has a null character at the end"
147 " to simplify lexing!");
148
149 // Check whether we have a BOM in the beginning of the buffer. If yes - act
150 // accordingly. Right now we support only UTF-8 with and without BOM, so, just
151 // skip the UTF-8 BOM if it's present.
152 if (BufferStart == BufferPtr) {
153 // Determine the size of the BOM.
154 StringRef Buf(BufferStart, BufferEnd - BufferStart);
155 size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
156 .StartsWith(S: "\xEF\xBB\xBF", Value: 3) // UTF-8 BOM
157 .Default(Value: 0);
158
159 // Skip the BOM.
160 BufferPtr += BOMLength;
161 }
162
163 Is_PragmaLexer = false;
164 CurrentConflictMarkerState = CMK_None;
165
166 // Start of the file is a start of line.
167 IsAtStartOfLine = true;
168 IsAtPhysicalStartOfLine = true;
169
170 HasLeadingSpace = false;
171 HasLeadingEmptyMacro = false;
172
173 // We are not after parsing a #.
174 ParsingPreprocessorDirective = false;
175
176 // We are not after parsing #include.
177 ParsingFilename = false;
178
179 // We are not in raw mode. Raw mode disables diagnostics and interpretation
180 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
181 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
182 // or otherwise skipping over tokens.
183 LexingRawMode = false;
184
185 // Default to not keeping comments.
186 ExtendedTokenMode = 0;
187
188 NewLinePtr = nullptr;
189}
190
191/// Lexer constructor - Create a new lexer object for the specified buffer
192/// with the specified preprocessor managing the lexing process. This lexer
193/// assumes that the associated file buffer and Preprocessor objects will
194/// outlive it, so it doesn't take ownership of either of them.
195Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
196 Preprocessor &PP, bool IsFirstIncludeOfFile)
197 : PreprocessorLexer(&PP, FID),
198 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
199 LangOpts(PP.getLangOpts()), LineComment(LangOpts.LineComment),
200 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
201 InitLexer(BufStart: InputFile.getBufferStart(), BufPtr: InputFile.getBufferStart(),
202 BufEnd: InputFile.getBufferEnd());
203
204 resetExtendedTokenMode();
205}
206
207/// Lexer constructor - Create a new raw lexer object. This object is only
208/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
209/// range will outlive it, so it doesn't take ownership of it.
210Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
211 const char *BufStart, const char *BufPtr, const char *BufEnd,
212 bool IsFirstIncludeOfFile)
213 : FileLoc(fileloc), LangOpts(langOpts), LineComment(LangOpts.LineComment),
214 IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
215 InitLexer(BufStart, BufPtr, BufEnd);
216
217 // We *are* in raw mode.
218 LexingRawMode = true;
219}
220
221/// Lexer constructor - Create a new raw lexer object. This object is only
222/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
223/// range will outlive it, so it doesn't take ownership of it.
224Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
225 const SourceManager &SM, const LangOptions &langOpts,
226 bool IsFirstIncludeOfFile)
227 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
228 FromFile.getBufferStart(), FromFile.getBufferEnd(),
229 IsFirstIncludeOfFile) {}
230
231void Lexer::resetExtendedTokenMode() {
232 assert(PP && "Cannot reset token mode without a preprocessor");
233 if (LangOpts.TraditionalCPP)
234 SetKeepWhitespaceMode(true);
235 else
236 SetCommentRetentionState(PP->getCommentRetentionState());
237}
238
239/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
240/// _Pragma expansion. This has a variety of magic semantics that this method
241/// sets up.
242///
243/// On entrance to this routine, TokStartLoc is a macro location which has a
244/// spelling loc that indicates the bytes to be lexed for the token and an
245/// expansion location that indicates where all lexed tokens should be
246/// "expanded from".
247///
248/// TODO: It would really be nice to make _Pragma just be a wrapper around a
249/// normal lexer that remaps tokens as they fly by. This would require making
250/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
251/// interface that could handle this stuff. This would pull GetMappedTokenLoc
252/// out of the critical path of the lexer!
253///
254std::unique_ptr<Lexer> Lexer::Create_PragmaLexer(
255 SourceLocation SpellingLoc, SourceLocation ExpansionLocStart,
256 SourceLocation ExpansionLocEnd, unsigned TokLen, Preprocessor &PP) {
257 SourceManager &SM = PP.getSourceManager();
258
259 // Create the lexer as if we were going to lex the file normally.
260 FileID SpellingFID = SM.getFileID(SpellingLoc);
261 llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(FID: SpellingFID);
262 auto L = std::make_unique<Lexer>(args&: SpellingFID, args&: InputFile, args&: PP);
263
264 // Now that the lexer is created, change the start/end locations so that we
265 // just lex the subsection of the file that we want. This is lexing from a
266 // scratch buffer.
267 const char *StrData = SM.getCharacterData(SL: SpellingLoc);
268
269 L->BufferPtr = StrData;
270 L->BufferEnd = StrData+TokLen;
271 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
272
273 // Set the SourceLocation with the remapping information. This ensures that
274 // GetMappedTokenLoc will remap the tokens as they are lexed.
275 L->FileLoc = SM.createExpansionLoc(SpellingLoc: SM.getLocForStartOfFile(FID: SpellingFID),
276 ExpansionLocStart,
277 ExpansionLocEnd, Length: TokLen);
278
279 // Ensure that the lexer thinks it is inside a directive, so that end \n will
280 // return an EOD token.
281 L->ParsingPreprocessorDirective = true;
282
283 // This lexer really is for _Pragma.
284 L->Is_PragmaLexer = true;
285 return L;
286}
287
288void Lexer::seek(unsigned Offset, bool IsAtStartOfLine) {
289 this->IsAtPhysicalStartOfLine = IsAtStartOfLine;
290 this->IsAtStartOfLine = IsAtStartOfLine;
291 assert((BufferStart + Offset) <= BufferEnd);
292 BufferPtr = BufferStart + Offset;
293}
294
295template <typename T> static void StringifyImpl(T &Str, char Quote) {
296 typename T::size_type i = 0, e = Str.size();
297 while (i < e) {
298 if (Str[i] == '\\' || Str[i] == Quote) {
299 Str.insert(Str.begin() + i, '\\');
300 i += 2;
301 ++e;
302 } else if (Str[i] == '\n' || Str[i] == '\r') {
303 // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
304 if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') &&
305 Str[i] != Str[i + 1]) {
306 Str[i] = '\\';
307 Str[i + 1] = 'n';
308 } else {
309 // Replace '\n' and '\r' to '\\' followed by 'n'.
310 Str[i] = '\\';
311 Str.insert(Str.begin() + i + 1, 'n');
312 ++e;
313 }
314 i += 2;
315 } else
316 ++i;
317 }
318}
319
320std::string Lexer::Stringify(StringRef Str, bool Charify) {
321 std::string Result = std::string(Str);
322 char Quote = Charify ? '\'' : '"';
323 StringifyImpl(Str&: Result, Quote);
324 return Result;
325}
326
327void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, Quote: '"'); }
328
329//===----------------------------------------------------------------------===//
330// Token Spelling
331//===----------------------------------------------------------------------===//
332
333/// Slow case of getSpelling. Extract the characters comprising the
334/// spelling of this token from the provided input buffer.
335static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
336 const LangOptions &LangOpts, char *Spelling) {
337 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
338
339 size_t Length = 0;
340 const char *BufEnd = BufPtr + Tok.getLength();
341
342 if (tok::isStringLiteral(K: Tok.getKind())) {
343 // Munch the encoding-prefix and opening double-quote.
344 while (BufPtr < BufEnd) {
345 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
346 Spelling[Length++] = CharAndSize.Char;
347 BufPtr += CharAndSize.Size;
348
349 if (Spelling[Length - 1] == '"')
350 break;
351 }
352
353 // Raw string literals need special handling; trigraph expansion and line
354 // splicing do not occur within their d-char-sequence nor within their
355 // r-char-sequence.
356 if (Length >= 2 &&
357 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') {
358 // Search backwards from the end of the token to find the matching closing
359 // quote.
360 const char *RawEnd = BufEnd;
361 do --RawEnd; while (*RawEnd != '"');
362 size_t RawLength = RawEnd - BufPtr + 1;
363
364 // Everything between the quotes is included verbatim in the spelling.
365 memcpy(dest: Spelling + Length, src: BufPtr, n: RawLength);
366 Length += RawLength;
367 BufPtr += RawLength;
368
369 // The rest of the token is lexed normally.
370 }
371 }
372
373 while (BufPtr < BufEnd) {
374 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
375 Spelling[Length++] = CharAndSize.Char;
376 BufPtr += CharAndSize.Size;
377 }
378
379 assert(Length < Tok.getLength() &&
380 "NeedsCleaning flag set on token that didn't need cleaning!");
381 return Length;
382}
383
384/// getSpelling() - Return the 'spelling' of this token. The spelling of a
385/// token are the characters used to represent the token in the source file
386/// after trigraph expansion and escaped-newline folding. In particular, this
387/// wants to get the true, uncanonicalized, spelling of things like digraphs
388/// UCNs, etc.
389StringRef Lexer::getSpelling(SourceLocation loc,
390 SmallVectorImpl<char> &buffer,
391 const SourceManager &SM,
392 const LangOptions &options,
393 bool *invalid) {
394 // Break down the source location.
395 FileIDAndOffset locInfo = SM.getDecomposedLoc(Loc: loc);
396
397 // Try to the load the file buffer.
398 bool invalidTemp = false;
399 StringRef file = SM.getBufferData(FID: locInfo.first, Invalid: &invalidTemp);
400 if (invalidTemp) {
401 if (invalid) *invalid = true;
402 return {};
403 }
404
405 const char *tokenBegin = file.data() + locInfo.second;
406
407 // Lex from the start of the given location.
408 Lexer lexer(SM.getLocForStartOfFile(FID: locInfo.first), options,
409 file.begin(), tokenBegin, file.end());
410 Token token;
411 lexer.LexFromRawLexer(Result&: token);
412
413 unsigned length = token.getLength();
414
415 // Common case: no need for cleaning.
416 if (!token.needsCleaning())
417 return StringRef(tokenBegin, length);
418
419 // Hard case, we need to relex the characters into the string.
420 buffer.resize(N: length);
421 buffer.resize(N: getSpellingSlow(Tok: token, BufPtr: tokenBegin, LangOpts: options, Spelling: buffer.data()));
422 return StringRef(buffer.data(), buffer.size());
423}
424
425/// getSpelling() - Return the 'spelling' of this token. The spelling of a
426/// token are the characters used to represent the token in the source file
427/// after trigraph expansion and escaped-newline folding. In particular, this
428/// wants to get the true, uncanonicalized, spelling of things like digraphs
429/// UCNs, etc.
430std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
431 const LangOptions &LangOpts, bool *Invalid) {
432 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
433
434 bool CharDataInvalid = false;
435 const char *TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(),
436 Invalid: &CharDataInvalid);
437 if (Invalid)
438 *Invalid = CharDataInvalid;
439 if (CharDataInvalid)
440 return {};
441
442 // If this token contains nothing interesting, return it directly.
443 if (!Tok.needsCleaning())
444 return std::string(TokStart, TokStart + Tok.getLength());
445
446 std::string Result;
447 Result.resize(n: Tok.getLength());
448 Result.resize(n: getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: &*Result.begin()));
449 return Result;
450}
451
452/// getSpelling - This method is used to get the spelling of a token into a
453/// preallocated buffer, instead of as an std::string. The caller is required
454/// to allocate enough space for the token, which is guaranteed to be at least
455/// Tok.getLength() bytes long. The actual length of the token is returned.
456///
457/// Note that this method may do two possible things: it may either fill in
458/// the buffer specified with characters, or it may *change the input pointer*
459/// to point to a constant buffer with the data already in it (avoiding a
460/// copy). The caller is not allowed to modify the returned buffer pointer
461/// if an internal buffer is returned.
462unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
463 const SourceManager &SourceMgr,
464 const LangOptions &LangOpts, bool *Invalid) {
465 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
466
467 const char *TokStart = nullptr;
468 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
469 if (Tok.is(K: tok::raw_identifier))
470 TokStart = Tok.getRawIdentifier().data();
471 else if (!Tok.hasUCN()) {
472 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
473 // Just return the string from the identifier table, which is very quick.
474 Buffer = II->getNameStart();
475 return II->getLength();
476 }
477 }
478
479 // NOTE: this can be checked even after testing for an IdentifierInfo.
480 if (Tok.isLiteral())
481 TokStart = Tok.getLiteralData();
482
483 if (!TokStart) {
484 // Compute the start of the token in the input lexer buffer.
485 bool CharDataInvalid = false;
486 TokStart = SourceMgr.getCharacterData(SL: Tok.getLocation(), Invalid: &CharDataInvalid);
487 if (Invalid)
488 *Invalid = CharDataInvalid;
489 if (CharDataInvalid) {
490 Buffer = "";
491 return 0;
492 }
493 }
494
495 // If this token contains nothing interesting, return it directly.
496 if (!Tok.needsCleaning()) {
497 Buffer = TokStart;
498 return Tok.getLength();
499 }
500
501 // Otherwise, hard case, relex the characters into the string.
502 return getSpellingSlow(Tok, BufPtr: TokStart, LangOpts, Spelling: const_cast<char*>(Buffer));
503}
504
505/// MeasureTokenLength - Relex the token at the specified location and return
506/// its length in bytes in the input file. If the token needs cleaning (e.g.
507/// includes a trigraph or an escaped newline) then this count includes bytes
508/// that are part of that.
509unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
510 const SourceManager &SM,
511 const LangOptions &LangOpts) {
512 Token TheTok;
513 if (getRawToken(Loc, Result&: TheTok, SM, LangOpts))
514 return 0;
515 return TheTok.getLength();
516}
517
518SourceLocation Lexer::findEndOfIdentifierContinuation(
519 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
520 Loc = SM.getExpansionLoc(Loc);
521 const FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
522 bool Invalid = false;
523 const StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
524 if (Invalid)
525 return Loc;
526
527 const char *StrData = Buffer.data() + LocInfo.second;
528 if (StrData >= Buffer.end())
529 return Loc;
530
531 // Use the lexer continuation rules directly, without requiring identifier
532 // start at Loc.
533 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
534 Buffer.begin(), StrData, Buffer.end());
535 Token Tok;
536 Tok.startToken();
537 TheLexer.LexIdentifierContinue(Result&: Tok, CurPtr: StrData);
538 return Loc.getLocWithOffset(Offset: Tok.getLength());
539}
540
541/// Relex the token at the specified location.
542/// \returns true if there was a failure, false on success.
543bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
544 const SourceManager &SM,
545 const LangOptions &LangOpts,
546 bool IgnoreWhiteSpace) {
547 // TODO: this could be special cased for common tokens like identifiers, ')',
548 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
549 // all obviously single-char tokens. This could use
550 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
551 // something.
552
553 // If this comes from a macro expansion, we really do want the macro name, not
554 // the token this macro expanded to.
555 Loc = SM.getExpansionLoc(Loc);
556 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
557 bool Invalid = false;
558 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
559 if (Invalid)
560 return true;
561
562 const char *StrData = Buffer.data()+LocInfo.second;
563
564 if (!IgnoreWhiteSpace && isWhitespace(c: SkipEscapedNewLines(P: StrData)[0]))
565 return true;
566
567 // Create a lexer starting at the beginning of this token.
568 Lexer TheLexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts,
569 Buffer.begin(), StrData, Buffer.end());
570 TheLexer.SetCommentRetentionState(true);
571 TheLexer.LexFromRawLexer(Result);
572 return false;
573}
574
575/// Returns the pointer that points to the beginning of line that contains
576/// the given offset, or null if the offset if invalid.
577static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
578 const char *BufStart = Buffer.data();
579 if (Offset >= Buffer.size())
580 return nullptr;
581
582 const char *LexStart = BufStart + Offset;
583 for (; LexStart != BufStart; --LexStart) {
584 if (isVerticalWhitespace(c: LexStart[0]) &&
585 !Lexer::isNewLineEscaped(BufferStart: BufStart, Str: LexStart)) {
586 // LexStart should point at first character of logical line.
587 ++LexStart;
588 break;
589 }
590 }
591 return LexStart;
592}
593
594static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
595 const SourceManager &SM,
596 const LangOptions &LangOpts) {
597 assert(Loc.isFileID());
598 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
599 if (LocInfo.first.isInvalid())
600 return Loc;
601
602 bool Invalid = false;
603 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
604 if (Invalid)
605 return Loc;
606
607 // Back up from the current location until we hit the beginning of a line
608 // (or the buffer). We'll relex from that point.
609 const char *StrData = Buffer.data() + LocInfo.second;
610 const char *LexStart = findBeginningOfLine(Buffer, Offset: LocInfo.second);
611 if (!LexStart || LexStart == StrData)
612 return Loc;
613
614 // Create a lexer starting at the beginning of this token.
615 SourceLocation LexerStartLoc = Loc.getLocWithOffset(Offset: -LocInfo.second);
616 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
617 Buffer.end());
618 TheLexer.SetCommentRetentionState(true);
619
620 // Lex tokens until we find the token that contains the source location.
621 Token TheTok;
622 do {
623 TheLexer.LexFromRawLexer(Result&: TheTok);
624
625 if (TheLexer.getBufferLocation() > StrData) {
626 // Lexing this token has taken the lexer past the source location we're
627 // looking for. If the current token encompasses our source location,
628 // return the beginning of that token.
629 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
630 return TheTok.getLocation();
631
632 // We ended up skipping over the source location entirely, which means
633 // that it points into whitespace. We're done here.
634 break;
635 }
636 } while (TheTok.getKind() != tok::eof);
637
638 // We've passed our source location; just return the original source location.
639 return Loc;
640}
641
642SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
643 const SourceManager &SM,
644 const LangOptions &LangOpts) {
645 if (Loc.isFileID())
646 return getBeginningOfFileToken(Loc, SM, LangOpts);
647
648 if (!SM.isMacroArgExpansion(Loc))
649 return Loc;
650
651 SourceLocation FileLoc = SM.getSpellingLoc(Loc);
652 SourceLocation BeginFileLoc = getBeginningOfFileToken(Loc: FileLoc, SM, LangOpts);
653 FileIDAndOffset FileLocInfo = SM.getDecomposedLoc(Loc: FileLoc);
654 FileIDAndOffset BeginFileLocInfo = SM.getDecomposedLoc(Loc: BeginFileLoc);
655 assert(FileLocInfo.first == BeginFileLocInfo.first &&
656 FileLocInfo.second >= BeginFileLocInfo.second);
657 return Loc.getLocWithOffset(Offset: BeginFileLocInfo.second - FileLocInfo.second);
658}
659
660namespace {
661
662enum PreambleDirectiveKind {
663 PDK_Skipped,
664 PDK_Unknown
665};
666
667} // namespace
668
669PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
670 const LangOptions &LangOpts,
671 unsigned MaxLines) {
672 // Create a lexer starting at the beginning of the file. Note that we use a
673 // "fake" file source location at offset 1 so that the lexer will track our
674 // position within the file.
675 const SourceLocation::UIntTy StartOffset = 1;
676 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(Encoding: StartOffset);
677 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
678 Buffer.end());
679 TheLexer.SetCommentRetentionState(true);
680
681 bool InPreprocessorDirective = false;
682 Token TheTok;
683 SourceLocation ActiveCommentLoc;
684
685 unsigned MaxLineOffset = 0;
686 if (MaxLines) {
687 const char *CurPtr = Buffer.begin();
688 unsigned CurLine = 0;
689 while (CurPtr != Buffer.end()) {
690 char ch = *CurPtr++;
691 if (ch == '\n') {
692 ++CurLine;
693 if (CurLine == MaxLines)
694 break;
695 }
696 }
697 if (CurPtr != Buffer.end())
698 MaxLineOffset = CurPtr - Buffer.begin();
699 }
700
701 do {
702 TheLexer.LexFromRawLexer(Result&: TheTok);
703
704 if (InPreprocessorDirective) {
705 // If we've hit the end of the file, we're done.
706 if (TheTok.getKind() == tok::eof) {
707 break;
708 }
709
710 // If we haven't hit the end of the preprocessor directive, skip this
711 // token.
712 if (!TheTok.isAtStartOfLine())
713 continue;
714
715 // We've passed the end of the preprocessor directive, and will look
716 // at this token again below.
717 InPreprocessorDirective = false;
718 }
719
720 // Keep track of the # of lines in the preamble.
721 if (TheTok.isAtStartOfLine()) {
722 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
723
724 // If we were asked to limit the number of lines in the preamble,
725 // and we're about to exceed that limit, we're done.
726 if (MaxLineOffset && TokOffset >= MaxLineOffset)
727 break;
728 }
729
730 // Comments are okay; skip over them.
731 if (TheTok.getKind() == tok::comment) {
732 if (ActiveCommentLoc.isInvalid())
733 ActiveCommentLoc = TheTok.getLocation();
734 continue;
735 }
736
737 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
738 // This is the start of a preprocessor directive.
739 Token HashTok = TheTok;
740 InPreprocessorDirective = true;
741 ActiveCommentLoc = SourceLocation();
742
743 // Figure out which directive this is. Since we're lexing raw tokens,
744 // we don't have an identifier table available. Instead, just look at
745 // the raw identifier to recognize and categorize preprocessor directives.
746 TheLexer.LexFromRawLexer(Result&: TheTok);
747 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
748 StringRef Keyword = TheTok.getRawIdentifier();
749 PreambleDirectiveKind PDK
750 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
751 .Case(S: "include", Value: PDK_Skipped)
752 .Case(S: "__include_macros", Value: PDK_Skipped)
753 .Case(S: "define", Value: PDK_Skipped)
754 .Case(S: "undef", Value: PDK_Skipped)
755 .Case(S: "line", Value: PDK_Skipped)
756 .Case(S: "error", Value: PDK_Skipped)
757 .Case(S: "pragma", Value: PDK_Skipped)
758 .Case(S: "import", Value: PDK_Skipped)
759 .Case(S: "include_next", Value: PDK_Skipped)
760 .Case(S: "warning", Value: PDK_Skipped)
761 .Case(S: "ident", Value: PDK_Skipped)
762 .Case(S: "sccs", Value: PDK_Skipped)
763 .Case(S: "assert", Value: PDK_Skipped)
764 .Case(S: "unassert", Value: PDK_Skipped)
765 .Case(S: "if", Value: PDK_Skipped)
766 .Case(S: "ifdef", Value: PDK_Skipped)
767 .Case(S: "ifndef", Value: PDK_Skipped)
768 .Case(S: "elif", Value: PDK_Skipped)
769 .Case(S: "elifdef", Value: PDK_Skipped)
770 .Case(S: "elifndef", Value: PDK_Skipped)
771 .Case(S: "else", Value: PDK_Skipped)
772 .Case(S: "endif", Value: PDK_Skipped)
773 .Default(Value: PDK_Unknown);
774
775 switch (PDK) {
776 case PDK_Skipped:
777 continue;
778
779 case PDK_Unknown:
780 // We don't know what this directive is; stop at the '#'.
781 break;
782 }
783 }
784
785 // We only end up here if we didn't recognize the preprocessor
786 // directive or it was one that can't occur in the preamble at this
787 // point. Roll back the current token to the location of the '#'.
788 TheTok = HashTok;
789 } else if (TheTok.isAtStartOfLine() &&
790 TheTok.getKind() == tok::raw_identifier &&
791 TheTok.getRawIdentifier() == "module" &&
792 LangOpts.CPlusPlusModules) {
793 // The initial global module fragment introducer "module;" is part of
794 // the preamble, which runs up to the module declaration "module foo;".
795 Token ModuleTok = TheTok;
796 do {
797 TheLexer.LexFromRawLexer(Result&: TheTok);
798 } while (TheTok.getKind() == tok::comment);
799 if (TheTok.getKind() != tok::semi) {
800 // Not global module fragment, roll back.
801 TheTok = ModuleTok;
802 break;
803 }
804 continue;
805 }
806
807 // We hit a token that we don't recognize as being in the
808 // "preprocessing only" part of the file, so we're no longer in
809 // the preamble.
810 break;
811 } while (true);
812
813 SourceLocation End;
814 if (ActiveCommentLoc.isValid())
815 End = ActiveCommentLoc; // don't truncate a decl comment.
816 else
817 End = TheTok.getLocation();
818
819 return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
820 TheTok.isAtStartOfLine());
821}
822
823unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
824 const SourceManager &SM,
825 const LangOptions &LangOpts) {
826 // Figure out how many physical characters away the specified expansion
827 // character is. This needs to take into consideration newlines and
828 // trigraphs.
829 bool Invalid = false;
830 const char *TokPtr = SM.getCharacterData(SL: TokStart, Invalid: &Invalid);
831
832 // If they request the first char of the token, we're trivially done.
833 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(C: *TokPtr)))
834 return 0;
835
836 unsigned PhysOffset = 0;
837
838 // The usual case is that tokens don't contain anything interesting. Skip
839 // over the uninteresting characters. If a token only consists of simple
840 // chars, this method is extremely fast.
841 while (Lexer::isObviouslySimpleCharacter(C: *TokPtr)) {
842 if (CharNo == 0)
843 return PhysOffset;
844 ++TokPtr;
845 --CharNo;
846 ++PhysOffset;
847 }
848
849 // If we have a character that may be a trigraph or escaped newline, use a
850 // lexer to parse it correctly.
851 for (; CharNo; --CharNo) {
852 auto CharAndSize = Lexer::getCharAndSizeNoWarn(Ptr: TokPtr, LangOpts);
853 TokPtr += CharAndSize.Size;
854 PhysOffset += CharAndSize.Size;
855 }
856
857 // Final detail: if we end up on an escaped newline, we want to return the
858 // location of the actual byte of the token. For example foo\<newline>bar
859 // advanced by 3 should return the location of b, not of \\. One compounding
860 // detail of this is that the escape may be made by a trigraph.
861 if (!Lexer::isObviouslySimpleCharacter(C: *TokPtr))
862 PhysOffset += Lexer::SkipEscapedNewLines(P: TokPtr)-TokPtr;
863
864 return PhysOffset;
865}
866
867/// Computes the source location just past the end of the
868/// token at this source location.
869///
870/// This routine can be used to produce a source location that
871/// points just past the end of the token referenced by \p Loc, and
872/// is generally used when a diagnostic needs to point just after a
873/// token where it expected something different that it received. If
874/// the returned source location would not be meaningful (e.g., if
875/// it points into a macro), this routine returns an invalid
876/// source location.
877///
878/// \param Offset an offset from the end of the token, where the source
879/// location should refer to. The default offset (0) produces a source
880/// location pointing just past the end of the token; an offset of 1 produces
881/// a source location pointing to the last character in the token, etc.
882SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
883 const SourceManager &SM,
884 const LangOptions &LangOpts) {
885 if (Loc.isInvalid())
886 return {};
887
888 if (Loc.isMacroID()) {
889 // Token split (for example, splitting '>>' into two '>' tokens) is
890 // represented in SourceManager as an ExpansionInfo (see
891 // createForTokenSplit), so these locations are MacroIDs even when no user
892 // macro is involved. For split expansions, the expansion end is already
893 // the correct insertion point.
894 const FileID LocFileID = SM.getFileID(SpellingLoc: Loc);
895 if (Offset > 0 || !isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
896 return {}; // Points inside the macro expansion.
897 if (!SM.getSLocEntry(FID: LocFileID).getExpansion().isExpansionTokenRange())
898 return Loc;
899 }
900
901 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
902 if (Len > Offset)
903 Len = Len - Offset;
904 else
905 return Loc;
906
907 return Loc.getLocWithOffset(Offset: Len);
908}
909
910/// Returns true if the given MacroID location points at the first
911/// token of the macro expansion.
912bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
913 const SourceManager &SM,
914 const LangOptions &LangOpts,
915 SourceLocation *MacroBegin) {
916 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
917
918 SourceLocation expansionLoc;
919 if (!SM.isAtStartOfImmediateMacroExpansion(Loc: loc, MacroBegin: &expansionLoc))
920 return false;
921
922 if (expansionLoc.isFileID()) {
923 // No other macro expansions, this is the first.
924 if (MacroBegin)
925 *MacroBegin = expansionLoc;
926 return true;
927 }
928
929 return isAtStartOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroBegin);
930}
931
932/// Returns true if the given MacroID location points at the last
933/// token of the macro expansion.
934bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
935 const SourceManager &SM,
936 const LangOptions &LangOpts,
937 SourceLocation *MacroEnd) {
938 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
939
940 SourceLocation spellLoc = SM.getSpellingLoc(Loc: loc);
941 unsigned tokLen = MeasureTokenLength(Loc: spellLoc, SM, LangOpts);
942 if (tokLen == 0)
943 return false;
944
945 SourceLocation afterLoc = loc.getLocWithOffset(Offset: tokLen);
946 SourceLocation expansionLoc;
947 FileID FID = SM.getFileID(SpellingLoc: loc);
948
949 if (SM.isInFileID(Loc: afterLoc, FID)) {
950 if (!SM.isAtEndOfImmediateMacroExpansion(Loc: afterLoc, MacroEnd: &expansionLoc))
951 return false;
952 } else {
953 // During error recovery, a zero-length synthetic token might be inserted
954 // past the end of the FileID, e.g. inserting ")" when a macro-arg
955 // containing a comma should be guarded by parentheses. In this case,
956 // afterLoc reaches the `NextLocalOffset` boundary, any operations on
957 // afterLoc will be invalid!
958 const SrcMgr::SLocEntry &Entry = SM.getSLocEntry(FID);
959 assert(Entry.isExpansion() && "Should be in an expansion");
960 expansionLoc = Entry.getExpansion().getExpansionLocEnd();
961 }
962
963 if (expansionLoc.isFileID()) {
964 // No other macro expansions.
965 if (MacroEnd)
966 *MacroEnd = expansionLoc;
967 return true;
968 }
969
970 return isAtEndOfMacroExpansion(loc: expansionLoc, SM, LangOpts, MacroEnd);
971}
972
973static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
974 const SourceManager &SM,
975 const LangOptions &LangOpts) {
976 SourceLocation Begin = Range.getBegin();
977 SourceLocation End = Range.getEnd();
978 assert(Begin.isFileID() && End.isFileID());
979 if (Range.isTokenRange()) {
980 End = Lexer::getLocForEndOfToken(Loc: End, Offset: 0, SM,LangOpts);
981 if (End.isInvalid())
982 return {};
983 }
984
985 // Break down the source locations.
986 auto [FID, BeginOffs] = SM.getDecomposedLoc(Loc: Begin);
987 if (FID.isInvalid())
988 return {};
989
990 unsigned EndOffs;
991 if (!SM.isInFileID(Loc: End, FID, RelativeOffset: &EndOffs) ||
992 BeginOffs > EndOffs)
993 return {};
994
995 return CharSourceRange::getCharRange(B: Begin, E: End);
996}
997
998// Assumes that `Loc` is in an expansion.
999static bool isInExpansionTokenRange(const SourceLocation Loc,
1000 const SourceManager &SM) {
1001 return SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Loc))
1002 .getExpansion()
1003 .isExpansionTokenRange();
1004}
1005
1006CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
1007 const SourceManager &SM,
1008 const LangOptions &LangOpts) {
1009 SourceLocation Begin = Range.getBegin();
1010 SourceLocation End = Range.getEnd();
1011 if (Begin.isInvalid() || End.isInvalid())
1012 return {};
1013
1014 if (Begin.isFileID() && End.isFileID())
1015 return makeRangeFromFileLocs(Range, SM, LangOpts);
1016
1017 if (Begin.isMacroID() && End.isFileID()) {
1018 if (!isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &Begin))
1019 return {};
1020 Range.setBegin(Begin);
1021 return makeRangeFromFileLocs(Range, SM, LangOpts);
1022 }
1023
1024 if (Begin.isFileID() && End.isMacroID()) {
1025 if (Range.isTokenRange()) {
1026 if (!isAtEndOfMacroExpansion(loc: End, SM, LangOpts, MacroEnd: &End))
1027 return {};
1028 // Use the *original* end, not the expanded one in `End`.
1029 Range.setTokenRange(isInExpansionTokenRange(Loc: Range.getEnd(), SM));
1030 } else if (!isAtStartOfMacroExpansion(loc: End, SM, LangOpts, MacroBegin: &End))
1031 return {};
1032 Range.setEnd(End);
1033 return makeRangeFromFileLocs(Range, SM, LangOpts);
1034 }
1035
1036 assert(Begin.isMacroID() && End.isMacroID());
1037 SourceLocation MacroBegin, MacroEnd;
1038 if (isAtStartOfMacroExpansion(loc: Begin, SM, LangOpts, MacroBegin: &MacroBegin) &&
1039 ((Range.isTokenRange() && isAtEndOfMacroExpansion(loc: End, SM, LangOpts,
1040 MacroEnd: &MacroEnd)) ||
1041 (Range.isCharRange() && isAtStartOfMacroExpansion(loc: End, SM, LangOpts,
1042 MacroBegin: &MacroEnd)))) {
1043 Range.setBegin(MacroBegin);
1044 Range.setEnd(MacroEnd);
1045 // Use the *original* `End`, not the expanded one in `MacroEnd`.
1046 if (Range.isTokenRange())
1047 Range.setTokenRange(isInExpansionTokenRange(Loc: End, SM));
1048 return makeRangeFromFileLocs(Range, SM, LangOpts);
1049 }
1050
1051 bool Invalid = false;
1052 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: Begin),
1053 Invalid: &Invalid);
1054 if (Invalid)
1055 return {};
1056
1057 if (BeginEntry.getExpansion().isMacroArgExpansion()) {
1058 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(FID: SM.getFileID(SpellingLoc: End),
1059 Invalid: &Invalid);
1060 if (Invalid)
1061 return {};
1062
1063 if (EndEntry.getExpansion().isMacroArgExpansion() &&
1064 BeginEntry.getExpansion().getExpansionLocStart() ==
1065 EndEntry.getExpansion().getExpansionLocStart()) {
1066 Range.setBegin(SM.getImmediateSpellingLoc(Loc: Begin));
1067 Range.setEnd(SM.getImmediateSpellingLoc(Loc: End));
1068 return makeFileCharRange(Range, SM, LangOpts);
1069 }
1070 }
1071
1072 return {};
1073}
1074
1075StringRef Lexer::getSourceText(CharSourceRange Range,
1076 const SourceManager &SM,
1077 const LangOptions &LangOpts,
1078 bool *Invalid) {
1079 Range = makeFileCharRange(Range, SM, LangOpts);
1080 if (Range.isInvalid()) {
1081 if (Invalid) *Invalid = true;
1082 return {};
1083 }
1084
1085 // Break down the source location.
1086 FileIDAndOffset beginInfo = SM.getDecomposedLoc(Loc: Range.getBegin());
1087 if (beginInfo.first.isInvalid()) {
1088 if (Invalid) *Invalid = true;
1089 return {};
1090 }
1091
1092 unsigned EndOffs;
1093 if (!SM.isInFileID(Loc: Range.getEnd(), FID: beginInfo.first, RelativeOffset: &EndOffs) ||
1094 beginInfo.second > EndOffs) {
1095 if (Invalid) *Invalid = true;
1096 return {};
1097 }
1098
1099 // Try to the load the file buffer.
1100 bool invalidTemp = false;
1101 StringRef file = SM.getBufferData(FID: beginInfo.first, Invalid: &invalidTemp);
1102 if (invalidTemp) {
1103 if (Invalid) *Invalid = true;
1104 return {};
1105 }
1106
1107 if (Invalid) *Invalid = false;
1108 return file.substr(Start: beginInfo.second, N: EndOffs - beginInfo.second);
1109}
1110
1111StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1112 const SourceManager &SM,
1113 const LangOptions &LangOpts) {
1114 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1115
1116 // Find the location of the immediate macro expansion.
1117 while (true) {
1118 FileID FID = SM.getFileID(SpellingLoc: Loc);
1119 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
1120 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
1121 Loc = Expansion.getExpansionLocStart();
1122 if (!Expansion.isMacroArgExpansion())
1123 break;
1124
1125 // For macro arguments we need to check that the argument did not come
1126 // from an inner macro, e.g: "MAC1( MAC2(foo) )"
1127
1128 // Loc points to the argument id of the macro definition, move to the
1129 // macro expansion.
1130 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1131 SourceLocation SpellLoc = Expansion.getSpellingLoc();
1132 if (SpellLoc.isFileID())
1133 break; // No inner macro.
1134
1135 // If spelling location resides in the same FileID as macro expansion
1136 // location, it means there is no inner macro.
1137 FileID MacroFID = SM.getFileID(SpellingLoc: Loc);
1138 if (SM.isInFileID(Loc: SpellLoc, FID: MacroFID))
1139 break;
1140
1141 // Argument came from inner macro.
1142 Loc = SpellLoc;
1143 }
1144
1145 // Find the spelling location of the start of the non-argument expansion
1146 // range. This is where the macro name was spelled in order to begin
1147 // expanding this macro.
1148 Loc = SM.getSpellingLoc(Loc);
1149
1150 // Dig out the buffer where the macro name was spelled and the extents of the
1151 // name so that we can render it into the expansion note.
1152 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1153 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1154 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1155 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1156}
1157
1158StringRef Lexer::getImmediateMacroNameForDiagnostics(
1159 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
1160 assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1161 // Walk past macro argument expansions.
1162 while (SM.isMacroArgExpansion(Loc))
1163 Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1164
1165 // If the macro's spelling isn't FileID or from scratch space, then it's
1166 // actually a token paste or stringization (or similar) and not a macro at
1167 // all.
1168 SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
1169 if (!SpellLoc.isFileID() || SM.isWrittenInScratchSpace(Loc: SpellLoc))
1170 return {};
1171
1172 // Find the spelling location of the start of the non-argument expansion
1173 // range. This is where the macro name was spelled in order to begin
1174 // expanding this macro.
1175 Loc = SM.getSpellingLoc(Loc: SM.getImmediateExpansionRange(Loc).getBegin());
1176
1177 // Dig out the buffer where the macro name was spelled and the extents of the
1178 // name so that we can render it into the expansion note.
1179 FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
1180 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1181 StringRef ExpansionBuffer = SM.getBufferData(FID: ExpansionInfo.first);
1182 return ExpansionBuffer.substr(Start: ExpansionInfo.second, N: MacroTokenLength);
1183}
1184
1185bool Lexer::isAsciiIdentifierContinueChar(char c, const LangOptions &LangOpts) {
1186 return isAsciiIdentifierContinue(c, AllowDollar: LangOpts.DollarIdents);
1187}
1188
1189bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1190 assert(isVerticalWhitespace(Str[0]));
1191 if (Str - 1 < BufferStart)
1192 return false;
1193
1194 if ((Str[0] == '\n' && Str[-1] == '\r') ||
1195 (Str[0] == '\r' && Str[-1] == '\n')) {
1196 if (Str - 2 < BufferStart)
1197 return false;
1198 --Str;
1199 }
1200 --Str;
1201
1202 // Rewind to first non-space character:
1203 while (Str > BufferStart && isHorizontalWhitespace(c: *Str))
1204 --Str;
1205
1206 return *Str == '\\';
1207}
1208
1209StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1210 const SourceManager &SM) {
1211 if (Loc.isInvalid() || Loc.isMacroID())
1212 return {};
1213 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1214 if (LocInfo.first.isInvalid())
1215 return {};
1216 bool Invalid = false;
1217 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
1218 if (Invalid)
1219 return {};
1220 const char *Line = findBeginningOfLine(Buffer, Offset: LocInfo.second);
1221 if (!Line)
1222 return {};
1223 StringRef Rest = Buffer.substr(Start: Line - Buffer.data());
1224 size_t NumWhitespaceChars = Rest.find_first_not_of(Chars: " \t");
1225 return NumWhitespaceChars == StringRef::npos
1226 ? ""
1227 : Rest.take_front(N: NumWhitespaceChars);
1228}
1229
1230//===----------------------------------------------------------------------===//
1231// Diagnostics forwarding code.
1232//===----------------------------------------------------------------------===//
1233
1234/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1235/// lexer buffer was all expanded at a single point, perform the mapping.
1236/// This is currently only used for _Pragma implementation, so it is the slow
1237/// path of the hot getSourceLocation method. Do not allow it to be inlined.
1238static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1239 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1240static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1241 SourceLocation FileLoc,
1242 unsigned CharNo, unsigned TokLen) {
1243 assert(FileLoc.isMacroID() && "Must be a macro expansion");
1244
1245 // Otherwise, we're lexing "mapped tokens". This is used for things like
1246 // _Pragma handling. Combine the expansion location of FileLoc with the
1247 // spelling location.
1248 SourceManager &SM = PP.getSourceManager();
1249
1250 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
1251 // characters come from spelling(FileLoc)+Offset.
1252 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc: FileLoc);
1253 SpellingLoc = SpellingLoc.getLocWithOffset(Offset: CharNo);
1254
1255 // Figure out the expansion loc range, which is the range covered by the
1256 // original _Pragma(...) sequence.
1257 CharSourceRange II = SM.getImmediateExpansionRange(Loc: FileLoc);
1258
1259 return SM.createExpansionLoc(SpellingLoc, ExpansionLocStart: II.getBegin(), ExpansionLocEnd: II.getEnd(), Length: TokLen);
1260}
1261
1262/// getSourceLocation - Return a source location identifier for the specified
1263/// offset in the current file.
1264SourceLocation Lexer::getSourceLocation(const char *Loc,
1265 unsigned TokLen) const {
1266 assert(Loc >= BufferStart && Loc <= BufferEnd &&
1267 "Location out of range for this buffer!");
1268
1269 // In the normal case, we're just lexing from a simple file buffer, return
1270 // the file id from FileLoc with the offset specified.
1271 unsigned CharNo = Loc-BufferStart;
1272 if (FileLoc.isFileID())
1273 return FileLoc.getLocWithOffset(Offset: CharNo);
1274
1275 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1276 // tokens are lexed from where the _Pragma was defined.
1277 assert(PP && "This doesn't work on raw lexers");
1278 return GetMappedTokenLoc(PP&: *PP, FileLoc, CharNo, TokLen);
1279}
1280
1281/// Diag - Forwarding function for diagnostics. This translate a source
1282/// position in the current buffer into a SourceLocation object for rendering.
1283DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1284 return PP->Diag(Loc: getSourceLocation(Loc), DiagID);
1285}
1286
1287DiagnosticBuilder Lexer::DiagCompat(const char *Loc,
1288 unsigned CompatDiagId) const {
1289 return Diag(Loc, DiagID: DiagnosticIDs::getCompatDiagId(LangOpts, CompatDiagId));
1290}
1291
1292//===----------------------------------------------------------------------===//
1293// Trigraph and Escaped Newline Handling Code.
1294//===----------------------------------------------------------------------===//
1295
1296/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1297/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1298static char GetTrigraphCharForLetter(char Letter) {
1299 switch (Letter) {
1300 default: return 0;
1301 case '=': return '#';
1302 case ')': return ']';
1303 case '(': return '[';
1304 case '!': return '|';
1305 case '\'': return '^';
1306 case '>': return '}';
1307 case '/': return '\\';
1308 case '<': return '{';
1309 case '-': return '~';
1310 }
1311}
1312
1313/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1314/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
1315/// return the result character. Finally, emit a warning about trigraph use
1316/// whether trigraphs are enabled or not.
1317static char DecodeTrigraphChar(const char *CP, Lexer *L, bool Trigraphs) {
1318 char Res = GetTrigraphCharForLetter(Letter: *CP);
1319 if (!Res)
1320 return Res;
1321
1322 if (!Trigraphs) {
1323 if (L && !L->isLexingRawMode())
1324 L->Diag(Loc: CP-2, DiagID: diag::trigraph_ignored);
1325 return 0;
1326 }
1327
1328 if (L && !L->isLexingRawMode())
1329 L->Diag(Loc: CP-2, DiagID: diag::trigraph_converted) << StringRef(&Res, 1);
1330 return Res;
1331}
1332
1333/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1334/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1335/// trigraph equivalent on entry to this function.
1336unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1337 unsigned Size = 0;
1338 while (isWhitespace(c: Ptr[Size])) {
1339 ++Size;
1340
1341 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
1342 continue;
1343
1344 // If this is a \r\n or \n\r, skip the other half.
1345 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
1346 Ptr[Size-1] != Ptr[Size])
1347 ++Size;
1348
1349 return Size;
1350 }
1351
1352 // Not an escaped newline, must be a \t or something else.
1353 return 0;
1354}
1355
1356/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1357/// them), skip over them and return the first non-escaped-newline found,
1358/// otherwise return P.
1359const char *Lexer::SkipEscapedNewLines(const char *P) {
1360 while (true) {
1361 const char *AfterEscape;
1362 if (*P == '\\') {
1363 AfterEscape = P+1;
1364 } else if (*P == '?') {
1365 // If not a trigraph for escape, bail out.
1366 if (P[1] != '?' || P[2] != '/')
1367 return P;
1368 // FIXME: Take LangOpts into account; the language might not
1369 // support trigraphs.
1370 AfterEscape = P+3;
1371 } else {
1372 return P;
1373 }
1374
1375 unsigned NewLineSize = Lexer::getEscapedNewLineSize(Ptr: AfterEscape);
1376 if (NewLineSize == 0) return P;
1377 P = AfterEscape+NewLineSize;
1378 }
1379}
1380
1381std::optional<Token> Lexer::findNextToken(SourceLocation Loc,
1382 const SourceManager &SM,
1383 const LangOptions &LangOpts,
1384 bool IncludeComments) {
1385 if (Loc.isMacroID()) {
1386 if (!Lexer::isAtEndOfMacroExpansion(loc: Loc, SM, LangOpts, MacroEnd: &Loc))
1387 return std::nullopt;
1388 }
1389 Loc = Lexer::getLocForEndOfToken(Loc, Offset: 0, SM, LangOpts);
1390
1391 // Break down the source location.
1392 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
1393
1394 // Try to load the file buffer.
1395 bool InvalidTemp = false;
1396 StringRef File = SM.getBufferData(FID: LocInfo.first, Invalid: &InvalidTemp);
1397 if (InvalidTemp)
1398 return std::nullopt;
1399
1400 const char *TokenBegin = File.data() + LocInfo.second;
1401
1402 // Lex from the start of the given location.
1403 Lexer lexer(SM.getLocForStartOfFile(FID: LocInfo.first), LangOpts, File.begin(),
1404 TokenBegin, File.end());
1405 lexer.SetCommentRetentionState(IncludeComments);
1406 // Find the token.
1407 Token Tok;
1408 lexer.LexFromRawLexer(Result&: Tok);
1409 return Tok;
1410}
1411
1412std::optional<Token> Lexer::findPreviousToken(SourceLocation Loc,
1413 const SourceManager &SM,
1414 const LangOptions &LangOpts,
1415 bool IncludeComments) {
1416 const auto StartOfFile = SM.getLocForStartOfFile(FID: SM.getFileID(SpellingLoc: Loc));
1417 while (Loc != StartOfFile) {
1418 Loc = Loc.getLocWithOffset(Offset: -1);
1419 if (Loc.isInvalid())
1420 return std::nullopt;
1421
1422 Loc = GetBeginningOfToken(Loc, SM, LangOpts);
1423 Token Tok;
1424 if (getRawToken(Loc, Result&: Tok, SM, LangOpts))
1425 continue; // Not a token, go to prev location.
1426 if (!Tok.is(K: tok::comment) || IncludeComments) {
1427 return Tok;
1428 }
1429 }
1430 return std::nullopt;
1431}
1432
1433/// Checks that the given token is the first token that occurs after the
1434/// given location (this excludes comments and whitespace). Returns the location
1435/// immediately after the specified token. If the token is not found or the
1436/// location is inside a macro, the returned source location will be invalid.
1437SourceLocation Lexer::findLocationAfterToken(
1438 SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
1439 const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
1440 std::optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
1441 if (!Tok || Tok->isNot(K: TKind))
1442 return {};
1443 SourceLocation TokenLoc = Tok->getLocation();
1444
1445 // Calculate how much whitespace needs to be skipped if any.
1446 unsigned NumWhitespaceChars = 0;
1447 if (SkipTrailingWhitespaceAndNewLine) {
1448 const char *TokenEnd = SM.getCharacterData(SL: TokenLoc) + Tok->getLength();
1449 unsigned char C = *TokenEnd;
1450 while (isHorizontalWhitespace(c: C)) {
1451 C = *(++TokenEnd);
1452 NumWhitespaceChars++;
1453 }
1454
1455 // Skip \r, \n, \r\n, or \n\r
1456 if (C == '\n' || C == '\r') {
1457 char PrevC = C;
1458 C = *(++TokenEnd);
1459 NumWhitespaceChars++;
1460 if ((C == '\n' || C == '\r') && C != PrevC)
1461 NumWhitespaceChars++;
1462 }
1463 }
1464
1465 return TokenLoc.getLocWithOffset(Offset: Tok->getLength() + NumWhitespaceChars);
1466}
1467
1468/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1469/// get its size, and return it. This is tricky in several cases:
1470/// 1. If currently at the start of a trigraph, we warn about the trigraph,
1471/// then either return the trigraph (skipping 3 chars) or the '?',
1472/// depending on whether trigraphs are enabled or not.
1473/// 2. If this is an escaped newline (potentially with whitespace between
1474/// the backslash and newline), implicitly skip the newline and return
1475/// the char after it.
1476///
1477/// This handles the slow/uncommon case of the getCharAndSize method. Here we
1478/// know that we can accumulate into Size, and that we have already incremented
1479/// Ptr by Size bytes.
1480///
1481/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1482/// be updated to match.
1483Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
1484 unsigned Size = 0;
1485 // If we have a slash, look for an escaped newline.
1486 if (Ptr[0] == '\\') {
1487 ++Size;
1488 ++Ptr;
1489Slash:
1490 // Common case, backslash-char where the char is not whitespace.
1491 if (!isWhitespace(c: Ptr[0]))
1492 return {.Char: '\\', .Size: Size};
1493
1494 // See if we have optional whitespace characters between the slash and
1495 // newline.
1496 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1497 // Remember that this token needs to be cleaned.
1498 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1499
1500 // Warn if there was whitespace between the backslash and newline.
1501 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1502 Diag(Loc: Ptr, DiagID: diag::backslash_newline_space);
1503
1504 // Found backslash<whitespace><newline>. Parse the char after it.
1505 Size += EscapedNewLineSize;
1506 Ptr += EscapedNewLineSize;
1507
1508 // Use slow version to accumulate a correct size field.
1509 auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
1510 CharAndSize.Size += Size;
1511 return CharAndSize;
1512 }
1513
1514 // Otherwise, this is not an escaped newline, just return the slash.
1515 return {.Char: '\\', .Size: Size};
1516 }
1517
1518 // If this is a trigraph, process it.
1519 if (Ptr[0] == '?' && Ptr[1] == '?') {
1520 // If this is actually a legal trigraph (not something like "??x"), emit
1521 // a trigraph warning. If so, and if trigraphs are enabled, return it.
1522 if (char C = DecodeTrigraphChar(CP: Ptr + 2, L: Tok ? this : nullptr,
1523 Trigraphs: LangOpts.Trigraphs)) {
1524 // Remember that this token needs to be cleaned.
1525 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1526
1527 Ptr += 3;
1528 Size += 3;
1529 if (C == '\\') goto Slash;
1530 return {.Char: C, .Size: Size};
1531 }
1532 }
1533
1534 // If this is neither, return a single character.
1535 return {.Char: *Ptr, .Size: Size + 1u};
1536}
1537
1538/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1539/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
1540/// and that we have already incremented Ptr by Size bytes.
1541///
1542/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1543/// be updated to match.
1544Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
1545 const LangOptions &LangOpts) {
1546
1547 unsigned Size = 0;
1548 // If we have a slash, look for an escaped newline.
1549 if (Ptr[0] == '\\') {
1550 ++Size;
1551 ++Ptr;
1552Slash:
1553 // Common case, backslash-char where the char is not whitespace.
1554 if (!isWhitespace(c: Ptr[0]))
1555 return {.Char: '\\', .Size: Size};
1556
1557 // See if we have optional whitespace characters followed by a newline.
1558 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1559 // Found backslash<whitespace><newline>. Parse the char after it.
1560 Size += EscapedNewLineSize;
1561 Ptr += EscapedNewLineSize;
1562
1563 // Use slow version to accumulate a correct size field.
1564 auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
1565 CharAndSize.Size += Size;
1566 return CharAndSize;
1567 }
1568
1569 // Otherwise, this is not an escaped newline, just return the slash.
1570 return {.Char: '\\', .Size: Size};
1571 }
1572
1573 // If this is a trigraph, process it.
1574 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1575 // If this is actually a legal trigraph (not something like "??x"), return
1576 // it.
1577 if (char C = GetTrigraphCharForLetter(Letter: Ptr[2])) {
1578 Ptr += 3;
1579 Size += 3;
1580 if (C == '\\') goto Slash;
1581 return {.Char: C, .Size: Size};
1582 }
1583 }
1584
1585 // If this is neither, return a single character.
1586 return {.Char: *Ptr, .Size: Size + 1u};
1587}
1588
1589//===----------------------------------------------------------------------===//
1590// Helper methods for lexing.
1591//===----------------------------------------------------------------------===//
1592
1593/// Routine that indiscriminately sets the offset into the source file.
1594void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1595 BufferPtr = BufferStart + Offset;
1596 if (BufferPtr > BufferEnd)
1597 BufferPtr = BufferEnd;
1598 // FIXME: What exactly does the StartOfLine bit mean? There are two
1599 // possible meanings for the "start" of the line: the first token on the
1600 // unexpanded line, or the first token on the expanded line.
1601 IsAtStartOfLine = StartOfLine;
1602 IsAtPhysicalStartOfLine = StartOfLine;
1603}
1604
1605static bool isUnicodeWhitespace(uint32_t Codepoint) {
1606 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
1607 UnicodeWhitespaceCharRanges);
1608 return UnicodeWhitespaceChars.contains(C: Codepoint);
1609}
1610
1611// The mathematical compatibility notation profile extends XID_Start and
1612// XID_Continue with mathematical symbols, superscript and subscript digits.
1613// https://www.unicode.org/reports/tr31/#Mathematical_Compatibility_Notation
1614static bool isMathematicalExtensionID(uint32_t C, const LangOptions &LangOpts,
1615 bool IsStart, bool &IsExtension) {
1616 static const llvm::sys::UnicodeCharSet MathStartChars(
1617 GeneratedMathematicalNotationProfileIDStartRanges);
1618 static const llvm::sys::UnicodeCharSet MathContinueChars(
1619 GeneratedMathematicalNotationProfileIDContinueRanges);
1620 if (MathStartChars.contains(C) ||
1621 (!IsStart && MathContinueChars.contains(C))) {
1622 IsExtension = true;
1623 return true;
1624 }
1625 return false;
1626}
1627
1628static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts,
1629 bool &IsExtension) {
1630 if (LangOpts.AsmPreprocessor) {
1631 return false;
1632 } else if (LangOpts.DollarIdents && '$' == C) {
1633 return true;
1634 } else if (LangOpts.CPlusPlus || LangOpts.C23) {
1635 // A non-leading codepoint must have the XID_Continue property.
1636 // GeneratedXIDContinueRanges doesn't contain characters also in
1637 // GeneratedXIDStartRanges, so we need to check both tables.
1638 // '_' doesn't have the XID_Continue property but is allowed in C and C++.
1639 static const llvm::sys::UnicodeCharSet XIDStartChars(
1640 GeneratedXIDStartRanges);
1641 static const llvm::sys::UnicodeCharSet XIDContinueChars(
1642 GeneratedXIDContinueRanges);
1643 if (C == '_' || XIDStartChars.contains(C) || XIDContinueChars.contains(C))
1644 return true;
1645 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/false,
1646 IsExtension);
1647 } else if (LangOpts.C11) {
1648 static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
1649 C11AllowedIDCharRanges);
1650 return C11AllowedIDChars.contains(C);
1651 } else {
1652 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1653 C99AllowedIDCharRanges);
1654 return C99AllowedIDChars.contains(C);
1655 }
1656}
1657
1658static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts,
1659 bool &IsExtension) {
1660 assert(C > 0x7F && "isAllowedInitiallyIDChar called with an ASCII codepoint");
1661 IsExtension = false;
1662 if (LangOpts.AsmPreprocessor) {
1663 return false;
1664 }
1665 if (LangOpts.CPlusPlus || LangOpts.C23) {
1666 static const llvm::sys::UnicodeCharSet XIDStartChars(
1667 GeneratedXIDStartRanges);
1668 if (XIDStartChars.contains(C))
1669 return true;
1670 return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/true,
1671 IsExtension);
1672 }
1673 if (!isAllowedIDChar(C, LangOpts, IsExtension))
1674 return false;
1675 if (LangOpts.C11) {
1676 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
1677 C11DisallowedInitialIDCharRanges);
1678 return !C11DisallowedInitialIDChars.contains(C);
1679 }
1680 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1681 C99DisallowedInitialIDCharRanges);
1682 return !C99DisallowedInitialIDChars.contains(C);
1683}
1684
1685static void
1686diagnoseMathematicalNotationInIdentifier(DiagnosticsEngine &Diags,
1687 const LangOptions &LangOpts,
1688 uint32_t C, CharSourceRange Range) {
1689
1690 static const llvm::sys::UnicodeCharSet MathStartChars(
1691 GeneratedMathematicalNotationProfileIDStartRanges);
1692 static const llvm::sys::UnicodeCharSet MathContinueChars(
1693 GeneratedMathematicalNotationProfileIDContinueRanges);
1694
1695 (void)MathStartChars;
1696 (void)MathContinueChars;
1697 assert((MathStartChars.contains(C) || MathContinueChars.contains(C)) &&
1698 "Unexpected mathematical notation codepoint");
1699 unsigned DiagID = LangOpts.CPlusPlus
1700 ? DiagnosticIDs::getCompatDiagId(
1701 LangOpts, CompatDiagId: diag_compat::mathematical_notation)
1702 : diag::ext_mathematical_notation;
1703 Diags.Report(Loc: Range.getBegin(), DiagID)
1704 << EscapeSingleCodepointForDiagnostic(CP: C) << Range;
1705}
1706
1707static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1708 const char *End) {
1709 return CharSourceRange::getCharRange(B: L.getSourceLocation(Loc: Begin),
1710 E: L.getSourceLocation(Loc: End));
1711}
1712
1713static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
1714 CharSourceRange Range, bool IsFirst) {
1715 // Check C99 compatibility.
1716 if (!Diags.isIgnored(DiagID: diag::warn_c99_compat_unicode_id, Loc: Range.getBegin())) {
1717 enum {
1718 CannotAppearInIdentifier = 0,
1719 CannotStartIdentifier
1720 };
1721
1722 static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1723 C99AllowedIDCharRanges);
1724 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1725 C99DisallowedInitialIDCharRanges);
1726 if (!C99AllowedIDChars.contains(C)) {
1727 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1728 << Range
1729 << CannotAppearInIdentifier;
1730 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) {
1731 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_c99_compat_unicode_id)
1732 << Range
1733 << CannotStartIdentifier;
1734 }
1735 }
1736}
1737
1738/// After encountering UTF-8 character C and interpreting it as an identifier
1739/// character, check whether it's a homoglyph for a common non-identifier
1740/// source character that is unlikely to be an intentional identifier
1741/// character and warn if so.
1742static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
1743 CharSourceRange Range) {
1744 // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
1745 struct HomoglyphPair {
1746 uint32_t Character;
1747 char LooksLike;
1748 bool operator<(HomoglyphPair R) const { return Character < R.Character; }
1749 };
1750 static constexpr HomoglyphPair SortedHomoglyphs[] = {
1751 {.Character: U'\u00ad', .LooksLike: 0}, // SOFT HYPHEN
1752 {.Character: U'\u01c3', .LooksLike: '!'}, // LATIN LETTER RETROFLEX CLICK
1753 {.Character: U'\u037e', .LooksLike: ';'}, // GREEK QUESTION MARK
1754 {.Character: U'\u200b', .LooksLike: 0}, // ZERO WIDTH SPACE
1755 {.Character: U'\u200c', .LooksLike: 0}, // ZERO WIDTH NON-JOINER
1756 {.Character: U'\u200d', .LooksLike: 0}, // ZERO WIDTH JOINER
1757 {.Character: U'\u2060', .LooksLike: 0}, // WORD JOINER
1758 {.Character: U'\u2061', .LooksLike: 0}, // FUNCTION APPLICATION
1759 {.Character: U'\u2062', .LooksLike: 0}, // INVISIBLE TIMES
1760 {.Character: U'\u2063', .LooksLike: 0}, // INVISIBLE SEPARATOR
1761 {.Character: U'\u2064', .LooksLike: 0}, // INVISIBLE PLUS
1762 {.Character: U'\u2212', .LooksLike: '-'}, // MINUS SIGN
1763 {.Character: U'\u2215', .LooksLike: '/'}, // DIVISION SLASH
1764 {.Character: U'\u2216', .LooksLike: '\\'}, // SET MINUS
1765 {.Character: U'\u2217', .LooksLike: '*'}, // ASTERISK OPERATOR
1766 {.Character: U'\u2223', .LooksLike: '|'}, // DIVIDES
1767 {.Character: U'\u2227', .LooksLike: '^'}, // LOGICAL AND
1768 {.Character: U'\u2236', .LooksLike: ':'}, // RATIO
1769 {.Character: U'\u223c', .LooksLike: '~'}, // TILDE OPERATOR
1770 {.Character: U'\ua789', .LooksLike: ':'}, // MODIFIER LETTER COLON
1771 {.Character: U'\ufeff', .LooksLike: 0}, // ZERO WIDTH NO-BREAK SPACE
1772 {.Character: U'\uff01', .LooksLike: '!'}, // FULLWIDTH EXCLAMATION MARK
1773 {.Character: U'\uff03', .LooksLike: '#'}, // FULLWIDTH NUMBER SIGN
1774 {.Character: U'\uff04', .LooksLike: '$'}, // FULLWIDTH DOLLAR SIGN
1775 {.Character: U'\uff05', .LooksLike: '%'}, // FULLWIDTH PERCENT SIGN
1776 {.Character: U'\uff06', .LooksLike: '&'}, // FULLWIDTH AMPERSAND
1777 {.Character: U'\uff08', .LooksLike: '('}, // FULLWIDTH LEFT PARENTHESIS
1778 {.Character: U'\uff09', .LooksLike: ')'}, // FULLWIDTH RIGHT PARENTHESIS
1779 {.Character: U'\uff0a', .LooksLike: '*'}, // FULLWIDTH ASTERISK
1780 {.Character: U'\uff0b', .LooksLike: '+'}, // FULLWIDTH ASTERISK
1781 {.Character: U'\uff0c', .LooksLike: ','}, // FULLWIDTH COMMA
1782 {.Character: U'\uff0d', .LooksLike: '-'}, // FULLWIDTH HYPHEN-MINUS
1783 {.Character: U'\uff0e', .LooksLike: '.'}, // FULLWIDTH FULL STOP
1784 {.Character: U'\uff0f', .LooksLike: '/'}, // FULLWIDTH SOLIDUS
1785 {.Character: U'\uff1a', .LooksLike: ':'}, // FULLWIDTH COLON
1786 {.Character: U'\uff1b', .LooksLike: ';'}, // FULLWIDTH SEMICOLON
1787 {.Character: U'\uff1c', .LooksLike: '<'}, // FULLWIDTH LESS-THAN SIGN
1788 {.Character: U'\uff1d', .LooksLike: '='}, // FULLWIDTH EQUALS SIGN
1789 {.Character: U'\uff1e', .LooksLike: '>'}, // FULLWIDTH GREATER-THAN SIGN
1790 {.Character: U'\uff1f', .LooksLike: '?'}, // FULLWIDTH QUESTION MARK
1791 {.Character: U'\uff20', .LooksLike: '@'}, // FULLWIDTH COMMERCIAL AT
1792 {.Character: U'\uff3b', .LooksLike: '['}, // FULLWIDTH LEFT SQUARE BRACKET
1793 {.Character: U'\uff3c', .LooksLike: '\\'}, // FULLWIDTH REVERSE SOLIDUS
1794 {.Character: U'\uff3d', .LooksLike: ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
1795 {.Character: U'\uff3e', .LooksLike: '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
1796 {.Character: U'\uff5b', .LooksLike: '{'}, // FULLWIDTH LEFT CURLY BRACKET
1797 {.Character: U'\uff5c', .LooksLike: '|'}, // FULLWIDTH VERTICAL LINE
1798 {.Character: U'\uff5d', .LooksLike: '}'}, // FULLWIDTH RIGHT CURLY BRACKET
1799 {.Character: U'\uff5e', .LooksLike: '~'}, // FULLWIDTH TILDE
1800 {.Character: 0, .LooksLike: 0}
1801 };
1802 auto Homoglyph =
1803 std::lower_bound(first: std::begin(arr: SortedHomoglyphs),
1804 last: std::end(arr: SortedHomoglyphs) - 1, val: HomoglyphPair{.Character: C, .LooksLike: '\0'});
1805 if (Homoglyph->Character == C) {
1806 if (Homoglyph->LooksLike) {
1807 const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
1808 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_homoglyph)
1809 << Range << EscapeSingleCodepointForDiagnostic(CP: C) << LooksLikeStr;
1810 } else {
1811 Diags.Report(Loc: Range.getBegin(), DiagID: diag::warn_utf8_symbol_zero_width)
1812 << Range << EscapeSingleCodepointForDiagnostic(CP: C);
1813 }
1814 }
1815}
1816
1817static bool CheckCodepointValidInIdentifier(const Preprocessor *PP,
1818 const LangOptions &LangOpts,
1819 uint32_t CodePoint,
1820 CharSourceRange Range, bool IsFirst,
1821 bool Diagnose) {
1822 if (isASCII(c: CodePoint))
1823 return true;
1824
1825 bool IsExtension;
1826 bool IsIDStart = isAllowedInitiallyIDChar(C: CodePoint, LangOpts, IsExtension);
1827 bool IsIDContinue =
1828 IsIDStart || isAllowedIDChar(C: CodePoint, LangOpts, IsExtension);
1829
1830 if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
1831 return true;
1832
1833 if (!Diagnose)
1834 return false;
1835
1836 bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
1837
1838 if (!IsFirst || InvalidOnlyAtStart) {
1839 PP->Diag(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed_identifier)
1840 << Range << EscapeSingleCodepointForDiagnostic(CP: CodePoint)
1841 << int(InvalidOnlyAtStart) << FixItHint::CreateRemoval(RemoveRange: Range);
1842 } else {
1843 PP->Diag(Loc: Range.getBegin(), DiagID: diag::err_character_not_allowed)
1844 << Range << EscapeSingleCodepointForDiagnostic(CP: CodePoint)
1845 << FixItHint::CreateRemoval(RemoveRange: Range);
1846 }
1847 return false;
1848}
1849
1850bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1851 Token &Result) {
1852 const char *UCNPtr = CurPtr + Size;
1853 uint32_t CodePoint = tryReadUCN(StartPtr&: UCNPtr, SlashLoc: CurPtr, /*Token=*/Result: nullptr);
1854 if (CodePoint == 0) {
1855 return false;
1856 }
1857 bool IsExtension = false;
1858 if (!isAllowedIDChar(C: CodePoint, LangOpts, IsExtension)) {
1859 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1860 return false;
1861
1862 bool DiagnoseAndContinue = !isLexingRawMode() &&
1863 !ParsingPreprocessorDirective &&
1864 !PP->isPreprocessedOutput();
1865 if (!CheckCodepointValidInIdentifier(
1866 PP, LangOpts, CodePoint, Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1867 /*IsFirst=*/false, Diagnose: DiagnoseAndContinue) &&
1868 !DiagnoseAndContinue)
1869 return false;
1870 // We got a unicode codepoint that is neither a space nor a
1871 // a valid identifier part.
1872 // Carry on as if the codepoint was valid for recovery purposes.
1873 } else if (!isLexingRawMode()) {
1874 if (IsExtension)
1875 diagnoseMathematicalNotationInIdentifier(
1876 Diags&: PP->getDiagnostics(), LangOpts, C: CodePoint,
1877 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr));
1878
1879 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1880 Range: makeCharRange(L&: *this, Begin: CurPtr, End: UCNPtr),
1881 /*IsFirst=*/false);
1882 }
1883
1884 Result.setFlag(Token::HasUCN);
1885 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') ||
1886 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
1887 CurPtr = UCNPtr;
1888 else
1889 while (CurPtr != UCNPtr)
1890 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
1891 return true;
1892}
1893
1894bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
1895 llvm::UTF32 CodePoint;
1896
1897 // If a UTF-8 codepoint appears immediately after an escaped new line,
1898 // CurPtr may point to the splicing \ on the preceding line,
1899 // so we need to skip it.
1900 unsigned FirstCodeUnitSize;
1901 getCharAndSize(Ptr: CurPtr, Size&: FirstCodeUnitSize);
1902 const char *CharStart = CurPtr + FirstCodeUnitSize - 1;
1903 const char *UnicodePtr = CharStart;
1904
1905 llvm::ConversionResult ConvResult = llvm::convertUTF8Sequence(
1906 source: (const llvm::UTF8 **)&UnicodePtr, sourceEnd: (const llvm::UTF8 *)BufferEnd,
1907 target: &CodePoint, flags: llvm::strictConversion);
1908 if (ConvResult != llvm::conversionOK)
1909 return false;
1910
1911 bool IsExtension = false;
1912 if (!isAllowedIDChar(C: static_cast<uint32_t>(CodePoint), LangOpts,
1913 IsExtension)) {
1914 if (isASCII(c: CodePoint) || isUnicodeWhitespace(Codepoint: CodePoint))
1915 return false;
1916
1917 bool DiagnoseAndContinue = !isLexingRawMode() &&
1918 !ParsingPreprocessorDirective &&
1919 !PP->isPreprocessedOutput();
1920
1921 if (!CheckCodepointValidInIdentifier(
1922 PP, LangOpts, CodePoint,
1923 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr), /*IsFirst=*/false,
1924 Diagnose: DiagnoseAndContinue) &&
1925 !DiagnoseAndContinue)
1926 return false;
1927 // We got a unicode codepoint that is neither a space nor a
1928 // a valid identifier part. Carry on as if the codepoint was
1929 // valid for recovery purposes.
1930 } else if (!isLexingRawMode()) {
1931 if (IsExtension)
1932 diagnoseMathematicalNotationInIdentifier(
1933 Diags&: PP->getDiagnostics(), LangOpts, C: CodePoint,
1934 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1935 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C: CodePoint,
1936 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr),
1937 /*IsFirst=*/false);
1938 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C: CodePoint,
1939 Range: makeCharRange(L&: *this, Begin: CharStart, End: UnicodePtr));
1940 }
1941
1942 // Once we sucessfully parsed some UTF-8,
1943 // calling ConsumeChar ensures the NeedsCleaning flag is set on the token
1944 // being lexed, and that warnings about trailing spaces are emitted.
1945 ConsumeChar(Ptr: CurPtr, Size: FirstCodeUnitSize, Tok&: Result);
1946 CurPtr = UnicodePtr;
1947 return true;
1948}
1949
1950bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
1951 const char *CurPtr) {
1952 bool IsExtension = false;
1953 if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
1954 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1955 !PP->isPreprocessedOutput()) {
1956 if (IsExtension)
1957 diagnoseMathematicalNotationInIdentifier(
1958 Diags&: PP->getDiagnostics(), LangOpts, C,
1959 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1960 maybeDiagnoseIDCharCompat(Diags&: PP->getDiagnostics(), C,
1961 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1962 /*IsFirst=*/true);
1963 maybeDiagnoseUTF8Homoglyph(Diags&: PP->getDiagnostics(), C,
1964 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr));
1965 }
1966
1967 MIOpt.ReadToken();
1968 return LexIdentifierContinue(Result, CurPtr);
1969 }
1970
1971 if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
1972 !PP->isPreprocessedOutput() && !isASCII(c: *BufferPtr) &&
1973 !isUnicodeWhitespace(Codepoint: C)) {
1974 // Non-ASCII characters tend to creep into source code unintentionally.
1975 // Instead of letting the parser complain about the unknown token,
1976 // just drop the character.
1977 // Note that we can /only/ do this when the non-ASCII character is actually
1978 // spelled as Unicode, not written as a UCN. The standard requires that
1979 // we not throw away any possible preprocessor tokens, but there's a
1980 // loophole in the mapping of Unicode characters to basic character set
1981 // characters that allows us to map these particular characters to, say,
1982 // whitespace.
1983 CheckCodepointValidInIdentifier(PP, LangOpts, CodePoint: C,
1984 Range: makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr),
1985 /*IsStart=*/IsFirst: true, /*Diagnose=*/true);
1986 BufferPtr = CurPtr;
1987 return false;
1988 }
1989
1990 // Otherwise, we have an explicit UCN or a character that's unlikely to show
1991 // up by accident.
1992 MIOpt.ReadToken();
1993 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
1994 return true;
1995}
1996
1997static const char *fastParseASCIIIdentifierScalar(const char *CurPtr) {
1998 unsigned char C = *CurPtr;
1999 while (isAsciiIdentifierContinue(c: C))
2000 C = *++CurPtr;
2001 return CurPtr;
2002}
2003
2004#if LLVM_IS_X86
2005// Fast path for lexing ASCII identifiers using SSE4.2 instructions.
2006LLVM_TARGET_SSE42 static const char *
2007fastParseASCIIIdentifierSSE42(const char *CurPtr, const char *BufferEnd) {
2008 alignas(16) static constexpr char AsciiIdentifierRange[16] = {
2009 '_', '_', 'A', 'Z', 'a', 'z', '0', '9',
2010 };
2011 constexpr ssize_t BytesPerRegister = 16;
2012
2013 __m128i AsciiIdentifierRangeV =
2014 _mm_load_si128(p: reinterpret_cast<const __m128i *>(AsciiIdentifierRange));
2015
2016 while (LLVM_LIKELY(BufferEnd - CurPtr >= BytesPerRegister)) {
2017 __m128i Cv = _mm_loadu_si128(p: reinterpret_cast<const __m128i *>(CurPtr));
2018
2019 const int Consumed =
2020 _mm_cmpistri(AsciiIdentifierRangeV, Cv,
2021 _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES |
2022 _SIDD_UBYTE_OPS | _SIDD_NEGATIVE_POLARITY);
2023 CurPtr += Consumed;
2024 if (Consumed == BytesPerRegister)
2025 continue;
2026 return CurPtr;
2027 }
2028
2029 return fastParseASCIIIdentifierScalar(CurPtr);
2030}
2031#endif
2032
2033static const char *fastParseASCIIIdentifier(const char *CurPtr,
2034 const char *BufferEnd) {
2035#if LLVM_IS_X86
2036 if (LLVM_LIKELY(LLVM_CPU_SUPPORTS_SSE42))
2037 return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
2038#endif
2039 return fastParseASCIIIdentifierScalar(CurPtr);
2040}
2041
2042bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
2043 // Match [_A-Za-z0-9]*, we have already matched an identifier start.
2044
2045 while (true) {
2046
2047 CurPtr = fastParseASCIIIdentifier(CurPtr, BufferEnd);
2048
2049 unsigned Size;
2050 // Slow path: handle trigraph, unicode codepoints, UCNs.
2051 unsigned char C = getCharAndSize(Ptr: CurPtr, Size);
2052 if (isAsciiIdentifierContinue(c: C)) {
2053 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2054 continue;
2055 }
2056 if (C == '$') {
2057 // If we hit a $ and they are not supported in identifiers, we are done.
2058 if (!LangOpts.DollarIdents)
2059 break;
2060 // Otherwise, emit a diagnostic and continue.
2061 if (!isLexingRawMode())
2062 Diag(Loc: CurPtr, DiagID: diag::ext_dollar_in_identifier);
2063 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2064 continue;
2065 }
2066 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2067 continue;
2068 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2069 continue;
2070 // Neither an expected Unicode codepoint nor a UCN.
2071 break;
2072 }
2073
2074 const char *IdStart = BufferPtr;
2075 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::raw_identifier);
2076 Result.setRawIdentifierData(IdStart);
2077
2078 // If we are in raw mode, return this identifier raw. There is no need to
2079 // look up identifier information or attempt to macro expand it.
2080 if (LexingRawMode)
2081 return true;
2082
2083 // Fill in Result.IdentifierInfo and update the token kind,
2084 // looking up the identifier in the identifier table.
2085 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
2086 // Note that we have to call PP->LookUpIdentifierInfo() even for code
2087 // completion, it writes IdentifierInfo into Result, and callers rely on it.
2088
2089 // If the completion point is at the end of an identifier, we want to treat
2090 // the identifier as incomplete even if it resolves to a macro or a keyword.
2091 // This allows e.g. 'class^' to complete to 'classifier'.
2092 if (isCodeCompletionPoint(CurPtr)) {
2093 // Return the code-completion token.
2094 Result.setKind(tok::code_completion);
2095 // Skip the code-completion char and all immediate identifier characters.
2096 // This ensures we get consistent behavior when completing at any point in
2097 // an identifier (i.e. at the start, in the middle, at the end). Note that
2098 // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
2099 // simpler.
2100 assert(*CurPtr == 0 && "Completion character must be 0");
2101 ++CurPtr;
2102 // Note that code completion token is not added as a separate character
2103 // when the completion point is at the end of the buffer. Therefore, we need
2104 // to check if the buffer has ended.
2105 if (CurPtr < BufferEnd) {
2106 while (isAsciiIdentifierContinue(c: *CurPtr))
2107 ++CurPtr;
2108 }
2109 BufferPtr = CurPtr;
2110 return true;
2111 }
2112
2113 // Finally, now that we know we have an identifier, pass this off to the
2114 // preprocessor, which may macro expand it or something.
2115 if (II->isHandleIdentifierCase() || II->isModuleKeyword() ||
2116 II->isImportKeyword() || II->getTokenID() == tok::kw_export)
2117 return PP->HandleIdentifier(Identifier&: Result);
2118
2119 return true;
2120}
2121
2122/// isHexaLiteral - Return true if Start points to a hex constant.
2123/// in microsoft mode (where this is supposed to be several different tokens).
2124bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
2125 auto CharAndSize1 = Lexer::getCharAndSizeNoWarn(Ptr: Start, LangOpts);
2126 char C1 = CharAndSize1.Char;
2127 if (C1 != '0')
2128 return false;
2129
2130 auto CharAndSize2 =
2131 Lexer::getCharAndSizeNoWarn(Ptr: Start + CharAndSize1.Size, LangOpts);
2132 char C2 = CharAndSize2.Char;
2133 return (C2 == 'x' || C2 == 'X');
2134}
2135
2136/// LexNumericConstant - Lex the remainder of a integer or floating point
2137/// constant. From[-1] is the first character lexed. Return the end of the
2138/// constant.
2139bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
2140 unsigned Size;
2141 char C = getCharAndSize(Ptr: CurPtr, Size);
2142 char PrevCh = 0;
2143 while (isPreprocessingNumberBody(c: C)) {
2144 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2145 PrevCh = C;
2146 if (LangOpts.HLSL && C == '.' && (*CurPtr == 'x' || *CurPtr == 'r')) {
2147 CurPtr -= Size;
2148 break;
2149 }
2150 C = getCharAndSize(Ptr: CurPtr, Size);
2151 }
2152
2153 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
2154 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
2155 // If we are in Microsoft mode, don't continue if the constant is hex.
2156 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
2157 if (!LangOpts.MicrosoftExt || !isHexaLiteral(Start: BufferPtr, LangOpts))
2158 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2159 }
2160
2161 // If we have a hex FP constant, continue.
2162 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
2163 // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
2164 // not-quite-conforming extension. Only do so if this looks like it's
2165 // actually meant to be a hexfloat, and not if it has a ud-suffix.
2166 bool IsHexFloat = true;
2167 if (!LangOpts.C99) {
2168 if (!isHexaLiteral(Start: BufferPtr, LangOpts))
2169 IsHexFloat = false;
2170 else if (!LangOpts.CPlusPlus17 &&
2171 std::find(first: BufferPtr, last: CurPtr, val: '_') != CurPtr)
2172 IsHexFloat = false;
2173 }
2174 if (IsHexFloat)
2175 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size, Tok&: Result));
2176 }
2177
2178 // If we have a digit separator, continue.
2179 if (C == '\'' && LangOpts.AllowLiteralDigitSeparator) {
2180 auto [Next, NextSize] = getCharAndSizeNoWarn(Ptr: CurPtr + Size, LangOpts);
2181 // A digit or non-digit.
2182 if (isAsciiIdentifierContinue(c: Next)) {
2183 if (!isLexingRawMode())
2184 Diag(Loc: CurPtr, DiagID: LangOpts.CPlusPlus
2185 ? diag::warn_cxx11_compat_digit_separator
2186 : diag::warn_c23_compat_digit_separator);
2187 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2188 CurPtr = ConsumeChar(Ptr: CurPtr, Size: NextSize, Tok&: Result);
2189 return LexNumericConstant(Result, CurPtr);
2190 }
2191 }
2192
2193 if (C == '$' && LangOpts.DollarIdents) {
2194 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2195 return LexNumericConstant(Result, CurPtr);
2196 }
2197
2198 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
2199 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2200 return LexNumericConstant(Result, CurPtr);
2201 if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2202 return LexNumericConstant(Result, CurPtr);
2203
2204 // Update the location of token as well as BufferPtr.
2205 const char *TokStart = BufferPtr;
2206 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::numeric_constant);
2207 Result.setLiteralData(TokStart);
2208 return true;
2209}
2210
2211/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
2212/// in C++11, or warn on a ud-suffix in C++98.
2213const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
2214 bool IsStringLiteral) {
2215 assert(LangOpts.CPlusPlus);
2216
2217 // Maximally munch an identifier.
2218 unsigned Size;
2219 char C = getCharAndSize(Ptr: CurPtr, Size);
2220 bool Consumed = false;
2221
2222 if (!isAsciiIdentifierStart(c: C, AllowDollar: LangOpts.DollarIdents)) {
2223 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
2224 Consumed = true;
2225 else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
2226 Consumed = true;
2227 else
2228 return CurPtr;
2229 }
2230
2231 if (!LangOpts.CPlusPlus11) {
2232 if (!isLexingRawMode())
2233 Diag(Loc: CurPtr,
2234 DiagID: C == '_' ? diag::warn_cxx11_compat_user_defined_literal
2235 : diag::warn_cxx11_compat_reserved_user_defined_literal)
2236 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2237 return CurPtr;
2238 }
2239
2240 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
2241 // that does not start with an underscore is ill-formed. As a conforming
2242 // extension, we treat all such suffixes as if they had whitespace before
2243 // them. We assume a suffix beginning with a UCN or UTF-8 character is more
2244 // likely to be a ud-suffix than a macro, however, and accept that.
2245 if (!Consumed) {
2246 bool IsUDSuffix = false;
2247 if (C == '_')
2248 IsUDSuffix = true;
2249 else if (IsStringLiteral && LangOpts.CPlusPlus14) {
2250 // In C++1y, we need to look ahead a few characters to see if this is a
2251 // valid suffix for a string literal or a numeric literal (this could be
2252 // the 'operator""if' defining a numeric literal operator).
2253 const unsigned MaxStandardSuffixLength = 3;
2254 char Buffer[MaxStandardSuffixLength] = { C };
2255 unsigned Consumed = Size;
2256 unsigned Chars = 1;
2257 while (true) {
2258 auto [Next, NextSize] =
2259 getCharAndSizeNoWarn(Ptr: CurPtr + Consumed, LangOpts);
2260 if (!isAsciiIdentifierContinue(c: Next, AllowDollar: LangOpts.DollarIdents)) {
2261 // End of suffix. Check whether this is on the allowed list.
2262 const StringRef CompleteSuffix(Buffer, Chars);
2263 IsUDSuffix =
2264 StringLiteralParser::isValidUDSuffix(LangOpts, Suffix: CompleteSuffix);
2265 break;
2266 }
2267
2268 if (Chars == MaxStandardSuffixLength)
2269 // Too long: can't be a standard suffix.
2270 break;
2271
2272 Buffer[Chars++] = Next;
2273 Consumed += NextSize;
2274 }
2275 }
2276
2277 if (!IsUDSuffix) {
2278 if (!isLexingRawMode())
2279 Diag(Loc: CurPtr, DiagID: LangOpts.MSVCCompat
2280 ? diag::ext_ms_reserved_user_defined_literal
2281 : diag::ext_reserved_user_defined_literal)
2282 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: CurPtr), Code: " ");
2283 return CurPtr;
2284 }
2285
2286 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2287 }
2288
2289 Result.setFlag(Token::HasUDSuffix);
2290 while (true) {
2291 C = getCharAndSize(Ptr: CurPtr, Size);
2292 if (isAsciiIdentifierContinue(c: C, AllowDollar: LangOpts.DollarIdents)) {
2293 CurPtr = ConsumeChar(Ptr: CurPtr, Size, Tok&: Result);
2294 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
2295 } else if (!isASCII(c: C) && tryConsumeIdentifierUTF8Char(CurPtr, Result)) {
2296 } else
2297 break;
2298 }
2299
2300 return CurPtr;
2301}
2302
2303/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
2304/// either " or L" or u8" or u" or U".
2305bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
2306 tok::TokenKind Kind) {
2307 const char *AfterQuote = CurPtr;
2308 // Does this string contain the \0 character?
2309 const char *NulCharacter = nullptr;
2310
2311 if (!isLexingRawMode() &&
2312 (Kind == tok::utf8_string_literal ||
2313 Kind == tok::utf16_string_literal ||
2314 Kind == tok::utf32_string_literal))
2315 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
2316 : diag::warn_c99_compat_unicode_literal);
2317
2318 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2319 while (C != '"') {
2320 // Skip escaped characters. Escaped newlines will already be processed by
2321 // getAndAdvanceChar.
2322 if (C == '\\') {
2323 const char *SavedCurPtr = CurPtr;
2324 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2325
2326 // lex.header
2327 //
2328 // header-name:
2329 // ...
2330 // " q-char-sequence "
2331 // ...
2332 // q-char-sequence:
2333 // q-char q-char-sequence[opt]
2334 // q-char:
2335 // any member of the translation character set except new-line and
2336 // U+0022 quotation mark
2337 //
2338 // The implementation-defined semantics cannot be taken as causing '\' to
2339 // "escape" the following " because there is no provision for " in a
2340 // q-char-sequence.
2341 if (ParsingFilename && C == '"')
2342 CurPtr = SavedCurPtr;
2343 }
2344
2345 if (C == '\n' || C == '\r' || // Newline.
2346 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2347 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2348 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 1;
2349 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2350 return true;
2351 }
2352
2353 if (C == 0) {
2354 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2355 if (ParsingFilename)
2356 codeCompleteIncludedFile(PathStart: AfterQuote, CompletionPoint: CurPtr - 1, /*IsAngled=*/false);
2357 else
2358 PP->CodeCompleteNaturalLanguage();
2359 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2360 cutOffLexing();
2361 return true;
2362 }
2363
2364 NulCharacter = CurPtr-1;
2365 }
2366 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2367 }
2368
2369 // If we are in C++11, lex the optional ud-suffix.
2370 if (LangOpts.CPlusPlus)
2371 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2372
2373 // If a nul character existed in the string, warn about it.
2374 if (NulCharacter && !isLexingRawMode())
2375 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2376
2377 // Update the location of the token as well as the BufferPtr instance var.
2378 const char *TokStart = BufferPtr;
2379 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2380 Result.setLiteralData(TokStart);
2381 return true;
2382}
2383
2384/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
2385/// having lexed R", LR", u8R", uR", or UR".
2386bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2387 tok::TokenKind Kind) {
2388 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
2389 // Between the initial and final double quote characters of the raw string,
2390 // any transformations performed in phases 1 and 2 (trigraphs,
2391 // universal-character-names, and line splicing) are reverted.
2392
2393 if (!isLexingRawMode())
2394 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_raw_string_literal);
2395
2396 unsigned PrefixLen = 0;
2397
2398 while (PrefixLen != 16 && isRawStringDelimBody(c: CurPtr[PrefixLen])) {
2399 if (!isLexingRawMode() &&
2400 llvm::is_contained(Set: {'$', '@', '`'}, Element: CurPtr[PrefixLen])) {
2401 const char *Pos = &CurPtr[PrefixLen];
2402 DiagCompat(Loc: Pos, CompatDiagId: diag_compat::raw_string_literal_character_set)
2403 << StringRef(Pos, 1);
2404 }
2405 ++PrefixLen;
2406 }
2407
2408 // If the last character was not a '(', then we didn't lex a valid delimiter.
2409 if (CurPtr[PrefixLen] != '(') {
2410 if (!isLexingRawMode()) {
2411 const char *PrefixEnd = &CurPtr[PrefixLen];
2412 if (PrefixLen == 16) {
2413 Diag(Loc: PrefixEnd, DiagID: diag::err_raw_delim_too_long);
2414 } else if (*PrefixEnd == '\n') {
2415 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_newline_raw_delim);
2416 } else {
2417 Diag(Loc: PrefixEnd, DiagID: diag::err_invalid_char_raw_delim)
2418 << StringRef(PrefixEnd, 1);
2419 }
2420 }
2421
2422 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2423 // it's possible the '"' was intended to be part of the raw string, but
2424 // there's not much we can do about that.
2425 while (true) {
2426 char C = *CurPtr++;
2427
2428 if (C == '"')
2429 break;
2430 if (C == 0 && CurPtr-1 == BufferEnd) {
2431 --CurPtr;
2432 break;
2433 }
2434 }
2435
2436 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2437 return true;
2438 }
2439
2440 // Save prefix and move CurPtr past it
2441 const char *Prefix = CurPtr;
2442 CurPtr += PrefixLen + 1; // skip over prefix and '('
2443
2444 while (true) {
2445 char C = *CurPtr++;
2446
2447 if (C == ')') {
2448 // Check for prefix match and closing quote.
2449 if (strncmp(s1: CurPtr, s2: Prefix, n: PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
2450 CurPtr += PrefixLen + 1; // skip over prefix and '"'
2451 break;
2452 }
2453 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
2454 if (!isLexingRawMode())
2455 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_raw_string)
2456 << StringRef(Prefix, PrefixLen);
2457 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2458 return true;
2459 }
2460 }
2461
2462 // If we are in C++11, lex the optional ud-suffix.
2463 if (LangOpts.CPlusPlus)
2464 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: true);
2465
2466 // Update the location of token as well as BufferPtr.
2467 const char *TokStart = BufferPtr;
2468 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2469 Result.setLiteralData(TokStart);
2470 return true;
2471}
2472
2473/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2474/// after having lexed the '<' character. This is used for #include filenames.
2475/// Returns false if failed to lex the angled string literal; so the caller can
2476/// lex the '<' normally.
2477bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2478 // Does this string contain the \0 character?
2479 const char *NulCharacter = nullptr;
2480 const char *AfterLessPos = CurPtr;
2481 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2482 while (C != '>') {
2483 // Skip escaped characters. Escaped newlines will already be processed by
2484 // getAndAdvanceChar.
2485 if (C == '\\')
2486 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2487
2488 if (isVerticalWhitespace(c: C) || // Newline.
2489 (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
2490 // If the filename is unterminated, let the caller lex the '<' normally.
2491 return false;
2492 }
2493
2494 if (C == 0) {
2495 if (isCodeCompletionPoint(CurPtr: CurPtr - 1)) {
2496 codeCompleteIncludedFile(PathStart: AfterLessPos, CompletionPoint: CurPtr - 1, /*IsAngled=*/true);
2497 cutOffLexing();
2498 FormTokenWithChars(Result, TokEnd: CurPtr - 1, Kind: tok::unknown);
2499 return true;
2500 }
2501 NulCharacter = CurPtr-1;
2502 }
2503 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2504 }
2505
2506 // If a nul character existed in the string, warn about it.
2507 if (NulCharacter && !isLexingRawMode())
2508 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 1;
2509
2510 // Update the location of token as well as BufferPtr.
2511 const char *TokStart = BufferPtr;
2512 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::header_name);
2513 Result.setLiteralData(TokStart);
2514 return true;
2515}
2516
2517void Lexer::codeCompleteIncludedFile(const char *PathStart,
2518 const char *CompletionPoint,
2519 bool IsAngled) {
2520 // Completion only applies to the filename, after the last slash.
2521 StringRef PartialPath(PathStart, CompletionPoint - PathStart);
2522 llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
2523 auto Slash = PartialPath.find_last_of(Chars: SlashChars);
2524 StringRef Dir =
2525 (Slash == StringRef::npos) ? "" : PartialPath.take_front(N: Slash);
2526 const char *StartOfFilename =
2527 (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
2528 // Code completion filter range is the filename only, up to completion point.
2529 PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
2530 Name: StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
2531 // We should replace the characters up to the closing quote or closest slash,
2532 // if any.
2533 while (CompletionPoint < BufferEnd) {
2534 char Next = *(CompletionPoint + 1);
2535 if (Next == 0 || Next == '\r' || Next == '\n')
2536 break;
2537 ++CompletionPoint;
2538 if (Next == (IsAngled ? '>' : '"'))
2539 break;
2540 if (SlashChars.contains(C: Next))
2541 break;
2542 }
2543
2544 PP->setCodeCompletionTokenRange(
2545 Start: FileLoc.getLocWithOffset(Offset: StartOfFilename - BufferStart),
2546 End: FileLoc.getLocWithOffset(Offset: CompletionPoint - BufferStart));
2547 PP->CodeCompleteIncludedFile(Dir, IsAngled);
2548}
2549
2550/// LexCharConstant - Lex the remainder of a character constant, after having
2551/// lexed either ' or L' or u8' or u' or U'.
2552bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2553 tok::TokenKind Kind) {
2554 // Does this character contain the \0 character?
2555 const char *NulCharacter = nullptr;
2556
2557 if (!isLexingRawMode()) {
2558 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
2559 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2560 ? diag::warn_cxx98_compat_unicode_literal
2561 : diag::warn_c99_compat_unicode_literal);
2562 else if (Kind == tok::utf8_char_constant)
2563 Diag(Loc: BufferPtr, DiagID: LangOpts.CPlusPlus
2564 ? diag::warn_cxx14_compat_u8_character_literal
2565 : diag::warn_c17_compat_u8_character_literal);
2566 }
2567
2568 char C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2569 if (C == '\'') {
2570 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2571 Diag(Loc: BufferPtr, DiagID: diag::ext_empty_character);
2572 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2573 return true;
2574 }
2575
2576 while (C != '\'') {
2577 // Skip escaped characters.
2578 if (C == '\\')
2579 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2580
2581 if (C == '\n' || C == '\r' || // Newline.
2582 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
2583 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
2584 Diag(Loc: BufferPtr, DiagID: diag::ext_unterminated_char_or_string) << 0;
2585 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2586 return true;
2587 }
2588
2589 if (C == 0) {
2590 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2591 PP->CodeCompleteNaturalLanguage();
2592 FormTokenWithChars(Result, TokEnd: CurPtr-1, Kind: tok::unknown);
2593 cutOffLexing();
2594 return true;
2595 }
2596
2597 NulCharacter = CurPtr-1;
2598 }
2599 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2600 }
2601
2602 // If we are in C++11, lex the optional ud-suffix.
2603 if (LangOpts.CPlusPlus)
2604 CurPtr = LexUDSuffix(Result, CurPtr, IsStringLiteral: false);
2605
2606 // If a nul character existed in the character, warn about it.
2607 if (NulCharacter && !isLexingRawMode())
2608 Diag(Loc: NulCharacter, DiagID: diag::null_in_char_or_string) << 0;
2609
2610 // Update the location of token as well as BufferPtr.
2611 const char *TokStart = BufferPtr;
2612 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
2613 Result.setLiteralData(TokStart);
2614 return true;
2615}
2616
2617/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2618/// Update BufferPtr to point to the next non-whitespace character and return.
2619///
2620/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2621bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
2622 // Whitespace - Skip it, then return the token after the whitespace.
2623 bool SawNewline = isVerticalWhitespace(c: CurPtr[-1]);
2624
2625 unsigned char Char = *CurPtr;
2626
2627 const char *lastNewLine = nullptr;
2628 auto setLastNewLine = [&](const char *Ptr) {
2629 lastNewLine = Ptr;
2630 if (!NewLinePtr)
2631 NewLinePtr = Ptr;
2632 };
2633 if (SawNewline)
2634 setLastNewLine(CurPtr - 1);
2635
2636 // Skip consecutive spaces efficiently.
2637 while (true) {
2638 // Skip horizontal whitespace, especially space, very aggressively.
2639 while (Char == ' ' || isHorizontalWhitespace(c: Char))
2640 Char = *++CurPtr;
2641
2642 // Otherwise if we have something other than whitespace, we're done.
2643 if (!isVerticalWhitespace(c: Char))
2644 break;
2645
2646 if (ParsingPreprocessorDirective) {
2647 // End of preprocessor directive line, let LexTokenInternal handle this.
2648 BufferPtr = CurPtr;
2649 return false;
2650 }
2651
2652 // OK, but handle newline.
2653 if (*CurPtr == '\n')
2654 setLastNewLine(CurPtr);
2655 SawNewline = true;
2656 Char = *++CurPtr;
2657 }
2658
2659 // If the client wants us to return whitespace, return it now.
2660 if (isKeepWhitespaceMode()) {
2661 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2662 if (SawNewline) {
2663 IsAtStartOfLine = true;
2664 IsAtPhysicalStartOfLine = true;
2665 }
2666 // FIXME: The next token will not have LeadingSpace set.
2667 return true;
2668 }
2669
2670 // If this isn't immediately after a newline, there is leading space.
2671 char PrevChar = CurPtr[-1];
2672 bool HasLeadingSpace = !isVerticalWhitespace(c: PrevChar);
2673
2674 Result.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace);
2675 if (SawNewline) {
2676 Result.setFlag(Token::StartOfLine);
2677 Result.setFlag(Token::PhysicalStartOfLine);
2678
2679 if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) {
2680 if (auto *Handler = PP->getEmptylineHandler())
2681 Handler->HandleEmptyline(Range: SourceRange(getSourceLocation(Loc: NewLinePtr + 1),
2682 getSourceLocation(Loc: lastNewLine)));
2683 }
2684 }
2685
2686 BufferPtr = CurPtr;
2687 return false;
2688}
2689
2690/// We have just read the // characters from input. Skip until we find the
2691/// newline character that terminates the comment. Then update BufferPtr and
2692/// return.
2693///
2694/// If we're in KeepCommentMode or any CommentHandler has inserted
2695/// some tokens, this will store the first token and return true.
2696bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) {
2697 // If Line comments aren't explicitly enabled for this language, emit an
2698 // extension warning.
2699 if (!LineComment) {
2700 if (!isLexingRawMode()) // There's no PP in raw mode, so can't emit diags.
2701 Diag(Loc: BufferPtr, DiagID: diag::ext_line_comment);
2702
2703 // Mark them enabled so we only emit one warning for this translation
2704 // unit.
2705 LineComment = true;
2706 }
2707
2708 // Scan over the body of the comment. The common case, when scanning, is that
2709 // the comment contains normal ascii characters with nothing interesting in
2710 // them. As such, optimize for this case with the inner loop.
2711 //
2712 // This loop terminates with CurPtr pointing at the newline (or end of buffer)
2713 // character that ends the line comment.
2714
2715 // C++23 [lex.phases] p1
2716 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
2717 // diagnostic only once per entire ill-formed subsequence to avoid
2718 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
2719 bool UnicodeDecodingAlreadyDiagnosed = false;
2720
2721 char C;
2722 while (true) {
2723 C = *CurPtr;
2724 // Skip over characters in the fast loop.
2725 while (isASCII(c: C) && C != 0 && // Potentially EOF.
2726 C != '\n' && C != '\r') { // Newline or DOS-style newline.
2727 C = *++CurPtr;
2728 UnicodeDecodingAlreadyDiagnosed = false;
2729 }
2730
2731 if (!isASCII(c: C)) {
2732 unsigned Length = llvm::getUTF8SequenceSize(
2733 source: (const llvm::UTF8 *)CurPtr, sourceEnd: (const llvm::UTF8 *)BufferEnd);
2734 if (Length == 0) {
2735 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
2736 Diag(Loc: CurPtr, DiagID: diag::warn_invalid_utf8_in_comment);
2737 UnicodeDecodingAlreadyDiagnosed = true;
2738 ++CurPtr;
2739 } else {
2740 UnicodeDecodingAlreadyDiagnosed = false;
2741 CurPtr += Length;
2742 }
2743 continue;
2744 }
2745
2746 const char *NextLine = CurPtr;
2747 if (C != 0) {
2748 // We found a newline, see if it's escaped.
2749 const char *EscapePtr = CurPtr-1;
2750 bool HasSpace = false;
2751 while (isHorizontalWhitespace(c: *EscapePtr)) { // Skip whitespace.
2752 --EscapePtr;
2753 HasSpace = true;
2754 }
2755
2756 if (*EscapePtr == '\\')
2757 // Escaped newline.
2758 CurPtr = EscapePtr;
2759 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
2760 EscapePtr[-2] == '?' && LangOpts.Trigraphs)
2761 // Trigraph-escaped newline.
2762 CurPtr = EscapePtr-2;
2763 else
2764 break; // This is a newline, we're done.
2765
2766 // If there was space between the backslash and newline, warn about it.
2767 if (HasSpace && !isLexingRawMode())
2768 Diag(Loc: EscapePtr, DiagID: diag::backslash_newline_space);
2769 }
2770
2771 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
2772 // properly decode the character. Read it in raw mode to avoid emitting
2773 // diagnostics about things like trigraphs. If we see an escaped newline,
2774 // we'll handle it below.
2775 const char *OldPtr = CurPtr;
2776 bool OldRawMode = isLexingRawMode();
2777 LexingRawMode = true;
2778 C = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
2779 LexingRawMode = OldRawMode;
2780
2781 // If we only read only one character, then no special handling is needed.
2782 // We're done and can skip forward to the newline.
2783 if (C != 0 && CurPtr == OldPtr+1) {
2784 CurPtr = NextLine;
2785 break;
2786 }
2787
2788 // If we read multiple characters, and one of those characters was a \r or
2789 // \n, then we had an escaped newline within the comment. Emit diagnostic
2790 // unless the next line is also a // comment.
2791 if (CurPtr != OldPtr + 1 && C != '/' &&
2792 (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) {
2793 for (; OldPtr != CurPtr; ++OldPtr)
2794 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
2795 // Okay, we found a // comment that ends in a newline, if the next
2796 // line is also a // comment, but has spaces, don't emit a diagnostic.
2797 if (isWhitespace(c: C)) {
2798 const char *ForwardPtr = CurPtr;
2799 while (isWhitespace(c: *ForwardPtr)) // Skip whitespace.
2800 ++ForwardPtr;
2801 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
2802 break;
2803 }
2804
2805 if (!isLexingRawMode())
2806 Diag(Loc: OldPtr-1, DiagID: diag::ext_multi_line_line_comment);
2807 break;
2808 }
2809 }
2810
2811 if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
2812 --CurPtr;
2813 break;
2814 }
2815
2816 if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
2817 PP->CodeCompleteNaturalLanguage();
2818 cutOffLexing();
2819 return false;
2820 }
2821 }
2822
2823 // Found but did not consume the newline. Notify comment handlers about the
2824 // comment unless we're in a #if 0 block.
2825 if (PP && !isLexingRawMode() &&
2826 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
2827 getSourceLocation(Loc: CurPtr)))) {
2828 BufferPtr = CurPtr;
2829 return true; // A token has to be returned.
2830 }
2831
2832 // If we are returning comments as tokens, return this comment as a token.
2833 if (inKeepCommentMode())
2834 return SaveLineComment(Result, CurPtr);
2835
2836 // If we are inside a preprocessor directive and we see the end of line,
2837 // return immediately, so that the lexer can return this as an EOD token.
2838 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
2839 BufferPtr = CurPtr;
2840 return false;
2841 }
2842
2843 // Otherwise, eat the \n character. We don't care if this is a \n\r or
2844 // \r\n sequence. This is an efficiency hack (because we know the \n can't
2845 // contribute to another token), it isn't needed for correctness. Note that
2846 // this is ok even in KeepWhitespaceMode, because we would have returned the
2847 // comment above in that mode.
2848 NewLinePtr = CurPtr++;
2849
2850 // The next returned token is at the start of the line.
2851 Result.setFlag(Token::StartOfLine);
2852 Result.setFlag(Token::PhysicalStartOfLine);
2853 // No leading whitespace seen so far.
2854 Result.clearFlag(Flag: Token::LeadingSpace);
2855 BufferPtr = CurPtr;
2856 return false;
2857}
2858
2859/// If in save-comment mode, package up this Line comment in an appropriate
2860/// way and return it.
2861bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2862 // If we're not in a preprocessor directive, just return the // comment
2863 // directly.
2864 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
2865
2866 if (!ParsingPreprocessorDirective || LexingRawMode)
2867 return true;
2868
2869 // If this Line-style comment is in a macro definition, transmogrify it into
2870 // a C-style block comment.
2871 bool Invalid = false;
2872 std::string Spelling = PP->getSpelling(Tok: Result, Invalid: &Invalid);
2873 if (Invalid)
2874 return true;
2875
2876 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
2877 Spelling[1] = '*'; // Change prefix to "/*".
2878 Spelling += "*/"; // add suffix.
2879
2880 Result.setKind(tok::comment);
2881 PP->CreateString(Str: Spelling, Tok&: Result,
2882 ExpansionLocStart: Result.getLocation(), ExpansionLocEnd: Result.getLocation());
2883 return true;
2884}
2885
2886/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2887/// character (either \\n or \\r) is part of an escaped newline sequence. Issue
2888/// a diagnostic if so. We know that the newline is inside of a block comment.
2889static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, Lexer *L,
2890 bool Trigraphs) {
2891 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
2892
2893 // Position of the first trigraph in the ending sequence.
2894 const char *TrigraphPos = nullptr;
2895 // Position of the first whitespace after a '\' in the ending sequence.
2896 const char *SpacePos = nullptr;
2897
2898 while (true) {
2899 // Back up off the newline.
2900 --CurPtr;
2901
2902 // If this is a two-character newline sequence, skip the other character.
2903 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
2904 // \n\n or \r\r -> not escaped newline.
2905 if (CurPtr[0] == CurPtr[1])
2906 return false;
2907 // \n\r or \r\n -> skip the newline.
2908 --CurPtr;
2909 }
2910
2911 // If we have horizontal whitespace, skip over it. We allow whitespace
2912 // between the slash and newline.
2913 while (isHorizontalWhitespace(c: *CurPtr) || *CurPtr == 0) {
2914 SpacePos = CurPtr;
2915 --CurPtr;
2916 }
2917
2918 // If we have a slash, this is an escaped newline.
2919 if (*CurPtr == '\\') {
2920 --CurPtr;
2921 } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') {
2922 // This is a trigraph encoding of a slash.
2923 TrigraphPos = CurPtr - 2;
2924 CurPtr -= 3;
2925 } else {
2926 return false;
2927 }
2928
2929 // If the character preceding the escaped newline is a '*', then after line
2930 // splicing we have a '*/' ending the comment.
2931 if (*CurPtr == '*')
2932 break;
2933
2934 if (*CurPtr != '\n' && *CurPtr != '\r')
2935 return false;
2936 }
2937
2938 if (TrigraphPos) {
2939 // If no trigraphs are enabled, warn that we ignored this trigraph and
2940 // ignore this * character.
2941 if (!Trigraphs) {
2942 if (!L->isLexingRawMode())
2943 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ignored_block_comment);
2944 return false;
2945 }
2946 if (!L->isLexingRawMode())
2947 L->Diag(Loc: TrigraphPos, DiagID: diag::trigraph_ends_block_comment);
2948 }
2949
2950 // Warn about having an escaped newline between the */ characters.
2951 if (!L->isLexingRawMode())
2952 L->Diag(Loc: CurPtr + 1, DiagID: diag::escaped_newline_block_comment_end);
2953
2954 // If there was space between the backslash and newline, warn about it.
2955 if (SpacePos && !L->isLexingRawMode())
2956 L->Diag(Loc: SpacePos, DiagID: diag::backslash_newline_space);
2957
2958 return true;
2959}
2960
2961#ifdef __SSE2__
2962#include <emmintrin.h>
2963#elif __ALTIVEC__
2964#include <altivec.h>
2965#undef bool
2966#endif
2967
2968/// We have just read from input the / and * characters that started a comment.
2969/// Read until we find the * and / characters that terminate the comment.
2970/// Note that we don't bother decoding trigraphs or escaped newlines in block
2971/// comments, because they cannot cause the comment to end. The only thing
2972/// that can happen is the comment could end with an escaped newline between
2973/// the terminating * and /.
2974///
2975/// If we're in KeepCommentMode or any CommentHandler has inserted
2976/// some tokens, this will store the first token and return true.
2977bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
2978 // Scan one character past where we should, looking for a '/' character. Once
2979 // we find it, check to see if it was preceded by a *. This common
2980 // optimization helps people who like to put a lot of * characters in their
2981 // comments.
2982
2983 // The first character we get with newlines and trigraphs skipped to handle
2984 // the degenerate /*/ case below correctly if the * has an escaped newline
2985 // after it.
2986 unsigned CharSize;
2987 unsigned char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
2988 CurPtr += CharSize;
2989 if (C == 0 && CurPtr == BufferEnd+1) {
2990 if (!isLexingRawMode())
2991 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
2992 --CurPtr;
2993
2994 // KeepWhitespaceMode should return this broken comment as a token. Since
2995 // it isn't a well formed comment, just return it as an 'unknown' token.
2996 if (isKeepWhitespaceMode()) {
2997 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
2998 return true;
2999 }
3000
3001 BufferPtr = CurPtr;
3002 return false;
3003 }
3004
3005 // Check to see if the first character after the '/*' is another /. If so,
3006 // then this slash does not end the block comment, it is part of it.
3007 if (C == '/')
3008 C = *CurPtr++;
3009
3010 // C++23 [lex.phases] p1
3011 // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
3012 // diagnostic only once per entire ill-formed subsequence to avoid
3013 // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
3014 bool UnicodeDecodingAlreadyDiagnosed = false;
3015
3016 while (true) {
3017 // Skip over all non-interesting characters until we find end of buffer or a
3018 // (probably ending) '/' character.
3019 if (CurPtr + 24 < BufferEnd &&
3020 // If there is a code-completion point avoid the fast scan because it
3021 // doesn't check for '\0'.
3022 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
3023 // While not aligned to a 16-byte boundary.
3024 while (C != '/' && (intptr_t)CurPtr % 16 != 0) {
3025 if (!isASCII(c: C))
3026 goto MultiByteUTF8;
3027 C = *CurPtr++;
3028 }
3029 if (C == '/') goto FoundSlash;
3030
3031#ifdef __SSE2__
3032 __m128i Slashes = _mm_set1_epi8(b: '/');
3033 while (CurPtr + 16 < BufferEnd) {
3034 int Mask = _mm_movemask_epi8(a: *(const __m128i *)CurPtr);
3035 if (LLVM_UNLIKELY(Mask != 0)) {
3036 goto MultiByteUTF8;
3037 }
3038 // look for slashes
3039 int cmp = _mm_movemask_epi8(a: _mm_cmpeq_epi8(a: *(const __m128i*)CurPtr,
3040 b: Slashes));
3041 if (cmp != 0) {
3042 // Adjust the pointer to point directly after the first slash. It's
3043 // not necessary to set C here, it will be overwritten at the end of
3044 // the outer loop.
3045 CurPtr += llvm::countr_zero<unsigned>(Val: cmp) + 1;
3046 goto FoundSlash;
3047 }
3048 CurPtr += 16;
3049 }
3050#elif __ALTIVEC__
3051 __vector unsigned char LongUTF = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
3052 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
3053 0x80, 0x80, 0x80, 0x80};
3054 __vector unsigned char Slashes = {
3055 '/', '/', '/', '/', '/', '/', '/', '/',
3056 '/', '/', '/', '/', '/', '/', '/', '/'
3057 };
3058 while (CurPtr + 16 < BufferEnd) {
3059 if (LLVM_UNLIKELY(
3060 vec_any_ge(*(const __vector unsigned char *)CurPtr, LongUTF)))
3061 goto MultiByteUTF8;
3062 if (vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) {
3063 break;
3064 }
3065 CurPtr += 16;
3066 }
3067
3068#else
3069 while (CurPtr + 16 < BufferEnd) {
3070 bool HasNonASCII = false;
3071 for (unsigned I = 0; I < 16; ++I)
3072 HasNonASCII |= !isASCII(CurPtr[I]);
3073
3074 if (LLVM_UNLIKELY(HasNonASCII))
3075 goto MultiByteUTF8;
3076
3077 bool HasSlash = false;
3078 for (unsigned I = 0; I < 16; ++I)
3079 HasSlash |= CurPtr[I] == '/';
3080 if (HasSlash)
3081 break;
3082 CurPtr += 16;
3083 }
3084#endif
3085
3086 // It has to be one of the bytes scanned, increment to it and read one.
3087 C = *CurPtr++;
3088 }
3089
3090 // Loop to scan the remainder, warning on invalid UTF-8
3091 // if the corresponding warning is enabled, emitting a diagnostic only once
3092 // per sequence that cannot be decoded.
3093 while (C != '/' && C != '\0') {
3094 if (isASCII(c: C)) {
3095 UnicodeDecodingAlreadyDiagnosed = false;
3096 C = *CurPtr++;
3097 continue;
3098 }
3099 MultiByteUTF8:
3100 // CurPtr is 1 code unit past C, so to decode
3101 // the codepoint, we need to read from the previous position.
3102 unsigned Length = llvm::getUTF8SequenceSize(
3103 source: (const llvm::UTF8 *)CurPtr - 1, sourceEnd: (const llvm::UTF8 *)BufferEnd);
3104 if (Length == 0) {
3105 if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
3106 Diag(Loc: CurPtr - 1, DiagID: diag::warn_invalid_utf8_in_comment);
3107 UnicodeDecodingAlreadyDiagnosed = true;
3108 } else {
3109 UnicodeDecodingAlreadyDiagnosed = false;
3110 CurPtr += Length - 1;
3111 }
3112 C = *CurPtr++;
3113 }
3114
3115 if (C == '/') {
3116 FoundSlash:
3117 if (CurPtr[-2] == '*') // We found the final */. We're done!
3118 break;
3119
3120 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
3121 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr: CurPtr - 2, L: this,
3122 Trigraphs: LangOpts.Trigraphs)) {
3123 // We found the final */, though it had an escaped newline between the
3124 // * and /. We're done!
3125 break;
3126 }
3127 }
3128 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
3129 // If this is a /* inside of the comment, emit a warning. Don't do this
3130 // if this is a /*/, which will end the comment. This misses cases with
3131 // embedded escaped newlines, but oh well.
3132 if (!isLexingRawMode())
3133 Diag(Loc: CurPtr-1, DiagID: diag::warn_nested_block_comment);
3134 }
3135 } else if (C == 0 && CurPtr == BufferEnd+1) {
3136 if (!isLexingRawMode())
3137 Diag(Loc: BufferPtr, DiagID: diag::err_unterminated_block_comment);
3138 // Note: the user probably forgot a */. We could continue immediately
3139 // after the /*, but this would involve lexing a lot of what really is the
3140 // comment, which surely would confuse the parser.
3141 --CurPtr;
3142
3143 // KeepWhitespaceMode should return this broken comment as a token. Since
3144 // it isn't a well formed comment, just return it as an 'unknown' token.
3145 if (isKeepWhitespaceMode()) {
3146 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3147 return true;
3148 }
3149
3150 BufferPtr = CurPtr;
3151 return false;
3152 } else if (C == '\0' && isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3153 PP->CodeCompleteNaturalLanguage();
3154 cutOffLexing();
3155 return false;
3156 }
3157
3158 C = *CurPtr++;
3159 }
3160
3161 // Notify comment handlers about the comment unless we're in a #if 0 block.
3162 if (PP && !isLexingRawMode() &&
3163 PP->HandleComment(result&: Result, Comment: SourceRange(getSourceLocation(Loc: BufferPtr),
3164 getSourceLocation(Loc: CurPtr)))) {
3165 BufferPtr = CurPtr;
3166 return true; // A token has to be returned.
3167 }
3168
3169 // If we are returning comments as tokens, return this comment as a token.
3170 if (inKeepCommentMode()) {
3171 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::comment);
3172 IsAtPhysicalStartOfLine = Result.isAtPhysicalStartOfLine();
3173 return true;
3174 }
3175
3176 // It is common for the tokens immediately after a /**/ comment to be
3177 // whitespace. Instead of going through the big switch, handle it
3178 // efficiently now. This is safe even in KeepWhitespaceMode because we would
3179 // have already returned above with the comment as a token.
3180 if (isHorizontalWhitespace(c: *CurPtr)) {
3181 SkipWhitespace(Result, CurPtr: CurPtr + 1);
3182 return false;
3183 }
3184
3185 // Otherwise, just return so that the next character will be lexed as a token.
3186 BufferPtr = CurPtr;
3187 Result.setFlag(Token::LeadingSpace);
3188 return false;
3189}
3190
3191//===----------------------------------------------------------------------===//
3192// Primary Lexing Entry Points
3193//===----------------------------------------------------------------------===//
3194
3195/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
3196/// uninterpreted string. This switches the lexer out of directive mode.
3197void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
3198 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
3199 "Must be in a preprocessing directive!");
3200 Token Tmp;
3201 Tmp.startToken();
3202
3203 // CurPtr - Cache BufferPtr in an automatic variable.
3204 const char *CurPtr = BufferPtr;
3205 while (true) {
3206 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Tmp);
3207 switch (Char) {
3208 default:
3209 if (Result)
3210 Result->push_back(Elt: Char);
3211 break;
3212 case 0: // Null.
3213 // Found end of file?
3214 if (CurPtr-1 != BufferEnd) {
3215 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3216 PP->CodeCompleteNaturalLanguage();
3217 cutOffLexing();
3218 return;
3219 }
3220
3221 // Nope, normal character, continue.
3222 if (Result)
3223 Result->push_back(Elt: Char);
3224 break;
3225 }
3226 // FALL THROUGH.
3227 [[fallthrough]];
3228 case '\r':
3229 case '\n':
3230 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
3231 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
3232 BufferPtr = CurPtr-1;
3233
3234 // Next, lex the character, which should handle the EOD transition.
3235 Lex(Result&: Tmp);
3236 if (Tmp.is(K: tok::code_completion)) {
3237 if (PP)
3238 PP->CodeCompleteNaturalLanguage();
3239 Lex(Result&: Tmp);
3240 }
3241 assert(Tmp.is(tok::eod) && "Unexpected token!");
3242
3243 // Finally, we're done;
3244 return;
3245 }
3246 }
3247}
3248
3249/// LexEndOfFile - CurPtr points to the end of this file. Handle this
3250/// condition, reporting diagnostics and handling other edge cases as required.
3251/// This returns true if Result contains a token, false if PP.Lex should be
3252/// called again.
3253bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
3254 // If we hit the end of the file while parsing a preprocessor directive,
3255 // end the preprocessor directive first. The next token returned will
3256 // then be the end of file.
3257 if (ParsingPreprocessorDirective) {
3258 // Done parsing the "line".
3259 ParsingPreprocessorDirective = false;
3260 // Update the location of token as well as BufferPtr.
3261 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::eod);
3262
3263 // Restore comment saving mode, in case it was disabled for directive.
3264 if (PP)
3265 resetExtendedTokenMode();
3266 return true; // Have a token.
3267 }
3268
3269 // If we are in raw mode, return this event as an EOF token. Let the caller
3270 // that put us in raw mode handle the event.
3271 if (isLexingRawMode()) {
3272 Result.startToken();
3273 BufferPtr = BufferEnd;
3274 FormTokenWithChars(Result, TokEnd: BufferEnd, Kind: tok::eof);
3275 return true;
3276 }
3277
3278 if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) {
3279 PP->setRecordedPreambleConditionalStack(ConditionalStack);
3280 // If the preamble cuts off the end of a header guard, consider it guarded.
3281 // The guard is valid for the preamble content itself, and for tools the
3282 // most useful answer is "yes, this file has a header guard".
3283 if (!ConditionalStack.empty())
3284 MIOpt.ExitTopLevelConditional();
3285 ConditionalStack.clear();
3286 }
3287
3288 // Issue diagnostics for unterminated #if and missing newline.
3289
3290 // If we are in a #if directive, emit an error.
3291 while (!ConditionalStack.empty()) {
3292 if (PP->getCodeCompletionFileLoc() != FileLoc)
3293 PP->Diag(Loc: ConditionalStack.back().IfLoc,
3294 DiagID: diag::err_pp_unterminated_conditional);
3295 ConditionalStack.pop_back();
3296 }
3297
3298 // Before C++11 and C2y, a file not ending with a newline was UB. Both
3299 // standards changed this behavior (as a DR or equivalent), but we still have
3300 // an opt-in diagnostic to warn about it.
3301 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
3302 Diag(Loc: BufferEnd, DiagID: diag::warn_no_newline_eof)
3303 << FixItHint::CreateInsertion(InsertionLoc: getSourceLocation(Loc: BufferEnd), Code: "\n");
3304
3305 BufferPtr = CurPtr;
3306
3307 // Finally, let the preprocessor handle this.
3308 return PP->HandleEndOfFile(Result, isEndOfMacro: isPragmaLexer());
3309}
3310
3311/// peekNextPPToken - Return std::nullopt if there are no more tokens in the
3312/// buffer controlled by this lexer, otherwise return the next unexpanded
3313/// token.
3314std::optional<Token> Lexer::peekNextPPToken() {
3315 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
3316
3317 if (isDependencyDirectivesLexer()) {
3318 if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size())
3319 return std::nullopt;
3320 Token Result;
3321 (void)convertDependencyDirectiveToken(
3322 DDTok: DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result);
3323 return Result;
3324 }
3325
3326 // Switch to 'skipping' mode. This will ensure that we can lex a token
3327 // without emitting diagnostics, disables macro expansion, and will cause EOF
3328 // to return an EOF token instead of popping the include stack.
3329 LexingRawMode = true;
3330
3331 // Save state that can be changed while lexing so that we can restore it.
3332 const char *TmpBufferPtr = BufferPtr;
3333 bool inPPDirectiveMode = ParsingPreprocessorDirective;
3334 bool atStartOfLine = IsAtStartOfLine;
3335 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3336 bool leadingSpace = HasLeadingSpace;
3337 MultipleIncludeOpt MIOptState = MIOpt;
3338
3339 Token Tok;
3340 Lex(Result&: Tok);
3341
3342 // Restore state that may have changed.
3343 BufferPtr = TmpBufferPtr;
3344 ParsingPreprocessorDirective = inPPDirectiveMode;
3345 HasLeadingSpace = leadingSpace;
3346 IsAtStartOfLine = atStartOfLine;
3347 IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
3348 MIOpt = MIOptState;
3349 // Restore the lexer back to non-skipping mode.
3350 LexingRawMode = false;
3351
3352 if (Tok.is(K: tok::eof))
3353 return std::nullopt;
3354 return Tok;
3355}
3356
3357/// Find the end of a version control conflict marker.
3358static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
3359 ConflictMarkerKind CMK) {
3360 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
3361 size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
3362 auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(Start: TermLen);
3363 size_t Pos = RestOfBuffer.find(Str: Terminator);
3364 while (Pos != StringRef::npos) {
3365 // Must occur at start of line.
3366 if (Pos == 0 ||
3367 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) {
3368 RestOfBuffer = RestOfBuffer.substr(Start: Pos+TermLen);
3369 Pos = RestOfBuffer.find(Str: Terminator);
3370 continue;
3371 }
3372 return RestOfBuffer.data()+Pos;
3373 }
3374 return nullptr;
3375}
3376
3377/// IsStartOfConflictMarker - If the specified pointer is the start of a version
3378/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
3379/// and recover nicely. This returns true if it is a conflict marker and false
3380/// if not.
3381bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
3382 // Only a conflict marker if it starts at the beginning of a line.
3383 if (CurPtr != BufferStart &&
3384 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3385 return false;
3386
3387 // Check to see if we have <<<<<<< or >>>>.
3388 if (!StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: "<<<<<<<") &&
3389 !StringRef(CurPtr, BufferEnd - CurPtr).starts_with(Prefix: ">>>> "))
3390 return false;
3391
3392 // If we have a situation where we don't care about conflict markers, ignore
3393 // it.
3394 if (CurrentConflictMarkerState || isLexingRawMode())
3395 return false;
3396
3397 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
3398
3399 // Check to see if there is an ending marker somewhere in the buffer at the
3400 // start of a line to terminate this conflict marker.
3401 if (FindConflictEnd(CurPtr, BufferEnd, CMK: Kind)) {
3402 // We found a match. We are really in a conflict marker.
3403 // Diagnose this, and ignore to the end of line.
3404 Diag(Loc: CurPtr, DiagID: diag::err_conflict_marker);
3405 CurrentConflictMarkerState = Kind;
3406
3407 // Skip ahead to the end of line. We know this exists because the
3408 // end-of-conflict marker starts with \r or \n.
3409 while (*CurPtr != '\r' && *CurPtr != '\n') {
3410 assert(CurPtr != BufferEnd && "Didn't find end of line");
3411 ++CurPtr;
3412 }
3413 BufferPtr = CurPtr;
3414 return true;
3415 }
3416
3417 // No end of conflict marker found.
3418 return false;
3419}
3420
3421/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
3422/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
3423/// is the end of a conflict marker. Handle it by ignoring up until the end of
3424/// the line. This returns true if it is a conflict marker and false if not.
3425bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
3426 // Only a conflict marker if it starts at the beginning of a line.
3427 if (CurPtr != BufferStart &&
3428 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
3429 return false;
3430
3431 // If we have a situation where we don't care about conflict markers, ignore
3432 // it.
3433 if (!CurrentConflictMarkerState || isLexingRawMode())
3434 return false;
3435
3436 // Check to see if we have the marker (4 characters in a row).
3437 for (unsigned i = 1; i != 4; ++i)
3438 if (CurPtr[i] != CurPtr[0])
3439 return false;
3440
3441 // If we do have it, search for the end of the conflict marker. This could
3442 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
3443 // be the end of conflict marker.
3444 if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
3445 CMK: CurrentConflictMarkerState)) {
3446 CurPtr = End;
3447
3448 // Skip ahead to the end of line.
3449 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
3450 ++CurPtr;
3451
3452 BufferPtr = CurPtr;
3453
3454 // No longer in the conflict marker.
3455 CurrentConflictMarkerState = CMK_None;
3456 return true;
3457 }
3458
3459 return false;
3460}
3461
3462static const char *findPlaceholderEnd(const char *CurPtr,
3463 const char *BufferEnd) {
3464 if (CurPtr == BufferEnd)
3465 return nullptr;
3466 BufferEnd -= 1; // Scan until the second last character.
3467 for (; CurPtr != BufferEnd; ++CurPtr) {
3468 if (CurPtr[0] == '#' && CurPtr[1] == '>')
3469 return CurPtr + 2;
3470 }
3471 return nullptr;
3472}
3473
3474bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
3475 assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!");
3476 if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
3477 return false;
3478 const char *End = findPlaceholderEnd(CurPtr: CurPtr + 1, BufferEnd);
3479 if (!End)
3480 return false;
3481 const char *Start = CurPtr - 1;
3482 if (!LangOpts.AllowEditorPlaceholders)
3483 Diag(Loc: Start, DiagID: diag::err_placeholder_in_source);
3484 Result.startToken();
3485 FormTokenWithChars(Result, TokEnd: End, Kind: tok::raw_identifier);
3486 Result.setRawIdentifierData(Start);
3487 PP->LookUpIdentifierInfo(Identifier&: Result);
3488 Result.setFlag(Token::IsEditorPlaceholder);
3489 BufferPtr = End;
3490 return true;
3491}
3492
3493bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
3494 if (PP && PP->isCodeCompletionEnabled()) {
3495 SourceLocation Loc = FileLoc.getLocWithOffset(Offset: CurPtr-BufferStart);
3496 return Loc == PP->getCodeCompletionLoc();
3497 }
3498
3499 return false;
3500}
3501
3502void Lexer::DiagnoseDelimitedOrNamedEscapeSequence(SourceLocation Loc,
3503 bool Named,
3504 const LangOptions &Opts,
3505 DiagnosticsEngine &Diags) {
3506 unsigned DiagId;
3507 if (Opts.CPlusPlus23)
3508 DiagId = diag::warn_cxx23_delimited_escape_sequence;
3509 else if (Opts.C2y && !Named)
3510 DiagId = diag::warn_c2y_delimited_escape_sequence;
3511 else
3512 DiagId = diag::ext_delimited_escape_sequence;
3513
3514 // The trailing arguments are only used by the extension warning; either this
3515 // is a C2y extension or a C++23 extension, unless it's a named escape
3516 // sequence in C, then it's a Clang extension.
3517 unsigned Ext;
3518 if (!Opts.CPlusPlus)
3519 Ext = Named ? 2 /* Clang extension */ : 1 /* C2y extension */;
3520 else
3521 Ext = 0; // C++23 extension
3522
3523 Diags.Report(Loc, DiagID: DiagId) << Named << Ext;
3524}
3525
3526std::optional<uint32_t> Lexer::tryReadNumericUCN(const char *&StartPtr,
3527 const char *SlashLoc,
3528 Token *Result) {
3529 unsigned CharSize;
3530 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3531 assert((Kind == 'u' || Kind == 'U') && "expected a UCN");
3532
3533 unsigned NumHexDigits;
3534 if (Kind == 'u')
3535 NumHexDigits = 4;
3536 else if (Kind == 'U')
3537 NumHexDigits = 8;
3538
3539 bool Delimited = false;
3540 bool FoundEndDelimiter = false;
3541 unsigned Count = 0;
3542 bool Diagnose = Result && !isLexingRawMode();
3543
3544 if (!LangOpts.CPlusPlus && !LangOpts.C99) {
3545 if (Diagnose)
3546 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_not_valid_in_c89);
3547 return std::nullopt;
3548 }
3549
3550 const char *CurPtr = StartPtr + CharSize;
3551 const char *KindLoc = &CurPtr[-1];
3552
3553 uint32_t CodePoint = 0;
3554 while (Count != NumHexDigits || Delimited) {
3555 char C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3556 if (!Delimited && Count == 0 && C == '{') {
3557 Delimited = true;
3558 CurPtr += CharSize;
3559 continue;
3560 }
3561
3562 if (Delimited && C == '}') {
3563 CurPtr += CharSize;
3564 FoundEndDelimiter = true;
3565 break;
3566 }
3567
3568 unsigned Value = llvm::hexDigitValue(C);
3569 if (Value == std::numeric_limits<unsigned>::max()) {
3570 if (!Delimited)
3571 break;
3572 if (Diagnose)
3573 Diag(Loc: SlashLoc, DiagID: diag::warn_delimited_ucn_incomplete)
3574 << StringRef(KindLoc, 1);
3575 return std::nullopt;
3576 }
3577
3578 if (CodePoint & 0xF000'0000) {
3579 if (Diagnose)
3580 Diag(Loc: KindLoc, DiagID: diag::err_escape_too_large) << 0;
3581 return std::nullopt;
3582 }
3583
3584 CodePoint <<= 4;
3585 CodePoint |= Value;
3586 CurPtr += CharSize;
3587 Count++;
3588 }
3589
3590 if (Count == 0) {
3591 if (Diagnose)
3592 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3593 : diag::warn_ucn_escape_no_digits)
3594 << StringRef(KindLoc, 1);
3595 return std::nullopt;
3596 }
3597
3598 if (Delimited && Kind == 'U') {
3599 if (Diagnose)
3600 Diag(Loc: SlashLoc, DiagID: diag::err_hex_escape_no_digits) << StringRef(KindLoc, 1);
3601 return std::nullopt;
3602 }
3603
3604 if (!Delimited && Count != NumHexDigits) {
3605 if (Diagnose) {
3606 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3607 // If the user wrote \U1234, suggest a fixit to \u.
3608 if (Count == 4 && NumHexDigits == 8) {
3609 CharSourceRange URange = makeCharRange(L&: *this, Begin: KindLoc, End: KindLoc + 1);
3610 Diag(Loc: KindLoc, DiagID: diag::note_ucn_four_not_eight)
3611 << FixItHint::CreateReplacement(RemoveRange: URange, Code: "u");
3612 }
3613 }
3614 return std::nullopt;
3615 }
3616
3617 if (Delimited && PP)
3618 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: false,
3619 Opts: PP->getLangOpts(),
3620 Diags&: PP->getDiagnostics());
3621
3622 if (Result) {
3623 Result->setFlag(Token::HasUCN);
3624 // If the UCN contains either a trigraph or a line splicing,
3625 // we need to call getAndAdvanceChar again to set the appropriate flags
3626 // on Result.
3627 if (CurPtr - StartPtr == (ptrdiff_t)(Count + 1 + (Delimited ? 2 : 0)))
3628 StartPtr = CurPtr;
3629 else
3630 while (StartPtr != CurPtr)
3631 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3632 } else {
3633 StartPtr = CurPtr;
3634 }
3635 return CodePoint;
3636}
3637
3638std::optional<uint32_t> Lexer::tryReadNamedUCN(const char *&StartPtr,
3639 const char *SlashLoc,
3640 Token *Result) {
3641 unsigned CharSize;
3642 bool Diagnose = Result && !isLexingRawMode();
3643
3644 char C = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3645 assert(C == 'N' && "expected \\N{...}");
3646
3647 const char *CurPtr = StartPtr + CharSize;
3648 const char *KindLoc = &CurPtr[-1];
3649
3650 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3651 if (C != '{') {
3652 if (Diagnose)
3653 Diag(Loc: SlashLoc, DiagID: diag::warn_ucn_escape_incomplete);
3654 return std::nullopt;
3655 }
3656 CurPtr += CharSize;
3657 const char *StartName = CurPtr;
3658 bool FoundEndDelimiter = false;
3659 llvm::SmallVector<char, 30> Buffer;
3660 while (C) {
3661 C = getCharAndSize(Ptr: CurPtr, Size&: CharSize);
3662 CurPtr += CharSize;
3663 if (C == '}') {
3664 FoundEndDelimiter = true;
3665 break;
3666 }
3667
3668 if (isVerticalWhitespace(c: C))
3669 break;
3670 Buffer.push_back(Elt: C);
3671 }
3672
3673 if (!FoundEndDelimiter || Buffer.empty()) {
3674 if (Diagnose)
3675 Diag(Loc: SlashLoc, DiagID: FoundEndDelimiter ? diag::warn_delimited_ucn_empty
3676 : diag::warn_delimited_ucn_incomplete)
3677 << StringRef(KindLoc, 1);
3678 return std::nullopt;
3679 }
3680
3681 StringRef Name(Buffer.data(), Buffer.size());
3682 std::optional<char32_t> Match =
3683 llvm::sys::unicode::nameToCodepointStrict(Name);
3684 std::optional<llvm::sys::unicode::LooseMatchingResult> LooseMatch;
3685 if (!Match) {
3686 LooseMatch = llvm::sys::unicode::nameToCodepointLooseMatching(Name);
3687 if (Diagnose) {
3688 Diag(Loc: StartName, DiagID: diag::err_invalid_ucn_name)
3689 << StringRef(Buffer.data(), Buffer.size())
3690 << makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize);
3691 if (LooseMatch) {
3692 Diag(Loc: StartName, DiagID: diag::note_invalid_ucn_name_loose_matching)
3693 << FixItHint::CreateReplacement(
3694 RemoveRange: makeCharRange(L&: *this, Begin: StartName, End: CurPtr - CharSize),
3695 Code: LooseMatch->Name);
3696 }
3697 }
3698 // We do not offer misspelled character names suggestions here
3699 // as the set of what would be a valid suggestion depends on context,
3700 // and we should not make invalid suggestions.
3701 }
3702
3703 if (Diagnose && Match)
3704 DiagnoseDelimitedOrNamedEscapeSequence(Loc: getSourceLocation(Loc: SlashLoc), Named: true,
3705 Opts: PP->getLangOpts(),
3706 Diags&: PP->getDiagnostics());
3707
3708 // If no diagnostic has been emitted yet, likely because we are doing a
3709 // tentative lexing, we do not want to recover here to make sure the token
3710 // will not be incorrectly considered valid. This function will be called
3711 // again and a diagnostic emitted then.
3712 if (LooseMatch && Diagnose)
3713 Match = LooseMatch->CodePoint;
3714
3715 if (Result) {
3716 Result->setFlag(Token::HasUCN);
3717 // If the UCN contains either a trigraph or a line splicing,
3718 // we need to call getAndAdvanceChar again to set the appropriate flags
3719 // on Result.
3720 if (CurPtr - StartPtr == (ptrdiff_t)(Buffer.size() + 3))
3721 StartPtr = CurPtr;
3722 else
3723 while (StartPtr != CurPtr)
3724 (void)getAndAdvanceChar(Ptr&: StartPtr, Tok&: *Result);
3725 } else {
3726 StartPtr = CurPtr;
3727 }
3728 return Match ? std::optional<uint32_t>(*Match) : std::nullopt;
3729}
3730
3731uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3732 Token *Result) {
3733
3734 unsigned CharSize;
3735 std::optional<uint32_t> CodePointOpt;
3736 char Kind = getCharAndSize(Ptr: StartPtr, Size&: CharSize);
3737 if (Kind == 'u' || Kind == 'U')
3738 CodePointOpt = tryReadNumericUCN(StartPtr, SlashLoc, Result);
3739 else if (Kind == 'N')
3740 CodePointOpt = tryReadNamedUCN(StartPtr, SlashLoc, Result);
3741
3742 if (!CodePointOpt)
3743 return 0;
3744
3745 uint32_t CodePoint = *CodePointOpt;
3746
3747 // Don't apply C family restrictions to UCNs in assembly mode
3748 if (LangOpts.AsmPreprocessor)
3749 return CodePoint;
3750
3751 // C23 6.4.3p2: A universal character name shall not designate a code point
3752 // where the hexadecimal value is:
3753 // - in the range D800 through DFFF inclusive; or
3754 // - greater than 10FFFF.
3755 // A universal-character-name outside the c-char-sequence of a character
3756 // constant, or the s-char-sequence of a string-literal shall not designate
3757 // a control character or a character in the basic character set.
3758
3759 // C++11 [lex.charset]p2: If the hexadecimal value for a
3760 // universal-character-name corresponds to a surrogate code point (in the
3761 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
3762 // if the hexadecimal value for a universal-character-name outside the
3763 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or
3764 // string literal corresponds to a control character (in either of the
3765 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
3766 // basic source character set, the program is ill-formed.
3767 if (CodePoint < 0xA0) {
3768 // We don't use isLexingRawMode() here because we need to warn about bad
3769 // UCNs even when skipping preprocessing tokens in a #if block.
3770 if (Result && PP) {
3771 if (CodePoint < 0x20 || CodePoint >= 0x7F)
3772 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_control_character);
3773 else {
3774 char C = static_cast<char>(CodePoint);
3775 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
3776 }
3777 }
3778
3779 return 0;
3780 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) {
3781 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
3782 // We don't use isLexingRawMode() here because we need to diagnose bad
3783 // UCNs even when skipping preprocessing tokens in a #if block.
3784 if (Result && PP) {
3785 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11)
3786 Diag(Loc: BufferPtr, DiagID: diag::warn_ucn_escape_surrogate);
3787 else
3788 Diag(Loc: BufferPtr, DiagID: diag::err_ucn_escape_invalid);
3789 }
3790 return 0;
3791 }
3792
3793 return CodePoint;
3794}
3795
3796bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3797 const char *CurPtr) {
3798 if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
3799 isUnicodeWhitespace(Codepoint: C)) {
3800 Diag(Loc: BufferPtr, DiagID: diag::ext_unicode_whitespace)
3801 << EscapeSingleCodepointForDiagnostic(CP: C)
3802 << makeCharRange(L&: *this, Begin: BufferPtr, End: CurPtr);
3803
3804 Result.setFlag(Token::LeadingSpace);
3805 return true;
3806 }
3807 return false;
3808}
3809
3810void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3811 IsAtStartOfLine = Result.isAtStartOfLine();
3812 HasLeadingSpace = Result.hasLeadingSpace();
3813 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
3814 // Note that this doesn't affect IsAtPhysicalStartOfLine.
3815}
3816
3817bool Lexer::Lex(Token &Result) {
3818 assert(!isDependencyDirectivesLexer());
3819
3820 // Start a new token.
3821 Result.startToken();
3822
3823 // Set up misc whitespace flags for LexTokenInternal.
3824 if (IsAtStartOfLine) {
3825 Result.setFlag(Token::StartOfLine);
3826 IsAtStartOfLine = false;
3827 }
3828
3829 if (IsAtPhysicalStartOfLine) {
3830 Result.setFlag(Token::PhysicalStartOfLine);
3831 IsAtPhysicalStartOfLine = false;
3832 }
3833
3834 if (HasLeadingSpace) {
3835 Result.setFlag(Token::LeadingSpace);
3836 HasLeadingSpace = false;
3837 }
3838
3839 if (HasLeadingEmptyMacro) {
3840 Result.setFlag(Token::LeadingEmptyMacro);
3841 HasLeadingEmptyMacro = false;
3842 }
3843
3844 bool isRawLex = isLexingRawMode();
3845 (void) isRawLex;
3846 bool returnedToken = LexTokenInternal(Result);
3847 // (After the LexTokenInternal call, the lexer might be destroyed.)
3848 assert((returnedToken || !isRawLex) && "Raw lex must succeed");
3849 return returnedToken;
3850}
3851
3852/// LexTokenInternal - This implements a simple C family lexer. It is an
3853/// extremely performance critical piece of code. This assumes that the buffer
3854/// has a null character at the end of the file. This returns a preprocessing
3855/// token, not a normal token, as such, it is an internal interface. It assumes
3856/// that the Flags of result have been cleared before calling this.
3857bool Lexer::LexTokenInternal(Token &Result) {
3858LexStart:
3859 assert(!Result.needsCleaning() && "Result needs cleaning");
3860 assert(!Result.hasPtrData() && "Result has not been reset");
3861
3862 // CurPtr - Cache BufferPtr in an automatic variable.
3863 const char *CurPtr = BufferPtr;
3864
3865 // Small amounts of horizontal whitespace is very common between tokens.
3866 // Check for space character separately to skip the expensive
3867 // isHorizontalWhitespace() check
3868 if (*CurPtr == ' ' || isHorizontalWhitespace(c: *CurPtr)) {
3869 do {
3870 ++CurPtr;
3871 } while (*CurPtr == ' ' || isHorizontalWhitespace(c: *CurPtr));
3872
3873 // If we are keeping whitespace and other tokens, just return what we just
3874 // skipped. The next lexer invocation will return the token after the
3875 // whitespace.
3876 if (isKeepWhitespaceMode()) {
3877 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::unknown);
3878 // FIXME: The next token will not have LeadingSpace set.
3879 return true;
3880 }
3881
3882 BufferPtr = CurPtr;
3883 Result.setFlag(Token::LeadingSpace);
3884 }
3885
3886 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
3887
3888 // Read a character, advancing over it.
3889 char Char = getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3890 tok::TokenKind Kind;
3891
3892 if (!isVerticalWhitespace(c: Char))
3893 NewLinePtr = nullptr;
3894
3895 switch (Char) {
3896 case 0: // Null.
3897 // Found end of file?
3898 if (CurPtr-1 == BufferEnd)
3899 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3900
3901 // Check if we are performing code completion.
3902 if (isCodeCompletionPoint(CurPtr: CurPtr-1)) {
3903 // Return the code-completion token.
3904 Result.startToken();
3905 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::code_completion);
3906 return true;
3907 }
3908
3909 if (!isLexingRawMode())
3910 Diag(Loc: CurPtr-1, DiagID: diag::null_in_file);
3911 Result.setFlag(Token::LeadingSpace);
3912 if (SkipWhitespace(Result, CurPtr))
3913 return true; // KeepWhitespaceMode
3914
3915 // We know the lexer hasn't changed, so just try again with this lexer.
3916 // (We manually eliminate the tail call to avoid recursion.)
3917 goto LexNextToken;
3918
3919 case 26: // DOS & CP/M EOF: "^Z".
3920 // If we're in Microsoft extensions mode, treat this as end of file.
3921 if (LangOpts.MicrosoftExt) {
3922 if (!isLexingRawMode())
3923 Diag(Loc: CurPtr-1, DiagID: diag::ext_ctrl_z_eof_microsoft);
3924 return LexEndOfFile(Result, CurPtr: CurPtr-1);
3925 }
3926
3927 // If Microsoft extensions are disabled, this is just random garbage.
3928 Kind = tok::unknown;
3929 break;
3930
3931 case '\r':
3932 if (CurPtr[0] == '\n')
3933 (void)getAndAdvanceChar(Ptr&: CurPtr, Tok&: Result);
3934 [[fallthrough]];
3935 case '\n':
3936 // If we are inside a preprocessor directive and we see the end of line,
3937 // we know we are done with the directive, so return an EOD token.
3938 if (ParsingPreprocessorDirective) {
3939 // Done parsing the "line".
3940 ParsingPreprocessorDirective = false;
3941
3942 // Restore comment saving mode, in case it was disabled for directive.
3943 if (PP)
3944 resetExtendedTokenMode();
3945
3946 // Since we consumed a newline, we are back at the start of a line.
3947 IsAtStartOfLine = true;
3948 IsAtPhysicalStartOfLine = true;
3949 NewLinePtr = CurPtr - 1;
3950
3951 Kind = tok::eod;
3952 break;
3953 }
3954
3955 // No leading whitespace seen so far.
3956 Result.clearFlag(Flag: Token::LeadingSpace);
3957
3958 if (SkipWhitespace(Result, CurPtr))
3959 return true; // KeepWhitespaceMode
3960
3961 // We only saw whitespace, so just try again with this lexer.
3962 // (We manually eliminate the tail call to avoid recursion.)
3963 goto LexNextToken;
3964 case ' ':
3965 case '\t':
3966 case '\f':
3967 case '\v':
3968 SkipHorizontalWhitespace:
3969 Result.setFlag(Token::LeadingSpace);
3970 if (SkipWhitespace(Result, CurPtr))
3971 return true; // KeepWhitespaceMode
3972
3973 SkipIgnoredUnits:
3974 CurPtr = BufferPtr;
3975
3976 // If the next token is obviously a // or /* */ comment, skip it efficiently
3977 // too (without going through the big switch stmt).
3978 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
3979 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
3980 if (SkipLineComment(Result, CurPtr: CurPtr + 2))
3981 return true; // There is a token to return.
3982 goto SkipIgnoredUnits;
3983 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
3984 if (SkipBlockComment(Result, CurPtr: CurPtr + 2))
3985 return true; // There is a token to return.
3986 goto SkipIgnoredUnits;
3987 } else if (isHorizontalWhitespace(c: *CurPtr)) {
3988 goto SkipHorizontalWhitespace;
3989 }
3990 // We only saw whitespace, so just try again with this lexer.
3991 // (We manually eliminate the tail call to avoid recursion.)
3992 goto LexNextToken;
3993
3994 // C99 6.4.4.1: Integer Constants.
3995 // C99 6.4.4.2: Floating Constants.
3996 case '0': case '1': case '2': case '3': case '4':
3997 case '5': case '6': case '7': case '8': case '9':
3998 // Notify MIOpt that we read a non-whitespace/non-comment token.
3999 MIOpt.ReadToken();
4000 return LexNumericConstant(Result, CurPtr);
4001
4002 // Identifier (e.g., uber), or
4003 // UTF-8 (C23/C++17) or UTF-16 (C11/C++11) character literal, or
4004 // UTF-8 or UTF-16 string literal (C11/C++11).
4005 case 'u':
4006 // Notify MIOpt that we read a non-whitespace/non-comment token.
4007 MIOpt.ReadToken();
4008
4009 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
4010 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4011
4012 // UTF-16 string literal
4013 if (Char == '"')
4014 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4015 Kind: tok::utf16_string_literal);
4016
4017 // UTF-16 character constant
4018 if (Char == '\'')
4019 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4020 Kind: tok::utf16_char_constant);
4021
4022 // UTF-16 raw string literal
4023 if (Char == 'R' && LangOpts.RawStringLiterals &&
4024 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4025 return LexRawStringLiteral(Result,
4026 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4027 Size: SizeTmp2, Tok&: Result),
4028 Kind: tok::utf16_string_literal);
4029
4030 if (Char == '8') {
4031 char Char2 = getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2);
4032
4033 // UTF-8 string literal
4034 if (Char2 == '"')
4035 return LexStringLiteral(Result,
4036 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4037 Size: SizeTmp2, Tok&: Result),
4038 Kind: tok::utf8_string_literal);
4039 if (Char2 == '\'' && (LangOpts.CPlusPlus17 || LangOpts.C23))
4040 return LexCharConstant(
4041 Result, CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4042 Size: SizeTmp2, Tok&: Result),
4043 Kind: tok::utf8_char_constant);
4044
4045 if (Char2 == 'R' && LangOpts.RawStringLiterals) {
4046 unsigned SizeTmp3;
4047 char Char3 = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4048 // UTF-8 raw string literal
4049 if (Char3 == '"') {
4050 return LexRawStringLiteral(Result,
4051 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4052 Size: SizeTmp2, Tok&: Result),
4053 Size: SizeTmp3, Tok&: Result),
4054 Kind: tok::utf8_string_literal);
4055 }
4056 }
4057 }
4058 }
4059
4060 // treat u like the start of an identifier.
4061 return LexIdentifierContinue(Result, CurPtr);
4062
4063 case 'U': // Identifier (e.g. Uber) or C11/C++11 UTF-32 string literal
4064 // Notify MIOpt that we read a non-whitespace/non-comment token.
4065 MIOpt.ReadToken();
4066
4067 if (LangOpts.CPlusPlus11 || LangOpts.C11) {
4068 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4069
4070 // UTF-32 string literal
4071 if (Char == '"')
4072 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4073 Kind: tok::utf32_string_literal);
4074
4075 // UTF-32 character constant
4076 if (Char == '\'')
4077 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4078 Kind: tok::utf32_char_constant);
4079
4080 // UTF-32 raw string literal
4081 if (Char == 'R' && LangOpts.RawStringLiterals &&
4082 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4083 return LexRawStringLiteral(Result,
4084 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4085 Size: SizeTmp2, Tok&: Result),
4086 Kind: tok::utf32_string_literal);
4087 }
4088
4089 // treat U like the start of an identifier.
4090 return LexIdentifierContinue(Result, CurPtr);
4091
4092 case 'R': // Identifier or C++0x raw string literal
4093 // Notify MIOpt that we read a non-whitespace/non-comment token.
4094 MIOpt.ReadToken();
4095
4096 if (LangOpts.RawStringLiterals) {
4097 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4098
4099 if (Char == '"')
4100 return LexRawStringLiteral(Result,
4101 CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4102 Kind: tok::string_literal);
4103 }
4104
4105 // treat R like the start of an identifier.
4106 return LexIdentifierContinue(Result, CurPtr);
4107
4108 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
4109 // Notify MIOpt that we read a non-whitespace/non-comment token.
4110 MIOpt.ReadToken();
4111 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4112
4113 // Wide string literal.
4114 if (Char == '"')
4115 return LexStringLiteral(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4116 Kind: tok::wide_string_literal);
4117
4118 // Wide raw string literal.
4119 if (LangOpts.RawStringLiterals && Char == 'R' &&
4120 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == '"')
4121 return LexRawStringLiteral(Result,
4122 CurPtr: ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4123 Size: SizeTmp2, Tok&: Result),
4124 Kind: tok::wide_string_literal);
4125
4126 // Wide character constant.
4127 if (Char == '\'')
4128 return LexCharConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4129 Kind: tok::wide_char_constant);
4130 // FALL THROUGH, treating L like the start of an identifier.
4131 [[fallthrough]];
4132
4133 // C99 6.4.2: Identifiers.
4134 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
4135 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
4136 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
4137 case 'V': case 'W': case 'X': case 'Y': case 'Z':
4138 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
4139 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
4140 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
4141 case 'v': case 'w': case 'x': case 'y': case 'z':
4142 case '_':
4143 // Notify MIOpt that we read a non-whitespace/non-comment token.
4144 MIOpt.ReadToken();
4145 return LexIdentifierContinue(Result, CurPtr);
4146 case '$': // $ in identifiers.
4147 if (LangOpts.DollarIdents) {
4148 if (!isLexingRawMode())
4149 Diag(Loc: CurPtr-1, DiagID: diag::ext_dollar_in_identifier);
4150 // Notify MIOpt that we read a non-whitespace/non-comment token.
4151 MIOpt.ReadToken();
4152 return LexIdentifierContinue(Result, CurPtr);
4153 }
4154
4155 Kind = tok::unknown;
4156 break;
4157
4158 // C99 6.4.4: Character Constants.
4159 case '\'':
4160 // Notify MIOpt that we read a non-whitespace/non-comment token.
4161 MIOpt.ReadToken();
4162 return LexCharConstant(Result, CurPtr, Kind: tok::char_constant);
4163
4164 // C99 6.4.5: String Literals.
4165 case '"':
4166 // Notify MIOpt that we read a non-whitespace/non-comment token.
4167 MIOpt.ReadToken();
4168 return LexStringLiteral(Result, CurPtr,
4169 Kind: ParsingFilename ? tok::header_name
4170 : tok::string_literal);
4171
4172 // C99 6.4.6: Punctuators.
4173 case '?':
4174 Kind = tok::question;
4175 break;
4176 case '[':
4177 Kind = tok::l_square;
4178 break;
4179 case ']':
4180 Kind = tok::r_square;
4181 break;
4182 case '(':
4183 Kind = tok::l_paren;
4184 break;
4185 case ')':
4186 Kind = tok::r_paren;
4187 break;
4188 case '{':
4189 Kind = tok::l_brace;
4190 break;
4191 case '}':
4192 Kind = tok::r_brace;
4193 break;
4194 case '.':
4195 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4196 if (Char >= '0' && Char <= '9') {
4197 // Notify MIOpt that we read a non-whitespace/non-comment token.
4198 MIOpt.ReadToken();
4199
4200 return LexNumericConstant(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result));
4201 } else if (LangOpts.CPlusPlus && Char == '*') {
4202 Kind = tok::periodstar;
4203 CurPtr += SizeTmp;
4204 } else if (Char == '.' &&
4205 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '.') {
4206 Kind = tok::ellipsis;
4207 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4208 Size: SizeTmp2, Tok&: Result);
4209 } else {
4210 Kind = tok::period;
4211 }
4212 break;
4213 case '&':
4214 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4215 if (Char == '&') {
4216 Kind = tok::ampamp;
4217 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4218 } else if (Char == '=') {
4219 Kind = tok::ampequal;
4220 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4221 } else {
4222 Kind = tok::amp;
4223 }
4224 break;
4225 case '*':
4226 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4227 Kind = tok::starequal;
4228 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4229 } else {
4230 Kind = tok::star;
4231 }
4232 break;
4233 case '+':
4234 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4235 if (Char == '+') {
4236 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4237 Kind = tok::plusplus;
4238 } else if (Char == '=') {
4239 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4240 Kind = tok::plusequal;
4241 } else {
4242 Kind = tok::plus;
4243 }
4244 break;
4245 case '-':
4246 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4247 if (Char == '-') { // --
4248 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4249 Kind = tok::minusminus;
4250 } else if (Char == '>' && LangOpts.CPlusPlus &&
4251 getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == '*') { // C++ ->*
4252 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4253 Size: SizeTmp2, Tok&: Result);
4254 Kind = tok::arrowstar;
4255 } else if (Char == '>') { // ->
4256 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4257 Kind = tok::arrow;
4258 } else if (Char == '=') { // -=
4259 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4260 Kind = tok::minusequal;
4261 } else {
4262 Kind = tok::minus;
4263 }
4264 break;
4265 case '~':
4266 Kind = tok::tilde;
4267 break;
4268 case '!':
4269 if (getCharAndSize(Ptr: CurPtr, Size&: SizeTmp) == '=') {
4270 Kind = tok::exclaimequal;
4271 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4272 } else {
4273 Kind = tok::exclaim;
4274 }
4275 break;
4276 case '/':
4277 // 6.4.9: Comments
4278 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4279 if (Char == '/') { // Line comment.
4280 // Even if Line comments are disabled (e.g. in C89 mode), we generally
4281 // want to lex this as a comment. There is one problem with this though,
4282 // that in one particular corner case, this can change the behavior of the
4283 // resultant program. For example, In "foo //**/ bar", C89 would lex
4284 // this as "foo / bar" and languages with Line comments would lex it as
4285 // "foo". Check to see if the character after the second slash is a '*'.
4286 // If so, we will lex that as a "/" instead of the start of a comment.
4287 // However, we never do this if we are just preprocessing.
4288 bool TreatAsComment =
4289 LineComment && (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP);
4290 if (!TreatAsComment)
4291 if (!(PP && PP->isPreprocessedOutput()))
4292 TreatAsComment = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) != '*';
4293
4294 if (TreatAsComment) {
4295 if (SkipLineComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result)))
4296 return true; // There is a token to return.
4297
4298 // It is common for the tokens immediately after a // comment to be
4299 // whitespace (indentation for the next line). Instead of going through
4300 // the big switch, handle it efficiently now.
4301 goto SkipIgnoredUnits;
4302 }
4303 }
4304
4305 if (Char == '*') { // /**/ comment.
4306 if (SkipBlockComment(Result, CurPtr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result)))
4307 return true; // There is a token to return.
4308
4309 // We only saw whitespace, so just try again with this lexer.
4310 // (We manually eliminate the tail call to avoid recursion.)
4311 goto LexNextToken;
4312 }
4313
4314 if (Char == '=') {
4315 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4316 Kind = tok::slashequal;
4317 } else {
4318 Kind = tok::slash;
4319 }
4320 break;
4321 case '%':
4322 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4323 if (Char == '=') {
4324 Kind = tok::percentequal;
4325 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4326 } else if (LangOpts.Digraphs && Char == '>') {
4327 Kind = tok::r_brace; // '%>' -> '}'
4328 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4329 } else if (LangOpts.Digraphs && Char == ':') {
4330 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4331 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4332 if (Char == '%' && getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2) == ':') {
4333 Kind = tok::hashhash; // '%:%:' -> '##'
4334 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4335 Size: SizeTmp2, Tok&: Result);
4336 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
4337 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4338 if (!isLexingRawMode())
4339 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4340 Kind = tok::hashat;
4341 } else { // '%:' -> '#'
4342 // We parsed a # character. If this occurs at the start of the line,
4343 // it's actually the start of a preprocessing directive. Callback to
4344 // the preprocessor to handle it.
4345 // TODO: -fpreprocessed mode??
4346 if (Result.isAtPhysicalStartOfLine() && !LexingRawMode &&
4347 !Is_PragmaLexer)
4348 goto HandleDirective;
4349
4350 Kind = tok::hash;
4351 }
4352 } else {
4353 Kind = tok::percent;
4354 }
4355 break;
4356 case '<':
4357 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4358 if (ParsingFilename && LexAngledStringLiteral(Result, CurPtr))
4359 return true;
4360
4361 if (Char == '<') {
4362 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4363 if (After == '=') {
4364 Kind = tok::lesslessequal;
4365 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4366 Size: SizeTmp2, Tok&: Result);
4367 } else if (After == '<' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4368 // If this is actually a '<<<<<<<' version control conflict marker,
4369 // recognize it as such and recover nicely.
4370 goto LexNextToken;
4371 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4372 // If this is '<<<<' and we're in a Perforce-style conflict marker,
4373 // ignore it.
4374 goto LexNextToken;
4375 } else if (LangOpts.CUDA && After == '<') {
4376 Kind = tok::lesslessless;
4377 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4378 Size: SizeTmp2, Tok&: Result);
4379 } else {
4380 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4381 Kind = tok::lessless;
4382 }
4383 } else if (Char == '=') {
4384 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4385 if (After == '>') {
4386 if (LangOpts.CPlusPlus20) {
4387 if (!isLexingRawMode())
4388 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx17_compat_spaceship);
4389 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4390 Size: SizeTmp2, Tok&: Result);
4391 Kind = tok::spaceship;
4392 break;
4393 }
4394 // Suggest adding a space between the '<=' and the '>' to avoid a
4395 // change in semantics if this turns up in C++ <=17 mode.
4396 if (LangOpts.CPlusPlus && !isLexingRawMode()) {
4397 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx20_compat_spaceship)
4398 << FixItHint::CreateInsertion(
4399 InsertionLoc: getSourceLocation(Loc: CurPtr + SizeTmp, TokLen: SizeTmp2), Code: " ");
4400 }
4401 }
4402 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4403 Kind = tok::lessequal;
4404 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '['
4405 if (LangOpts.CPlusPlus11 &&
4406 getCharAndSize(Ptr: CurPtr + SizeTmp, Size&: SizeTmp2) == ':') {
4407 // C++0x [lex.pptoken]p3:
4408 // Otherwise, if the next three characters are <:: and the subsequent
4409 // character is neither : nor >, the < is treated as a preprocessor
4410 // token by itself and not as the first character of the alternative
4411 // token <:.
4412 unsigned SizeTmp3;
4413 char After = getCharAndSize(Ptr: CurPtr + SizeTmp + SizeTmp2, Size&: SizeTmp3);
4414 if (After != ':' && After != '>') {
4415 Kind = tok::less;
4416 if (!isLexingRawMode())
4417 Diag(Loc: BufferPtr, DiagID: diag::warn_cxx98_compat_less_colon_colon);
4418 break;
4419 }
4420 }
4421
4422 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4423 Kind = tok::l_square;
4424 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{'
4425 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4426 Kind = tok::l_brace;
4427 } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 &&
4428 lexEditorPlaceholder(Result, CurPtr)) {
4429 return true;
4430 } else {
4431 Kind = tok::less;
4432 }
4433 break;
4434 case '>':
4435 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4436 if (Char == '=') {
4437 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4438 Kind = tok::greaterequal;
4439 } else if (Char == '>') {
4440 char After = getCharAndSize(Ptr: CurPtr+SizeTmp, Size&: SizeTmp2);
4441 if (After == '=') {
4442 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4443 Size: SizeTmp2, Tok&: Result);
4444 Kind = tok::greatergreaterequal;
4445 } else if (After == '>' && IsStartOfConflictMarker(CurPtr: CurPtr-1)) {
4446 // If this is actually a '>>>>' conflict marker, recognize it as such
4447 // and recover nicely.
4448 goto LexNextToken;
4449 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr: CurPtr-1)) {
4450 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
4451 goto LexNextToken;
4452 } else if (LangOpts.CUDA && After == '>') {
4453 Kind = tok::greatergreatergreater;
4454 CurPtr = ConsumeChar(Ptr: ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result),
4455 Size: SizeTmp2, Tok&: Result);
4456 } else {
4457 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4458 Kind = tok::greatergreater;
4459 }
4460 } else {
4461 Kind = tok::greater;
4462 }
4463 break;
4464 case '^':
4465 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4466 if (Char == '=') {
4467 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4468 Kind = tok::caretequal;
4469 } else if (LangOpts.Reflection && Char == '^') {
4470 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4471 Kind = tok::caretcaret;
4472 } else {
4473 if (LangOpts.OpenCL && Char == '^')
4474 Diag(Loc: CurPtr, DiagID: diag::err_opencl_logical_exclusive_or);
4475 Kind = tok::caret;
4476 }
4477 break;
4478 case '|':
4479 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4480 if (Char == '=') {
4481 Kind = tok::pipeequal;
4482 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4483 } else if (Char == '|') {
4484 // If this is '|||||||' and we're in a conflict marker, ignore it.
4485 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4486 goto LexNextToken;
4487 Kind = tok::pipepipe;
4488 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4489 } else {
4490 Kind = tok::pipe;
4491 }
4492 break;
4493 case ':':
4494 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4495 if (LangOpts.Digraphs && Char == '>') {
4496 Kind = tok::r_square; // ':>' -> ']'
4497 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4498 } else if (Char == ':') {
4499 Kind = tok::coloncolon;
4500 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4501 } else {
4502 Kind = tok::colon;
4503 }
4504 break;
4505 case ';':
4506 Kind = tok::semi;
4507 break;
4508 case '=':
4509 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4510 if (Char == '=') {
4511 // If this is '====' and we're in a conflict marker, ignore it.
4512 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr: CurPtr-1))
4513 goto LexNextToken;
4514
4515 Kind = tok::equalequal;
4516 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4517 } else {
4518 Kind = tok::equal;
4519 }
4520 break;
4521 case ',':
4522 Kind = tok::comma;
4523 break;
4524 case '#':
4525 Char = getCharAndSize(Ptr: CurPtr, Size&: SizeTmp);
4526 if (Char == '#') {
4527 Kind = tok::hashhash;
4528 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4529 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize
4530 Kind = tok::hashat;
4531 if (!isLexingRawMode())
4532 Diag(Loc: BufferPtr, DiagID: diag::ext_charize_microsoft);
4533 CurPtr = ConsumeChar(Ptr: CurPtr, Size: SizeTmp, Tok&: Result);
4534 } else {
4535 // We parsed a # character. If this occurs at the start of the line,
4536 // it's actually the start of a preprocessing directive. Callback to
4537 // the preprocessor to handle it.
4538 // TODO: -fpreprocessed mode??
4539 if (Result.isAtPhysicalStartOfLine() && !LexingRawMode && !Is_PragmaLexer)
4540 goto HandleDirective;
4541
4542 Kind = tok::hash;
4543 }
4544 break;
4545
4546 case '@':
4547 // Objective C support.
4548 if (CurPtr[-1] == '@' && LangOpts.ObjC) {
4549 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::at);
4550 if (PP && Result.isAtPhysicalStartOfLine() && !LexingRawMode &&
4551 !Is_PragmaLexer) {
4552 Token NextPPTok;
4553 NextPPTok.startToken();
4554 {
4555 llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective(
4556 this->ParsingPreprocessorDirective, true);
4557 auto NextTokOr = peekNextPPToken();
4558 if (NextTokOr.has_value()) {
4559 NextPPTok = *NextTokOr;
4560 }
4561 }
4562 if (NextPPTok.is(K: tok::raw_identifier) &&
4563 NextPPTok.getRawIdentifier() == "import") {
4564 PP->HandleDirective(Result);
4565 return false;
4566 }
4567 }
4568 return true;
4569 } else
4570 Kind = tok::unknown;
4571 break;
4572
4573 // UCNs (C99 6.4.3, C++11 [lex.charset]p2)
4574 case '\\':
4575 if (!LangOpts.AsmPreprocessor) {
4576 if (uint32_t CodePoint = tryReadUCN(StartPtr&: CurPtr, SlashLoc: BufferPtr, Result: &Result)) {
4577 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4578 if (SkipWhitespace(Result, CurPtr))
4579 return true; // KeepWhitespaceMode
4580
4581 // We only saw whitespace, so just try again with this lexer.
4582 // (We manually eliminate the tail call to avoid recursion.)
4583 goto LexNextToken;
4584 }
4585
4586 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4587 }
4588 }
4589
4590 Kind = tok::unknown;
4591 break;
4592
4593 default: {
4594 if (isASCII(c: Char)) {
4595 Kind = tok::unknown;
4596 break;
4597 }
4598
4599 llvm::UTF32 CodePoint;
4600
4601 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
4602 // an escaped newline.
4603 --CurPtr;
4604 llvm::ConversionResult Status =
4605 llvm::convertUTF8Sequence(source: (const llvm::UTF8 **)&CurPtr,
4606 sourceEnd: (const llvm::UTF8 *)BufferEnd,
4607 target: &CodePoint,
4608 flags: llvm::strictConversion);
4609 if (Status == llvm::conversionOK) {
4610 if (CheckUnicodeWhitespace(Result, C: CodePoint, CurPtr)) {
4611 if (SkipWhitespace(Result, CurPtr))
4612 return true; // KeepWhitespaceMode
4613
4614 // We only saw whitespace, so just try again with this lexer.
4615 // (We manually eliminate the tail call to avoid recursion.)
4616 goto LexNextToken;
4617 }
4618 return LexUnicodeIdentifierStart(Result, C: CodePoint, CurPtr);
4619 }
4620
4621 if (isLexingRawMode() || ParsingPreprocessorDirective ||
4622 PP->isPreprocessedOutput()) {
4623 ++CurPtr;
4624 Kind = tok::unknown;
4625 break;
4626 }
4627
4628 // Non-ASCII characters tend to creep into source code unintentionally.
4629 // Instead of letting the parser complain about the unknown token,
4630 // just diagnose the invalid UTF-8, then drop the character.
4631 Diag(Loc: CurPtr, DiagID: diag::err_invalid_utf8);
4632
4633 BufferPtr = CurPtr+1;
4634 // We're pretending the character didn't exist, so just try again with
4635 // this lexer.
4636 // (We manually eliminate the tail call to avoid recursion.)
4637 goto LexNextToken;
4638 }
4639 }
4640
4641 // Notify MIOpt that we read a non-whitespace/non-comment token.
4642 MIOpt.ReadToken();
4643
4644 // Update the location of token as well as BufferPtr.
4645 FormTokenWithChars(Result, TokEnd: CurPtr, Kind);
4646 return true;
4647
4648HandleDirective:
4649
4650 // We parsed a # character and it's the start of a preprocessing directive.
4651 FormTokenWithChars(Result, TokEnd: CurPtr, Kind: tok::hash);
4652 PP->HandleDirective(Result);
4653
4654 if (PP->hadModuleLoaderFatalFailure())
4655 // With a fatal failure in the module loader, we abort parsing.
4656 return true;
4657
4658 // We parsed the directive; lex a token with the new state.
4659 return false;
4660
4661LexNextToken:
4662 Result.clearFlag(Flag: Token::NeedsCleaning);
4663 goto LexStart;
4664}
4665
4666const char *Lexer::convertDependencyDirectiveToken(
4667 const dependency_directives_scan::Token &DDTok, Token &Result) {
4668 const char *TokPtr = BufferStart + DDTok.Offset;
4669 Result = Token::create(Kind: DDTok.Kind, Loc: getSourceLocation(Loc: TokPtr), Length: DDTok.Length);
4670 Result.setFlag((Token::TokenFlags)DDTok.Flags);
4671 if (Result.is(K: tok::raw_identifier))
4672 Result.setRawIdentifierData(TokPtr);
4673 else if (Result.isLiteral())
4674 Result.setLiteralData(TokPtr);
4675 BufferPtr = TokPtr + DDTok.Length;
4676 return TokPtr;
4677}
4678
4679bool Lexer::LexDependencyDirectiveToken(Token &Result) {
4680 assert(isDependencyDirectivesLexer());
4681
4682 Result.startToken();
4683
4684 using namespace dependency_directives_scan;
4685
4686 if (BufferPtr == BufferEnd)
4687 return LexEndOfFile(Result, CurPtr: BufferPtr);
4688
4689 while (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) {
4690 if (DepDirectives.front().Kind == pp_eof)
4691 return LexEndOfFile(Result, CurPtr: BufferEnd);
4692 if (DepDirectives.front().Kind == tokens_present_before_eof)
4693 MIOpt.ReadToken();
4694 NextDepDirectiveTokenIndex = 0;
4695 DepDirectives = DepDirectives.drop_front();
4696 }
4697
4698 const dependency_directives_scan::Token &DDTok =
4699 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex++];
4700 if (NextDepDirectiveTokenIndex > 1 || DDTok.Kind != tok::hash) {
4701 // Read something other than a preprocessor directive hash.
4702 MIOpt.ReadToken();
4703 }
4704
4705 const char *DDTokPtr = BufferStart + DDTok.Offset;
4706 if (ParsingFilename && *DDTokPtr == '<') {
4707 Result.setFlag((clang::Token::TokenFlags)DDTok.Flags);
4708 Result.clearFlag(Flag: clang::Token::NeedsCleaning);
4709 BufferPtr = DDTokPtr;
4710 if (!LexAngledStringLiteral(Result, CurPtr: BufferPtr + 1)) {
4711 convertDependencyDirectiveToken(DDTok, Result);
4712 return true;
4713 }
4714
4715 // Advance the index of lexed tokens.
4716 // FIXME: This will skip too many tokens if the header-name ended in the
4717 // middle of a token, such as in '<foo>='.
4718 while (true) {
4719 const dependency_directives_scan::Token &NextTok =
4720 DepDirectives.front().Tokens[NextDepDirectiveTokenIndex];
4721 if (BufferStart + NextTok.Offset >= BufferPtr)
4722 break;
4723 ++NextDepDirectiveTokenIndex;
4724 }
4725 return true;
4726 }
4727
4728 const char *TokPtr = convertDependencyDirectiveToken(DDTok, Result);
4729
4730 if (Result.is(K: tok::hash) && Result.isAtStartOfLine()) {
4731 PP->HandleDirective(Result);
4732 if (PP->hadModuleLoaderFatalFailure())
4733 // With a fatal failure in the module loader, we abort parsing.
4734 return true;
4735 return false;
4736 }
4737 if (Result.is(K: tok::at) && Result.isAtStartOfLine()) {
4738 auto NextTok = peekNextPPToken();
4739 if (NextTok && NextTok->is(K: tok::raw_identifier) &&
4740 NextTok->getRawIdentifier() == "import") {
4741 PP->HandleDirective(Result);
4742 if (PP->hadModuleLoaderFatalFailure())
4743 return true;
4744 return false;
4745 }
4746 }
4747 if (Result.is(K: tok::raw_identifier)) {
4748 Result.setRawIdentifierData(TokPtr);
4749 if (!isLexingRawMode()) {
4750 const IdentifierInfo *II = PP->LookUpIdentifierInfo(Identifier&: Result);
4751 if (LangOpts.CPlusPlusModules && Result.isModuleContextualKeyword() &&
4752 PP->HandleModuleContextualKeyword(Result)) {
4753 PP->HandleDirective(Result);
4754 return false;
4755 }
4756 if (II->isHandleIdentifierCase())
4757 return PP->HandleIdentifier(Identifier&: Result);
4758 }
4759 return true;
4760 }
4761 if (Result.isLiteral())
4762 return true;
4763 if (Result.is(K: tok::colon)) {
4764 // Convert consecutive colons to 'tok::coloncolon'.
4765 if (*BufferPtr == ':') {
4766 assert(DepDirectives.front().Tokens[NextDepDirectiveTokenIndex].is(
4767 tok::colon));
4768 ++NextDepDirectiveTokenIndex;
4769 Result.setKind(tok::coloncolon);
4770 }
4771 return true;
4772 }
4773 if (Result.is(K: tok::eod))
4774 ParsingPreprocessorDirective = false;
4775
4776 return true;
4777}
4778
4779bool Lexer::LexDependencyDirectiveTokenWhileSkipping(Token &Result) {
4780 assert(isDependencyDirectivesLexer());
4781
4782 using namespace dependency_directives_scan;
4783
4784 bool Stop = false;
4785 unsigned NestedIfs = 0;
4786 do {
4787 DepDirectives = DepDirectives.drop_front();
4788 switch (DepDirectives.front().Kind) {
4789 case pp_none:
4790 llvm_unreachable("unexpected 'pp_none'");
4791 case pp_include:
4792 case pp___include_macros:
4793 case pp_define:
4794 case pp_undef:
4795 case pp_import:
4796 case pp_pragma_import:
4797 case pp_pragma_once:
4798 case pp_pragma_push_macro:
4799 case pp_pragma_pop_macro:
4800 case pp_pragma_include_alias:
4801 case pp_pragma_system_header:
4802 case pp_include_next:
4803 case decl_at_import:
4804 case cxx_module_decl:
4805 case cxx_import_decl:
4806 case cxx_export_module_decl:
4807 case cxx_export_import_decl:
4808 case tokens_present_before_eof:
4809 break;
4810 case pp_if:
4811 case pp_ifdef:
4812 case pp_ifndef:
4813 ++NestedIfs;
4814 break;
4815 case pp_elif:
4816 case pp_elifdef:
4817 case pp_elifndef:
4818 case pp_else:
4819 if (!NestedIfs) {
4820 Stop = true;
4821 }
4822 break;
4823 case pp_endif:
4824 if (!NestedIfs) {
4825 Stop = true;
4826 } else {
4827 --NestedIfs;
4828 }
4829 break;
4830 case pp_eof:
4831 NextDepDirectiveTokenIndex = 0;
4832 return LexEndOfFile(Result, CurPtr: BufferEnd);
4833 }
4834 } while (!Stop);
4835
4836 const dependency_directives_scan::Token &DDTok =
4837 DepDirectives.front().Tokens.front();
4838 assert(DDTok.is(tok::hash));
4839 NextDepDirectiveTokenIndex = 1;
4840
4841 convertDependencyDirectiveToken(DDTok, Result);
4842 return false;
4843}
4844