1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the NumericLiteralParser, CharLiteralParser, and
10// StringLiteralParser interfaces.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Lex/LiteralSupport.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/LangOptions.h"
17#include "clang/Basic/SourceLocation.h"
18#include "clang/Basic/TargetInfo.h"
19#include "clang/Lex/LexDiagnostic.h"
20#include "clang/Lex/Lexer.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/Token.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ScopeExit.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/ADT/StringSwitch.h"
28#include "llvm/Support/ConvertUTF.h"
29#include "llvm/Support/Error.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/Unicode.h"
32#include <algorithm>
33#include <cassert>
34#include <cstddef>
35#include <cstdint>
36#include <cstring>
37#include <string>
38
39using namespace clang;
40
41static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
42 switch (kind) {
43 default: llvm_unreachable("Unknown token type!");
44 case tok::char_constant:
45 case tok::string_literal:
46 case tok::utf8_char_constant:
47 case tok::utf8_string_literal:
48 return Target.getCharWidth();
49 case tok::wide_char_constant:
50 case tok::wide_string_literal:
51 return Target.getWCharWidth();
52 case tok::utf16_char_constant:
53 case tok::utf16_string_literal:
54 return Target.getChar16Width();
55 case tok::utf32_char_constant:
56 case tok::utf32_string_literal:
57 return Target.getChar32Width();
58 }
59}
60
61static unsigned getEncodingPrefixLen(tok::TokenKind kind) {
62 switch (kind) {
63 default:
64 llvm_unreachable("Unknown token type!");
65 case tok::char_constant:
66 case tok::string_literal:
67 return 0;
68 case tok::utf8_char_constant:
69 case tok::utf8_string_literal:
70 return 2;
71 case tok::wide_char_constant:
72 case tok::wide_string_literal:
73 case tok::utf16_char_constant:
74 case tok::utf16_string_literal:
75 case tok::utf32_char_constant:
76 case tok::utf32_string_literal:
77 return 1;
78 }
79}
80
81static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
82 FullSourceLoc TokLoc,
83 const char *TokBegin,
84 const char *TokRangeBegin,
85 const char *TokRangeEnd) {
86 SourceLocation Begin =
87 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
88 SM: TokLoc.getManager(), LangOpts: Features);
89 SourceLocation End =
90 Lexer::AdvanceToTokenCharacter(TokStart: Begin, Characters: TokRangeEnd - TokRangeBegin,
91 SM: TokLoc.getManager(), LangOpts: Features);
92 return CharSourceRange::getCharRange(B: Begin, E: End);
93}
94
95/// Produce a diagnostic highlighting some portion of a literal.
96///
97/// Emits the diagnostic \p DiagID, highlighting the range of characters from
98/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
99/// a substring of a spelling buffer for the token beginning at \p TokBegin.
100static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
101 const LangOptions &Features, FullSourceLoc TokLoc,
102 const char *TokBegin, const char *TokRangeBegin,
103 const char *TokRangeEnd, unsigned DiagID) {
104 SourceLocation Begin =
105 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
106 SM: TokLoc.getManager(), LangOpts: Features);
107 return Diags->Report(Loc: Begin, DiagID) <<
108 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
109}
110
111static bool IsEscapeValidInUnevaluatedStringLiteral(char Escape) {
112 switch (Escape) {
113 case '\'':
114 case '"':
115 case '?':
116 case '\\':
117 case 'a':
118 case 'b':
119 case 'f':
120 case 'n':
121 case 'r':
122 case 't':
123 case 'v':
124 return true;
125 }
126 return false;
127}
128
129static llvm::ErrorOr<char>
130convertCharacter(StringRef Char, const llvm::TextEncodingConverter &Converter) {
131 SmallString<8> ResultCharConv;
132 std::error_code EC = Converter.convert(Source: Char, Result&: ResultCharConv);
133 if (EC)
134 return EC;
135 else if (ResultCharConv.size() > 1)
136 return std::error_code(E2BIG, std::generic_category());
137 return ResultCharConv[0];
138}
139
140/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
141/// either a character or a string literal.
142static unsigned ProcessCharEscape(const char *ThisTokBegin,
143 const char *&ThisTokBuf,
144 const char *ThisTokEnd, bool &HadError,
145 FullSourceLoc Loc, unsigned CharWidth,
146 DiagnosticsEngine *Diags,
147 const LangOptions &Features,
148 StringLiteralEvalMethod EvalMethod,
149 llvm::TextEncodingConverter *Converter) {
150 const char *EscapeBegin = ThisTokBuf;
151 bool Delimited = false;
152 bool EndDelimiterFound = false;
153
154 // Skip the '\' char.
155 ++ThisTokBuf;
156
157 // We know that this character can't be off the end of the buffer, because
158 // that would have been \", which would not have been the end of string.
159 unsigned ResultChar = *ThisTokBuf++;
160 char Escape = ResultChar;
161 bool Transcode = true;
162 bool Invalid = false;
163 switch (ResultChar) {
164 // These map to themselves.
165 case '\\': case '\'': case '"': case '?': break;
166
167 // These have fixed mappings.
168 case 'a':
169 // TODO: K&R: the meaning of '\\a' is different in traditional C
170 ResultChar = 7;
171 break;
172 case 'b':
173 ResultChar = 8;
174 break;
175 case 'e':
176 if (Diags)
177 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
178 DiagID: diag::ext_nonstandard_escape) << "e";
179 ResultChar = 27;
180 break;
181 case 'E':
182 if (Diags)
183 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
184 DiagID: diag::ext_nonstandard_escape) << "E";
185 ResultChar = 27;
186 break;
187 case 'f':
188 ResultChar = 12;
189 break;
190 case 'n':
191 ResultChar = 10;
192 break;
193 case 'r':
194 ResultChar = 13;
195 break;
196 case 't':
197 ResultChar = 9;
198 break;
199 case 'v':
200 ResultChar = 11;
201 break;
202 case 'x': { // Hex escape.
203 Transcode = false;
204 ResultChar = 0;
205 if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
206 Delimited = true;
207 ThisTokBuf++;
208 if (*ThisTokBuf == '}') {
209 HadError = true;
210 if (Diags)
211 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
212 DiagID: diag::err_delimited_escape_empty);
213 }
214 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
215 if (Diags)
216 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
217 DiagID: diag::err_hex_escape_no_digits) << "x";
218 return ResultChar;
219 }
220
221 // Hex escapes are a maximal series of hex digits.
222 bool Overflow = false;
223 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
224 if (Delimited && *ThisTokBuf == '}') {
225 ThisTokBuf++;
226 EndDelimiterFound = true;
227 break;
228 }
229 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
230 if (CharVal == -1) {
231 // Non delimited hex escape sequences stop at the first non-hex digit.
232 if (!Delimited)
233 break;
234 HadError = true;
235 if (Diags)
236 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
237 DiagID: diag::err_delimited_escape_invalid)
238 << StringRef(ThisTokBuf, 1);
239 continue;
240 }
241 // About to shift out a digit?
242 if (ResultChar & 0xF0000000)
243 Overflow = true;
244 ResultChar <<= 4;
245 ResultChar |= CharVal;
246 }
247 // See if any bits will be truncated when evaluated as a character.
248 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
249 Overflow = true;
250 ResultChar &= ~0U >> (32-CharWidth);
251 }
252
253 // Check for overflow.
254 if (!HadError && Overflow) { // Too many digits to fit in
255 HadError = true;
256 if (Diags)
257 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
258 DiagID: diag::err_escape_too_large)
259 << 0;
260 }
261 break;
262 }
263 case '0': case '1': case '2': case '3':
264 case '4': case '5': case '6': case '7': {
265 // Octal escapes.
266 --ThisTokBuf;
267 Transcode = false;
268 ResultChar = 0;
269
270 // Octal escapes are a series of octal digits with maximum length 3.
271 // "\0123" is a two digit sequence equal to "\012" "3".
272 unsigned NumDigits = 0;
273 do {
274 ResultChar <<= 3;
275 ResultChar |= *ThisTokBuf++ - '0';
276 ++NumDigits;
277 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
278 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
279
280 // Check for overflow. Reject '\777', but not L'\777'.
281 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
282 if (Diags)
283 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
284 DiagID: diag::err_escape_too_large) << 1;
285 ResultChar &= ~0U >> (32-CharWidth);
286 }
287 break;
288 }
289 case 'o': {
290 bool Overflow = false;
291 Transcode = false;
292 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
293 HadError = true;
294 if (Diags)
295 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
296 DiagID: diag::err_delimited_escape_missing_brace)
297 << "o";
298
299 break;
300 }
301 ResultChar = 0;
302 Delimited = true;
303 ++ThisTokBuf;
304 if (*ThisTokBuf == '}') {
305 HadError = true;
306 if (Diags)
307 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
308 DiagID: diag::err_delimited_escape_empty);
309 }
310
311 while (ThisTokBuf != ThisTokEnd) {
312 if (*ThisTokBuf == '}') {
313 EndDelimiterFound = true;
314 ThisTokBuf++;
315 break;
316 }
317 if (*ThisTokBuf < '0' || *ThisTokBuf > '7') {
318 HadError = true;
319 if (Diags)
320 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
321 DiagID: diag::err_delimited_escape_invalid)
322 << StringRef(ThisTokBuf, 1);
323 ThisTokBuf++;
324 continue;
325 }
326 // Check if one of the top three bits is set before shifting them out.
327 if (ResultChar & 0xE0000000)
328 Overflow = true;
329
330 ResultChar <<= 3;
331 ResultChar |= *ThisTokBuf++ - '0';
332 }
333 // Check for overflow. Reject '\777', but not L'\777'.
334 if (!HadError &&
335 (Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) {
336 HadError = true;
337 if (Diags)
338 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
339 DiagID: diag::err_escape_too_large)
340 << 1;
341 ResultChar &= ~0U >> (32 - CharWidth);
342 }
343 break;
344 }
345 // Otherwise, these are not valid escapes.
346 case '(': case '{': case '[': case '%':
347 // GCC accepts these as extensions. We warn about them as such though.
348 if (Diags)
349 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
350 DiagID: diag::ext_nonstandard_escape)
351 << std::string(1, ResultChar);
352 break;
353 default:
354 Invalid = true;
355 if (!Diags)
356 break;
357
358 if (isPrintable(c: ResultChar))
359 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
360 DiagID: diag::ext_unknown_escape)
361 << std::string(1, ResultChar);
362 else
363 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
364 DiagID: diag::ext_unknown_escape)
365 << "x" + llvm::utohexstr(X: ResultChar);
366 break;
367 }
368
369 if (Delimited && Diags) {
370 if (!EndDelimiterFound)
371 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
372 DiagID: diag::err_expected)
373 << tok::r_brace;
374 else if (!HadError) {
375 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: false, Opts: Features,
376 Diags&: *Diags);
377 }
378 }
379
380 if (EvalMethod == StringLiteralEvalMethod::Unevaluated &&
381 !IsEscapeValidInUnevaluatedStringLiteral(Escape)) {
382 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
383 DiagID: diag::err_unevaluated_string_invalid_escape_sequence)
384 << StringRef(EscapeBegin, ThisTokBuf - EscapeBegin);
385 HadError = true;
386 }
387
388 if (!HadError && EvalMethod != StringLiteralEvalMethod::Unevaluated &&
389 Transcode && Converter) {
390 // Invalid escapes are written as '?' and then translated.
391 assert(ResultChar <=
392 static_cast<unsigned>(std::numeric_limits<char>::max()));
393 char ByteChar = Invalid ? '?' : ResultChar;
394 auto ErrorOrChar = convertCharacter(Char: StringRef(&ByteChar, 1), Converter: *Converter);
395 if (ErrorOrChar)
396 ResultChar = *ErrorOrChar;
397 else {
398 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
399 DiagID: diag::err_exec_charset_conversion_failed)
400 << ErrorOrChar.getError().message();
401 HadError = true;
402 }
403 }
404 return ResultChar;
405}
406
407static void appendCodePoint(unsigned Codepoint,
408 llvm::SmallVectorImpl<char> &Str) {
409 char ResultBuf[4];
410 char *ResultPtr = ResultBuf;
411 if (llvm::ConvertCodePointToUTF8(Source: Codepoint, ResultPtr))
412 Str.append(in_start: ResultBuf, in_end: ResultPtr);
413}
414
415void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
416 for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
417 if (*I != '\\') {
418 Buf.push_back(Elt: *I);
419 continue;
420 }
421
422 ++I;
423 char Kind = *I;
424 ++I;
425
426 assert(Kind == 'u' || Kind == 'U' || Kind == 'N');
427 uint32_t CodePoint = 0;
428
429 if (Kind == 'u' && *I == '{') {
430 for (++I; *I != '}'; ++I) {
431 unsigned Value = llvm::hexDigitValue(C: *I);
432 assert(Value != -1U);
433 CodePoint <<= 4;
434 CodePoint += Value;
435 }
436 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
437 continue;
438 }
439
440 if (Kind == 'N') {
441 assert(*I == '{');
442 ++I;
443 auto Delim = std::find(first: I, last: Input.end(), val: '}');
444 assert(Delim != Input.end());
445 StringRef Name(I, std::distance(first: I, last: Delim));
446 std::optional<llvm::sys::unicode::LooseMatchingResult> Res =
447 llvm::sys::unicode::nameToCodepointLooseMatching(Name);
448 assert(Res && "could not find a codepoint that was previously found");
449 CodePoint = Res->CodePoint;
450 assert(CodePoint != 0xFFFFFFFF);
451 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
452 I = Delim;
453 continue;
454 }
455
456 unsigned NumHexDigits;
457 if (Kind == 'u')
458 NumHexDigits = 4;
459 else
460 NumHexDigits = 8;
461
462 assert(I + NumHexDigits <= E);
463
464 for (; NumHexDigits != 0; ++I, --NumHexDigits) {
465 unsigned Value = llvm::hexDigitValue(C: *I);
466 assert(Value != -1U);
467
468 CodePoint <<= 4;
469 CodePoint += Value;
470 }
471
472 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
473 --I;
474 }
475}
476
477bool clang::isFunctionLocalStringLiteralMacro(tok::TokenKind K,
478 const LangOptions &LO) {
479 return LO.MicrosoftExt &&
480 (K == tok::kw___FUNCTION__ || K == tok::kw_L__FUNCTION__ ||
481 K == tok::kw___FUNCSIG__ || K == tok::kw_L__FUNCSIG__ ||
482 K == tok::kw___FUNCDNAME__);
483}
484
485bool clang::tokenIsLikeStringLiteral(const Token &Tok, const LangOptions &LO) {
486 return tok::isStringLiteral(K: Tok.getKind()) ||
487 isFunctionLocalStringLiteralMacro(K: Tok.getKind(), LO);
488}
489
490static bool ProcessNumericUCNEscape(const char *ThisTokBegin,
491 const char *&ThisTokBuf,
492 const char *ThisTokEnd, uint32_t &UcnVal,
493 unsigned short &UcnLen, bool &Delimited,
494 FullSourceLoc Loc, DiagnosticsEngine *Diags,
495 const LangOptions &Features,
496 bool in_char_string_literal = false) {
497 const char *UcnBegin = ThisTokBuf;
498 bool HasError = false;
499 bool EndDelimiterFound = false;
500
501 // Skip the '\u' char's.
502 ThisTokBuf += 2;
503 Delimited = false;
504 if (UcnBegin[1] == 'u' && in_char_string_literal &&
505 ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
506 Delimited = true;
507 ThisTokBuf++;
508 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
509 if (Diags)
510 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
511 DiagID: diag::err_hex_escape_no_digits)
512 << StringRef(&ThisTokBuf[-1], 1);
513 return false;
514 }
515 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
516
517 bool Overflow = false;
518 unsigned short Count = 0;
519 for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen);
520 ++ThisTokBuf) {
521 if (Delimited && *ThisTokBuf == '}') {
522 ++ThisTokBuf;
523 EndDelimiterFound = true;
524 break;
525 }
526 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
527 if (CharVal == -1) {
528 HasError = true;
529 if (!Delimited)
530 break;
531 if (Diags) {
532 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
533 DiagID: diag::err_delimited_escape_invalid)
534 << StringRef(ThisTokBuf, 1);
535 }
536 Count++;
537 continue;
538 }
539 if (UcnVal & 0xF0000000) {
540 Overflow = true;
541 continue;
542 }
543 UcnVal <<= 4;
544 UcnVal |= CharVal;
545 Count++;
546 }
547
548 if (Overflow) {
549 if (Diags)
550 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
551 DiagID: diag::err_escape_too_large)
552 << 0;
553 return false;
554 }
555
556 if (Delimited && !EndDelimiterFound) {
557 if (Diags) {
558 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
559 DiagID: diag::err_expected)
560 << tok::r_brace;
561 }
562 return false;
563 }
564
565 // If we didn't consume the proper number of digits, there is a problem.
566 if (Count == 0 || (!Delimited && Count != UcnLen)) {
567 if (Diags)
568 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
569 DiagID: Delimited ? diag::err_delimited_escape_empty
570 : diag::err_ucn_escape_incomplete);
571 return false;
572 }
573 return !HasError;
574}
575
576static bool allowedInCharacterName(char C) {
577 return (C >= 'A' && C <= 'Z') || (C >= '0' && C <= '9') || C == '-' ||
578 C == ' ';
579}
580
581static void DiagnoseInvalidUnicodeCharacterName(
582 DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc,
583 const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd,
584 llvm::StringRef Name) {
585
586 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
587 DiagID: diag::err_invalid_ucn_name)
588 << Name;
589
590 namespace u = llvm::sys::unicode;
591
592 bool HasIllegalCharacter = false;
593 for (const char *P = Name.begin(), *E = Name.end(); P != E;) {
594 if (allowedInCharacterName(C: *P)) {
595 ++P;
596 continue;
597 }
598 SourceLocation CharLoc = Lexer::AdvanceToTokenCharacter(
599 TokStart: Loc, Characters: (TokRangeBegin - TokBegin) + (P - Name.begin()), SM: Loc.getManager(),
600 LangOpts: Features);
601 Diags->Report(Loc: CharLoc, DiagID: diag::note_invalid_ucn_name_character)
602 << EscapeSingleCodepointForDiagnostic(
603 Str: StringRef(P, std::distance(first: P, last: E)));
604 HasIllegalCharacter = true;
605 break;
606 }
607
608 std::optional<u::LooseMatchingResult> Res =
609 u::nameToCodepointLooseMatching(Name);
610 if (Res) {
611 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
612 DiagID: diag::note_invalid_ucn_name_loose_matching)
613 << FixItHint::CreateReplacement(
614 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
615 TokRangeEnd),
616 Code: Res->Name);
617 return;
618 }
619
620 // Providing illegal characters suggests a fundamental misuse of the feature,
621 // like providing emoji in \N{}. Offering alternative suggestions is often
622 // unhelpful in that scenario.
623 if (HasIllegalCharacter)
624 return;
625
626 unsigned Distance = 0;
627 SmallVector<u::MatchForCodepointName> Matches =
628 u::nearestMatchesForCodepointName(Pattern: Name, MaxMatchesCount: 5);
629 assert(!Matches.empty() && "No unicode characters found");
630
631 for (const auto &Match : Matches) {
632 if (Distance == 0)
633 Distance = Match.Distance;
634 if (std::max(a: Distance, b: Match.Distance) -
635 std::min(a: Distance, b: Match.Distance) >
636 3)
637 break;
638 Distance = Match.Distance;
639 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
640 DiagID: diag::note_invalid_ucn_name_candidate)
641 << Match.Name << EscapeSingleCodepointForDiagnostic(CP: Match.Value)
642 << FixItHint::CreateReplacement(
643 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
644 TokRangeEnd),
645 Code: Match.Name);
646 }
647}
648
649static bool ProcessNamedUCNEscape(const char *ThisTokBegin,
650 const char *&ThisTokBuf,
651 const char *ThisTokEnd, uint32_t &UcnVal,
652 unsigned short &UcnLen, FullSourceLoc Loc,
653 DiagnosticsEngine *Diags,
654 const LangOptions &Features) {
655 const char *UcnBegin = ThisTokBuf;
656 assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N');
657 ThisTokBuf += 2;
658 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
659 if (Diags) {
660 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
661 DiagID: diag::err_delimited_escape_missing_brace)
662 << StringRef(&ThisTokBuf[-1], 1);
663 }
664 return false;
665 }
666 ThisTokBuf++;
667 const char *ClosingBrace = std::find_if(first: ThisTokBuf, last: ThisTokEnd, pred: [](char C) {
668 return C == '}' || isVerticalWhitespace(c: C);
669 });
670 bool Incomplete = ClosingBrace == ThisTokEnd;
671 bool Empty = ClosingBrace == ThisTokBuf;
672 if (Incomplete || Empty) {
673 if (Diags) {
674 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
675 DiagID: Incomplete ? diag::err_ucn_escape_incomplete
676 : diag::err_delimited_escape_empty)
677 << StringRef(&UcnBegin[1], 1);
678 }
679 ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1;
680 return false;
681 }
682 StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf);
683 ThisTokBuf = ClosingBrace + 1;
684 std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name);
685 if (!Res) {
686 if (Diags)
687 DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, TokBegin: ThisTokBegin,
688 TokRangeBegin: &UcnBegin[3], TokRangeEnd: ClosingBrace, Name);
689 return false;
690 }
691 UcnVal = *Res;
692 UcnLen = UcnVal > 0xFFFF ? 8 : 4;
693 return true;
694}
695
696/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
697/// return the UTF32.
698static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
699 const char *ThisTokEnd, uint32_t &UcnVal,
700 unsigned short &UcnLen, FullSourceLoc Loc,
701 DiagnosticsEngine *Diags,
702 const LangOptions &Features,
703 bool in_char_string_literal = false) {
704
705 bool HasError;
706 const char *UcnBegin = ThisTokBuf;
707 bool IsDelimitedEscapeSequence = false;
708 bool IsNamedEscapeSequence = false;
709 if (ThisTokBuf[1] == 'N') {
710 IsNamedEscapeSequence = true;
711 HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
712 UcnVal, UcnLen, Loc, Diags, Features);
713 } else {
714 HasError =
715 !ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
716 UcnLen, Delimited&: IsDelimitedEscapeSequence, Loc, Diags,
717 Features, in_char_string_literal);
718 }
719 if (HasError)
720 return false;
721
722 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
723 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
724 UcnVal > 0x10FFFF) { // maximum legal UTF32 value
725 if (Diags)
726 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
727 DiagID: diag::err_ucn_escape_invalid);
728 return false;
729 }
730
731 // C23 and C++11 allow UCNs that refer to control characters
732 // and basic source characters inside character and string literals
733 if (UcnVal < 0xa0 &&
734 // $, @, ` are allowed in all language modes
735 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {
736 bool IsError =
737 (!(Features.CPlusPlus11 || Features.C23) || !in_char_string_literal);
738 if (Diags) {
739 char BasicSCSChar = UcnVal;
740 if (UcnVal >= 0x20 && UcnVal < 0x7f)
741 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
742 DiagID: IsError ? diag::err_ucn_escape_basic_scs
743 : Features.CPlusPlus
744 ? diag::warn_cxx98_compat_literal_ucn_escape_basic_scs
745 : diag::warn_c23_compat_literal_ucn_escape_basic_scs)
746 << StringRef(&BasicSCSChar, 1);
747 else
748 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
749 DiagID: IsError ? diag::err_ucn_control_character
750 : Features.CPlusPlus
751 ? diag::warn_cxx98_compat_literal_ucn_control_character
752 : diag::warn_c23_compat_literal_ucn_control_character);
753 }
754 if (IsError)
755 return false;
756 }
757
758 if (!Features.CPlusPlus && !Features.C99 && Diags)
759 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
760 DiagID: diag::warn_ucn_not_valid_in_c89_literal);
761
762 if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
763 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: IsNamedEscapeSequence,
764 Opts: Features, Diags&: *Diags);
765 return true;
766}
767
768/// MeasureUCNEscape - Determine the number of bytes within the resulting string
769/// which this UCN will occupy.
770static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
771 const char *ThisTokEnd, unsigned CharByteWidth,
772 const LangOptions &Features, bool &HadError) {
773 // UTF-32: 4 bytes per escape.
774 if (CharByteWidth == 4)
775 return 4;
776
777 uint32_t UcnVal = 0;
778 unsigned short UcnLen = 0;
779 FullSourceLoc Loc;
780
781 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
782 UcnLen, Loc, Diags: nullptr, Features, in_char_string_literal: true)) {
783 HadError = true;
784 return 0;
785 }
786
787 // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
788 if (CharByteWidth == 2)
789 return UcnVal <= 0xFFFF ? 2 : 4;
790
791 // UTF-8.
792 if (UcnVal < 0x80)
793 return 1;
794 if (UcnVal < 0x800)
795 return 2;
796 if (UcnVal < 0x10000)
797 return 3;
798 return 4;
799}
800
801/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
802/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
803/// StringLiteralParser. When we decide to implement UCN's for identifiers,
804/// we will likely rework our support for UCN's.
805static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
806 const char *ThisTokEnd, char *&ResultBuf,
807 bool &HadError, FullSourceLoc Loc,
808 unsigned CharByteWidth, DiagnosticsEngine *Diags,
809 const LangOptions &Features,
810 llvm::TextEncodingConverter *Converter) {
811 typedef uint32_t UTF32;
812 UTF32 UcnVal = 0;
813 unsigned short UcnLen = 0;
814 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
815 Loc, Diags, Features, in_char_string_literal: true)) {
816 HadError = true;
817 return;
818 }
819
820 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
821 "only character widths of 1, 2, or 4 bytes supported");
822
823 (void)UcnLen;
824 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
825
826 if (CharByteWidth == 4) {
827 // FIXME: Make the type of the result buffer correct instead of
828 // using reinterpret_cast.
829 llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
830 *ResultPtr = UcnVal;
831 ResultBuf += 4;
832 return;
833 }
834
835 if (CharByteWidth == 2) {
836 // FIXME: Make the type of the result buffer correct instead of
837 // using reinterpret_cast.
838 llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
839
840 if (UcnVal <= (UTF32)0xFFFF) {
841 *ResultPtr = UcnVal;
842 ResultBuf += 2;
843 return;
844 }
845
846 // Convert to UTF16.
847 UcnVal -= 0x10000;
848 *ResultPtr = 0xD800 + (UcnVal >> 10);
849 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
850 ResultBuf += 4;
851 return;
852 }
853
854 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
855
856 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
857 // The conversion below was inspired by:
858 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
859 // First, we determine how many bytes the result will require.
860 typedef uint8_t UTF8;
861
862 unsigned short bytesToWrite = 0;
863 if (UcnVal < (UTF32)0x80)
864 bytesToWrite = 1;
865 else if (UcnVal < (UTF32)0x800)
866 bytesToWrite = 2;
867 else if (UcnVal < (UTF32)0x10000)
868 bytesToWrite = 3;
869 else
870 bytesToWrite = 4;
871
872 const unsigned byteMask = 0xBF;
873 const unsigned byteMark = 0x80;
874
875 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
876 // into the first byte, depending on how many bytes follow.
877 static const UTF8 firstByteMark[5] = {
878 0x00, 0x00, 0xC0, 0xE0, 0xF0
879 };
880 // Finally, we write the bytes into ResultBuf.
881 ResultBuf += bytesToWrite;
882 switch (bytesToWrite) { // note: everything falls through.
883 case 4:
884 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
885 [[fallthrough]];
886 case 3:
887 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
888 [[fallthrough]];
889 case 2:
890 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
891 [[fallthrough]];
892 case 1:
893 *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
894 }
895 // Update the buffer.
896 ResultBuf += bytesToWrite;
897
898 if (Converter) {
899 SmallString<4> CpConv;
900 char *Cp = ResultBuf - bytesToWrite;
901 auto EC = Converter->convert(Source: StringRef(Cp, bytesToWrite), Result&: CpConv);
902 if (!EC) {
903 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
904 ResultBuf = Cp + CpConv.size();
905 } else {
906 Diags->Report(Loc, DiagID: diag::err_exec_charset_conversion_failed)
907 << EC.message();
908 HadError = true;
909 }
910 }
911}
912
913/// integer-constant: [C99 6.4.4.1]
914/// decimal-constant integer-suffix
915/// octal-constant integer-suffix
916/// hexadecimal-constant integer-suffix
917/// binary-literal integer-suffix [GNU, C++1y]
918/// user-defined-integer-literal: [C++11 lex.ext]
919/// decimal-literal ud-suffix
920/// octal-literal ud-suffix
921/// hexadecimal-literal ud-suffix
922/// binary-literal ud-suffix [GNU, C++1y]
923/// decimal-constant:
924/// nonzero-digit
925/// decimal-constant digit
926/// octal-constant:
927/// 0
928/// octal-constant octal-digit
929/// hexadecimal-constant:
930/// hexadecimal-prefix hexadecimal-digit
931/// hexadecimal-constant hexadecimal-digit
932/// hexadecimal-prefix: one of
933/// 0x 0X
934/// binary-literal:
935/// 0b binary-digit
936/// 0B binary-digit
937/// binary-literal binary-digit
938/// integer-suffix:
939/// unsigned-suffix [long-suffix]
940/// unsigned-suffix [long-long-suffix]
941/// long-suffix [unsigned-suffix]
942/// long-long-suffix [unsigned-sufix]
943/// nonzero-digit:
944/// 1 2 3 4 5 6 7 8 9
945/// octal-digit:
946/// 0 1 2 3 4 5 6 7
947/// hexadecimal-digit:
948/// 0 1 2 3 4 5 6 7 8 9
949/// a b c d e f
950/// A B C D E F
951/// binary-digit:
952/// 0
953/// 1
954/// unsigned-suffix: one of
955/// u U
956/// long-suffix: one of
957/// l L
958/// long-long-suffix: one of
959/// ll LL
960///
961/// floating-constant: [C99 6.4.4.2]
962/// TODO: add rules...
963///
964NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
965 SourceLocation TokLoc,
966 const SourceManager &SM,
967 const LangOptions &LangOpts,
968 const TargetInfo &Target,
969 DiagnosticsEngine &Diags)
970 : SM(SM), LangOpts(LangOpts), Diags(Diags),
971 ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
972
973 s = DigitsBegin = ThisTokBegin;
974 saw_exponent = false;
975 saw_period = false;
976 saw_ud_suffix = false;
977 saw_fixed_point_suffix = false;
978 isLong = false;
979 isUnsigned = false;
980 isLongLong = false;
981 isSizeT = false;
982 isHalf = false;
983 isFloat = false;
984 isImaginary = false;
985 isFloat16 = false;
986 isFloat128 = false;
987 MicrosoftInteger = 0;
988 isFract = false;
989 isAccum = false;
990 hadError = false;
991 isBitInt = false;
992
993 // This routine assumes that the range begin/end matches the regex for integer
994 // and FP constants (specifically, the 'pp-number' regex), and assumes that
995 // the byte at "*end" is both valid and not part of the regex. Because of
996 // this, it doesn't have to check for 'overscan' in various places.
997 // Note: For HLSL, the end token is allowed to be '.' which would be in the
998 // 'pp-number' regex. This is required to support vector swizzles on numeric
999 // constants (i.e. 1.xx or 1.5f.rrr).
1000 if (isPreprocessingNumberBody(c: *ThisTokEnd) &&
1001 !(LangOpts.HLSL && *ThisTokEnd == '.')) {
1002 Diags.Report(Loc: TokLoc, DiagID: diag::err_lexing_numeric);
1003 hadError = true;
1004 return;
1005 }
1006
1007 if (*s == '0') { // parse radix
1008 ParseNumberStartingWithZero(TokLoc);
1009 if (hadError)
1010 return;
1011 } else { // the first digit is non-zero
1012 radix = 10;
1013 s = SkipDigits(ptr: s);
1014 if (s == ThisTokEnd) {
1015 // Done.
1016 } else {
1017 ParseDecimalOrOctalCommon(TokLoc);
1018 if (hadError)
1019 return;
1020 }
1021 }
1022
1023 SuffixBegin = s;
1024 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1025
1026 // Initial scan to lookahead for fixed point suffix.
1027 if (LangOpts.FixedPoint) {
1028 for (const char *c = s; c != ThisTokEnd; ++c) {
1029 if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
1030 saw_fixed_point_suffix = true;
1031 break;
1032 }
1033 }
1034 }
1035
1036 // Parse the suffix. At this point we can classify whether we have an FP or
1037 // integer constant.
1038 bool isFixedPointConstant = isFixedPointLiteral();
1039 bool isFPConstant = isFloatingLiteral();
1040 bool HasSize = false;
1041 bool DoubleUnderscore = false;
1042
1043 // Loop over all of the characters of the suffix. If we see something bad,
1044 // we break out of the loop.
1045 for (; s != ThisTokEnd; ++s) {
1046 switch (*s) {
1047 case 'R':
1048 case 'r':
1049 if (!LangOpts.FixedPoint)
1050 break;
1051 if (isFract || isAccum) break;
1052 if (!(saw_period || saw_exponent)) break;
1053 isFract = true;
1054 continue;
1055 case 'K':
1056 case 'k':
1057 if (!LangOpts.FixedPoint)
1058 break;
1059 if (isFract || isAccum) break;
1060 if (!(saw_period || saw_exponent)) break;
1061 isAccum = true;
1062 continue;
1063 case 'h': // FP Suffix for "half".
1064 case 'H':
1065 // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
1066 if (!(LangOpts.Half || LangOpts.FixedPoint))
1067 break;
1068 if (isIntegerLiteral()) break; // Error for integer constant.
1069 if (HasSize)
1070 break;
1071 HasSize = true;
1072 isHalf = true;
1073 continue; // Success.
1074 case 'f': // FP Suffix for "float"
1075 case 'F':
1076 if (!isFPConstant) break; // Error for integer constant.
1077 if (HasSize)
1078 break;
1079 HasSize = true;
1080
1081 // CUDA host and device may have different _Float16 support, therefore
1082 // allows f16 literals to avoid false alarm.
1083 // When we compile for OpenMP target offloading on NVPTX, f16 suffix
1084 // should also be supported.
1085 // ToDo: more precise check for CUDA.
1086 // TODO: AMDGPU might also support it in the future.
1087 if ((Target.hasFloat16Type() || LangOpts.CUDA ||
1088 (LangOpts.OpenMPIsTargetDevice && Target.getTriple().isNVPTX())) &&
1089 s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
1090 s += 2; // success, eat up 2 characters.
1091 isFloat16 = true;
1092 continue;
1093 }
1094
1095 isFloat = true;
1096 continue; // Success.
1097 case 'q': // FP Suffix for "__float128"
1098 case 'Q':
1099 if (!isFPConstant) break; // Error for integer constant.
1100 if (HasSize)
1101 break;
1102 HasSize = true;
1103 isFloat128 = true;
1104 continue; // Success.
1105 case 'u':
1106 case 'U':
1107 if (isFPConstant) break; // Error for floating constant.
1108 if (isUnsigned) break; // Cannot be repeated.
1109 isUnsigned = true;
1110 continue; // Success.
1111 case 'l':
1112 case 'L':
1113 if (HasSize)
1114 break;
1115 HasSize = true;
1116
1117 // Check for long long. The L's need to be adjacent and the same case.
1118 if (s[1] == s[0]) {
1119 assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
1120 if (isFPConstant) break; // long long invalid for floats.
1121 isLongLong = true;
1122 ++s; // Eat both of them.
1123 } else {
1124 isLong = true;
1125 }
1126 continue; // Success.
1127 case 'z':
1128 case 'Z':
1129 if (isFPConstant)
1130 break; // Invalid for floats.
1131 if (HasSize)
1132 break;
1133 HasSize = true;
1134 isSizeT = true;
1135 continue;
1136 case 'i':
1137 case 'I':
1138 if (LangOpts.MicrosoftExt && s + 1 < ThisTokEnd && !isFPConstant) {
1139 // Allow i8, i16, i32, i64, and i128. First, look ahead and check if
1140 // suffixes are Microsoft integers and not the imaginary unit.
1141 uint8_t Bits = 0;
1142 size_t ToSkip = 0;
1143 switch (s[1]) {
1144 case '8': // i8 suffix
1145 Bits = 8;
1146 ToSkip = 2;
1147 break;
1148 case '1':
1149 if (s + 2 < ThisTokEnd && s[2] == '6') { // i16 suffix
1150 Bits = 16;
1151 ToSkip = 3;
1152 } else if (s + 3 < ThisTokEnd && s[2] == '2' &&
1153 s[3] == '8') { // i128 suffix
1154 Bits = 128;
1155 ToSkip = 4;
1156 }
1157 break;
1158 case '3':
1159 if (s + 2 < ThisTokEnd && s[2] == '2') { // i32 suffix
1160 Bits = 32;
1161 ToSkip = 3;
1162 }
1163 break;
1164 case '6':
1165 if (s + 2 < ThisTokEnd && s[2] == '4') { // i64 suffix
1166 Bits = 64;
1167 ToSkip = 3;
1168 }
1169 break;
1170 default:
1171 break;
1172 }
1173 if (Bits) {
1174 if (HasSize)
1175 break;
1176 HasSize = true;
1177 MicrosoftInteger = Bits;
1178 s += ToSkip;
1179 assert(s <= ThisTokEnd && "didn't maximally munch?");
1180 break;
1181 }
1182 }
1183 [[fallthrough]];
1184 case 'j':
1185 case 'J':
1186 if (isImaginary) break; // Cannot be repeated.
1187 isImaginary = true;
1188 continue; // Success.
1189 case '_':
1190 if (isFPConstant)
1191 break; // Invalid for floats
1192 if (HasSize)
1193 break;
1194 // There is currently no way to reach this with DoubleUnderscore set.
1195 // If new double underscope literals are added handle it here as above.
1196 assert(!DoubleUnderscore && "unhandled double underscore case");
1197 if (LangOpts.CPlusPlus && s + 2 < ThisTokEnd &&
1198 s[1] == '_') { // s + 2 < ThisTokEnd to ensure some character exists
1199 // after __
1200 DoubleUnderscore = true;
1201 s += 2; // Skip both '_'
1202 if (s + 1 < ThisTokEnd &&
1203 (*s == 'u' || *s == 'U')) { // Ensure some character after 'u'/'U'
1204 isUnsigned = true;
1205 ++s;
1206 }
1207 if (s + 1 < ThisTokEnd &&
1208 ((*s == 'w' && *(++s) == 'b') || (*s == 'W' && *(++s) == 'B'))) {
1209 isBitInt = true;
1210 HasSize = true;
1211 continue;
1212 }
1213 }
1214 break;
1215 case 'w':
1216 case 'W':
1217 if (isFPConstant)
1218 break; // Invalid for floats.
1219 if (HasSize)
1220 break; // Invalid if we already have a size for the literal.
1221
1222 // wb and WB are allowed, but a mixture of cases like Wb or wB is not. We
1223 // explicitly do not support the suffix in C++ as an extension because a
1224 // library-based UDL that resolves to a library type may be more
1225 // appropriate there. The same rules apply for __wb/__WB.
1226 if ((!LangOpts.CPlusPlus || DoubleUnderscore) && s + 1 < ThisTokEnd &&
1227 ((s[0] == 'w' && s[1] == 'b') || (s[0] == 'W' && s[1] == 'B'))) {
1228 isBitInt = true;
1229 HasSize = true;
1230 ++s; // Skip both characters (2nd char skipped on continue).
1231 continue; // Success.
1232 }
1233 }
1234 // If we reached here, there was an error or a ud-suffix.
1235 break;
1236 }
1237
1238 // "i", "if", and "il" are user-defined suffixes in C++1y.
1239 if (s != ThisTokEnd || isImaginary) {
1240 // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
1241 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
1242 if (isValidUDSuffix(LangOpts, Suffix: UDSuffixBuf)) {
1243 if (!isImaginary) {
1244 // Any suffix pieces we might have parsed are actually part of the
1245 // ud-suffix.
1246 isLong = false;
1247 isUnsigned = false;
1248 isLongLong = false;
1249 isSizeT = false;
1250 isFloat = false;
1251 isFloat16 = false;
1252 isHalf = false;
1253 isImaginary = false;
1254 isBitInt = false;
1255 MicrosoftInteger = 0;
1256 saw_fixed_point_suffix = false;
1257 isFract = false;
1258 isAccum = false;
1259 }
1260
1261 saw_ud_suffix = true;
1262 return;
1263 }
1264
1265 if (s != ThisTokEnd) {
1266 // Report an error if there are any.
1267 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1268 TokStart: TokLoc, Characters: SuffixBegin - ThisTokBegin, SM, LangOpts),
1269 DiagID: diag::err_invalid_suffix_constant)
1270 << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
1271 << (isFixedPointConstant ? 2 : isFPConstant);
1272 hadError = true;
1273 }
1274 }
1275
1276 if (!hadError && saw_fixed_point_suffix) {
1277 assert(isFract || isAccum);
1278 }
1279}
1280
1281/// ParseDecimalOrOctalCommon - This method is called for decimal or octal
1282/// numbers. It issues an error for illegal digits, and handles floating point
1283/// parsing. If it detects a floating point number, the radix is set to 10.
1284void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
1285 assert((radix == 8 || radix == 10) && "Unexpected radix");
1286
1287 // If we have a hex digit other than 'e' (which denotes a FP exponent) then
1288 // the code is using an incorrect base.
1289 if (isHexDigit(c: *s) && *s != 'e' && *s != 'E' &&
1290 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1291 Diags.Report(
1292 Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts),
1293 DiagID: diag::err_invalid_digit)
1294 << StringRef(s, 1) << (radix == 8 ? 1 : 0);
1295 hadError = true;
1296 return;
1297 }
1298
1299 if (*s == '.') {
1300 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1301 s++;
1302 radix = 10;
1303 saw_period = true;
1304 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1305 s = SkipDigits(ptr: s); // Skip suffix.
1306 }
1307 if (*s == 'e' || *s == 'E') { // exponent
1308 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1309 const char *Exponent = s;
1310 s++;
1311 radix = 10;
1312 saw_exponent = true;
1313 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1314 const char *first_non_digit = SkipDigits(ptr: s);
1315 if (containsDigits(Start: s, End: first_non_digit)) {
1316 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1317 s = first_non_digit;
1318 } else {
1319 if (!hadError) {
1320 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1321 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1322 DiagID: diag::err_exponent_has_no_digits);
1323 hadError = true;
1324 }
1325 return;
1326 }
1327 }
1328}
1329
1330/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1331/// suffixes as ud-suffixes, because the diagnostic experience is better if we
1332/// treat it as an invalid suffix.
1333bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1334 StringRef Suffix) {
1335 if (!LangOpts.CPlusPlus11 || Suffix.empty())
1336 return false;
1337
1338 // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
1339 // Suffixes starting with '__' (double underscore) are for use by
1340 // the implementation.
1341 if (Suffix.starts_with(Prefix: "_") && !Suffix.starts_with(Prefix: "__"))
1342 return true;
1343
1344 // In C++11, there are no library suffixes.
1345 if (!LangOpts.CPlusPlus14)
1346 return false;
1347
1348 // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
1349 // Per tweaked N3660, "il", "i", and "if" are also used in the library.
1350 // In C++2a "d" and "y" are used in the library.
1351 return llvm::StringSwitch<bool>(Suffix)
1352 .Cases(CaseStrings: {"h", "min", "s"}, Value: true)
1353 .Cases(CaseStrings: {"ms", "us", "ns"}, Value: true)
1354 .Cases(CaseStrings: {"il", "i", "if"}, Value: true)
1355 .Cases(CaseStrings: {"d", "y"}, Value: LangOpts.CPlusPlus20)
1356 .Default(Value: false);
1357}
1358
1359void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
1360 const char *Pos,
1361 CheckSeparatorKind IsAfterDigits) {
1362 if (IsAfterDigits == CSK_AfterDigits) {
1363 if (Pos == ThisTokBegin)
1364 return;
1365 --Pos;
1366 } else if (Pos == ThisTokEnd)
1367 return;
1368
1369 if (isDigitSeparator(C: *Pos)) {
1370 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: Pos - ThisTokBegin, SM,
1371 LangOpts),
1372 DiagID: diag::err_digit_separator_not_between_digits)
1373 << IsAfterDigits;
1374 hadError = true;
1375 }
1376}
1377
1378/// ParseNumberStartingWithZero - This method is called when the first character
1379/// of the number is found to be a zero. This means it is either an octal
1380/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
1381/// a floating point number (01239.123e4). Eat the prefix, determining the
1382/// radix etc.
1383void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
1384 assert(s[0] == '0' && "Invalid method call");
1385 s++;
1386
1387 int c1 = s[0];
1388
1389 // Handle a hex number like 0x1234.
1390 if ((c1 == 'x' || c1 == 'X') && (isHexDigit(c: s[1]) || s[1] == '.')) {
1391 s++;
1392 assert(s < ThisTokEnd && "didn't maximally munch?");
1393 radix = 16;
1394 DigitsBegin = s;
1395 s = SkipHexDigits(ptr: s);
1396 bool HasSignificandDigits = containsDigits(Start: DigitsBegin, End: s);
1397 if (s == ThisTokEnd) {
1398 // Done.
1399 } else if (*s == '.') {
1400 s++;
1401 saw_period = true;
1402 const char *floatDigitsBegin = s;
1403 s = SkipHexDigits(ptr: s);
1404 if (containsDigits(Start: floatDigitsBegin, End: s))
1405 HasSignificandDigits = true;
1406 if (HasSignificandDigits)
1407 checkSeparator(TokLoc, Pos: floatDigitsBegin, IsAfterDigits: CSK_BeforeDigits);
1408 }
1409
1410 if (!HasSignificandDigits) {
1411 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1412 LangOpts),
1413 DiagID: diag::err_hex_constant_requires)
1414 << LangOpts.CPlusPlus << 1;
1415 hadError = true;
1416 return;
1417 }
1418
1419 // A binary exponent can appear with or with a '.'. If dotted, the
1420 // binary exponent is required.
1421 if (*s == 'p' || *s == 'P') {
1422 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1423 const char *Exponent = s;
1424 s++;
1425 saw_exponent = true;
1426 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1427 const char *first_non_digit = SkipDigits(ptr: s);
1428 if (!containsDigits(Start: s, End: first_non_digit)) {
1429 if (!hadError) {
1430 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1431 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1432 DiagID: diag::err_exponent_has_no_digits);
1433 hadError = true;
1434 }
1435 return;
1436 }
1437 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1438 s = first_non_digit;
1439
1440 if (!LangOpts.HexFloats)
1441 Diags.Report(Loc: TokLoc, DiagID: LangOpts.CPlusPlus
1442 ? diag::ext_hex_literal_invalid
1443 : diag::ext_hex_constant_invalid);
1444 else if (LangOpts.CPlusPlus17)
1445 Diags.Report(Loc: TokLoc, DiagID: diag::warn_cxx17_hex_literal);
1446 } else if (saw_period) {
1447 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1448 LangOpts),
1449 DiagID: diag::err_hex_constant_requires)
1450 << LangOpts.CPlusPlus << 0;
1451 hadError = true;
1452 }
1453 return;
1454 }
1455
1456 // Handle simple binary numbers 0b01010
1457 if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
1458 // 0b101010 is a C++14 and C23 extension.
1459 unsigned DiagId;
1460 if (LangOpts.CPlusPlus14)
1461 DiagId = diag::warn_cxx11_compat_binary_literal;
1462 else if (LangOpts.C23)
1463 DiagId = diag::warn_c23_compat_binary_literal;
1464 else if (LangOpts.CPlusPlus)
1465 DiagId = diag::ext_binary_literal_cxx14;
1466 else
1467 DiagId = diag::ext_binary_literal;
1468 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1469 ++s;
1470 assert(s < ThisTokEnd && "didn't maximally munch?");
1471 radix = 2;
1472 DigitsBegin = s;
1473 s = SkipBinaryDigits(ptr: s);
1474 if (s == ThisTokEnd) {
1475 // Done.
1476 } else if (isHexDigit(c: *s) &&
1477 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1478 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1479 LangOpts),
1480 DiagID: diag::err_invalid_digit)
1481 << StringRef(s, 1) << 2;
1482 hadError = true;
1483 }
1484 // Other suffixes will be diagnosed by the caller.
1485 return;
1486 }
1487
1488 // Parse a potential octal literal prefix.
1489 bool IsSingleZero = false;
1490 if ((c1 == 'O' || c1 == 'o') && (s[1] >= '0' && s[1] <= '7')) {
1491 unsigned DiagId;
1492 if (LangOpts.C2y)
1493 DiagId = diag::warn_c2y_compat_octal_literal;
1494 else if (LangOpts.CPlusPlus)
1495 DiagId = diag::ext_cpp_octal_literal;
1496 else
1497 DiagId = diag::ext_octal_literal;
1498 // If the token location is from a macro expansion where the macro was
1499 // defined in a system header, suppress the diagnostic.
1500 // FIXME: this is actually a more general issue, for example we have a
1501 // similar need for binary literals above. It would be best for this to be
1502 // handled by the diagnostics engine instead of with ad hoc solutions. This
1503 // same concern exists below for issuing the deprecation warning.
1504 if (!SM.isInSystemMacro(loc: TokLoc))
1505 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1506
1507 ++s;
1508 DigitsBegin = s;
1509 radix = 8;
1510 s = SkipOctalDigits(ptr: s);
1511 if (s == ThisTokEnd) {
1512 // Done
1513 } else if ((isHexDigit(c: *s) && *s != 'e' && *s != 'E' && *s != '.') &&
1514 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1515 auto InvalidDigitLoc = Lexer::AdvanceToTokenCharacter(
1516 TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts);
1517 Diags.Report(Loc: InvalidDigitLoc, DiagID: diag::err_invalid_digit)
1518 << StringRef(s, 1) << 1;
1519 hadError = true;
1520 }
1521 // Other suffixes will be diagnosed by the caller.
1522 return;
1523 }
1524
1525 llvm::scope_exit _([&] {
1526 // If we still have an octal value but we did not see an octal prefix,
1527 // diagnose as being an obsolescent feature starting in C2y. If the token
1528 // location is from a macro expansion where the macro was defined in a
1529 // system header, suppress the diagnostic.
1530 if (radix == 8 && LangOpts.C2y && !hadError && !IsSingleZero &&
1531 !SM.isInSystemMacro(loc: TokLoc))
1532 Diags.Report(Loc: TokLoc, DiagID: diag::warn_unprefixed_octal_deprecated);
1533 });
1534
1535 // For now, the radix is set to 8. If we discover that we have a
1536 // floating point constant, the radix will change to 10. Octal floating
1537 // point constants are not permitted (only decimal and hexadecimal).
1538 radix = 8;
1539 const char *PossibleNewDigitStart = s;
1540 s = SkipOctalDigits(ptr: s);
1541 // When the value is 0 followed by a suffix (like 0wb), we want to leave 0
1542 // as the start of the digits. So if skipping octal digits does not skip
1543 // anything, we leave the digit start where it was.
1544 if (s != PossibleNewDigitStart)
1545 DigitsBegin = PossibleNewDigitStart;
1546 else
1547 IsSingleZero = (s == ThisTokBegin + 1);
1548
1549 if (s == ThisTokEnd)
1550 return; // Done, simple octal number like 01234
1551
1552 // If we have some other non-octal digit that *is* a decimal digit, see if
1553 // this is part of a floating point number like 094.123 or 09e1.
1554 if (isDigit(c: *s)) {
1555 const char *EndDecimal = SkipDigits(ptr: s);
1556 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
1557 s = EndDecimal;
1558 radix = 10;
1559 }
1560 }
1561
1562 ParseDecimalOrOctalCommon(TokLoc);
1563}
1564
1565static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
1566 switch (Radix) {
1567 case 2:
1568 return NumDigits <= 64;
1569 case 8:
1570 return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
1571 case 10:
1572 return NumDigits <= 19; // floor(log10(2^64))
1573 case 16:
1574 return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
1575 default:
1576 llvm_unreachable("impossible Radix");
1577 }
1578}
1579
1580/// GetIntegerValue - Convert this numeric literal value to an APInt that
1581/// matches Val's input width. If there is an overflow, set Val to the low bits
1582/// of the result and return true. Otherwise, return false.
1583bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1584 // Fast path: Compute a conservative bound on the maximum number of
1585 // bits per digit in this radix. If we can't possibly overflow a
1586 // uint64 based on that bound then do the simple conversion to
1587 // integer. This avoids the expensive overflow checking below, and
1588 // handles the common cases that matter (small decimal integers and
1589 // hex/octal values which don't overflow).
1590 const unsigned NumDigits = SuffixBegin - DigitsBegin;
1591 if (alwaysFitsInto64Bits(Radix: radix, NumDigits)) {
1592 uint64_t N = 0;
1593 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
1594 if (!isDigitSeparator(C: *Ptr))
1595 N = N * radix + llvm::hexDigitValue(C: *Ptr);
1596
1597 // This will truncate the value to Val's input width. Simply check
1598 // for overflow by comparing.
1599 Val = N;
1600 return Val.getZExtValue() != N;
1601 }
1602
1603 Val = 0;
1604 const char *Ptr = DigitsBegin;
1605
1606 llvm::APInt RadixVal(Val.getBitWidth(), radix);
1607 llvm::APInt CharVal(Val.getBitWidth(), 0);
1608 llvm::APInt OldVal = Val;
1609
1610 bool OverflowOccurred = false;
1611 while (Ptr < SuffixBegin) {
1612 if (isDigitSeparator(C: *Ptr)) {
1613 ++Ptr;
1614 continue;
1615 }
1616
1617 unsigned C = llvm::hexDigitValue(C: *Ptr++);
1618
1619 // If this letter is out of bound for this radix, reject it.
1620 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1621
1622 CharVal = C;
1623
1624 // Add the digit to the value in the appropriate radix. If adding in digits
1625 // made the value smaller, then this overflowed.
1626 OldVal = Val;
1627
1628 // Multiply by radix, did overflow occur on the multiply?
1629 Val *= RadixVal;
1630 OverflowOccurred |= Val.udiv(RHS: RadixVal) != OldVal;
1631
1632 // Add value, did overflow occur on the value?
1633 // (a + b) ult b <=> overflow
1634 Val += CharVal;
1635 OverflowOccurred |= Val.ult(RHS: CharVal);
1636 }
1637 return OverflowOccurred;
1638}
1639
1640llvm::APFloat::opStatus
1641NumericLiteralParser::GetFloatValue(llvm::APFloat &Result,
1642 llvm::RoundingMode RM) {
1643 using llvm::APFloat;
1644
1645 unsigned n = std::min(a: SuffixBegin - ThisTokBegin, b: ThisTokEnd - ThisTokBegin);
1646
1647 llvm::SmallString<16> Buffer;
1648 StringRef Str(ThisTokBegin, n);
1649 if (Str.contains(C: '\'')) {
1650 Buffer.reserve(N: n);
1651 std::remove_copy_if(first: Str.begin(), last: Str.end(), result: std::back_inserter(x&: Buffer),
1652 pred: &isDigitSeparator);
1653 Str = Buffer;
1654 }
1655
1656 auto StatusOrErr = Result.convertFromString(Str, RM);
1657 assert(StatusOrErr && "Invalid floating point representation");
1658 return !errorToBool(Err: StatusOrErr.takeError()) ? *StatusOrErr
1659 : APFloat::opInvalidOp;
1660}
1661
1662static inline bool IsExponentPart(char c, bool isHex) {
1663 if (isHex)
1664 return c == 'p' || c == 'P';
1665 return c == 'e' || c == 'E';
1666}
1667
1668bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1669 assert(radix == 16 || radix == 10);
1670
1671 // Find how many digits are needed to store the whole literal.
1672 unsigned NumDigits = SuffixBegin - DigitsBegin;
1673 if (saw_period) --NumDigits;
1674
1675 // Initial scan of the exponent if it exists
1676 bool ExpOverflowOccurred = false;
1677 bool NegativeExponent = false;
1678 const char *ExponentBegin;
1679 uint64_t Exponent = 0;
1680 int64_t BaseShift = 0;
1681 if (saw_exponent) {
1682 const char *Ptr = DigitsBegin;
1683
1684 while (!IsExponentPart(c: *Ptr, isHex: radix == 16))
1685 ++Ptr;
1686 ExponentBegin = Ptr;
1687 ++Ptr;
1688 NegativeExponent = *Ptr == '-';
1689 if (NegativeExponent) ++Ptr;
1690
1691 unsigned NumExpDigits = SuffixBegin - Ptr;
1692 if (alwaysFitsInto64Bits(Radix: radix, NumDigits: NumExpDigits)) {
1693 llvm::StringRef ExpStr(Ptr, NumExpDigits);
1694 llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1695 Exponent = ExpInt.getZExtValue();
1696 } else {
1697 ExpOverflowOccurred = true;
1698 }
1699
1700 if (NegativeExponent) BaseShift -= Exponent;
1701 else BaseShift += Exponent;
1702 }
1703
1704 // Number of bits needed for decimal literal is
1705 // ceil(NumDigits * log2(10)) Integral part
1706 // + Scale Fractional part
1707 // + ceil(Exponent * log2(10)) Exponent
1708 // --------------------------------------------------
1709 // ceil((NumDigits + Exponent) * log2(10)) + Scale
1710 //
1711 // But for simplicity in handling integers, we can round up log2(10) to 4,
1712 // making:
1713 // 4 * (NumDigits + Exponent) + Scale
1714 //
1715 // Number of digits needed for hexadecimal literal is
1716 // 4 * NumDigits Integral part
1717 // + Scale Fractional part
1718 // + Exponent Exponent
1719 // --------------------------------------------------
1720 // (4 * NumDigits) + Scale + Exponent
1721 uint64_t NumBitsNeeded;
1722 if (radix == 10)
1723 NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1724 else
1725 NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1726
1727 if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1728 ExpOverflowOccurred = true;
1729 llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1730
1731 bool FoundDecimal = false;
1732
1733 int64_t FractBaseShift = 0;
1734 const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1735 for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1736 if (*Ptr == '.') {
1737 FoundDecimal = true;
1738 continue;
1739 }
1740
1741 // Normal reading of an integer
1742 unsigned C = llvm::hexDigitValue(C: *Ptr);
1743 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1744
1745 Val *= radix;
1746 Val += C;
1747
1748 if (FoundDecimal)
1749 // Keep track of how much we will need to adjust this value by from the
1750 // number of digits past the radix point.
1751 --FractBaseShift;
1752 }
1753
1754 // For a radix of 16, we will be multiplying by 2 instead of 16.
1755 if (radix == 16) FractBaseShift *= 4;
1756 BaseShift += FractBaseShift;
1757
1758 Val <<= Scale;
1759
1760 uint64_t Base = (radix == 16) ? 2 : 10;
1761 if (BaseShift > 0) {
1762 for (int64_t i = 0; i < BaseShift; ++i) {
1763 Val *= Base;
1764 }
1765 } else if (BaseShift < 0) {
1766 for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i)
1767 Val = Val.udiv(RHS: Base);
1768 }
1769
1770 bool IntOverflowOccurred = false;
1771 auto MaxVal = llvm::APInt::getMaxValue(numBits: StoreVal.getBitWidth());
1772 if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1773 IntOverflowOccurred |= Val.ugt(RHS: MaxVal.zext(width: Val.getBitWidth()));
1774 StoreVal = Val.trunc(width: StoreVal.getBitWidth());
1775 } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1776 IntOverflowOccurred |= Val.zext(width: MaxVal.getBitWidth()).ugt(RHS: MaxVal);
1777 StoreVal = Val.zext(width: StoreVal.getBitWidth());
1778 } else {
1779 StoreVal = std::move(Val);
1780 }
1781
1782 return IntOverflowOccurred || ExpOverflowOccurred;
1783}
1784
1785/// \verbatim
1786/// user-defined-character-literal: [C++11 lex.ext]
1787/// character-literal ud-suffix
1788/// ud-suffix:
1789/// identifier
1790/// character-literal: [C++11 lex.ccon]
1791/// ' c-char-sequence '
1792/// u' c-char-sequence '
1793/// U' c-char-sequence '
1794/// L' c-char-sequence '
1795/// u8' c-char-sequence ' [C++1z lex.ccon]
1796/// c-char-sequence:
1797/// c-char
1798/// c-char-sequence c-char
1799/// c-char:
1800/// any member of the source character set except the single-quote ',
1801/// backslash \, or new-line character
1802/// escape-sequence
1803/// universal-character-name
1804/// escape-sequence:
1805/// simple-escape-sequence
1806/// octal-escape-sequence
1807/// hexadecimal-escape-sequence
1808/// simple-escape-sequence:
1809/// one of \' \" \? \\ \a \b \f \n \r \t \v
1810/// octal-escape-sequence:
1811/// \ octal-digit
1812/// \ octal-digit octal-digit
1813/// \ octal-digit octal-digit octal-digit
1814/// hexadecimal-escape-sequence:
1815/// \x hexadecimal-digit
1816/// hexadecimal-escape-sequence hexadecimal-digit
1817/// universal-character-name: [C++11 lex.charset]
1818/// \u hex-quad
1819/// \U hex-quad hex-quad
1820/// hex-quad:
1821/// hex-digit hex-digit hex-digit hex-digit
1822/// \endverbatim
1823///
1824CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1825 SourceLocation Loc, Preprocessor &PP,
1826 tok::TokenKind kind) {
1827 // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1828 HadError = false;
1829
1830 Kind = kind;
1831
1832 const char *TokBegin = begin;
1833
1834 // Skip over wide character determinant.
1835 if (Kind != tok::char_constant)
1836 ++begin;
1837 if (Kind == tok::utf8_char_constant)
1838 ++begin;
1839
1840 // Skip over the entry quote.
1841 if (begin[0] != '\'') {
1842 PP.Diag(Loc, DiagID: diag::err_lexing_char);
1843 HadError = true;
1844 return;
1845 }
1846
1847 ++begin;
1848
1849 // Remove an optional ud-suffix.
1850 if (end[-1] != '\'') {
1851 const char *UDSuffixEnd = end;
1852 do {
1853 --end;
1854 } while (end[-1] != '\'');
1855 // FIXME: Don't bother with this if !tok.hasUCN().
1856 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(end, UDSuffixEnd - end));
1857 UDSuffixOffset = end - TokBegin;
1858 }
1859
1860 // Trim the ending quote.
1861 assert(end != begin && "Invalid token lexed");
1862 --end;
1863
1864 // FIXME: The "Value" is an uint64_t so we can handle char literals of
1865 // up to 64-bits.
1866 // FIXME: This extensively assumes that 'char' is 8-bits.
1867 assert(PP.getTargetInfo().getCharWidth() == 8 &&
1868 "Assumes char is 8 bits");
1869 assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1870 (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1871 "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1872 assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1873 "Assumes sizeof(wchar) on target is <= 64");
1874
1875 SmallVector<uint32_t, 4> codepoint_buffer;
1876 codepoint_buffer.resize(N: end - begin);
1877 uint32_t *buffer_begin = &codepoint_buffer.front();
1878 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1879
1880 const TextEncoding &TE = PP.getTextEncoding();
1881 llvm::TextEncodingConverter *Converter = nullptr;
1882 if (isOrdinary())
1883 Converter = TE.getConverter(Action: CA_ToLiteralEncoding);
1884
1885 // Unicode escapes representing characters that cannot be correctly
1886 // represented in a single code unit are disallowed in character literals
1887 // by this implementation.
1888 uint32_t largest_character_for_kind;
1889 if (tok::wide_char_constant == Kind) {
1890 largest_character_for_kind =
1891 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1892 } else if (tok::utf8_char_constant == Kind) {
1893 largest_character_for_kind = 0x7F;
1894 } else if (tok::utf16_char_constant == Kind) {
1895 largest_character_for_kind = 0xFFFF;
1896 } else if (tok::utf32_char_constant == Kind) {
1897 largest_character_for_kind = 0x10FFFF;
1898 } else {
1899 largest_character_for_kind = (Converter == nullptr) ? 0x7Fu : 0xFFu;
1900 }
1901
1902 while (begin != end) {
1903 // Is this a span of non-escape characters?
1904 if (begin[0] != '\\') {
1905 char const *start = begin;
1906 do {
1907 ++begin;
1908 } while (begin != end && *begin != '\\');
1909
1910 char const *tmp_in_start = start;
1911 uint32_t *tmp_out_start = buffer_begin;
1912 std::string UTF8String(start, begin);
1913 llvm::ConversionResult res =
1914 llvm::ConvertUTF8toUTF32(sourceStart: reinterpret_cast<llvm::UTF8 const **>(&start),
1915 sourceEnd: reinterpret_cast<llvm::UTF8 const *>(begin),
1916 targetStart: &buffer_begin, targetEnd: buffer_end, flags: llvm::strictConversion);
1917 if (res != llvm::conversionOK) {
1918 // If we see bad encoding for unprefixed character literals, warn and
1919 // simply copy the byte values, for compatibility with gcc and
1920 // older versions of clang.
1921 bool NoErrorOnBadEncoding = isOrdinary();
1922 unsigned Msg = diag::err_bad_character_encoding;
1923 if (NoErrorOnBadEncoding)
1924 Msg = diag::warn_bad_character_encoding;
1925 PP.Diag(Loc, DiagID: Msg);
1926 if (NoErrorOnBadEncoding) {
1927 start = tmp_in_start;
1928 buffer_begin = tmp_out_start;
1929 for (; start != begin; ++start, ++buffer_begin)
1930 *buffer_begin = static_cast<uint8_t>(*start);
1931 } else {
1932 HadError = true;
1933 }
1934 } else {
1935 uint32_t *validation_ptr = tmp_out_start;
1936 for (; validation_ptr < buffer_begin; ++validation_ptr) {
1937 if (*validation_ptr > largest_character_for_kind) {
1938 HadError = true;
1939 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1940 }
1941 }
1942
1943 // Convert to execution character set if needed
1944 if (!HadError && Converter) {
1945 assert(isOrdinary() && "Only ordinary characters are supported");
1946 SmallString<4> Converted;
1947 auto ErrorOrChar = Converter->convert(Source: UTF8String, Result&: Converted);
1948 if (!ErrorOrChar) {
1949 for (int i = 0; tmp_out_start < buffer_begin;
1950 ++tmp_out_start, ++i) {
1951 *tmp_out_start = Converted[i];
1952 }
1953 } else {
1954 HadError = true;
1955 PP.Diag(Loc, DiagID: diag::err_exec_charset_conversion_failed)
1956 << ErrorOrChar.message();
1957 }
1958 }
1959 }
1960
1961 continue;
1962 }
1963 // Is this a Universal Character Name escape?
1964 if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') {
1965 if (Converter == nullptr) {
1966 unsigned short UcnLen = 0;
1967 if (!ProcessUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, UcnVal&: *buffer_begin, UcnLen,
1968 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1969 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(), in_char_string_literal: true)) {
1970 HadError = true;
1971 } else if (*buffer_begin > largest_character_for_kind) {
1972 HadError = true;
1973 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1974 }
1975 } else {
1976 char Cp[5];
1977 char *ResultPtr = Cp;
1978 EncodeUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, ResultBuf&: ResultPtr, HadError,
1979 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1980 /*CharByteWidth=*/1u, Diags: &PP.getDiagnostics(),
1981 Features: PP.getLangOpts(), Converter);
1982 if (!HadError)
1983 *buffer_begin = *Cp;
1984 }
1985 ++buffer_begin;
1986 continue;
1987 }
1988 unsigned CharWidth = getCharWidth(kind: Kind, Target: PP.getTargetInfo());
1989 uint64_t result =
1990 ProcessCharEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, HadError,
1991 Loc: FullSourceLoc(Loc, PP.getSourceManager()), CharWidth,
1992 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(),
1993 EvalMethod: StringLiteralEvalMethod::Evaluated, Converter: nullptr);
1994 *buffer_begin++ = result;
1995 }
1996
1997 unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1998
1999 if (NumCharsSoFar > 1) {
2000 if (isOrdinary() && NumCharsSoFar == 4)
2001 PP.Diag(Loc, DiagID: diag::warn_four_char_character_literal);
2002 else if (isOrdinary())
2003 PP.Diag(Loc, DiagID: diag::warn_multichar_character_literal);
2004 else {
2005 PP.Diag(Loc, DiagID: diag::err_multichar_character_literal) << (isWide() ? 0 : 1);
2006 HadError = true;
2007 }
2008 IsMultiChar = true;
2009 } else {
2010 IsMultiChar = false;
2011 }
2012
2013 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
2014
2015 // Narrow character literals act as though their value is concatenated
2016 // in this implementation, but warn on overflow.
2017 bool multi_char_too_long = false;
2018 if (isOrdinary() && isMultiChar()) {
2019 LitVal = 0;
2020 for (size_t i = 0; i < NumCharsSoFar; ++i) {
2021 // check for enough leading zeros to shift into
2022 multi_char_too_long |= (LitVal.countl_zero() < 8);
2023 LitVal <<= 8;
2024 LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
2025 }
2026 } else if (NumCharsSoFar > 0) {
2027 // otherwise just take the last character
2028 LitVal = buffer_begin[-1];
2029 }
2030
2031 if (!HadError && multi_char_too_long) {
2032 PP.Diag(Loc, DiagID: diag::warn_char_constant_too_large);
2033 }
2034
2035 // Transfer the value from APInt to uint64_t
2036 Value = LitVal.getZExtValue();
2037
2038 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
2039 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple
2040 // character constants are not sign extended in the this implementation:
2041 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
2042 if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) &&
2043 PP.getLangOpts().CharIsSigned)
2044 Value = (signed char)Value;
2045}
2046
2047/// \verbatim
2048/// string-literal: [C++0x lex.string]
2049/// encoding-prefix " [s-char-sequence] "
2050/// encoding-prefix R raw-string
2051/// encoding-prefix:
2052/// u8
2053/// u
2054/// U
2055/// L
2056/// s-char-sequence:
2057/// s-char
2058/// s-char-sequence s-char
2059/// s-char:
2060/// any member of the source character set except the double-quote ",
2061/// backslash \, or new-line character
2062/// escape-sequence
2063/// universal-character-name
2064/// raw-string:
2065/// " d-char-sequence ( r-char-sequence ) d-char-sequence "
2066/// r-char-sequence:
2067/// r-char
2068/// r-char-sequence r-char
2069/// r-char:
2070/// any member of the source character set, except a right parenthesis )
2071/// followed by the initial d-char-sequence (which may be empty)
2072/// followed by a double quote ".
2073/// d-char-sequence:
2074/// d-char
2075/// d-char-sequence d-char
2076/// d-char:
2077/// any member of the basic source character set except:
2078/// space, the left parenthesis (, the right parenthesis ),
2079/// the backslash \, and the control characters representing horizontal
2080/// tab, vertical tab, form feed, and newline.
2081/// escape-sequence: [C++0x lex.ccon]
2082/// simple-escape-sequence
2083/// octal-escape-sequence
2084/// hexadecimal-escape-sequence
2085/// simple-escape-sequence:
2086/// one of \' \" \? \\ \a \b \f \n \r \t \v
2087/// octal-escape-sequence:
2088/// \ octal-digit
2089/// \ octal-digit octal-digit
2090/// \ octal-digit octal-digit octal-digit
2091/// hexadecimal-escape-sequence:
2092/// \x hexadecimal-digit
2093/// hexadecimal-escape-sequence hexadecimal-digit
2094/// universal-character-name:
2095/// \u hex-quad
2096/// \U hex-quad hex-quad
2097/// hex-quad:
2098/// hex-digit hex-digit hex-digit hex-digit
2099/// \endverbatim
2100///
2101StringLiteralParser::StringLiteralParser(ArrayRef<Token> StringToks,
2102 Preprocessor &PP,
2103 StringLiteralEvalMethod EvalMethod,
2104 ConversionAction Action)
2105 : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
2106 Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()),
2107 TE(&PP.getTextEncoding()), MaxTokenLength(0), SizeBound(0),
2108 CharByteWidth(0), Kind(tok::unknown), ResultPtr(ResultBuf.data()),
2109 EvalMethod(EvalMethod), hadError(false), Pascal(false) {
2110 init(StringToks, Action);
2111}
2112
2113void StringLiteralParser::init(ArrayRef<Token> StringToks,
2114 ConversionAction Action) {
2115 // The literal token may have come from an invalid source location (e.g. due
2116 // to a PCH error), in which case the token length will be 0.
2117 if (StringToks.empty() || StringToks[0].getLength() < 2)
2118 return DiagnoseLexingError(Loc: SourceLocation());
2119
2120 // Scan all of the string portions, remember the max individual token length,
2121 // computing a bound on the concatenated string length, and see whether any
2122 // piece is a wide-string. If any of the string portions is a wide-string
2123 // literal, the result is a wide-string literal [C99 6.4.5p4].
2124 assert(!StringToks.empty() && "expected at least one token");
2125 MaxTokenLength = StringToks[0].getLength();
2126 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
2127 SizeBound = StringToks[0].getLength() - 2; // -2 for "".
2128 hadError = false;
2129
2130 // Determines the kind of string from the prefix
2131 Kind = tok::string_literal;
2132
2133 /// (C99 5.1.1.2p1). The common case is only one string fragment.
2134 for (const Token &Tok : StringToks) {
2135 if (Tok.getLength() < 2)
2136 return DiagnoseLexingError(Loc: Tok.getLocation());
2137
2138 // The string could be shorter than this if it needs cleaning, but this is a
2139 // reasonable bound, which is all we need.
2140 assert(Tok.getLength() >= 2 && "literal token is invalid!");
2141 SizeBound += Tok.getLength() - 2; // -2 for "".
2142
2143 // Remember maximum string piece length.
2144 if (Tok.getLength() > MaxTokenLength)
2145 MaxTokenLength = Tok.getLength();
2146
2147 // Remember if we see any wide or utf-8/16/32 strings.
2148 // Also check for illegal concatenations.
2149 if (isUnevaluated() && Tok.getKind() != tok::string_literal) {
2150 if (Diags) {
2151 SourceLocation PrefixEndLoc = Lexer::AdvanceToTokenCharacter(
2152 TokStart: Tok.getLocation(), Characters: getEncodingPrefixLen(kind: Tok.getKind()), SM,
2153 LangOpts: Features);
2154 CharSourceRange Range =
2155 CharSourceRange::getCharRange(R: {Tok.getLocation(), PrefixEndLoc});
2156 StringRef Prefix(SM.getCharacterData(SL: Tok.getLocation()),
2157 getEncodingPrefixLen(kind: Tok.getKind()));
2158 Diags->Report(Loc: Tok.getLocation(),
2159 DiagID: Features.CPlusPlus26
2160 ? diag::err_unevaluated_string_prefix
2161 : diag::warn_unevaluated_string_prefix)
2162 << Prefix << Features.CPlusPlus << FixItHint::CreateRemoval(RemoveRange: Range);
2163 }
2164 if (Features.CPlusPlus26)
2165 hadError = true;
2166 } else if (Tok.isNot(K: Kind) && Tok.isNot(K: tok::string_literal)) {
2167 if (isOrdinary()) {
2168 Kind = Tok.getKind();
2169 } else {
2170 if (Diags)
2171 Diags->Report(Loc: Tok.getLocation(), DiagID: diag::err_unsupported_string_concat);
2172 hadError = true;
2173 }
2174 }
2175 }
2176
2177 // Include space for the null terminator.
2178 ++SizeBound;
2179
2180 // TODO: K&R warning: "traditional C rejects string constant concatenation"
2181
2182 // Get the width in bytes of char/wchar_t/char16_t/char32_t
2183 CharByteWidth = getCharWidth(kind: Kind, Target);
2184 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
2185 CharByteWidth /= 8;
2186
2187 // The output buffer size needs to be large enough to hold wide characters.
2188 // This is a worst-case assumption which basically corresponds to L"" "long".
2189 SizeBound *= CharByteWidth;
2190
2191 // Size the temporary buffer to hold the result string data.
2192 ResultBuf.resize(N: SizeBound);
2193
2194 // Likewise, but for each string piece.
2195 SmallString<512> TokenBuf;
2196 TokenBuf.resize(N: MaxTokenLength);
2197
2198 // Loop over all the strings, getting their spelling, and expanding them to
2199 // wide strings as appropriate.
2200 ResultPtr = &ResultBuf[0]; // Next byte to fill in.
2201
2202 Pascal = false;
2203
2204 SourceLocation UDSuffixTokLoc;
2205
2206 llvm::TextEncodingConverter *Converter = nullptr;
2207 if (isOrdinary() && TE)
2208 Converter = TE->getConverter(Action);
2209
2210 for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
2211 const char *ThisTokBuf = &TokenBuf[0];
2212 // Get the spelling of the token, which eliminates trigraphs, etc. We know
2213 // that ThisTokBuf points to a buffer that is big enough for the whole token
2214 // and 'spelled' tokens can only shrink.
2215 bool StringInvalid = false;
2216 unsigned ThisTokLen =
2217 Lexer::getSpelling(Tok: StringToks[i], Buffer&: ThisTokBuf, SourceMgr: SM, LangOpts: Features,
2218 Invalid: &StringInvalid);
2219 if (StringInvalid)
2220 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2221
2222 const char *ThisTokBegin = ThisTokBuf;
2223 const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
2224
2225 // Remove an optional ud-suffix.
2226 if (ThisTokEnd[-1] != '"') {
2227 const char *UDSuffixEnd = ThisTokEnd;
2228 do {
2229 --ThisTokEnd;
2230 } while (ThisTokEnd[-1] != '"');
2231
2232 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
2233
2234 if (UDSuffixBuf.empty()) {
2235 if (StringToks[i].hasUCN())
2236 expandUCNs(Buf&: UDSuffixBuf, Input: UDSuffix);
2237 else
2238 UDSuffixBuf.assign(RHS: UDSuffix);
2239 UDSuffixToken = i;
2240 UDSuffixOffset = ThisTokEnd - ThisTokBuf;
2241 UDSuffixTokLoc = StringToks[i].getLocation();
2242 } else {
2243 SmallString<32> ExpandedUDSuffix;
2244 if (StringToks[i].hasUCN()) {
2245 expandUCNs(Buf&: ExpandedUDSuffix, Input: UDSuffix);
2246 UDSuffix = ExpandedUDSuffix;
2247 }
2248
2249 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
2250 // result of a concatenation involving at least one user-defined-string-
2251 // literal, all the participating user-defined-string-literals shall
2252 // have the same ud-suffix.
2253 bool UnevaluatedStringHasUDL = isUnevaluated() && !UDSuffix.empty();
2254 if (UDSuffixBuf != UDSuffix || UnevaluatedStringHasUDL) {
2255 if (Diags) {
2256 SourceLocation TokLoc = StringToks[i].getLocation();
2257 if (UnevaluatedStringHasUDL) {
2258 Diags->Report(Loc: TokLoc, DiagID: diag::err_unevaluated_string_udl)
2259 << SourceRange(TokLoc, TokLoc);
2260 } else {
2261 Diags->Report(Loc: TokLoc, DiagID: diag::err_string_concat_mixed_suffix)
2262 << UDSuffixBuf << UDSuffix
2263 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc);
2264 }
2265 }
2266 hadError = true;
2267 }
2268 }
2269 }
2270
2271 // Strip the end quote.
2272 --ThisTokEnd;
2273
2274 // TODO: Input character set mapping support.
2275
2276 // Skip marker for wide or unicode strings.
2277 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
2278 ++ThisTokBuf;
2279 // Skip 8 of u8 marker for utf8 strings.
2280 if (ThisTokBuf[0] == '8')
2281 ++ThisTokBuf;
2282 }
2283
2284 // Check for raw string
2285 if (ThisTokBuf[0] == 'R') {
2286 if (ThisTokBuf[1] != '"') {
2287 // The file may have come from PCH and then changed after loading the
2288 // PCH; Fail gracefully.
2289 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2290 }
2291 ThisTokBuf += 2; // skip R"
2292
2293 // C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16
2294 // characters.
2295 constexpr unsigned MaxRawStrDelimLen = 16;
2296
2297 const char *Prefix = ThisTokBuf;
2298 while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen &&
2299 ThisTokBuf[0] != '(')
2300 ++ThisTokBuf;
2301 if (ThisTokBuf[0] != '(')
2302 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2303 ++ThisTokBuf; // skip '('
2304
2305 // Remove same number of characters from the end
2306 ThisTokEnd -= ThisTokBuf - Prefix;
2307 if (ThisTokEnd < ThisTokBuf)
2308 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2309
2310 // C++14 [lex.string]p4: A source-file new-line in a raw string literal
2311 // results in a new-line in the resulting execution string-literal.
2312 StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
2313 while (!RemainingTokenSpan.empty()) {
2314 // Split the string literal on \r\n boundaries.
2315 size_t CRLFPos = RemainingTokenSpan.find(Str: "\r\n");
2316 StringRef BeforeCRLF = RemainingTokenSpan.substr(Start: 0, N: CRLFPos);
2317 StringRef AfterCRLF = RemainingTokenSpan.substr(Start: CRLFPos);
2318
2319 // Copy everything before the \r\n sequence into the string literal.
2320 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin, Fragment: BeforeCRLF,
2321 Converter))
2322 hadError = true;
2323
2324 // Point into the \n inside the \r\n sequence and operate on the
2325 // remaining portion of the literal.
2326 RemainingTokenSpan = AfterCRLF.substr(Start: 1);
2327 }
2328 } else {
2329 if (ThisTokBuf[0] != '"') {
2330 // The file may have come from PCH and then changed after loading the
2331 // PCH; Fail gracefully.
2332 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2333 }
2334 ++ThisTokBuf; // skip "
2335
2336 // Check if this is a pascal string
2337 if (!isUnevaluated() && Features.PascalStrings &&
2338 ThisTokBuf + 1 != ThisTokEnd && ThisTokBuf[0] == '\\' &&
2339 ThisTokBuf[1] == 'p') {
2340
2341 // If the \p sequence is found in the first token, we have a pascal string
2342 // Otherwise, if we already have a pascal string, ignore the first \p
2343 if (i == 0) {
2344 ++ThisTokBuf;
2345 Pascal = true;
2346 } else if (Pascal)
2347 ThisTokBuf += 2;
2348 }
2349
2350 while (ThisTokBuf != ThisTokEnd) {
2351 // Is this a span of non-escape characters?
2352 if (ThisTokBuf[0] != '\\') {
2353 const char *InStart = ThisTokBuf;
2354 do {
2355 ++ThisTokBuf;
2356 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
2357
2358 // Copy the character span over.
2359 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin,
2360 Fragment: StringRef(InStart, ThisTokBuf - InStart),
2361 Converter))
2362 hadError = true;
2363 continue;
2364 }
2365 // Is this a Universal Character Name escape?
2366 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' ||
2367 ThisTokBuf[1] == 'N') {
2368 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, ResultBuf&: ResultPtr,
2369 HadError&: hadError,
2370 Loc: FullSourceLoc(StringToks[i].getLocation(), SM),
2371 CharByteWidth, Diags, Features, Converter);
2372 continue;
2373 }
2374 // Otherwise, this is a non-UCN escape character. Process it.
2375 unsigned ResultChar = ProcessCharEscape(
2376 ThisTokBegin, ThisTokBuf, ThisTokEnd, HadError&: hadError,
2377 Loc: FullSourceLoc(StringToks[i].getLocation(), SM), CharWidth: CharByteWidth * 8,
2378 Diags, Features, EvalMethod, Converter);
2379
2380 if (CharByteWidth == 4) {
2381 // FIXME: Make the type of the result buffer correct instead of
2382 // using reinterpret_cast.
2383 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
2384 *ResultWidePtr = ResultChar;
2385 ResultPtr += 4;
2386 } else if (CharByteWidth == 2) {
2387 // FIXME: Make the type of the result buffer correct instead of
2388 // using reinterpret_cast.
2389 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
2390 *ResultWidePtr = ResultChar & 0xFFFF;
2391 ResultPtr += 2;
2392 } else {
2393 assert(CharByteWidth == 1 && "Unexpected char width");
2394 *ResultPtr++ = ResultChar & 0xFF;
2395 }
2396 }
2397 }
2398 }
2399
2400 assert((!Pascal || !isUnevaluated()) &&
2401 "Pascal string in unevaluated context");
2402 if (Pascal) {
2403 if (CharByteWidth == 4) {
2404 // FIXME: Make the type of the result buffer correct instead of
2405 // using reinterpret_cast.
2406 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
2407 ResultWidePtr[0] = GetNumStringChars() - 1;
2408 } else if (CharByteWidth == 2) {
2409 // FIXME: Make the type of the result buffer correct instead of
2410 // using reinterpret_cast.
2411 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
2412 ResultWidePtr[0] = GetNumStringChars() - 1;
2413 } else {
2414 assert(CharByteWidth == 1 && "Unexpected char width");
2415 ResultBuf[0] = GetNumStringChars() - 1;
2416 }
2417
2418 // Verify that pascal strings aren't too large.
2419 if (GetStringLength() > 256) {
2420 if (Diags)
2421 Diags->Report(Loc: StringToks.front().getLocation(),
2422 DiagID: diag::err_pascal_string_too_long)
2423 << SourceRange(StringToks.front().getLocation(),
2424 StringToks.back().getLocation());
2425 hadError = true;
2426 return;
2427 }
2428 } else if (Diags) {
2429 // Complain if this string literal has too many characters.
2430 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
2431
2432 if (GetNumStringChars() > MaxChars)
2433 Diags->Report(Loc: StringToks.front().getLocation(),
2434 DiagID: diag::ext_string_too_long)
2435 << GetNumStringChars() << MaxChars
2436 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
2437 << SourceRange(StringToks.front().getLocation(),
2438 StringToks.back().getLocation());
2439 }
2440}
2441
2442static const char *resyncUTF8(const char *Err, const char *End) {
2443 if (Err == End)
2444 return End;
2445 End = Err + std::min<unsigned>(a: llvm::getNumBytesForUTF8(firstByte: *Err), b: End-Err);
2446 while (++Err != End && (*Err & 0xC0) == 0x80)
2447 ;
2448 return Err;
2449}
2450
2451/// This function copies from Fragment, which is a sequence of bytes
2452/// within Tok's contents (which begin at TokBegin) into ResultPtr.
2453/// Performs widening for multi-byte characters.
2454bool StringLiteralParser::CopyStringFragment(
2455 const Token &Tok, const char *TokBegin, StringRef Fragment,
2456 llvm::TextEncodingConverter *Converter) {
2457
2458 const llvm::UTF8 *ErrorPtrTmp;
2459 if (ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: Fragment, ResultPtr, ErrorPtr&: ErrorPtrTmp)) {
2460 if (Converter) {
2461 assert(isOrdinary() && "Only ordinary literals are supported");
2462 SmallString<64> CpConv;
2463 char *Cp = ResultPtr - Fragment.size();
2464 auto EC = Converter->convert(Source: Fragment, Result&: CpConv);
2465 if (!EC) {
2466 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
2467 ResultPtr = Cp + CpConv.size();
2468 } else { // there was a conversion error
2469 if (Diags)
2470 Diags->Report(Loc: Tok.getLocation(),
2471 DiagID: diag::err_exec_charset_conversion_failed)
2472 << EC.message();
2473 return true;
2474 }
2475 }
2476 return false;
2477 }
2478
2479 // If we see bad encoding for unprefixed string literals, warn and
2480 // simply copy the byte values, for compatibility with gcc and older
2481 // versions of clang.
2482 bool NoErrorOnBadEncoding = isOrdinary();
2483 if (NoErrorOnBadEncoding) {
2484 memcpy(dest: ResultPtr, src: Fragment.data(), n: Fragment.size());
2485 ResultPtr += Fragment.size();
2486 }
2487
2488 if (Diags) {
2489 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2490
2491 FullSourceLoc SourceLoc(Tok.getLocation(), SM);
2492 const DiagnosticBuilder &Builder =
2493 Diag(Diags, Features, TokLoc: SourceLoc, TokBegin,
2494 TokRangeBegin: ErrorPtr, TokRangeEnd: resyncUTF8(Err: ErrorPtr, End: Fragment.end()),
2495 DiagID: NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
2496 : diag::err_bad_string_encoding);
2497
2498 const char *NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2499 StringRef NextFragment(NextStart, Fragment.end()-NextStart);
2500
2501 // Decode into a dummy buffer.
2502 SmallString<512> Dummy;
2503 Dummy.reserve(N: Fragment.size() * CharByteWidth);
2504 char *Ptr = Dummy.data();
2505
2506 while (!ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: NextFragment, ResultPtr&: Ptr, ErrorPtr&: ErrorPtrTmp)) {
2507 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2508 NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2509 Builder << MakeCharSourceRange(Features, TokLoc: SourceLoc, TokBegin,
2510 TokRangeBegin: ErrorPtr, TokRangeEnd: NextStart);
2511 NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
2512 }
2513 }
2514 return !NoErrorOnBadEncoding;
2515}
2516
2517void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
2518 hadError = true;
2519 if (Diags)
2520 Diags->Report(Loc, DiagID: diag::err_lexing_string);
2521}
2522
2523/// getOffsetOfStringByte - This function returns the offset of the
2524/// specified byte of the string data represented by Token. This handles
2525/// advancing over escape sequences in the string.
2526unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
2527 unsigned ByteNo) const {
2528 // Get the spelling of the token.
2529 SmallString<32> SpellingBuffer;
2530 SpellingBuffer.resize(N: Tok.getLength());
2531
2532 bool StringInvalid = false;
2533 const char *SpellingPtr = &SpellingBuffer[0];
2534 unsigned TokLen = Lexer::getSpelling(Tok, Buffer&: SpellingPtr, SourceMgr: SM, LangOpts: Features,
2535 Invalid: &StringInvalid);
2536 if (StringInvalid)
2537 return 0;
2538
2539 const char *SpellingStart = SpellingPtr;
2540 const char *SpellingEnd = SpellingPtr+TokLen;
2541
2542 // Handle UTF-8 strings just like narrow strings.
2543 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
2544 SpellingPtr += 2;
2545
2546 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
2547 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
2548
2549 // For raw string literals, this is easy.
2550 if (SpellingPtr[0] == 'R') {
2551 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
2552 // Skip 'R"'.
2553 SpellingPtr += 2;
2554 while (*SpellingPtr != '(') {
2555 ++SpellingPtr;
2556 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
2557 }
2558 // Skip '('.
2559 ++SpellingPtr;
2560 return SpellingPtr - SpellingStart + ByteNo;
2561 }
2562
2563 // Skip over the leading quote
2564 assert(SpellingPtr[0] == '"' && "Should be a string literal!");
2565 ++SpellingPtr;
2566
2567 // Skip over bytes until we find the offset we're looking for.
2568 while (ByteNo) {
2569 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
2570
2571 // Step over non-escapes simply.
2572 if (*SpellingPtr != '\\') {
2573 ++SpellingPtr;
2574 --ByteNo;
2575 continue;
2576 }
2577
2578 // Otherwise, this is an escape character. Advance over it.
2579 bool HadError = false;
2580 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' ||
2581 SpellingPtr[1] == 'N') {
2582 const char *EscapePtr = SpellingPtr;
2583 unsigned Len = MeasureUCNEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd,
2584 CharByteWidth: 1, Features, HadError);
2585 if (Len > ByteNo) {
2586 // ByteNo is somewhere within the escape sequence.
2587 SpellingPtr = EscapePtr;
2588 break;
2589 }
2590 ByteNo -= Len;
2591 } else {
2592 ProcessCharEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd, HadError,
2593 Loc: FullSourceLoc(Tok.getLocation(), SM), CharWidth: CharByteWidth * 8,
2594 Diags, Features, EvalMethod: StringLiteralEvalMethod::Evaluated,
2595 /*TextEncoding=*/Converter: nullptr);
2596 --ByteNo;
2597 }
2598 assert(!HadError && "This method isn't valid on erroneous strings");
2599 }
2600
2601 return SpellingPtr-SpellingStart;
2602}
2603
2604/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
2605/// suffixes as ud-suffixes, because the diagnostic experience is better if we
2606/// treat it as an invalid suffix.
2607bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
2608 StringRef Suffix) {
2609 return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
2610 Suffix == "sv";
2611}
2612