1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the NumericLiteralParser, CharLiteralParser, and
10// StringLiteralParser interfaces.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Lex/LiteralSupport.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/LangOptions.h"
17#include "clang/Basic/SourceLocation.h"
18#include "clang/Basic/TargetInfo.h"
19#include "clang/Lex/LexDiagnostic.h"
20#include "clang/Lex/Lexer.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/Token.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ScopeExit.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/ADT/StringSwitch.h"
28#include "llvm/Support/ConvertUTF.h"
29#include "llvm/Support/Error.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/Unicode.h"
32#include <algorithm>
33#include <cassert>
34#include <cstddef>
35#include <cstdint>
36#include <cstring>
37#include <string>
38
39using namespace clang;
40
41static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
42 switch (kind) {
43 default: llvm_unreachable("Unknown token type!");
44 case tok::char_constant:
45 case tok::string_literal:
46 case tok::utf8_char_constant:
47 case tok::utf8_string_literal:
48 return Target.getCharWidth();
49 case tok::wide_char_constant:
50 case tok::wide_string_literal:
51 return Target.getWCharWidth();
52 case tok::utf16_char_constant:
53 case tok::utf16_string_literal:
54 return Target.getChar16Width();
55 case tok::utf32_char_constant:
56 case tok::utf32_string_literal:
57 return Target.getChar32Width();
58 }
59}
60
61static unsigned getEncodingPrefixLen(tok::TokenKind kind) {
62 switch (kind) {
63 default:
64 llvm_unreachable("Unknown token type!");
65 case tok::char_constant:
66 case tok::string_literal:
67 return 0;
68 case tok::utf8_char_constant:
69 case tok::utf8_string_literal:
70 return 2;
71 case tok::wide_char_constant:
72 case tok::wide_string_literal:
73 case tok::utf16_char_constant:
74 case tok::utf16_string_literal:
75 case tok::utf32_char_constant:
76 case tok::utf32_string_literal:
77 return 1;
78 }
79}
80
81static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
82 FullSourceLoc TokLoc,
83 const char *TokBegin,
84 const char *TokRangeBegin,
85 const char *TokRangeEnd) {
86 SourceLocation Begin =
87 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
88 SM: TokLoc.getManager(), LangOpts: Features);
89 SourceLocation End =
90 Lexer::AdvanceToTokenCharacter(TokStart: Begin, Characters: TokRangeEnd - TokRangeBegin,
91 SM: TokLoc.getManager(), LangOpts: Features);
92 return CharSourceRange::getCharRange(B: Begin, E: End);
93}
94
95/// Produce a diagnostic highlighting some portion of a literal.
96///
97/// Emits the diagnostic \p DiagID, highlighting the range of characters from
98/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
99/// a substring of a spelling buffer for the token beginning at \p TokBegin.
100static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
101 const LangOptions &Features, FullSourceLoc TokLoc,
102 const char *TokBegin, const char *TokRangeBegin,
103 const char *TokRangeEnd, unsigned DiagID) {
104 SourceLocation Begin =
105 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
106 SM: TokLoc.getManager(), LangOpts: Features);
107 return Diags->Report(Loc: Begin, DiagID) <<
108 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
109}
110
111static bool IsEscapeValidInUnevaluatedStringLiteral(char Escape) {
112 switch (Escape) {
113 case '\'':
114 case '"':
115 case '?':
116 case '\\':
117 case 'a':
118 case 'b':
119 case 'f':
120 case 'n':
121 case 'r':
122 case 't':
123 case 'v':
124 return true;
125 }
126 return false;
127}
128
129static llvm::ErrorOr<char>
130convertCharacter(StringRef Char, const llvm::TextEncodingConverter &Converter) {
131 SmallString<8> ResultCharConv;
132 std::error_code EC = Converter.convert(Source: Char, Result&: ResultCharConv);
133 if (EC)
134 return EC;
135 if (ResultCharConv.size() > 1)
136 return std::error_code(E2BIG, std::generic_category());
137 return ResultCharConv[0];
138}
139
140/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
141/// either a character or a string literal.
142static unsigned ProcessCharEscape(
143 const char *ThisTokBegin, const char *&ThisTokBuf, const char *ThisTokEnd,
144 bool &HadError, FullSourceLoc Loc, unsigned CharWidth,
145 DiagnosticsEngine *Diags, const LangOptions &Features,
146 StringLiteralEvalMethod EvalMethod, llvm::TextEncodingConverter *Converter,
147 ConversionAction Action) {
148 const char *EscapeBegin = ThisTokBuf;
149 bool Delimited = false;
150 bool EndDelimiterFound = false;
151
152 // Skip the '\' char.
153 ++ThisTokBuf;
154
155 // We know that this character can't be off the end of the buffer, because
156 // that would have been \", which would not have been the end of string.
157 unsigned ResultChar = *ThisTokBuf++;
158 char Escape = ResultChar;
159 bool Transcode = true;
160 bool Invalid = false;
161 switch (ResultChar) {
162 // These map to themselves.
163 case '\\': case '\'': case '"': case '?': break;
164
165 // These have fixed mappings.
166 case 'a':
167 // TODO: K&R: the meaning of '\\a' is different in traditional C
168 ResultChar = 7;
169 break;
170 case 'b':
171 ResultChar = 8;
172 break;
173 case 'e':
174 if (Diags)
175 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
176 DiagID: diag::ext_nonstandard_escape) << "e";
177 ResultChar = 27;
178 break;
179 case 'E':
180 if (Diags)
181 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
182 DiagID: diag::ext_nonstandard_escape) << "E";
183 ResultChar = 27;
184 break;
185 case 'f':
186 ResultChar = 12;
187 break;
188 case 'n':
189 ResultChar = 10;
190 break;
191 case 'r':
192 ResultChar = 13;
193 break;
194 case 't':
195 ResultChar = 9;
196 break;
197 case 'v':
198 ResultChar = 11;
199 break;
200 case 'x': { // Hex escape.
201 Transcode = false;
202 ResultChar = 0;
203 if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
204 Delimited = true;
205 ThisTokBuf++;
206 if (*ThisTokBuf == '}') {
207 HadError = true;
208 if (Diags)
209 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
210 DiagID: diag::err_delimited_escape_empty);
211 }
212 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
213 if (Diags)
214 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
215 DiagID: diag::err_hex_escape_no_digits) << "x";
216 return ResultChar;
217 }
218
219 // Hex escapes are a maximal series of hex digits.
220 bool Overflow = false;
221 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
222 if (Delimited && *ThisTokBuf == '}') {
223 ThisTokBuf++;
224 EndDelimiterFound = true;
225 break;
226 }
227 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
228 if (CharVal == -1) {
229 // Non delimited hex escape sequences stop at the first non-hex digit.
230 if (!Delimited)
231 break;
232 HadError = true;
233 if (Diags)
234 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
235 DiagID: diag::err_delimited_escape_invalid)
236 << StringRef(ThisTokBuf, 1);
237 continue;
238 }
239 // About to shift out a digit?
240 if (ResultChar & 0xF0000000)
241 Overflow = true;
242 ResultChar <<= 4;
243 ResultChar |= CharVal;
244 }
245 // See if any bits will be truncated when evaluated as a character.
246 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
247 Overflow = true;
248 ResultChar &= ~0U >> (32-CharWidth);
249 }
250
251 // Check for overflow.
252 if (!HadError && Overflow) { // Too many digits to fit in
253 HadError = true;
254 if (Diags)
255 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
256 DiagID: diag::err_escape_too_large)
257 << 0;
258 }
259 break;
260 }
261 case '0': case '1': case '2': case '3':
262 case '4': case '5': case '6': case '7': {
263 // Octal escapes.
264 --ThisTokBuf;
265 Transcode = false;
266 ResultChar = 0;
267
268 // Octal escapes are a series of octal digits with maximum length 3.
269 // "\0123" is a two digit sequence equal to "\012" "3".
270 unsigned NumDigits = 0;
271 do {
272 ResultChar <<= 3;
273 ResultChar |= *ThisTokBuf++ - '0';
274 ++NumDigits;
275 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
276 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
277
278 // Check for overflow. Reject '\777', but not L'\777'.
279 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
280 if (Diags)
281 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
282 DiagID: diag::err_escape_too_large) << 1;
283 ResultChar &= ~0U >> (32-CharWidth);
284 }
285 break;
286 }
287 case 'o': {
288 bool Overflow = false;
289 Transcode = false;
290 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
291 HadError = true;
292 if (Diags)
293 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
294 DiagID: diag::err_delimited_escape_missing_brace)
295 << "o";
296
297 break;
298 }
299 ResultChar = 0;
300 Delimited = true;
301 ++ThisTokBuf;
302 if (*ThisTokBuf == '}') {
303 HadError = true;
304 if (Diags)
305 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
306 DiagID: diag::err_delimited_escape_empty);
307 }
308
309 while (ThisTokBuf != ThisTokEnd) {
310 if (*ThisTokBuf == '}') {
311 EndDelimiterFound = true;
312 ThisTokBuf++;
313 break;
314 }
315 if (*ThisTokBuf < '0' || *ThisTokBuf > '7') {
316 HadError = true;
317 if (Diags)
318 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
319 DiagID: diag::err_delimited_escape_invalid)
320 << StringRef(ThisTokBuf, 1);
321 ThisTokBuf++;
322 continue;
323 }
324 // Check if one of the top three bits is set before shifting them out.
325 if (ResultChar & 0xE0000000)
326 Overflow = true;
327
328 ResultChar <<= 3;
329 ResultChar |= *ThisTokBuf++ - '0';
330 }
331 // Check for overflow. Reject '\777', but not L'\777'.
332 if (!HadError &&
333 (Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) {
334 HadError = true;
335 if (Diags)
336 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
337 DiagID: diag::err_escape_too_large)
338 << 1;
339 ResultChar &= ~0U >> (32 - CharWidth);
340 }
341 break;
342 }
343 // Otherwise, these are not valid escapes.
344 case '(': case '{': case '[': case '%':
345 // GCC accepts these as extensions. We warn about them as such though.
346 if (Diags)
347 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
348 DiagID: diag::ext_nonstandard_escape)
349 << std::string(1, ResultChar);
350 break;
351 default:
352 Invalid = true;
353 if (!Diags)
354 break;
355
356 if (isPrintable(c: ResultChar))
357 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
358 DiagID: diag::ext_unknown_escape)
359 << std::string(1, ResultChar);
360 else
361 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
362 DiagID: diag::ext_unknown_escape)
363 << "x" + llvm::utohexstr(X: ResultChar);
364 break;
365 }
366
367 if (Delimited && Diags) {
368 if (!EndDelimiterFound)
369 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
370 DiagID: diag::err_expected)
371 << tok::r_brace;
372 else if (!HadError) {
373 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: false, Opts: Features,
374 Diags&: *Diags);
375 }
376 }
377
378 if (EvalMethod == StringLiteralEvalMethod::Unevaluated &&
379 !IsEscapeValidInUnevaluatedStringLiteral(Escape)) {
380 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
381 DiagID: diag::err_unevaluated_string_invalid_escape_sequence)
382 << StringRef(EscapeBegin, ThisTokBuf - EscapeBegin);
383 HadError = true;
384 }
385
386 if (!HadError && EvalMethod != StringLiteralEvalMethod::Unevaluated &&
387 Transcode && Converter) {
388 // Invalid escapes are written as '?' and then translated.
389 assert(ResultChar <=
390 static_cast<unsigned>(std::numeric_limits<char>::max()));
391 char ByteChar = Invalid ? '?' : ResultChar;
392 auto ErrorOrChar = convertCharacter(Char: StringRef(&ByteChar, 1), Converter: *Converter);
393 if (ErrorOrChar)
394 ResultChar = *ErrorOrChar;
395 else {
396 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
397 DiagID: diag::err_exec_charset_conversion_failed)
398 << ErrorOrChar.getError().message()
399 << (Action == CA_ToLiteralEncodingForAsm ? 1 : 0);
400 HadError = true;
401 }
402 }
403 return ResultChar;
404}
405
406static void appendCodePoint(unsigned Codepoint,
407 llvm::SmallVectorImpl<char> &Str) {
408 char ResultBuf[4];
409 char *ResultPtr = ResultBuf;
410 if (llvm::ConvertCodePointToUTF8(Source: Codepoint, ResultPtr))
411 Str.append(in_start: ResultBuf, in_end: ResultPtr);
412}
413
414void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
415 for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
416 if (*I != '\\') {
417 Buf.push_back(Elt: *I);
418 continue;
419 }
420
421 ++I;
422 char Kind = *I;
423 ++I;
424
425 assert(Kind == 'u' || Kind == 'U' || Kind == 'N');
426 uint32_t CodePoint = 0;
427
428 if (Kind == 'u' && *I == '{') {
429 for (++I; *I != '}'; ++I) {
430 unsigned Value = llvm::hexDigitValue(C: *I);
431 assert(Value != -1U);
432 CodePoint <<= 4;
433 CodePoint += Value;
434 }
435 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
436 continue;
437 }
438
439 if (Kind == 'N') {
440 assert(*I == '{');
441 ++I;
442 const auto *Delim = std::find(first: I, last: Input.end(), val: '}');
443 assert(Delim != Input.end());
444 StringRef Name(I, std::distance(first: I, last: Delim));
445 std::optional<llvm::sys::unicode::LooseMatchingResult> Res =
446 llvm::sys::unicode::nameToCodepointLooseMatching(Name);
447 assert(Res && "could not find a codepoint that was previously found");
448 CodePoint = Res->CodePoint;
449 assert(CodePoint != 0xFFFFFFFF);
450 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
451 I = Delim;
452 continue;
453 }
454
455 unsigned NumHexDigits;
456 if (Kind == 'u')
457 NumHexDigits = 4;
458 else
459 NumHexDigits = 8;
460
461 assert(I + NumHexDigits <= E);
462
463 for (; NumHexDigits != 0; ++I, --NumHexDigits) {
464 unsigned Value = llvm::hexDigitValue(C: *I);
465 assert(Value != -1U);
466
467 CodePoint <<= 4;
468 CodePoint += Value;
469 }
470
471 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
472 --I;
473 }
474}
475
476bool clang::isFunctionLocalStringLiteralMacro(tok::TokenKind K,
477 const LangOptions &LO) {
478 return LO.MicrosoftExt &&
479 (K == tok::kw___FUNCTION__ || K == tok::kw_L__FUNCTION__ ||
480 K == tok::kw___FUNCSIG__ || K == tok::kw_L__FUNCSIG__ ||
481 K == tok::kw___FUNCDNAME__);
482}
483
484bool clang::tokenIsLikeStringLiteral(const Token &Tok, const LangOptions &LO) {
485 return tok::isStringLiteral(K: Tok.getKind()) ||
486 isFunctionLocalStringLiteralMacro(K: Tok.getKind(), LO);
487}
488
489static bool ProcessNumericUCNEscape(const char *ThisTokBegin,
490 const char *&ThisTokBuf,
491 const char *ThisTokEnd, uint32_t &UcnVal,
492 unsigned short &UcnLen, bool &Delimited,
493 FullSourceLoc Loc, DiagnosticsEngine *Diags,
494 const LangOptions &Features,
495 bool in_char_string_literal = false) {
496 const char *UcnBegin = ThisTokBuf;
497 bool HasError = false;
498 bool EndDelimiterFound = false;
499
500 // Skip the '\u' char's.
501 ThisTokBuf += 2;
502 Delimited = false;
503 if (UcnBegin[1] == 'u' && in_char_string_literal &&
504 ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
505 Delimited = true;
506 ThisTokBuf++;
507 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
508 if (Diags)
509 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
510 DiagID: diag::err_hex_escape_no_digits)
511 << StringRef(&ThisTokBuf[-1], 1);
512 return false;
513 }
514 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
515
516 bool Overflow = false;
517 unsigned short Count = 0;
518 for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen);
519 ++ThisTokBuf) {
520 if (Delimited && *ThisTokBuf == '}') {
521 ++ThisTokBuf;
522 EndDelimiterFound = true;
523 break;
524 }
525 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
526 if (CharVal == -1) {
527 HasError = true;
528 if (!Delimited)
529 break;
530 if (Diags) {
531 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
532 DiagID: diag::err_delimited_escape_invalid)
533 << StringRef(ThisTokBuf, 1);
534 }
535 Count++;
536 continue;
537 }
538 if (UcnVal & 0xF0000000) {
539 Overflow = true;
540 continue;
541 }
542 UcnVal <<= 4;
543 UcnVal |= CharVal;
544 Count++;
545 }
546
547 if (Overflow) {
548 if (Diags)
549 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
550 DiagID: diag::err_escape_too_large)
551 << 0;
552 return false;
553 }
554
555 if (Delimited && !EndDelimiterFound) {
556 if (Diags) {
557 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
558 DiagID: diag::err_expected)
559 << tok::r_brace;
560 }
561 return false;
562 }
563
564 // If we didn't consume the proper number of digits, there is a problem.
565 if (Count == 0 || (!Delimited && Count != UcnLen)) {
566 if (Diags)
567 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
568 DiagID: Delimited ? diag::err_delimited_escape_empty
569 : diag::err_ucn_escape_incomplete);
570 return false;
571 }
572 return !HasError;
573}
574
575static bool allowedInCharacterName(char C) {
576 return (C >= 'A' && C <= 'Z') || (C >= '0' && C <= '9') || C == '-' ||
577 C == ' ';
578}
579
580static void DiagnoseInvalidUnicodeCharacterName(
581 DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc,
582 const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd,
583 llvm::StringRef Name) {
584
585 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
586 DiagID: diag::err_invalid_ucn_name)
587 << Name;
588
589 namespace u = llvm::sys::unicode;
590
591 bool HasIllegalCharacter = false;
592 for (const char *P = Name.begin(), *E = Name.end(); P != E;) {
593 if (allowedInCharacterName(C: *P)) {
594 ++P;
595 continue;
596 }
597 SourceLocation CharLoc = Lexer::AdvanceToTokenCharacter(
598 TokStart: Loc, Characters: (TokRangeBegin - TokBegin) + (P - Name.begin()), SM: Loc.getManager(),
599 LangOpts: Features);
600 Diags->Report(Loc: CharLoc, DiagID: diag::note_invalid_ucn_name_character)
601 << EscapeSingleCodepointForDiagnostic(
602 Str: StringRef(P, std::distance(first: P, last: E)));
603 HasIllegalCharacter = true;
604 break;
605 }
606
607 std::optional<u::LooseMatchingResult> Res =
608 u::nameToCodepointLooseMatching(Name);
609 if (Res) {
610 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
611 DiagID: diag::note_invalid_ucn_name_loose_matching)
612 << FixItHint::CreateReplacement(
613 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
614 TokRangeEnd),
615 Code: Res->Name);
616 return;
617 }
618
619 // Providing illegal characters suggests a fundamental misuse of the feature,
620 // like providing emoji in \N{}. Offering alternative suggestions is often
621 // unhelpful in that scenario.
622 if (HasIllegalCharacter)
623 return;
624
625 unsigned Distance = 0;
626 SmallVector<u::MatchForCodepointName> Matches =
627 u::nearestMatchesForCodepointName(Pattern: Name, MaxMatchesCount: 5);
628 assert(!Matches.empty() && "No unicode characters found");
629
630 for (const auto &Match : Matches) {
631 if (Distance == 0)
632 Distance = Match.Distance;
633 if (std::max(a: Distance, b: Match.Distance) -
634 std::min(a: Distance, b: Match.Distance) >
635 3)
636 break;
637 Distance = Match.Distance;
638 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
639 DiagID: diag::note_invalid_ucn_name_candidate)
640 << Match.Name << EscapeSingleCodepointForDiagnostic(CP: Match.Value)
641 << FixItHint::CreateReplacement(
642 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
643 TokRangeEnd),
644 Code: Match.Name);
645 }
646}
647
648static bool ProcessNamedUCNEscape(const char *ThisTokBegin,
649 const char *&ThisTokBuf,
650 const char *ThisTokEnd, uint32_t &UcnVal,
651 unsigned short &UcnLen, FullSourceLoc Loc,
652 DiagnosticsEngine *Diags,
653 const LangOptions &Features) {
654 const char *UcnBegin = ThisTokBuf;
655 assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N');
656 ThisTokBuf += 2;
657 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
658 if (Diags) {
659 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
660 DiagID: diag::err_delimited_escape_missing_brace)
661 << StringRef(&ThisTokBuf[-1], 1);
662 }
663 return false;
664 }
665 ThisTokBuf++;
666 const char *ClosingBrace = std::find_if(first: ThisTokBuf, last: ThisTokEnd, pred: [](char C) {
667 return C == '}' || isVerticalWhitespace(c: C);
668 });
669 bool Incomplete = ClosingBrace == ThisTokEnd;
670 bool Empty = ClosingBrace == ThisTokBuf;
671 if (Incomplete || Empty) {
672 if (Diags) {
673 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
674 DiagID: Incomplete ? diag::err_ucn_escape_incomplete
675 : diag::err_delimited_escape_empty)
676 << StringRef(&UcnBegin[1], 1);
677 }
678 ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1;
679 return false;
680 }
681 StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf);
682 ThisTokBuf = ClosingBrace + 1;
683 std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name);
684 if (!Res) {
685 if (Diags)
686 DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, TokBegin: ThisTokBegin,
687 TokRangeBegin: &UcnBegin[3], TokRangeEnd: ClosingBrace, Name);
688 return false;
689 }
690 UcnVal = *Res;
691 UcnLen = UcnVal > 0xFFFF ? 8 : 4;
692 return true;
693}
694
695/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
696/// return the UTF32.
697static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
698 const char *ThisTokEnd, uint32_t &UcnVal,
699 unsigned short &UcnLen, FullSourceLoc Loc,
700 DiagnosticsEngine *Diags,
701 const LangOptions &Features,
702 bool in_char_string_literal = false) {
703
704 bool HasError;
705 const char *UcnBegin = ThisTokBuf;
706 bool IsDelimitedEscapeSequence = false;
707 bool IsNamedEscapeSequence = false;
708 if (ThisTokBuf[1] == 'N') {
709 IsNamedEscapeSequence = true;
710 HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
711 UcnVal, UcnLen, Loc, Diags, Features);
712 } else {
713 HasError =
714 !ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
715 UcnLen, Delimited&: IsDelimitedEscapeSequence, Loc, Diags,
716 Features, in_char_string_literal);
717 }
718 if (HasError)
719 return false;
720
721 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
722 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
723 UcnVal > 0x10FFFF) { // maximum legal UTF32 value
724 if (Diags)
725 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
726 DiagID: diag::err_ucn_escape_invalid);
727 return false;
728 }
729
730 // C23 and C++11 allow UCNs that refer to control characters
731 // and basic source characters inside character and string literals
732 if (UcnVal < 0xa0 &&
733 // $, @, ` are allowed in all language modes
734 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {
735 bool IsError =
736 (!(Features.CPlusPlus11 || Features.C23) || !in_char_string_literal);
737 if (Diags) {
738 char BasicSCSChar = UcnVal;
739 if (UcnVal >= 0x20 && UcnVal < 0x7f)
740 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
741 DiagID: IsError ? diag::err_ucn_escape_basic_scs
742 : Features.CPlusPlus
743 ? diag::warn_cxx98_compat_literal_ucn_escape_basic_scs
744 : diag::warn_c23_compat_literal_ucn_escape_basic_scs)
745 << StringRef(&BasicSCSChar, 1);
746 else
747 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
748 DiagID: IsError ? diag::err_ucn_control_character
749 : Features.CPlusPlus
750 ? diag::warn_cxx98_compat_literal_ucn_control_character
751 : diag::warn_c23_compat_literal_ucn_control_character);
752 }
753 if (IsError)
754 return false;
755 }
756
757 if (!Features.CPlusPlus && !Features.C99 && Diags)
758 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
759 DiagID: diag::warn_ucn_not_valid_in_c89_literal);
760
761 if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
762 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: IsNamedEscapeSequence,
763 Opts: Features, Diags&: *Diags);
764 return true;
765}
766
767/// MeasureUCNEscape - Determine the number of bytes within the resulting string
768/// which this UCN will occupy.
769static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
770 const char *ThisTokEnd, unsigned CharByteWidth,
771 const LangOptions &Features, bool &HadError) {
772 // UTF-32: 4 bytes per escape.
773 if (CharByteWidth == 4)
774 return 4;
775
776 uint32_t UcnVal = 0;
777 unsigned short UcnLen = 0;
778 FullSourceLoc Loc;
779
780 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
781 UcnLen, Loc, Diags: nullptr, Features, in_char_string_literal: true)) {
782 HadError = true;
783 return 0;
784 }
785
786 // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
787 if (CharByteWidth == 2)
788 return UcnVal <= 0xFFFF ? 2 : 4;
789
790 // UTF-8.
791 if (UcnVal < 0x80)
792 return 1;
793 if (UcnVal < 0x800)
794 return 2;
795 if (UcnVal < 0x10000)
796 return 3;
797 return 4;
798}
799
800/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
801/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
802/// StringLiteralParser. When we decide to implement UCN's for identifiers,
803/// we will likely rework our support for UCN's.
804static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
805 const char *ThisTokEnd, char *&ResultBuf,
806 bool &HadError, FullSourceLoc Loc,
807 unsigned CharByteWidth, DiagnosticsEngine *Diags,
808 const LangOptions &Features,
809 llvm::TextEncodingConverter *Converter,
810 ConversionAction Action) {
811 typedef uint32_t UTF32;
812 UTF32 UcnVal = 0;
813 unsigned short UcnLen = 0;
814 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
815 Loc, Diags, Features, in_char_string_literal: true)) {
816 HadError = true;
817 return;
818 }
819
820 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
821 "only character widths of 1, 2, or 4 bytes supported");
822
823 (void)UcnLen;
824 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
825
826 if (CharByteWidth == 4) {
827 // FIXME: Make the type of the result buffer correct instead of
828 // using reinterpret_cast.
829 llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
830 *ResultPtr = UcnVal;
831 ResultBuf += 4;
832 return;
833 }
834
835 if (CharByteWidth == 2) {
836 // FIXME: Make the type of the result buffer correct instead of
837 // using reinterpret_cast.
838 llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
839
840 if (UcnVal <= (UTF32)0xFFFF) {
841 *ResultPtr = UcnVal;
842 ResultBuf += 2;
843 return;
844 }
845
846 // Convert to UTF16.
847 UcnVal -= 0x10000;
848 *ResultPtr = 0xD800 + (UcnVal >> 10);
849 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
850 ResultBuf += 4;
851 return;
852 }
853
854 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
855
856 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
857 // The conversion below was inspired by:
858 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
859 // First, we determine how many bytes the result will require.
860 typedef uint8_t UTF8;
861
862 unsigned short bytesToWrite = 0;
863 if (UcnVal < (UTF32)0x80)
864 bytesToWrite = 1;
865 else if (UcnVal < (UTF32)0x800)
866 bytesToWrite = 2;
867 else if (UcnVal < (UTF32)0x10000)
868 bytesToWrite = 3;
869 else
870 bytesToWrite = 4;
871
872 const unsigned byteMask = 0xBF;
873 const unsigned byteMark = 0x80;
874
875 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
876 // into the first byte, depending on how many bytes follow.
877 static const UTF8 firstByteMark[5] = {
878 0x00, 0x00, 0xC0, 0xE0, 0xF0
879 };
880 // Finally, we write the bytes into ResultBuf.
881 ResultBuf += bytesToWrite;
882 switch (bytesToWrite) { // note: everything falls through.
883 case 4:
884 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
885 [[fallthrough]];
886 case 3:
887 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
888 [[fallthrough]];
889 case 2:
890 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
891 [[fallthrough]];
892 case 1:
893 *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
894 }
895 // Update the buffer.
896 ResultBuf += bytesToWrite;
897
898 if (Converter) {
899 SmallString<4> CpConv;
900 char *Cp = ResultBuf - bytesToWrite;
901 auto EC = Converter->convert(Source: StringRef(Cp, bytesToWrite), Result&: CpConv);
902 if (!EC) {
903 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
904 ResultBuf = Cp + CpConv.size();
905 } else {
906 Diags->Report(Loc, DiagID: diag::err_exec_charset_conversion_failed)
907 << EC.message() << (Action == CA_ToLiteralEncodingForAsm ? 1 : 0);
908 HadError = true;
909 }
910 }
911}
912
913/// integer-constant: [C99 6.4.4.1]
914/// decimal-constant integer-suffix
915/// octal-constant integer-suffix
916/// hexadecimal-constant integer-suffix
917/// binary-literal integer-suffix [GNU, C++1y]
918/// user-defined-integer-literal: [C++11 lex.ext]
919/// decimal-literal ud-suffix
920/// octal-literal ud-suffix
921/// hexadecimal-literal ud-suffix
922/// binary-literal ud-suffix [GNU, C++1y]
923/// decimal-constant:
924/// nonzero-digit
925/// decimal-constant digit
926/// octal-constant:
927/// 0
928/// octal-constant octal-digit
929/// hexadecimal-constant:
930/// hexadecimal-prefix hexadecimal-digit
931/// hexadecimal-constant hexadecimal-digit
932/// hexadecimal-prefix: one of
933/// 0x 0X
934/// binary-literal:
935/// 0b binary-digit
936/// 0B binary-digit
937/// binary-literal binary-digit
938/// integer-suffix:
939/// unsigned-suffix [long-suffix]
940/// unsigned-suffix [long-long-suffix]
941/// long-suffix [unsigned-suffix]
942/// long-long-suffix [unsigned-sufix]
943/// nonzero-digit:
944/// 1 2 3 4 5 6 7 8 9
945/// octal-digit:
946/// 0 1 2 3 4 5 6 7
947/// hexadecimal-digit:
948/// 0 1 2 3 4 5 6 7 8 9
949/// a b c d e f
950/// A B C D E F
951/// binary-digit:
952/// 0
953/// 1
954/// unsigned-suffix: one of
955/// u U
956/// long-suffix: one of
957/// l L
958/// long-long-suffix: one of
959/// ll LL
960///
961/// floating-constant: [C99 6.4.4.2]
962/// TODO: add rules...
963///
964NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
965 SourceLocation TokLoc,
966 const SourceManager &SM,
967 const LangOptions &LangOpts,
968 const TargetInfo &Target,
969 DiagnosticsEngine &Diags)
970 : SM(SM), LangOpts(LangOpts), Diags(Diags),
971 ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
972
973 s = DigitsBegin = ThisTokBegin;
974 saw_exponent = false;
975 saw_period = false;
976 saw_ud_suffix = false;
977 saw_fixed_point_suffix = false;
978 isLong = false;
979 isUnsigned = false;
980 isLongLong = false;
981 isSizeT = false;
982 isHalf = false;
983 isFloat = false;
984 isImaginary = false;
985 isFloat16 = false;
986 isFloat128 = false;
987 MicrosoftInteger = 0;
988 isFract = false;
989 isAccum = false;
990 hadError = false;
991 isBitInt = false;
992
993 // This routine assumes that the range begin/end matches the regex for integer
994 // and FP constants (specifically, the 'pp-number' regex), and assumes that
995 // the byte at "*end" is both valid and not part of the regex. Because of
996 // this, it doesn't have to check for 'overscan' in various places.
997 // Note: For HLSL, the end token is allowed to be '.' which would be in the
998 // 'pp-number' regex. This is required to support vector swizzles on numeric
999 // constants (i.e. 1.xx or 1.5f.rrr).
1000 if (isPreprocessingNumberBody(c: *ThisTokEnd) &&
1001 !(LangOpts.HLSL && *ThisTokEnd == '.')) {
1002 Diags.Report(Loc: TokLoc, DiagID: diag::err_lexing_numeric);
1003 hadError = true;
1004 return;
1005 }
1006
1007 if (*s == '0') { // parse radix
1008 ParseNumberStartingWithZero(TokLoc);
1009 if (hadError)
1010 return;
1011 } else { // the first digit is non-zero
1012 radix = 10;
1013 s = SkipDigits(ptr: s);
1014 if (s == ThisTokEnd) {
1015 // Done.
1016 } else {
1017 ParseDecimalOrOctalCommon(TokLoc);
1018 if (hadError)
1019 return;
1020 }
1021 }
1022
1023 SuffixBegin = s;
1024 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1025
1026 // Initial scan to lookahead for fixed point suffix.
1027 if (LangOpts.FixedPoint) {
1028 for (const char *c = s; c != ThisTokEnd; ++c) {
1029 if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
1030 saw_fixed_point_suffix = true;
1031 break;
1032 }
1033 }
1034 }
1035
1036 // Parse the suffix. At this point we can classify whether we have an FP or
1037 // integer constant.
1038 bool isFixedPointConstant = isFixedPointLiteral();
1039 bool isFPConstant = isFloatingLiteral();
1040 bool HasSize = false;
1041 bool DoubleUnderscore = false;
1042
1043 // Loop over all of the characters of the suffix. If we see something bad,
1044 // we break out of the loop.
1045 for (; s != ThisTokEnd; ++s) {
1046 switch (*s) {
1047 case 'R':
1048 case 'r':
1049 if (!LangOpts.FixedPoint)
1050 break;
1051 if (isFract || isAccum) break;
1052 if (!(saw_period || saw_exponent)) break;
1053 isFract = true;
1054 continue;
1055 case 'K':
1056 case 'k':
1057 if (!LangOpts.FixedPoint)
1058 break;
1059 if (isFract || isAccum) break;
1060 if (!(saw_period || saw_exponent)) break;
1061 isAccum = true;
1062 continue;
1063 case 'h': // FP Suffix for "half".
1064 case 'H':
1065 // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
1066 if (!(LangOpts.Half || LangOpts.FixedPoint))
1067 break;
1068 if (isIntegerLiteral()) break; // Error for integer constant.
1069 if (HasSize)
1070 break;
1071 HasSize = true;
1072 isHalf = true;
1073 continue; // Success.
1074 case 'f': // FP Suffix for "float"
1075 case 'F':
1076 if (!isFPConstant) break; // Error for integer constant.
1077 if (HasSize)
1078 break;
1079 HasSize = true;
1080
1081 // CUDA host and device may have different _Float16 support, therefore
1082 // allows f16 literals to avoid false alarm.
1083 // When we compile for OpenMP target offloading on NVPTX, f16 suffix
1084 // should also be supported.
1085 // ToDo: more precise check for CUDA.
1086 // TODO: AMDGPU might also support it in the future.
1087 if ((Target.hasFloat16Type() || LangOpts.CUDA ||
1088 (LangOpts.OpenMPIsTargetDevice && Target.getTriple().isNVPTX())) &&
1089 s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
1090 s += 2; // success, eat up 2 characters.
1091 isFloat16 = true;
1092 continue;
1093 }
1094
1095 isFloat = true;
1096 continue; // Success.
1097 case 'q': // FP Suffix for "__float128"
1098 case 'Q':
1099 if (!isFPConstant) break; // Error for integer constant.
1100 if (HasSize)
1101 break;
1102 HasSize = true;
1103 isFloat128 = true;
1104 continue; // Success.
1105 case 'u':
1106 case 'U':
1107 if (isFPConstant) break; // Error for floating constant.
1108 if (isUnsigned) break; // Cannot be repeated.
1109 isUnsigned = true;
1110 continue; // Success.
1111 case 'l':
1112 case 'L':
1113 if (HasSize)
1114 break;
1115 HasSize = true;
1116
1117 // Check for long long. The L's need to be adjacent and the same case.
1118 if (s[1] == s[0]) {
1119 assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
1120 if (isFPConstant) break; // long long invalid for floats.
1121 isLongLong = true;
1122 ++s; // Eat both of them.
1123 } else {
1124 isLong = true;
1125 }
1126 continue; // Success.
1127 case 'z':
1128 case 'Z':
1129 if (isFPConstant)
1130 break; // Invalid for floats.
1131 if (HasSize)
1132 break;
1133 HasSize = true;
1134 isSizeT = true;
1135 continue;
1136 case 'i':
1137 case 'I':
1138 if (LangOpts.MicrosoftExt && s + 1 < ThisTokEnd && !isFPConstant) {
1139 // Allow i8, i16, i32, i64, and i128. First, look ahead and check if
1140 // suffixes are Microsoft integers and not the imaginary unit.
1141 uint8_t Bits = 0;
1142 size_t ToSkip = 0;
1143 switch (s[1]) {
1144 case '8': // i8 suffix
1145 Bits = 8;
1146 ToSkip = 2;
1147 break;
1148 case '1':
1149 if (s + 2 < ThisTokEnd && s[2] == '6') { // i16 suffix
1150 Bits = 16;
1151 ToSkip = 3;
1152 } else if (s + 3 < ThisTokEnd && s[2] == '2' &&
1153 s[3] == '8') { // i128 suffix
1154 Bits = 128;
1155 ToSkip = 4;
1156 }
1157 break;
1158 case '3':
1159 if (s + 2 < ThisTokEnd && s[2] == '2') { // i32 suffix
1160 Bits = 32;
1161 ToSkip = 3;
1162 }
1163 break;
1164 case '6':
1165 if (s + 2 < ThisTokEnd && s[2] == '4') { // i64 suffix
1166 Bits = 64;
1167 ToSkip = 3;
1168 }
1169 break;
1170 default:
1171 break;
1172 }
1173 if (Bits) {
1174 if (HasSize)
1175 break;
1176 HasSize = true;
1177 MicrosoftInteger = Bits;
1178 s += ToSkip;
1179 assert(s <= ThisTokEnd && "didn't maximally munch?");
1180 break;
1181 }
1182 }
1183 [[fallthrough]];
1184 case 'j':
1185 case 'J':
1186 if (isImaginary) break; // Cannot be repeated.
1187 isImaginary = true;
1188 continue; // Success.
1189 case '_':
1190 if (isFPConstant)
1191 break; // Invalid for floats
1192 if (HasSize)
1193 break;
1194 // There is currently no way to reach this with DoubleUnderscore set.
1195 // If new double underscope literals are added handle it here as above.
1196 assert(!DoubleUnderscore && "unhandled double underscore case");
1197 if (LangOpts.CPlusPlus && s + 2 < ThisTokEnd &&
1198 s[1] == '_') { // s + 2 < ThisTokEnd to ensure some character exists
1199 // after __
1200 DoubleUnderscore = true;
1201 s += 2; // Skip both '_'
1202 if (s + 1 < ThisTokEnd &&
1203 (*s == 'u' || *s == 'U')) { // Ensure some character after 'u'/'U'
1204 isUnsigned = true;
1205 ++s;
1206 }
1207 if (s + 1 < ThisTokEnd &&
1208 ((*s == 'w' && *(++s) == 'b') || (*s == 'W' && *(++s) == 'B'))) {
1209 isBitInt = true;
1210 HasSize = true;
1211 continue;
1212 }
1213 }
1214 break;
1215 case 'w':
1216 case 'W':
1217 if (isFPConstant)
1218 break; // Invalid for floats.
1219 if (HasSize)
1220 break; // Invalid if we already have a size for the literal.
1221
1222 // wb and WB are allowed, but a mixture of cases like Wb or wB is not. We
1223 // explicitly do not support the suffix in C++ as an extension because a
1224 // library-based UDL that resolves to a library type may be more
1225 // appropriate there. The same rules apply for __wb/__WB.
1226 if ((!LangOpts.CPlusPlus || DoubleUnderscore) && s + 1 < ThisTokEnd &&
1227 ((s[0] == 'w' && s[1] == 'b') || (s[0] == 'W' && s[1] == 'B'))) {
1228 isBitInt = true;
1229 HasSize = true;
1230 ++s; // Skip both characters (2nd char skipped on continue).
1231 continue; // Success.
1232 }
1233 }
1234 // If we reached here, there was an error or a ud-suffix.
1235 break;
1236 }
1237
1238 // "i", "if", and "il" are user-defined suffixes in C++1y.
1239 if (s != ThisTokEnd || isImaginary) {
1240 // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
1241 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
1242 if (isValidUDSuffix(LangOpts, Suffix: UDSuffixBuf)) {
1243 if (!isImaginary) {
1244 // Any suffix pieces we might have parsed are actually part of the
1245 // ud-suffix.
1246 isLong = false;
1247 isUnsigned = false;
1248 isLongLong = false;
1249 isSizeT = false;
1250 isFloat = false;
1251 isFloat16 = false;
1252 isHalf = false;
1253 isImaginary = false;
1254 isBitInt = false;
1255 MicrosoftInteger = 0;
1256 saw_fixed_point_suffix = false;
1257 isFract = false;
1258 isAccum = false;
1259 }
1260
1261 saw_ud_suffix = true;
1262 return;
1263 }
1264
1265 if (s != ThisTokEnd) {
1266 // Report an error if there are any.
1267 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1268 TokStart: TokLoc, Characters: SuffixBegin - ThisTokBegin, SM, LangOpts),
1269 DiagID: diag::err_invalid_suffix_constant)
1270 << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
1271 << (isFixedPointConstant ? 2 : isFPConstant);
1272 hadError = true;
1273 }
1274 }
1275
1276 if (!hadError && saw_fixed_point_suffix) {
1277 assert(isFract || isAccum);
1278 }
1279}
1280
1281/// ParseDecimalOrOctalCommon - This method is called for decimal or octal
1282/// numbers. It issues an error for illegal digits, and handles floating point
1283/// parsing. If it detects a floating point number, the radix is set to 10.
1284void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
1285 assert((radix == 8 || radix == 10) && "Unexpected radix");
1286
1287 // If we have a hex digit other than 'e' (which denotes a FP exponent) then
1288 // the code is using an incorrect base.
1289 if (isHexDigit(c: *s) && *s != 'e' && *s != 'E' &&
1290 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1291 Diags.Report(
1292 Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts),
1293 DiagID: diag::err_invalid_digit)
1294 << StringRef(s, 1) << (radix == 8 ? 1 : 0);
1295 hadError = true;
1296 return;
1297 }
1298
1299 if (*s == '.') {
1300 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1301 s++;
1302 radix = 10;
1303 saw_period = true;
1304 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1305 s = SkipDigits(ptr: s); // Skip suffix.
1306 }
1307 if (*s == 'e' || *s == 'E') { // exponent
1308 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1309 const char *Exponent = s;
1310 s++;
1311 radix = 10;
1312 saw_exponent = true;
1313 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1314 const char *first_non_digit = SkipDigits(ptr: s);
1315 if (containsDigits(Start: s, End: first_non_digit)) {
1316 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1317 s = first_non_digit;
1318 } else {
1319 if (!hadError) {
1320 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1321 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1322 DiagID: diag::err_exponent_has_no_digits);
1323 hadError = true;
1324 }
1325 return;
1326 }
1327 }
1328}
1329
1330/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1331/// suffixes as ud-suffixes, because the diagnostic experience is better if we
1332/// treat it as an invalid suffix.
1333bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1334 StringRef Suffix) {
1335 if (!LangOpts.CPlusPlus11 || Suffix.empty())
1336 return false;
1337
1338 // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
1339 // Suffixes starting with '__' (double underscore) are for use by
1340 // the implementation.
1341 if (Suffix.starts_with(Prefix: "_") && !Suffix.starts_with(Prefix: "__"))
1342 return true;
1343
1344 // In C++11, there are no library suffixes.
1345 if (!LangOpts.CPlusPlus14)
1346 return false;
1347
1348 // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
1349 // Per tweaked N3660, "il", "i", and "if" are also used in the library.
1350 // In C++2a "d" and "y" are used in the library.
1351 return llvm::StringSwitch<bool>(Suffix)
1352 .Cases(CaseStrings: {"h", "min", "s"}, Value: true)
1353 .Cases(CaseStrings: {"ms", "us", "ns"}, Value: true)
1354 .Cases(CaseStrings: {"il", "i", "if"}, Value: true)
1355 .Cases(CaseStrings: {"d", "y"}, Value: LangOpts.CPlusPlus20)
1356 .Default(Value: false);
1357}
1358
1359void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
1360 const char *Pos,
1361 CheckSeparatorKind IsAfterDigits) {
1362 if (IsAfterDigits == CSK_AfterDigits) {
1363 if (Pos == ThisTokBegin)
1364 return;
1365 --Pos;
1366 } else if (Pos == ThisTokEnd)
1367 return;
1368
1369 if (isDigitSeparator(C: *Pos)) {
1370 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: Pos - ThisTokBegin, SM,
1371 LangOpts),
1372 DiagID: diag::err_digit_separator_not_between_digits)
1373 << IsAfterDigits;
1374 hadError = true;
1375 }
1376}
1377
1378/// ParseNumberStartingWithZero - This method is called when the first character
1379/// of the number is found to be a zero. This means it is either an octal
1380/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
1381/// a floating point number (01239.123e4). Eat the prefix, determining the
1382/// radix etc.
1383void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
1384 assert(s[0] == '0' && "Invalid method call");
1385 s++;
1386
1387 int c1 = s[0];
1388
1389 // Handle a hex number like 0x1234.
1390 if ((c1 == 'x' || c1 == 'X') && (isHexDigit(c: s[1]) || s[1] == '.')) {
1391 s++;
1392 assert(s < ThisTokEnd && "didn't maximally munch?");
1393 radix = 16;
1394 DigitsBegin = s;
1395 s = SkipHexDigits(ptr: s);
1396 bool HasSignificandDigits = containsDigits(Start: DigitsBegin, End: s);
1397 if (s == ThisTokEnd) {
1398 // Done.
1399 } else if (*s == '.') {
1400 s++;
1401 saw_period = true;
1402 const char *floatDigitsBegin = s;
1403 s = SkipHexDigits(ptr: s);
1404 if (containsDigits(Start: floatDigitsBegin, End: s))
1405 HasSignificandDigits = true;
1406 if (HasSignificandDigits)
1407 checkSeparator(TokLoc, Pos: floatDigitsBegin, IsAfterDigits: CSK_BeforeDigits);
1408 }
1409
1410 if (!HasSignificandDigits) {
1411 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1412 LangOpts),
1413 DiagID: diag::err_hex_constant_requires)
1414 << LangOpts.CPlusPlus << 1;
1415 hadError = true;
1416 return;
1417 }
1418
1419 // A binary exponent can appear with or with a '.'. If dotted, the
1420 // binary exponent is required.
1421 if (*s == 'p' || *s == 'P') {
1422 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1423 const char *Exponent = s;
1424 s++;
1425 saw_exponent = true;
1426 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1427 const char *first_non_digit = SkipDigits(ptr: s);
1428 if (!containsDigits(Start: s, End: first_non_digit)) {
1429 if (!hadError) {
1430 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1431 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1432 DiagID: diag::err_exponent_has_no_digits);
1433 hadError = true;
1434 }
1435 return;
1436 }
1437 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1438 s = first_non_digit;
1439
1440 if (!LangOpts.HexFloats)
1441 Diags.Report(Loc: TokLoc, DiagID: LangOpts.CPlusPlus
1442 ? diag::compat_pre_cxx17_hex_literal
1443 : diag::ext_hex_constant_invalid);
1444 else if (LangOpts.CPlusPlus17)
1445 Diags.Report(Loc: TokLoc, DiagID: diag::compat_cxx17_hex_literal);
1446 } else if (saw_period) {
1447 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1448 LangOpts),
1449 DiagID: diag::err_hex_constant_requires)
1450 << LangOpts.CPlusPlus << 0;
1451 hadError = true;
1452 }
1453 return;
1454 }
1455
1456 // Handle simple binary numbers 0b01010
1457 if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
1458 // 0b101010 is a C++14 and C23 extension.
1459 unsigned DiagId;
1460 if (LangOpts.CPlusPlus14)
1461 DiagId = diag::warn_cxx11_compat_binary_literal;
1462 else if (LangOpts.C23)
1463 DiagId = diag::warn_c23_compat_binary_literal;
1464 else if (LangOpts.CPlusPlus)
1465 DiagId = diag::ext_binary_literal_cxx14;
1466 else
1467 DiagId = diag::ext_binary_literal;
1468 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1469 ++s;
1470 assert(s < ThisTokEnd && "didn't maximally munch?");
1471 radix = 2;
1472 DigitsBegin = s;
1473 s = SkipBinaryDigits(ptr: s);
1474 if (s == ThisTokEnd) {
1475 // Done.
1476 } else if (isHexDigit(c: *s) &&
1477 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1478 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1479 LangOpts),
1480 DiagID: diag::err_invalid_digit)
1481 << StringRef(s, 1) << 2;
1482 hadError = true;
1483 }
1484 // Other suffixes will be diagnosed by the caller.
1485 return;
1486 }
1487
1488 // Parse a potential octal literal prefix.
1489 bool IsSingleZero = false;
1490 if ((c1 == 'O' || c1 == 'o') && (s[1] >= '0' && s[1] <= '7')) {
1491 unsigned DiagId;
1492 if (LangOpts.C2y)
1493 DiagId = diag::warn_c2y_compat_octal_literal;
1494 else if (LangOpts.CPlusPlus)
1495 DiagId = diag::ext_cpp_octal_literal;
1496 else
1497 DiagId = diag::ext_octal_literal;
1498 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1499
1500 ++s;
1501 DigitsBegin = s;
1502 radix = 8;
1503 s = SkipOctalDigits(ptr: s);
1504 if (s == ThisTokEnd) {
1505 // Done
1506 } else if ((isHexDigit(c: *s) && *s != 'e' && *s != 'E' && *s != '.') &&
1507 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1508 auto InvalidDigitLoc = Lexer::AdvanceToTokenCharacter(
1509 TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts);
1510 Diags.Report(Loc: InvalidDigitLoc, DiagID: diag::err_invalid_digit)
1511 << StringRef(s, 1) << 1;
1512 hadError = true;
1513 }
1514 // Other suffixes will be diagnosed by the caller.
1515 return;
1516 }
1517
1518 llvm::scope_exit _([&] {
1519 // If we still have an octal value but we did not see an octal prefix,
1520 // diagnose as being an obsolescent feature starting in C2y.
1521 if (radix == 8 && LangOpts.C2y && !hadError && !IsSingleZero)
1522 Diags.Report(Loc: TokLoc, DiagID: diag::warn_unprefixed_octal_deprecated);
1523 });
1524
1525 // For now, the radix is set to 8. If we discover that we have a
1526 // floating point constant, the radix will change to 10. Octal floating
1527 // point constants are not permitted (only decimal and hexadecimal).
1528 radix = 8;
1529 const char *PossibleNewDigitStart = s;
1530 s = SkipOctalDigits(ptr: s);
1531 // When the value is 0 followed by a suffix (like 0wb), we want to leave 0
1532 // as the start of the digits. So if skipping octal digits does not skip
1533 // anything, we leave the digit start where it was.
1534 if (s != PossibleNewDigitStart)
1535 DigitsBegin = PossibleNewDigitStart;
1536 else
1537 IsSingleZero = (s == ThisTokBegin + 1);
1538
1539 if (s == ThisTokEnd)
1540 return; // Done, simple octal number like 01234
1541
1542 // If we have some other non-octal digit that *is* a decimal digit, see if
1543 // this is part of a floating point number like 094.123 or 09e1.
1544 if (isDigit(c: *s)) {
1545 const char *EndDecimal = SkipDigits(ptr: s);
1546 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
1547 s = EndDecimal;
1548 radix = 10;
1549 }
1550 }
1551
1552 ParseDecimalOrOctalCommon(TokLoc);
1553}
1554
1555static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
1556 switch (Radix) {
1557 case 2:
1558 return NumDigits <= 64;
1559 case 8:
1560 return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
1561 case 10:
1562 return NumDigits <= 19; // floor(log10(2^64))
1563 case 16:
1564 return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
1565 default:
1566 llvm_unreachable("impossible Radix");
1567 }
1568}
1569
1570/// GetIntegerValue - Convert this numeric literal value to an APInt that
1571/// matches Val's input width. If there is an overflow, set Val to the low bits
1572/// of the result and return true. Otherwise, return false.
1573bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1574 // Fast path: Compute a conservative bound on the maximum number of
1575 // bits per digit in this radix. If we can't possibly overflow a
1576 // uint64 based on that bound then do the simple conversion to
1577 // integer. This avoids the expensive overflow checking below, and
1578 // handles the common cases that matter (small decimal integers and
1579 // hex/octal values which don't overflow).
1580 const unsigned NumDigits = SuffixBegin - DigitsBegin;
1581 if (alwaysFitsInto64Bits(Radix: radix, NumDigits)) {
1582 uint64_t N = 0;
1583 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
1584 if (!isDigitSeparator(C: *Ptr))
1585 N = N * radix + llvm::hexDigitValue(C: *Ptr);
1586
1587 // This will truncate the value to Val's input width. Simply check
1588 // for overflow by comparing.
1589 Val = N;
1590 return Val.getZExtValue() != N;
1591 }
1592
1593 Val = 0;
1594 const char *Ptr = DigitsBegin;
1595
1596 llvm::APInt RadixVal(Val.getBitWidth(), radix);
1597 llvm::APInt CharVal(Val.getBitWidth(), 0);
1598 llvm::APInt OldVal = Val;
1599
1600 bool OverflowOccurred = false;
1601 while (Ptr < SuffixBegin) {
1602 if (isDigitSeparator(C: *Ptr)) {
1603 ++Ptr;
1604 continue;
1605 }
1606
1607 unsigned C = llvm::hexDigitValue(C: *Ptr++);
1608
1609 // If this letter is out of bound for this radix, reject it.
1610 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1611
1612 CharVal = C;
1613
1614 // Add the digit to the value in the appropriate radix. If adding in digits
1615 // made the value smaller, then this overflowed.
1616 OldVal = Val;
1617
1618 // Multiply by radix, did overflow occur on the multiply?
1619 Val *= RadixVal;
1620 OverflowOccurred |= Val.udiv(RHS: RadixVal) != OldVal;
1621
1622 // Add value, did overflow occur on the value?
1623 // (a + b) ult b <=> overflow
1624 Val += CharVal;
1625 OverflowOccurred |= Val.ult(RHS: CharVal);
1626 }
1627 return OverflowOccurred;
1628}
1629
1630llvm::APFloat::opStatus
1631NumericLiteralParser::GetFloatValue(llvm::APFloat &Result,
1632 llvm::RoundingMode RM) {
1633 using llvm::APFloat;
1634
1635 unsigned n = std::min(a: SuffixBegin - ThisTokBegin, b: ThisTokEnd - ThisTokBegin);
1636
1637 llvm::SmallString<16> Buffer;
1638 StringRef Str(ThisTokBegin, n);
1639 if (Str.contains(C: '\'')) {
1640 Buffer.reserve(N: n);
1641 std::remove_copy_if(first: Str.begin(), last: Str.end(), result: std::back_inserter(x&: Buffer),
1642 pred: &isDigitSeparator);
1643 Str = Buffer;
1644 }
1645
1646 auto StatusOrErr = Result.convertFromString(Str, RM);
1647 assert(StatusOrErr && "Invalid floating point representation");
1648 return !errorToBool(Err: StatusOrErr.takeError()) ? *StatusOrErr
1649 : APFloat::opInvalidOp;
1650}
1651
1652static inline bool IsExponentPart(char c, bool isHex) {
1653 if (isHex)
1654 return c == 'p' || c == 'P';
1655 return c == 'e' || c == 'E';
1656}
1657
1658bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1659 assert(radix == 16 || radix == 10);
1660
1661 // Find how many digits are needed to store the whole literal.
1662 unsigned NumDigits = SuffixBegin - DigitsBegin;
1663 if (saw_period) --NumDigits;
1664
1665 // Initial scan of the exponent if it exists
1666 bool ExpOverflowOccurred = false;
1667 bool NegativeExponent = false;
1668 const char *ExponentBegin;
1669 uint64_t Exponent = 0;
1670 int64_t BaseShift = 0;
1671 if (saw_exponent) {
1672 const char *Ptr = DigitsBegin;
1673
1674 while (!IsExponentPart(c: *Ptr, isHex: radix == 16))
1675 ++Ptr;
1676 ExponentBegin = Ptr;
1677 ++Ptr;
1678 NegativeExponent = *Ptr == '-';
1679 if (NegativeExponent) ++Ptr;
1680
1681 unsigned NumExpDigits = SuffixBegin - Ptr;
1682 if (alwaysFitsInto64Bits(Radix: radix, NumDigits: NumExpDigits)) {
1683 llvm::StringRef ExpStr(Ptr, NumExpDigits);
1684 llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1685 Exponent = ExpInt.getZExtValue();
1686 } else {
1687 ExpOverflowOccurred = true;
1688 }
1689
1690 if (NegativeExponent) BaseShift -= Exponent;
1691 else BaseShift += Exponent;
1692 }
1693
1694 // Number of bits needed for decimal literal is
1695 // ceil(NumDigits * log2(10)) Integral part
1696 // + Scale Fractional part
1697 // + ceil(Exponent * log2(10)) Exponent
1698 // --------------------------------------------------
1699 // ceil((NumDigits + Exponent) * log2(10)) + Scale
1700 //
1701 // But for simplicity in handling integers, we can round up log2(10) to 4,
1702 // making:
1703 // 4 * (NumDigits + Exponent) + Scale
1704 //
1705 // Number of digits needed for hexadecimal literal is
1706 // 4 * NumDigits Integral part
1707 // + Scale Fractional part
1708 // + Exponent Exponent
1709 // --------------------------------------------------
1710 // (4 * NumDigits) + Scale + Exponent
1711 uint64_t NumBitsNeeded;
1712 if (radix == 10)
1713 NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1714 else
1715 NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1716
1717 if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1718 ExpOverflowOccurred = true;
1719 llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1720
1721 bool FoundDecimal = false;
1722
1723 int64_t FractBaseShift = 0;
1724 const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1725 for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1726 if (*Ptr == '.') {
1727 FoundDecimal = true;
1728 continue;
1729 }
1730
1731 // Normal reading of an integer
1732 unsigned C = llvm::hexDigitValue(C: *Ptr);
1733 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1734
1735 Val *= radix;
1736 Val += C;
1737
1738 if (FoundDecimal)
1739 // Keep track of how much we will need to adjust this value by from the
1740 // number of digits past the radix point.
1741 --FractBaseShift;
1742 }
1743
1744 // For a radix of 16, we will be multiplying by 2 instead of 16.
1745 if (radix == 16) FractBaseShift *= 4;
1746 BaseShift += FractBaseShift;
1747
1748 Val <<= Scale;
1749
1750 uint64_t Base = (radix == 16) ? 2 : 10;
1751 if (BaseShift > 0) {
1752 for (int64_t i = 0; i < BaseShift; ++i) {
1753 Val *= Base;
1754 }
1755 } else if (BaseShift < 0) {
1756 for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i)
1757 Val = Val.udiv(RHS: Base);
1758 }
1759
1760 bool IntOverflowOccurred = false;
1761 auto MaxVal = llvm::APInt::getMaxValue(numBits: StoreVal.getBitWidth());
1762 if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1763 IntOverflowOccurred |= Val.ugt(RHS: MaxVal.zext(width: Val.getBitWidth()));
1764 StoreVal = Val.trunc(width: StoreVal.getBitWidth());
1765 } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1766 IntOverflowOccurred |= Val.zext(width: MaxVal.getBitWidth()).ugt(RHS: MaxVal);
1767 StoreVal = Val.zext(width: StoreVal.getBitWidth());
1768 } else {
1769 StoreVal = std::move(Val);
1770 }
1771
1772 return IntOverflowOccurred || ExpOverflowOccurred;
1773}
1774
1775/// \verbatim
1776/// user-defined-character-literal: [C++11 lex.ext]
1777/// character-literal ud-suffix
1778/// ud-suffix:
1779/// identifier
1780/// character-literal: [C++11 lex.ccon]
1781/// ' c-char-sequence '
1782/// u' c-char-sequence '
1783/// U' c-char-sequence '
1784/// L' c-char-sequence '
1785/// u8' c-char-sequence ' [C++1z lex.ccon]
1786/// c-char-sequence:
1787/// c-char
1788/// c-char-sequence c-char
1789/// c-char:
1790/// any member of the source character set except the single-quote ',
1791/// backslash \, or new-line character
1792/// escape-sequence
1793/// universal-character-name
1794/// escape-sequence:
1795/// simple-escape-sequence
1796/// octal-escape-sequence
1797/// hexadecimal-escape-sequence
1798/// simple-escape-sequence:
1799/// one of \' \" \? \\ \a \b \f \n \r \t \v
1800/// octal-escape-sequence:
1801/// \ octal-digit
1802/// \ octal-digit octal-digit
1803/// \ octal-digit octal-digit octal-digit
1804/// hexadecimal-escape-sequence:
1805/// \x hexadecimal-digit
1806/// hexadecimal-escape-sequence hexadecimal-digit
1807/// universal-character-name: [C++11 lex.charset]
1808/// \u hex-quad
1809/// \U hex-quad hex-quad
1810/// hex-quad:
1811/// hex-digit hex-digit hex-digit hex-digit
1812/// \endverbatim
1813///
1814CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1815 SourceLocation Loc, Preprocessor &PP,
1816 tok::TokenKind kind) {
1817 // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1818 HadError = false;
1819
1820 Kind = kind;
1821
1822 const char *TokBegin = begin;
1823
1824 // Skip over wide character determinant.
1825 if (Kind != tok::char_constant)
1826 ++begin;
1827 if (Kind == tok::utf8_char_constant)
1828 ++begin;
1829
1830 // Skip over the entry quote.
1831 if (begin[0] != '\'') {
1832 PP.Diag(Loc, DiagID: diag::err_lexing_char);
1833 HadError = true;
1834 return;
1835 }
1836
1837 ++begin;
1838
1839 // Remove an optional ud-suffix.
1840 if (end[-1] != '\'') {
1841 const char *UDSuffixEnd = end;
1842 do {
1843 --end;
1844 } while (end[-1] != '\'');
1845 // FIXME: Don't bother with this if !tok.hasUCN().
1846 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(end, UDSuffixEnd - end));
1847 UDSuffixOffset = end - TokBegin;
1848 }
1849
1850 // Trim the ending quote.
1851 assert(end != begin && "Invalid token lexed");
1852 --end;
1853
1854 // FIXME: The "Value" is an uint64_t so we can handle char literals of
1855 // up to 64-bits.
1856 // FIXME: This extensively assumes that 'char' is 8-bits.
1857 assert(PP.getTargetInfo().getCharWidth() == 8 &&
1858 "Assumes char is 8 bits");
1859 assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1860 (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1861 "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1862 assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1863 "Assumes sizeof(wchar) on target is <= 64");
1864
1865 SmallVector<uint32_t, 4> codepoint_buffer;
1866 codepoint_buffer.resize(N: end - begin);
1867 uint32_t *buffer_begin = &codepoint_buffer.front();
1868 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1869
1870 const TextEncoding &TE = PP.getTextEncoding();
1871 llvm::TextEncodingConverter *Converter = nullptr;
1872 if (isOrdinary())
1873 Converter = TE.getConverter(Action: CA_ToLiteralEncoding);
1874
1875 // Unicode escapes representing characters that cannot be correctly
1876 // represented in a single code unit are disallowed in character literals
1877 // by this implementation.
1878 uint32_t largest_character_for_kind;
1879 if (tok::wide_char_constant == Kind) {
1880 largest_character_for_kind =
1881 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1882 } else if (tok::utf8_char_constant == Kind) {
1883 largest_character_for_kind = 0x7F;
1884 } else if (tok::utf16_char_constant == Kind) {
1885 largest_character_for_kind = 0xFFFF;
1886 } else if (tok::utf32_char_constant == Kind) {
1887 largest_character_for_kind = 0x10FFFF;
1888 } else {
1889 largest_character_for_kind = (Converter == nullptr) ? 0x7Fu : 0xFFu;
1890 }
1891
1892 while (begin != end) {
1893 // Is this a span of non-escape characters?
1894 if (begin[0] != '\\') {
1895 char const *start = begin;
1896 do {
1897 ++begin;
1898 } while (begin != end && *begin != '\\');
1899
1900 char const *tmp_in_start = start;
1901 uint32_t *tmp_out_start = buffer_begin;
1902 std::string UTF8String(start, begin);
1903 llvm::ConversionResult res =
1904 llvm::ConvertUTF8toUTF32(sourceStart: reinterpret_cast<llvm::UTF8 const **>(&start),
1905 sourceEnd: reinterpret_cast<llvm::UTF8 const *>(begin),
1906 targetStart: &buffer_begin, targetEnd: buffer_end, flags: llvm::strictConversion);
1907 if (res != llvm::conversionOK) {
1908 // If we see bad encoding for unprefixed character literals, warn and
1909 // simply copy the byte values, for compatibility with gcc and
1910 // older versions of clang.
1911 bool NoErrorOnBadEncoding = isOrdinary();
1912 unsigned Msg = diag::err_bad_character_encoding;
1913 if (NoErrorOnBadEncoding)
1914 Msg = diag::warn_bad_character_encoding;
1915 PP.Diag(Loc, DiagID: Msg);
1916 if (NoErrorOnBadEncoding) {
1917 start = tmp_in_start;
1918 buffer_begin = tmp_out_start;
1919 for (; start != begin; ++start, ++buffer_begin)
1920 *buffer_begin = static_cast<uint8_t>(*start);
1921 } else {
1922 HadError = true;
1923 }
1924 } else {
1925 uint32_t *validation_ptr = tmp_out_start;
1926 for (; validation_ptr < buffer_begin; ++validation_ptr) {
1927 if (*validation_ptr > largest_character_for_kind) {
1928 HadError = true;
1929 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1930 }
1931 }
1932
1933 // Convert to execution character set if needed
1934 if (!HadError && Converter) {
1935 assert(isOrdinary() && "Only ordinary characters are supported");
1936 SmallString<4> Converted;
1937 auto ErrorOrChar = Converter->convert(Source: UTF8String, Result&: Converted);
1938 if (!ErrorOrChar) {
1939 for (int i = 0; tmp_out_start < buffer_begin;
1940 ++tmp_out_start, ++i) {
1941 *tmp_out_start = Converted[i];
1942 }
1943 } else {
1944 HadError = true;
1945 PP.Diag(Loc, DiagID: diag::err_exec_charset_conversion_failed)
1946 << ErrorOrChar.message() << 0;
1947 }
1948 }
1949 }
1950
1951 continue;
1952 }
1953 // Is this a Universal Character Name escape?
1954 if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') {
1955 if (Converter == nullptr) {
1956 unsigned short UcnLen = 0;
1957 if (!ProcessUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, UcnVal&: *buffer_begin, UcnLen,
1958 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1959 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(), in_char_string_literal: true)) {
1960 HadError = true;
1961 } else if (*buffer_begin > largest_character_for_kind) {
1962 HadError = true;
1963 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1964 }
1965 } else {
1966 char Cp[5];
1967 char *ResultPtr = Cp;
1968 EncodeUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, ResultBuf&: ResultPtr, HadError,
1969 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1970 /*CharByteWidth=*/1u, Diags: &PP.getDiagnostics(),
1971 Features: PP.getLangOpts(), Converter, Action: CA_ToLiteralEncoding);
1972 if (!HadError)
1973 *buffer_begin = *Cp;
1974 }
1975 ++buffer_begin;
1976 continue;
1977 }
1978 unsigned CharWidth = getCharWidth(kind: Kind, Target: PP.getTargetInfo());
1979 uint64_t result = ProcessCharEscape(
1980 ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, HadError,
1981 Loc: FullSourceLoc(Loc, PP.getSourceManager()), CharWidth,
1982 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(),
1983 EvalMethod: StringLiteralEvalMethod::Evaluated, Converter: nullptr, Action: CA_NoConversion);
1984 *buffer_begin++ = result;
1985 }
1986
1987 unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1988
1989 if (NumCharsSoFar > 1) {
1990 if (isOrdinary() && NumCharsSoFar == 4)
1991 PP.Diag(Loc, DiagID: diag::warn_four_char_character_literal);
1992 else if (isOrdinary())
1993 PP.Diag(Loc, DiagID: diag::warn_multichar_character_literal);
1994 else {
1995 PP.Diag(Loc, DiagID: diag::err_multichar_character_literal) << (isWide() ? 0 : 1);
1996 HadError = true;
1997 }
1998 IsMultiChar = true;
1999 } else {
2000 IsMultiChar = false;
2001 }
2002
2003 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
2004
2005 // Narrow character literals act as though their value is concatenated
2006 // in this implementation, but warn on overflow.
2007 bool multi_char_too_long = false;
2008 if (isOrdinary() && isMultiChar()) {
2009 LitVal = 0;
2010 for (size_t i = 0; i < NumCharsSoFar; ++i) {
2011 // check for enough leading zeros to shift into
2012 multi_char_too_long |= (LitVal.countl_zero() < 8);
2013 LitVal <<= 8;
2014 LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
2015 }
2016 } else if (NumCharsSoFar > 0) {
2017 // otherwise just take the last character
2018 LitVal = buffer_begin[-1];
2019 }
2020
2021 if (!HadError && multi_char_too_long) {
2022 PP.Diag(Loc, DiagID: diag::warn_char_constant_too_large);
2023 }
2024
2025 // Transfer the value from APInt to uint64_t
2026 Value = LitVal.getZExtValue();
2027
2028 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
2029 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple
2030 // character constants are not sign extended in the this implementation:
2031 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
2032 if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) &&
2033 PP.getLangOpts().CharIsSigned)
2034 Value = (signed char)Value;
2035}
2036
2037/// \verbatim
2038/// string-literal: [C++0x lex.string]
2039/// encoding-prefix " [s-char-sequence] "
2040/// encoding-prefix R raw-string
2041/// encoding-prefix:
2042/// u8
2043/// u
2044/// U
2045/// L
2046/// s-char-sequence:
2047/// s-char
2048/// s-char-sequence s-char
2049/// s-char:
2050/// any member of the source character set except the double-quote ",
2051/// backslash \, or new-line character
2052/// escape-sequence
2053/// universal-character-name
2054/// raw-string:
2055/// " d-char-sequence ( r-char-sequence ) d-char-sequence "
2056/// r-char-sequence:
2057/// r-char
2058/// r-char-sequence r-char
2059/// r-char:
2060/// any member of the source character set, except a right parenthesis )
2061/// followed by the initial d-char-sequence (which may be empty)
2062/// followed by a double quote ".
2063/// d-char-sequence:
2064/// d-char
2065/// d-char-sequence d-char
2066/// d-char:
2067/// any member of the basic source character set except:
2068/// space, the left parenthesis (, the right parenthesis ),
2069/// the backslash \, and the control characters representing horizontal
2070/// tab, vertical tab, form feed, and newline.
2071/// escape-sequence: [C++0x lex.ccon]
2072/// simple-escape-sequence
2073/// octal-escape-sequence
2074/// hexadecimal-escape-sequence
2075/// simple-escape-sequence:
2076/// one of \' \" \? \\ \a \b \f \n \r \t \v
2077/// octal-escape-sequence:
2078/// \ octal-digit
2079/// \ octal-digit octal-digit
2080/// \ octal-digit octal-digit octal-digit
2081/// hexadecimal-escape-sequence:
2082/// \x hexadecimal-digit
2083/// hexadecimal-escape-sequence hexadecimal-digit
2084/// universal-character-name:
2085/// \u hex-quad
2086/// \U hex-quad hex-quad
2087/// hex-quad:
2088/// hex-digit hex-digit hex-digit hex-digit
2089/// \endverbatim
2090///
2091StringLiteralParser::StringLiteralParser(ArrayRef<Token> StringToks,
2092 Preprocessor &PP,
2093 StringLiteralEvalMethod EvalMethod,
2094 ConversionAction Action)
2095 : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
2096 Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()),
2097 TE(&PP.getTextEncoding()), MaxTokenLength(0), SizeBound(0),
2098 CharByteWidth(0), Kind(tok::unknown), ResultPtr(ResultBuf.data()),
2099 EvalMethod(EvalMethod), Action(Action), hadError(false), Pascal(false) {
2100 init(StringToks);
2101}
2102
2103void StringLiteralParser::init(ArrayRef<Token> StringToks) {
2104 // The literal token may have come from an invalid source location (e.g. due
2105 // to a PCH error), in which case the token length will be 0.
2106 if (StringToks.empty() || StringToks[0].getLength() < 2)
2107 return DiagnoseLexingError(Loc: SourceLocation());
2108
2109 // Scan all of the string portions, remember the max individual token length,
2110 // computing a bound on the concatenated string length, and see whether any
2111 // piece is a wide-string. If any of the string portions is a wide-string
2112 // literal, the result is a wide-string literal [C99 6.4.5p4].
2113 assert(!StringToks.empty() && "expected at least one token");
2114 MaxTokenLength = StringToks[0].getLength();
2115 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
2116 SizeBound = StringToks[0].getLength() - 2; // -2 for "".
2117 hadError = false;
2118
2119 // Determines the kind of string from the prefix
2120 Kind = tok::string_literal;
2121
2122 /// (C99 5.1.1.2p1). The common case is only one string fragment.
2123 for (const Token &Tok : StringToks) {
2124 if (Tok.getLength() < 2)
2125 return DiagnoseLexingError(Loc: Tok.getLocation());
2126
2127 // The string could be shorter than this if it needs cleaning, but this is a
2128 // reasonable bound, which is all we need.
2129 assert(Tok.getLength() >= 2 && "literal token is invalid!");
2130 SizeBound += Tok.getLength() - 2; // -2 for "".
2131
2132 // Remember maximum string piece length.
2133 if (Tok.getLength() > MaxTokenLength)
2134 MaxTokenLength = Tok.getLength();
2135
2136 // Remember if we see any wide or utf-8/16/32 strings.
2137 // Also check for illegal concatenations.
2138 if (isUnevaluated() && Tok.getKind() != tok::string_literal) {
2139 if (Diags) {
2140 SourceLocation PrefixEndLoc = Lexer::AdvanceToTokenCharacter(
2141 TokStart: Tok.getLocation(), Characters: getEncodingPrefixLen(kind: Tok.getKind()), SM,
2142 LangOpts: Features);
2143 CharSourceRange Range =
2144 CharSourceRange::getCharRange(R: {Tok.getLocation(), PrefixEndLoc});
2145 StringRef Prefix(SM.getCharacterData(SL: Tok.getLocation()),
2146 getEncodingPrefixLen(kind: Tok.getKind()));
2147 Diags->Report(Loc: Tok.getLocation(),
2148 DiagID: Features.CPlusPlus26
2149 ? diag::err_unevaluated_string_prefix
2150 : diag::warn_unevaluated_string_prefix)
2151 << Prefix << Features.CPlusPlus << FixItHint::CreateRemoval(RemoveRange: Range);
2152 }
2153 if (Features.CPlusPlus26)
2154 hadError = true;
2155 } else if (Tok.isNot(K: Kind) && Tok.isNot(K: tok::string_literal)) {
2156 if (isOrdinary()) {
2157 Kind = Tok.getKind();
2158 } else {
2159 if (Diags)
2160 Diags->Report(Loc: Tok.getLocation(), DiagID: diag::err_unsupported_string_concat);
2161 hadError = true;
2162 }
2163 }
2164 }
2165
2166 // Include space for the null terminator.
2167 ++SizeBound;
2168
2169 // TODO: K&R warning: "traditional C rejects string constant concatenation"
2170
2171 // Get the width in bytes of char/wchar_t/char16_t/char32_t
2172 CharByteWidth = getCharWidth(kind: Kind, Target);
2173 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
2174 CharByteWidth /= 8;
2175
2176 // The output buffer size needs to be large enough to hold wide characters.
2177 // This is a worst-case assumption which basically corresponds to L"" "long".
2178 SizeBound *= CharByteWidth;
2179
2180 // Size the temporary buffer to hold the result string data.
2181 ResultBuf.resize(N: SizeBound);
2182
2183 // Likewise, but for each string piece.
2184 SmallString<512> TokenBuf;
2185 TokenBuf.resize(N: MaxTokenLength);
2186
2187 // Loop over all the strings, getting their spelling, and expanding them to
2188 // wide strings as appropriate.
2189 ResultPtr = &ResultBuf[0]; // Next byte to fill in.
2190
2191 Pascal = false;
2192
2193 SourceLocation UDSuffixTokLoc;
2194
2195 llvm::TextEncodingConverter *Converter = nullptr;
2196 if (isOrdinary() && TE)
2197 Converter = TE->getConverter(Action);
2198
2199 for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
2200 const char *ThisTokBuf = &TokenBuf[0];
2201 // Get the spelling of the token, which eliminates trigraphs, etc. We know
2202 // that ThisTokBuf points to a buffer that is big enough for the whole token
2203 // and 'spelled' tokens can only shrink.
2204 bool StringInvalid = false;
2205 unsigned ThisTokLen =
2206 Lexer::getSpelling(Tok: StringToks[i], Buffer&: ThisTokBuf, SourceMgr: SM, LangOpts: Features,
2207 Invalid: &StringInvalid);
2208 if (StringInvalid)
2209 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2210
2211 const char *ThisTokBegin = ThisTokBuf;
2212 const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
2213
2214 // Remove an optional ud-suffix.
2215 if (ThisTokEnd[-1] != '"') {
2216 const char *UDSuffixEnd = ThisTokEnd;
2217 do {
2218 --ThisTokEnd;
2219 } while (ThisTokEnd[-1] != '"');
2220
2221 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
2222
2223 if (UDSuffixBuf.empty()) {
2224 if (StringToks[i].hasUCN())
2225 expandUCNs(Buf&: UDSuffixBuf, Input: UDSuffix);
2226 else
2227 UDSuffixBuf.assign(RHS: UDSuffix);
2228 UDSuffixToken = i;
2229 UDSuffixOffset = ThisTokEnd - ThisTokBuf;
2230 UDSuffixTokLoc = StringToks[i].getLocation();
2231 } else {
2232 SmallString<32> ExpandedUDSuffix;
2233 if (StringToks[i].hasUCN()) {
2234 expandUCNs(Buf&: ExpandedUDSuffix, Input: UDSuffix);
2235 UDSuffix = ExpandedUDSuffix;
2236 }
2237
2238 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
2239 // result of a concatenation involving at least one user-defined-string-
2240 // literal, all the participating user-defined-string-literals shall
2241 // have the same ud-suffix.
2242 bool UnevaluatedStringHasUDL = isUnevaluated() && !UDSuffix.empty();
2243 if (UDSuffixBuf != UDSuffix || UnevaluatedStringHasUDL) {
2244 if (Diags) {
2245 SourceLocation TokLoc = StringToks[i].getLocation();
2246 if (UnevaluatedStringHasUDL) {
2247 Diags->Report(Loc: TokLoc, DiagID: diag::err_unevaluated_string_udl)
2248 << SourceRange(TokLoc, TokLoc);
2249 } else {
2250 Diags->Report(Loc: TokLoc, DiagID: diag::err_string_concat_mixed_suffix)
2251 << UDSuffixBuf << UDSuffix
2252 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc);
2253 }
2254 }
2255 hadError = true;
2256 }
2257 }
2258 }
2259
2260 // Strip the end quote.
2261 --ThisTokEnd;
2262
2263 // TODO: Input character set mapping support.
2264
2265 // Skip marker for wide or unicode strings.
2266 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
2267 ++ThisTokBuf;
2268 // Skip 8 of u8 marker for utf8 strings.
2269 if (ThisTokBuf[0] == '8')
2270 ++ThisTokBuf;
2271 }
2272
2273 // Check for raw string
2274 if (ThisTokBuf[0] == 'R') {
2275 if (ThisTokBuf[1] != '"') {
2276 // The file may have come from PCH and then changed after loading the
2277 // PCH; Fail gracefully.
2278 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2279 }
2280 ThisTokBuf += 2; // skip R"
2281
2282 // C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16
2283 // characters.
2284 constexpr unsigned MaxRawStrDelimLen = 16;
2285
2286 const char *Prefix = ThisTokBuf;
2287 while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen &&
2288 ThisTokBuf[0] != '(')
2289 ++ThisTokBuf;
2290 if (ThisTokBuf[0] != '(')
2291 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2292 ++ThisTokBuf; // skip '('
2293
2294 // Remove same number of characters from the end
2295 ThisTokEnd -= ThisTokBuf - Prefix;
2296 if (ThisTokEnd < ThisTokBuf)
2297 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2298
2299 // C++14 [lex.string]p4: A source-file new-line in a raw string literal
2300 // results in a new-line in the resulting execution string-literal.
2301 StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
2302 while (!RemainingTokenSpan.empty()) {
2303 // Split the string literal on \r\n boundaries.
2304 size_t CRLFPos = RemainingTokenSpan.find(Str: "\r\n");
2305 StringRef BeforeCRLF = RemainingTokenSpan.substr(Start: 0, N: CRLFPos);
2306 StringRef AfterCRLF = RemainingTokenSpan.substr(Start: CRLFPos);
2307
2308 // Copy everything before the \r\n sequence into the string literal.
2309 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin, Fragment: BeforeCRLF,
2310 Converter))
2311 hadError = true;
2312
2313 // Point into the \n inside the \r\n sequence and operate on the
2314 // remaining portion of the literal.
2315 RemainingTokenSpan = AfterCRLF.substr(Start: 1);
2316 }
2317 } else {
2318 if (ThisTokBuf[0] != '"') {
2319 // The file may have come from PCH and then changed after loading the
2320 // PCH; Fail gracefully.
2321 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2322 }
2323 ++ThisTokBuf; // skip "
2324
2325 // Check if this is a pascal string
2326 if (!isUnevaluated() && Features.PascalStrings &&
2327 ThisTokBuf + 1 != ThisTokEnd && ThisTokBuf[0] == '\\' &&
2328 ThisTokBuf[1] == 'p') {
2329
2330 // If the \p sequence is found in the first token, we have a pascal string
2331 // Otherwise, if we already have a pascal string, ignore the first \p
2332 if (i == 0) {
2333 ++ThisTokBuf;
2334 Pascal = true;
2335 } else if (Pascal)
2336 ThisTokBuf += 2;
2337 }
2338
2339 while (ThisTokBuf != ThisTokEnd) {
2340 // Is this a span of non-escape characters?
2341 if (ThisTokBuf[0] != '\\') {
2342 const char *InStart = ThisTokBuf;
2343 do {
2344 ++ThisTokBuf;
2345 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
2346
2347 // Copy the character span over.
2348 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin,
2349 Fragment: StringRef(InStart, ThisTokBuf - InStart),
2350 Converter))
2351 hadError = true;
2352 continue;
2353 }
2354 // Is this a Universal Character Name escape?
2355 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' ||
2356 ThisTokBuf[1] == 'N') {
2357 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, ResultBuf&: ResultPtr,
2358 HadError&: hadError,
2359 Loc: FullSourceLoc(StringToks[i].getLocation(), SM),
2360 CharByteWidth, Diags, Features, Converter, Action);
2361 continue;
2362 }
2363 // Otherwise, this is a non-UCN escape character. Process it.
2364 unsigned ResultChar = ProcessCharEscape(
2365 ThisTokBegin, ThisTokBuf, ThisTokEnd, HadError&: hadError,
2366 Loc: FullSourceLoc(StringToks[i].getLocation(), SM), CharWidth: CharByteWidth * 8,
2367 Diags, Features, EvalMethod, Converter, Action);
2368
2369 if (CharByteWidth == 4) {
2370 // FIXME: Make the type of the result buffer correct instead of
2371 // using reinterpret_cast.
2372 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
2373 *ResultWidePtr = ResultChar;
2374 ResultPtr += 4;
2375 } else if (CharByteWidth == 2) {
2376 // FIXME: Make the type of the result buffer correct instead of
2377 // using reinterpret_cast.
2378 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
2379 *ResultWidePtr = ResultChar & 0xFFFF;
2380 ResultPtr += 2;
2381 } else {
2382 assert(CharByteWidth == 1 && "Unexpected char width");
2383 *ResultPtr++ = ResultChar & 0xFF;
2384 }
2385 }
2386 }
2387 }
2388
2389 assert((!Pascal || !isUnevaluated()) &&
2390 "Pascal string in unevaluated context");
2391 if (Pascal) {
2392 if (CharByteWidth == 4) {
2393 // FIXME: Make the type of the result buffer correct instead of
2394 // using reinterpret_cast.
2395 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
2396 ResultWidePtr[0] = GetNumStringChars() - 1;
2397 } else if (CharByteWidth == 2) {
2398 // FIXME: Make the type of the result buffer correct instead of
2399 // using reinterpret_cast.
2400 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
2401 ResultWidePtr[0] = GetNumStringChars() - 1;
2402 } else {
2403 assert(CharByteWidth == 1 && "Unexpected char width");
2404 ResultBuf[0] = GetNumStringChars() - 1;
2405 }
2406
2407 // Verify that pascal strings aren't too large.
2408 if (GetStringLength() > 256) {
2409 if (Diags)
2410 Diags->Report(Loc: StringToks.front().getLocation(),
2411 DiagID: diag::err_pascal_string_too_long)
2412 << SourceRange(StringToks.front().getLocation(),
2413 StringToks.back().getLocation());
2414 hadError = true;
2415 return;
2416 }
2417 } else if (Diags) {
2418 // Complain if this string literal has too many characters.
2419 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
2420
2421 if (GetNumStringChars() > MaxChars)
2422 Diags->Report(Loc: StringToks.front().getLocation(),
2423 DiagID: diag::ext_string_too_long)
2424 << GetNumStringChars() << MaxChars
2425 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
2426 << SourceRange(StringToks.front().getLocation(),
2427 StringToks.back().getLocation());
2428 }
2429}
2430
2431static const char *resyncUTF8(const char *Err, const char *End) {
2432 if (Err == End)
2433 return End;
2434 End = Err + std::min<unsigned>(a: llvm::getNumBytesForUTF8(firstByte: *Err), b: End-Err);
2435 while (++Err != End && (*Err & 0xC0) == 0x80)
2436 ;
2437 return Err;
2438}
2439
2440/// This function copies from Fragment, which is a sequence of bytes
2441/// within Tok's contents (which begin at TokBegin) into ResultPtr.
2442/// Performs widening for multi-byte characters.
2443bool StringLiteralParser::CopyStringFragment(
2444 const Token &Tok, const char *TokBegin, StringRef Fragment,
2445 llvm::TextEncodingConverter *Converter) {
2446
2447 const llvm::UTF8 *ErrorPtrTmp;
2448 if (ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: Fragment, ResultPtr, ErrorPtr&: ErrorPtrTmp)) {
2449 if (Converter) {
2450 assert(isOrdinary() && "Only ordinary literals are supported");
2451 SmallString<64> CpConv;
2452 char *Cp = ResultPtr - Fragment.size();
2453 auto EC = Converter->convert(Source: Fragment, Result&: CpConv);
2454 if (!EC) {
2455 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
2456 ResultPtr = Cp + CpConv.size();
2457 } else { // there was a conversion error
2458 if (Diags)
2459 Diags->Report(Loc: Tok.getLocation(),
2460 DiagID: diag::err_exec_charset_conversion_failed)
2461 << EC.message() << (Action == CA_ToLiteralEncodingForAsm ? 1 : 0);
2462 return true;
2463 }
2464 }
2465 return false;
2466 }
2467
2468 // If we see bad encoding for unprefixed string literals, warn and
2469 // simply copy the byte values, for compatibility with gcc and older
2470 // versions of clang.
2471 bool NoErrorOnBadEncoding = isOrdinary();
2472 if (NoErrorOnBadEncoding) {
2473 memcpy(dest: ResultPtr, src: Fragment.data(), n: Fragment.size());
2474 ResultPtr += Fragment.size();
2475 }
2476
2477 if (Diags) {
2478 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2479
2480 FullSourceLoc SourceLoc(Tok.getLocation(), SM);
2481 const DiagnosticBuilder &Builder =
2482 Diag(Diags, Features, TokLoc: SourceLoc, TokBegin,
2483 TokRangeBegin: ErrorPtr, TokRangeEnd: resyncUTF8(Err: ErrorPtr, End: Fragment.end()),
2484 DiagID: NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
2485 : diag::err_bad_string_encoding);
2486
2487 const char *NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2488 StringRef NextFragment(NextStart, Fragment.end()-NextStart);
2489
2490 // Decode into a dummy buffer.
2491 SmallString<512> Dummy;
2492 Dummy.reserve(N: Fragment.size() * CharByteWidth);
2493 char *Ptr = Dummy.data();
2494
2495 while (!ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: NextFragment, ResultPtr&: Ptr, ErrorPtr&: ErrorPtrTmp)) {
2496 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2497 NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2498 Builder << MakeCharSourceRange(Features, TokLoc: SourceLoc, TokBegin,
2499 TokRangeBegin: ErrorPtr, TokRangeEnd: NextStart);
2500 NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
2501 }
2502 }
2503 return !NoErrorOnBadEncoding;
2504}
2505
2506void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
2507 hadError = true;
2508 if (Diags)
2509 Diags->Report(Loc, DiagID: diag::err_lexing_string);
2510}
2511
2512/// getOffsetOfStringByte - This function returns the offset of the
2513/// specified byte of the string data represented by Token. This handles
2514/// advancing over escape sequences in the string.
2515unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
2516 unsigned ByteNo) const {
2517 // Get the spelling of the token.
2518 SmallString<32> SpellingBuffer;
2519 SpellingBuffer.resize(N: Tok.getLength());
2520
2521 bool StringInvalid = false;
2522 const char *SpellingPtr = &SpellingBuffer[0];
2523 unsigned TokLen = Lexer::getSpelling(Tok, Buffer&: SpellingPtr, SourceMgr: SM, LangOpts: Features,
2524 Invalid: &StringInvalid);
2525 if (StringInvalid)
2526 return 0;
2527
2528 const char *SpellingStart = SpellingPtr;
2529 const char *SpellingEnd = SpellingPtr+TokLen;
2530
2531 // Handle UTF-8 strings just like narrow strings.
2532 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
2533 SpellingPtr += 2;
2534
2535 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
2536 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
2537
2538 // For raw string literals, this is easy.
2539 if (SpellingPtr[0] == 'R') {
2540 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
2541 // Skip 'R"'.
2542 SpellingPtr += 2;
2543 while (*SpellingPtr != '(') {
2544 ++SpellingPtr;
2545 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
2546 }
2547 // Skip '('.
2548 ++SpellingPtr;
2549 return SpellingPtr - SpellingStart + ByteNo;
2550 }
2551
2552 // Skip over the leading quote
2553 assert(SpellingPtr[0] == '"' && "Should be a string literal!");
2554 ++SpellingPtr;
2555
2556 // Skip over bytes until we find the offset we're looking for.
2557 while (ByteNo) {
2558 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
2559
2560 // Step over non-escapes simply.
2561 if (*SpellingPtr != '\\') {
2562 ++SpellingPtr;
2563 --ByteNo;
2564 continue;
2565 }
2566
2567 // Otherwise, this is an escape character. Advance over it.
2568 bool HadError = false;
2569 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' ||
2570 SpellingPtr[1] == 'N') {
2571 const char *EscapePtr = SpellingPtr;
2572 unsigned Len = MeasureUCNEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd,
2573 CharByteWidth: 1, Features, HadError);
2574 if (Len > ByteNo) {
2575 // ByteNo is somewhere within the escape sequence.
2576 SpellingPtr = EscapePtr;
2577 break;
2578 }
2579 ByteNo -= Len;
2580 } else {
2581 ProcessCharEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd, HadError,
2582 Loc: FullSourceLoc(Tok.getLocation(), SM), CharWidth: CharByteWidth * 8,
2583 Diags, Features, EvalMethod: StringLiteralEvalMethod::Evaluated,
2584 /*TextEncoding=*/Converter: nullptr, Action: CA_NoConversion);
2585 --ByteNo;
2586 }
2587 assert(!HadError && "This method isn't valid on erroneous strings");
2588 }
2589
2590 return SpellingPtr-SpellingStart;
2591}
2592
2593/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
2594/// suffixes as ud-suffixes, because the diagnostic experience is better if we
2595/// treat it as an invalid suffix.
2596bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
2597 StringRef Suffix) {
2598 return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
2599 Suffix == "sv";
2600}
2601