1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the NumericLiteralParser, CharLiteralParser, and
10// StringLiteralParser interfaces.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Lex/LiteralSupport.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/LangOptions.h"
17#include "clang/Basic/SourceLocation.h"
18#include "clang/Basic/TargetInfo.h"
19#include "clang/Lex/LexDiagnostic.h"
20#include "clang/Lex/Lexer.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/Token.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ScopeExit.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/ADT/StringSwitch.h"
28#include "llvm/Support/ConvertUTF.h"
29#include "llvm/Support/Error.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/Unicode.h"
32#include <algorithm>
33#include <cassert>
34#include <cstddef>
35#include <cstdint>
36#include <cstring>
37#include <string>
38
39using namespace clang;
40
41static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
42 switch (kind) {
43 default: llvm_unreachable("Unknown token type!");
44 case tok::char_constant:
45 case tok::string_literal:
46 case tok::utf8_char_constant:
47 case tok::utf8_string_literal:
48 return Target.getCharWidth();
49 case tok::wide_char_constant:
50 case tok::wide_string_literal:
51 return Target.getWCharWidth();
52 case tok::utf16_char_constant:
53 case tok::utf16_string_literal:
54 return Target.getChar16Width();
55 case tok::utf32_char_constant:
56 case tok::utf32_string_literal:
57 return Target.getChar32Width();
58 }
59}
60
61static unsigned getEncodingPrefixLen(tok::TokenKind kind) {
62 switch (kind) {
63 default:
64 llvm_unreachable("Unknown token type!");
65 case tok::char_constant:
66 case tok::string_literal:
67 return 0;
68 case tok::utf8_char_constant:
69 case tok::utf8_string_literal:
70 return 2;
71 case tok::wide_char_constant:
72 case tok::wide_string_literal:
73 case tok::utf16_char_constant:
74 case tok::utf16_string_literal:
75 case tok::utf32_char_constant:
76 case tok::utf32_string_literal:
77 return 1;
78 }
79}
80
81static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
82 FullSourceLoc TokLoc,
83 const char *TokBegin,
84 const char *TokRangeBegin,
85 const char *TokRangeEnd) {
86 SourceLocation Begin =
87 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
88 SM: TokLoc.getManager(), LangOpts: Features);
89 SourceLocation End =
90 Lexer::AdvanceToTokenCharacter(TokStart: Begin, Characters: TokRangeEnd - TokRangeBegin,
91 SM: TokLoc.getManager(), LangOpts: Features);
92 return CharSourceRange::getCharRange(B: Begin, E: End);
93}
94
95/// Produce a diagnostic highlighting some portion of a literal.
96///
97/// Emits the diagnostic \p DiagID, highlighting the range of characters from
98/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
99/// a substring of a spelling buffer for the token beginning at \p TokBegin.
100static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
101 const LangOptions &Features, FullSourceLoc TokLoc,
102 const char *TokBegin, const char *TokRangeBegin,
103 const char *TokRangeEnd, unsigned DiagID) {
104 SourceLocation Begin =
105 Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
106 SM: TokLoc.getManager(), LangOpts: Features);
107 return Diags->Report(Loc: Begin, DiagID) <<
108 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
109}
110
111static bool IsEscapeValidInUnevaluatedStringLiteral(char Escape) {
112 switch (Escape) {
113 case '\'':
114 case '"':
115 case '?':
116 case '\\':
117 case 'a':
118 case 'b':
119 case 'f':
120 case 'n':
121 case 'r':
122 case 't':
123 case 'v':
124 return true;
125 }
126 return false;
127}
128
129static llvm::ErrorOr<char>
130convertCharacter(StringRef Char, const llvm::TextEncodingConverter &Converter) {
131 SmallString<8> ResultCharConv;
132 std::error_code EC = Converter.convert(Source: Char, Result&: ResultCharConv);
133 if (EC)
134 return EC;
135 else if (ResultCharConv.size() > 1)
136 return std::error_code(E2BIG, std::generic_category());
137 return ResultCharConv[0];
138}
139
140/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
141/// either a character or a string literal.
142static unsigned ProcessCharEscape(const char *ThisTokBegin,
143 const char *&ThisTokBuf,
144 const char *ThisTokEnd, bool &HadError,
145 FullSourceLoc Loc, unsigned CharWidth,
146 DiagnosticsEngine *Diags,
147 const LangOptions &Features,
148 StringLiteralEvalMethod EvalMethod,
149 llvm::TextEncodingConverter *Converter) {
150 const char *EscapeBegin = ThisTokBuf;
151 bool Delimited = false;
152 bool EndDelimiterFound = false;
153
154 // Skip the '\' char.
155 ++ThisTokBuf;
156
157 // We know that this character can't be off the end of the buffer, because
158 // that would have been \", which would not have been the end of string.
159 unsigned ResultChar = *ThisTokBuf++;
160 char Escape = ResultChar;
161 bool Transcode = true;
162 bool Invalid = false;
163 switch (ResultChar) {
164 // These map to themselves.
165 case '\\': case '\'': case '"': case '?': break;
166
167 // These have fixed mappings.
168 case 'a':
169 // TODO: K&R: the meaning of '\\a' is different in traditional C
170 ResultChar = 7;
171 break;
172 case 'b':
173 ResultChar = 8;
174 break;
175 case 'e':
176 if (Diags)
177 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
178 DiagID: diag::ext_nonstandard_escape) << "e";
179 ResultChar = 27;
180 break;
181 case 'E':
182 if (Diags)
183 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
184 DiagID: diag::ext_nonstandard_escape) << "E";
185 ResultChar = 27;
186 break;
187 case 'f':
188 ResultChar = 12;
189 break;
190 case 'n':
191 ResultChar = 10;
192 break;
193 case 'r':
194 ResultChar = 13;
195 break;
196 case 't':
197 ResultChar = 9;
198 break;
199 case 'v':
200 ResultChar = 11;
201 break;
202 case 'x': { // Hex escape.
203 Transcode = false;
204 ResultChar = 0;
205 if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
206 Delimited = true;
207 ThisTokBuf++;
208 if (*ThisTokBuf == '}') {
209 HadError = true;
210 if (Diags)
211 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
212 DiagID: diag::err_delimited_escape_empty);
213 }
214 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
215 if (Diags)
216 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
217 DiagID: diag::err_hex_escape_no_digits) << "x";
218 return ResultChar;
219 }
220
221 // Hex escapes are a maximal series of hex digits.
222 bool Overflow = false;
223 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
224 if (Delimited && *ThisTokBuf == '}') {
225 ThisTokBuf++;
226 EndDelimiterFound = true;
227 break;
228 }
229 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
230 if (CharVal == -1) {
231 // Non delimited hex escape sequences stop at the first non-hex digit.
232 if (!Delimited)
233 break;
234 HadError = true;
235 if (Diags)
236 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
237 DiagID: diag::err_delimited_escape_invalid)
238 << StringRef(ThisTokBuf, 1);
239 continue;
240 }
241 // About to shift out a digit?
242 if (ResultChar & 0xF0000000)
243 Overflow = true;
244 ResultChar <<= 4;
245 ResultChar |= CharVal;
246 }
247 // See if any bits will be truncated when evaluated as a character.
248 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
249 Overflow = true;
250 ResultChar &= ~0U >> (32-CharWidth);
251 }
252
253 // Check for overflow.
254 if (!HadError && Overflow) { // Too many digits to fit in
255 HadError = true;
256 if (Diags)
257 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
258 DiagID: diag::err_escape_too_large)
259 << 0;
260 }
261 break;
262 }
263 case '0': case '1': case '2': case '3':
264 case '4': case '5': case '6': case '7': {
265 // Octal escapes.
266 --ThisTokBuf;
267 Transcode = false;
268 ResultChar = 0;
269
270 // Octal escapes are a series of octal digits with maximum length 3.
271 // "\0123" is a two digit sequence equal to "\012" "3".
272 unsigned NumDigits = 0;
273 do {
274 ResultChar <<= 3;
275 ResultChar |= *ThisTokBuf++ - '0';
276 ++NumDigits;
277 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
278 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
279
280 // Check for overflow. Reject '\777', but not L'\777'.
281 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
282 if (Diags)
283 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
284 DiagID: diag::err_escape_too_large) << 1;
285 ResultChar &= ~0U >> (32-CharWidth);
286 }
287 break;
288 }
289 case 'o': {
290 bool Overflow = false;
291 Transcode = false;
292 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
293 HadError = true;
294 if (Diags)
295 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
296 DiagID: diag::err_delimited_escape_missing_brace)
297 << "o";
298
299 break;
300 }
301 ResultChar = 0;
302 Delimited = true;
303 ++ThisTokBuf;
304 if (*ThisTokBuf == '}') {
305 HadError = true;
306 if (Diags)
307 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
308 DiagID: diag::err_delimited_escape_empty);
309 }
310
311 while (ThisTokBuf != ThisTokEnd) {
312 if (*ThisTokBuf == '}') {
313 EndDelimiterFound = true;
314 ThisTokBuf++;
315 break;
316 }
317 if (*ThisTokBuf < '0' || *ThisTokBuf > '7') {
318 HadError = true;
319 if (Diags)
320 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
321 DiagID: diag::err_delimited_escape_invalid)
322 << StringRef(ThisTokBuf, 1);
323 ThisTokBuf++;
324 continue;
325 }
326 // Check if one of the top three bits is set before shifting them out.
327 if (ResultChar & 0xE0000000)
328 Overflow = true;
329
330 ResultChar <<= 3;
331 ResultChar |= *ThisTokBuf++ - '0';
332 }
333 // Check for overflow. Reject '\777', but not L'\777'.
334 if (!HadError &&
335 (Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) {
336 HadError = true;
337 if (Diags)
338 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
339 DiagID: diag::err_escape_too_large)
340 << 1;
341 ResultChar &= ~0U >> (32 - CharWidth);
342 }
343 break;
344 }
345 // Otherwise, these are not valid escapes.
346 case '(': case '{': case '[': case '%':
347 // GCC accepts these as extensions. We warn about them as such though.
348 if (Diags)
349 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
350 DiagID: diag::ext_nonstandard_escape)
351 << std::string(1, ResultChar);
352 break;
353 default:
354 Invalid = true;
355 if (!Diags)
356 break;
357
358 if (isPrintable(c: ResultChar))
359 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
360 DiagID: diag::ext_unknown_escape)
361 << std::string(1, ResultChar);
362 else
363 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
364 DiagID: diag::ext_unknown_escape)
365 << "x" + llvm::utohexstr(X: ResultChar);
366 break;
367 }
368
369 if (Delimited && Diags) {
370 if (!EndDelimiterFound)
371 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
372 DiagID: diag::err_expected)
373 << tok::r_brace;
374 else if (!HadError) {
375 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: false, Opts: Features,
376 Diags&: *Diags);
377 }
378 }
379
380 if (EvalMethod == StringLiteralEvalMethod::Unevaluated &&
381 !IsEscapeValidInUnevaluatedStringLiteral(Escape)) {
382 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
383 DiagID: diag::err_unevaluated_string_invalid_escape_sequence)
384 << StringRef(EscapeBegin, ThisTokBuf - EscapeBegin);
385 HadError = true;
386 }
387
388 if (!HadError && EvalMethod != StringLiteralEvalMethod::Unevaluated &&
389 Transcode && Converter) {
390 // Invalid escapes are written as '?' and then translated.
391 assert(ResultChar <=
392 static_cast<unsigned>(std::numeric_limits<char>::max()));
393 char ByteChar = Invalid ? '?' : ResultChar;
394 auto ErrorOrChar = convertCharacter(Char: StringRef(&ByteChar, 1), Converter: *Converter);
395 if (ErrorOrChar)
396 ResultChar = *ErrorOrChar;
397 else {
398 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: EscapeBegin, TokRangeEnd: ThisTokBuf,
399 DiagID: diag::err_exec_charset_conversion_failed)
400 << ErrorOrChar.getError().message();
401 HadError = true;
402 }
403 }
404 return ResultChar;
405}
406
407static void appendCodePoint(unsigned Codepoint,
408 llvm::SmallVectorImpl<char> &Str) {
409 char ResultBuf[4];
410 char *ResultPtr = ResultBuf;
411 if (llvm::ConvertCodePointToUTF8(Source: Codepoint, ResultPtr))
412 Str.append(in_start: ResultBuf, in_end: ResultPtr);
413}
414
415void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
416 for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
417 if (*I != '\\') {
418 Buf.push_back(Elt: *I);
419 continue;
420 }
421
422 ++I;
423 char Kind = *I;
424 ++I;
425
426 assert(Kind == 'u' || Kind == 'U' || Kind == 'N');
427 uint32_t CodePoint = 0;
428
429 if (Kind == 'u' && *I == '{') {
430 for (++I; *I != '}'; ++I) {
431 unsigned Value = llvm::hexDigitValue(C: *I);
432 assert(Value != -1U);
433 CodePoint <<= 4;
434 CodePoint += Value;
435 }
436 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
437 continue;
438 }
439
440 if (Kind == 'N') {
441 assert(*I == '{');
442 ++I;
443 auto Delim = std::find(first: I, last: Input.end(), val: '}');
444 assert(Delim != Input.end());
445 StringRef Name(I, std::distance(first: I, last: Delim));
446 std::optional<llvm::sys::unicode::LooseMatchingResult> Res =
447 llvm::sys::unicode::nameToCodepointLooseMatching(Name);
448 assert(Res && "could not find a codepoint that was previously found");
449 CodePoint = Res->CodePoint;
450 assert(CodePoint != 0xFFFFFFFF);
451 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
452 I = Delim;
453 continue;
454 }
455
456 unsigned NumHexDigits;
457 if (Kind == 'u')
458 NumHexDigits = 4;
459 else
460 NumHexDigits = 8;
461
462 assert(I + NumHexDigits <= E);
463
464 for (; NumHexDigits != 0; ++I, --NumHexDigits) {
465 unsigned Value = llvm::hexDigitValue(C: *I);
466 assert(Value != -1U);
467
468 CodePoint <<= 4;
469 CodePoint += Value;
470 }
471
472 appendCodePoint(Codepoint: CodePoint, Str&: Buf);
473 --I;
474 }
475}
476
477bool clang::isFunctionLocalStringLiteralMacro(tok::TokenKind K,
478 const LangOptions &LO) {
479 return LO.MicrosoftExt &&
480 (K == tok::kw___FUNCTION__ || K == tok::kw_L__FUNCTION__ ||
481 K == tok::kw___FUNCSIG__ || K == tok::kw_L__FUNCSIG__ ||
482 K == tok::kw___FUNCDNAME__);
483}
484
485bool clang::tokenIsLikeStringLiteral(const Token &Tok, const LangOptions &LO) {
486 return tok::isStringLiteral(K: Tok.getKind()) ||
487 isFunctionLocalStringLiteralMacro(K: Tok.getKind(), LO);
488}
489
490static bool ProcessNumericUCNEscape(const char *ThisTokBegin,
491 const char *&ThisTokBuf,
492 const char *ThisTokEnd, uint32_t &UcnVal,
493 unsigned short &UcnLen, bool &Delimited,
494 FullSourceLoc Loc, DiagnosticsEngine *Diags,
495 const LangOptions &Features,
496 bool in_char_string_literal = false) {
497 const char *UcnBegin = ThisTokBuf;
498 bool HasError = false;
499 bool EndDelimiterFound = false;
500
501 // Skip the '\u' char's.
502 ThisTokBuf += 2;
503 Delimited = false;
504 if (UcnBegin[1] == 'u' && in_char_string_literal &&
505 ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
506 Delimited = true;
507 ThisTokBuf++;
508 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
509 if (Diags)
510 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
511 DiagID: diag::err_hex_escape_no_digits)
512 << StringRef(&ThisTokBuf[-1], 1);
513 return false;
514 }
515 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
516
517 bool Overflow = false;
518 unsigned short Count = 0;
519 for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen);
520 ++ThisTokBuf) {
521 if (Delimited && *ThisTokBuf == '}') {
522 ++ThisTokBuf;
523 EndDelimiterFound = true;
524 break;
525 }
526 int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
527 if (CharVal == -1) {
528 HasError = true;
529 if (!Delimited)
530 break;
531 if (Diags) {
532 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
533 DiagID: diag::err_delimited_escape_invalid)
534 << StringRef(ThisTokBuf, 1);
535 }
536 Count++;
537 continue;
538 }
539 if (UcnVal & 0xF0000000) {
540 Overflow = true;
541 continue;
542 }
543 UcnVal <<= 4;
544 UcnVal |= CharVal;
545 Count++;
546 }
547
548 if (Overflow) {
549 if (Diags)
550 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
551 DiagID: diag::err_escape_too_large)
552 << 0;
553 return false;
554 }
555
556 if (Delimited && !EndDelimiterFound) {
557 if (Diags) {
558 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
559 DiagID: diag::err_expected)
560 << tok::r_brace;
561 }
562 return false;
563 }
564
565 // If we didn't consume the proper number of digits, there is a problem.
566 if (Count == 0 || (!Delimited && Count != UcnLen)) {
567 if (Diags)
568 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
569 DiagID: Delimited ? diag::err_delimited_escape_empty
570 : diag::err_ucn_escape_incomplete);
571 return false;
572 }
573 return !HasError;
574}
575
576static bool allowedInCharacterName(char C) {
577 return (C >= 'A' && C <= 'Z') || (C >= '0' && C <= '9') || C == '-' ||
578 C == ' ';
579}
580
581static void DiagnoseInvalidUnicodeCharacterName(
582 DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc,
583 const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd,
584 llvm::StringRef Name) {
585
586 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
587 DiagID: diag::err_invalid_ucn_name)
588 << Name;
589
590 namespace u = llvm::sys::unicode;
591
592 bool HasIllegalCharacter = false;
593 for (const char *P = Name.begin(), *E = Name.end(); P != E;) {
594 if (allowedInCharacterName(C: *P)) {
595 ++P;
596 continue;
597 }
598 SourceLocation CharLoc = Lexer::AdvanceToTokenCharacter(
599 TokStart: Loc, Characters: (TokRangeBegin - TokBegin) + (P - Name.begin()), SM: Loc.getManager(),
600 LangOpts: Features);
601 Diags->Report(Loc: CharLoc, DiagID: diag::note_invalid_ucn_name_character)
602 << EscapeSingleCodepointForDiagnostic(
603 Str: StringRef(P, std::distance(first: P, last: E)));
604 HasIllegalCharacter = true;
605 break;
606 }
607
608 std::optional<u::LooseMatchingResult> Res =
609 u::nameToCodepointLooseMatching(Name);
610 if (Res) {
611 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
612 DiagID: diag::note_invalid_ucn_name_loose_matching)
613 << FixItHint::CreateReplacement(
614 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
615 TokRangeEnd),
616 Code: Res->Name);
617 return;
618 }
619
620 // Providing illegal characters suggests a fundamental misuse of the feature,
621 // like providing emoji in \N{}. Offering alternative suggestions is often
622 // unhelpful in that scenario.
623 if (HasIllegalCharacter)
624 return;
625
626 unsigned Distance = 0;
627 SmallVector<u::MatchForCodepointName> Matches =
628 u::nearestMatchesForCodepointName(Pattern: Name, MaxMatchesCount: 5);
629 assert(!Matches.empty() && "No unicode characters found");
630
631 for (const auto &Match : Matches) {
632 if (Distance == 0)
633 Distance = Match.Distance;
634 if (std::max(a: Distance, b: Match.Distance) -
635 std::min(a: Distance, b: Match.Distance) >
636 3)
637 break;
638 Distance = Match.Distance;
639 Diag(Diags, Features, TokLoc: Loc, TokBegin, TokRangeBegin, TokRangeEnd,
640 DiagID: diag::note_invalid_ucn_name_candidate)
641 << Match.Name << EscapeSingleCodepointForDiagnostic(CP: Match.Value)
642 << FixItHint::CreateReplacement(
643 RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
644 TokRangeEnd),
645 Code: Match.Name);
646 }
647}
648
649static bool ProcessNamedUCNEscape(const char *ThisTokBegin,
650 const char *&ThisTokBuf,
651 const char *ThisTokEnd, uint32_t &UcnVal,
652 unsigned short &UcnLen, FullSourceLoc Loc,
653 DiagnosticsEngine *Diags,
654 const LangOptions &Features) {
655 const char *UcnBegin = ThisTokBuf;
656 assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N');
657 ThisTokBuf += 2;
658 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
659 if (Diags) {
660 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
661 DiagID: diag::err_delimited_escape_missing_brace)
662 << StringRef(&ThisTokBuf[-1], 1);
663 }
664 return false;
665 }
666 ThisTokBuf++;
667 const char *ClosingBrace = std::find_if(first: ThisTokBuf, last: ThisTokEnd, pred: [](char C) {
668 return C == '}' || isVerticalWhitespace(c: C);
669 });
670 bool Incomplete = ClosingBrace == ThisTokEnd;
671 bool Empty = ClosingBrace == ThisTokBuf;
672 if (Incomplete || Empty) {
673 if (Diags) {
674 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
675 DiagID: Incomplete ? diag::err_ucn_escape_incomplete
676 : diag::err_delimited_escape_empty)
677 << StringRef(&UcnBegin[1], 1);
678 }
679 ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1;
680 return false;
681 }
682 StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf);
683 ThisTokBuf = ClosingBrace + 1;
684 std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name);
685 if (!Res) {
686 if (Diags)
687 DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, TokBegin: ThisTokBegin,
688 TokRangeBegin: &UcnBegin[3], TokRangeEnd: ClosingBrace, Name);
689 return false;
690 }
691 UcnVal = *Res;
692 UcnLen = UcnVal > 0xFFFF ? 8 : 4;
693 return true;
694}
695
696/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
697/// return the UTF32.
698static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
699 const char *ThisTokEnd, uint32_t &UcnVal,
700 unsigned short &UcnLen, FullSourceLoc Loc,
701 DiagnosticsEngine *Diags,
702 const LangOptions &Features,
703 bool in_char_string_literal = false) {
704
705 bool HasError;
706 const char *UcnBegin = ThisTokBuf;
707 bool IsDelimitedEscapeSequence = false;
708 bool IsNamedEscapeSequence = false;
709 if (ThisTokBuf[1] == 'N') {
710 IsNamedEscapeSequence = true;
711 HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
712 UcnVal, UcnLen, Loc, Diags, Features);
713 } else {
714 HasError =
715 !ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
716 UcnLen, Delimited&: IsDelimitedEscapeSequence, Loc, Diags,
717 Features, in_char_string_literal);
718 }
719 if (HasError)
720 return false;
721
722 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
723 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
724 UcnVal > 0x10FFFF) { // maximum legal UTF32 value
725 if (Diags)
726 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
727 DiagID: diag::err_ucn_escape_invalid);
728 return false;
729 }
730
731 // C23 and C++11 allow UCNs that refer to control characters
732 // and basic source characters inside character and string literals
733 if (UcnVal < 0xa0 &&
734 // $, @, ` are allowed in all language modes
735 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {
736 bool IsError =
737 (!(Features.CPlusPlus11 || Features.C23) || !in_char_string_literal);
738 if (Diags) {
739 char BasicSCSChar = UcnVal;
740 if (UcnVal >= 0x20 && UcnVal < 0x7f)
741 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
742 DiagID: IsError ? diag::err_ucn_escape_basic_scs
743 : Features.CPlusPlus
744 ? diag::warn_cxx98_compat_literal_ucn_escape_basic_scs
745 : diag::warn_c23_compat_literal_ucn_escape_basic_scs)
746 << StringRef(&BasicSCSChar, 1);
747 else
748 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
749 DiagID: IsError ? diag::err_ucn_control_character
750 : Features.CPlusPlus
751 ? diag::warn_cxx98_compat_literal_ucn_control_character
752 : diag::warn_c23_compat_literal_ucn_control_character);
753 }
754 if (IsError)
755 return false;
756 }
757
758 if (!Features.CPlusPlus && !Features.C99 && Diags)
759 Diag(Diags, Features, TokLoc: Loc, TokBegin: ThisTokBegin, TokRangeBegin: UcnBegin, TokRangeEnd: ThisTokBuf,
760 DiagID: diag::warn_ucn_not_valid_in_c89_literal);
761
762 if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
763 Lexer::DiagnoseDelimitedOrNamedEscapeSequence(Loc, Named: IsNamedEscapeSequence,
764 Opts: Features, Diags&: *Diags);
765 return true;
766}
767
768/// MeasureUCNEscape - Determine the number of bytes within the resulting string
769/// which this UCN will occupy.
770static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
771 const char *ThisTokEnd, unsigned CharByteWidth,
772 const LangOptions &Features, bool &HadError) {
773 // UTF-32: 4 bytes per escape.
774 if (CharByteWidth == 4)
775 return 4;
776
777 uint32_t UcnVal = 0;
778 unsigned short UcnLen = 0;
779 FullSourceLoc Loc;
780
781 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
782 UcnLen, Loc, Diags: nullptr, Features, in_char_string_literal: true)) {
783 HadError = true;
784 return 0;
785 }
786
787 // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
788 if (CharByteWidth == 2)
789 return UcnVal <= 0xFFFF ? 2 : 4;
790
791 // UTF-8.
792 if (UcnVal < 0x80)
793 return 1;
794 if (UcnVal < 0x800)
795 return 2;
796 if (UcnVal < 0x10000)
797 return 3;
798 return 4;
799}
800
801/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
802/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
803/// StringLiteralParser. When we decide to implement UCN's for identifiers,
804/// we will likely rework our support for UCN's.
805static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
806 const char *ThisTokEnd, char *&ResultBuf,
807 bool &HadError, FullSourceLoc Loc,
808 unsigned CharByteWidth, DiagnosticsEngine *Diags,
809 const LangOptions &Features,
810 llvm::TextEncodingConverter *Converter) {
811 typedef uint32_t UTF32;
812 UTF32 UcnVal = 0;
813 unsigned short UcnLen = 0;
814 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
815 Loc, Diags, Features, in_char_string_literal: true)) {
816 HadError = true;
817 return;
818 }
819
820 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
821 "only character widths of 1, 2, or 4 bytes supported");
822
823 (void)UcnLen;
824 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
825
826 if (CharByteWidth == 4) {
827 // FIXME: Make the type of the result buffer correct instead of
828 // using reinterpret_cast.
829 llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
830 *ResultPtr = UcnVal;
831 ResultBuf += 4;
832 return;
833 }
834
835 if (CharByteWidth == 2) {
836 // FIXME: Make the type of the result buffer correct instead of
837 // using reinterpret_cast.
838 llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
839
840 if (UcnVal <= (UTF32)0xFFFF) {
841 *ResultPtr = UcnVal;
842 ResultBuf += 2;
843 return;
844 }
845
846 // Convert to UTF16.
847 UcnVal -= 0x10000;
848 *ResultPtr = 0xD800 + (UcnVal >> 10);
849 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
850 ResultBuf += 4;
851 return;
852 }
853
854 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
855
856 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
857 // The conversion below was inspired by:
858 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
859 // First, we determine how many bytes the result will require.
860 typedef uint8_t UTF8;
861
862 unsigned short bytesToWrite = 0;
863 if (UcnVal < (UTF32)0x80)
864 bytesToWrite = 1;
865 else if (UcnVal < (UTF32)0x800)
866 bytesToWrite = 2;
867 else if (UcnVal < (UTF32)0x10000)
868 bytesToWrite = 3;
869 else
870 bytesToWrite = 4;
871
872 const unsigned byteMask = 0xBF;
873 const unsigned byteMark = 0x80;
874
875 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
876 // into the first byte, depending on how many bytes follow.
877 static const UTF8 firstByteMark[5] = {
878 0x00, 0x00, 0xC0, 0xE0, 0xF0
879 };
880 // Finally, we write the bytes into ResultBuf.
881 ResultBuf += bytesToWrite;
882 switch (bytesToWrite) { // note: everything falls through.
883 case 4:
884 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
885 [[fallthrough]];
886 case 3:
887 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
888 [[fallthrough]];
889 case 2:
890 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
891 [[fallthrough]];
892 case 1:
893 *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
894 }
895 // Update the buffer.
896 ResultBuf += bytesToWrite;
897
898 if (Converter) {
899 SmallString<4> CpConv;
900 char *Cp = ResultBuf - bytesToWrite;
901 auto EC = Converter->convert(Source: StringRef(Cp, bytesToWrite), Result&: CpConv);
902 if (!EC) {
903 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
904 ResultBuf = Cp + CpConv.size();
905 } else {
906 Diags->Report(Loc, DiagID: diag::err_exec_charset_conversion_failed)
907 << EC.message();
908 HadError = true;
909 }
910 }
911}
912
913/// integer-constant: [C99 6.4.4.1]
914/// decimal-constant integer-suffix
915/// octal-constant integer-suffix
916/// hexadecimal-constant integer-suffix
917/// binary-literal integer-suffix [GNU, C++1y]
918/// user-defined-integer-literal: [C++11 lex.ext]
919/// decimal-literal ud-suffix
920/// octal-literal ud-suffix
921/// hexadecimal-literal ud-suffix
922/// binary-literal ud-suffix [GNU, C++1y]
923/// decimal-constant:
924/// nonzero-digit
925/// decimal-constant digit
926/// octal-constant:
927/// 0
928/// octal-constant octal-digit
929/// hexadecimal-constant:
930/// hexadecimal-prefix hexadecimal-digit
931/// hexadecimal-constant hexadecimal-digit
932/// hexadecimal-prefix: one of
933/// 0x 0X
934/// binary-literal:
935/// 0b binary-digit
936/// 0B binary-digit
937/// binary-literal binary-digit
938/// integer-suffix:
939/// unsigned-suffix [long-suffix]
940/// unsigned-suffix [long-long-suffix]
941/// long-suffix [unsigned-suffix]
942/// long-long-suffix [unsigned-sufix]
943/// nonzero-digit:
944/// 1 2 3 4 5 6 7 8 9
945/// octal-digit:
946/// 0 1 2 3 4 5 6 7
947/// hexadecimal-digit:
948/// 0 1 2 3 4 5 6 7 8 9
949/// a b c d e f
950/// A B C D E F
951/// binary-digit:
952/// 0
953/// 1
954/// unsigned-suffix: one of
955/// u U
956/// long-suffix: one of
957/// l L
958/// long-long-suffix: one of
959/// ll LL
960///
961/// floating-constant: [C99 6.4.4.2]
962/// TODO: add rules...
963///
964NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
965 SourceLocation TokLoc,
966 const SourceManager &SM,
967 const LangOptions &LangOpts,
968 const TargetInfo &Target,
969 DiagnosticsEngine &Diags)
970 : SM(SM), LangOpts(LangOpts), Diags(Diags),
971 ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
972
973 s = DigitsBegin = ThisTokBegin;
974 saw_exponent = false;
975 saw_period = false;
976 saw_ud_suffix = false;
977 saw_fixed_point_suffix = false;
978 isLong = false;
979 isUnsigned = false;
980 isLongLong = false;
981 isSizeT = false;
982 isHalf = false;
983 isFloat = false;
984 isImaginary = false;
985 isFloat16 = false;
986 isFloat128 = false;
987 MicrosoftInteger = 0;
988 isFract = false;
989 isAccum = false;
990 hadError = false;
991 isBitInt = false;
992
993 // This routine assumes that the range begin/end matches the regex for integer
994 // and FP constants (specifically, the 'pp-number' regex), and assumes that
995 // the byte at "*end" is both valid and not part of the regex. Because of
996 // this, it doesn't have to check for 'overscan' in various places.
997 // Note: For HLSL, the end token is allowed to be '.' which would be in the
998 // 'pp-number' regex. This is required to support vector swizzles on numeric
999 // constants (i.e. 1.xx or 1.5f.rrr).
1000 if (isPreprocessingNumberBody(c: *ThisTokEnd) &&
1001 !(LangOpts.HLSL && *ThisTokEnd == '.')) {
1002 Diags.Report(Loc: TokLoc, DiagID: diag::err_lexing_numeric);
1003 hadError = true;
1004 return;
1005 }
1006
1007 if (*s == '0') { // parse radix
1008 ParseNumberStartingWithZero(TokLoc);
1009 if (hadError)
1010 return;
1011 } else { // the first digit is non-zero
1012 radix = 10;
1013 s = SkipDigits(ptr: s);
1014 if (s == ThisTokEnd) {
1015 // Done.
1016 } else {
1017 ParseDecimalOrOctalCommon(TokLoc);
1018 if (hadError)
1019 return;
1020 }
1021 }
1022
1023 SuffixBegin = s;
1024 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1025
1026 // Initial scan to lookahead for fixed point suffix.
1027 if (LangOpts.FixedPoint) {
1028 for (const char *c = s; c != ThisTokEnd; ++c) {
1029 if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
1030 saw_fixed_point_suffix = true;
1031 break;
1032 }
1033 }
1034 }
1035
1036 // Parse the suffix. At this point we can classify whether we have an FP or
1037 // integer constant.
1038 bool isFixedPointConstant = isFixedPointLiteral();
1039 bool isFPConstant = isFloatingLiteral();
1040 bool HasSize = false;
1041 bool DoubleUnderscore = false;
1042
1043 // Loop over all of the characters of the suffix. If we see something bad,
1044 // we break out of the loop.
1045 for (; s != ThisTokEnd; ++s) {
1046 switch (*s) {
1047 case 'R':
1048 case 'r':
1049 if (!LangOpts.FixedPoint)
1050 break;
1051 if (isFract || isAccum) break;
1052 if (!(saw_period || saw_exponent)) break;
1053 isFract = true;
1054 continue;
1055 case 'K':
1056 case 'k':
1057 if (!LangOpts.FixedPoint)
1058 break;
1059 if (isFract || isAccum) break;
1060 if (!(saw_period || saw_exponent)) break;
1061 isAccum = true;
1062 continue;
1063 case 'h': // FP Suffix for "half".
1064 case 'H':
1065 // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
1066 if (!(LangOpts.Half || LangOpts.FixedPoint))
1067 break;
1068 if (isIntegerLiteral()) break; // Error for integer constant.
1069 if (HasSize)
1070 break;
1071 HasSize = true;
1072 isHalf = true;
1073 continue; // Success.
1074 case 'f': // FP Suffix for "float"
1075 case 'F':
1076 if (!isFPConstant) break; // Error for integer constant.
1077 if (HasSize)
1078 break;
1079 HasSize = true;
1080
1081 // CUDA host and device may have different _Float16 support, therefore
1082 // allows f16 literals to avoid false alarm.
1083 // When we compile for OpenMP target offloading on NVPTX, f16 suffix
1084 // should also be supported.
1085 // ToDo: more precise check for CUDA.
1086 // TODO: AMDGPU might also support it in the future.
1087 if ((Target.hasFloat16Type() || LangOpts.CUDA ||
1088 (LangOpts.OpenMPIsTargetDevice && Target.getTriple().isNVPTX())) &&
1089 s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
1090 s += 2; // success, eat up 2 characters.
1091 isFloat16 = true;
1092 continue;
1093 }
1094
1095 isFloat = true;
1096 continue; // Success.
1097 case 'q': // FP Suffix for "__float128"
1098 case 'Q':
1099 if (!isFPConstant) break; // Error for integer constant.
1100 if (HasSize)
1101 break;
1102 HasSize = true;
1103 isFloat128 = true;
1104 continue; // Success.
1105 case 'u':
1106 case 'U':
1107 if (isFPConstant) break; // Error for floating constant.
1108 if (isUnsigned) break; // Cannot be repeated.
1109 isUnsigned = true;
1110 continue; // Success.
1111 case 'l':
1112 case 'L':
1113 if (HasSize)
1114 break;
1115 HasSize = true;
1116
1117 // Check for long long. The L's need to be adjacent and the same case.
1118 if (s[1] == s[0]) {
1119 assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
1120 if (isFPConstant) break; // long long invalid for floats.
1121 isLongLong = true;
1122 ++s; // Eat both of them.
1123 } else {
1124 isLong = true;
1125 }
1126 continue; // Success.
1127 case 'z':
1128 case 'Z':
1129 if (isFPConstant)
1130 break; // Invalid for floats.
1131 if (HasSize)
1132 break;
1133 HasSize = true;
1134 isSizeT = true;
1135 continue;
1136 case 'i':
1137 case 'I':
1138 if (LangOpts.MicrosoftExt && s + 1 < ThisTokEnd && !isFPConstant) {
1139 // Allow i8, i16, i32, i64, and i128. First, look ahead and check if
1140 // suffixes are Microsoft integers and not the imaginary unit.
1141 uint8_t Bits = 0;
1142 size_t ToSkip = 0;
1143 switch (s[1]) {
1144 case '8': // i8 suffix
1145 Bits = 8;
1146 ToSkip = 2;
1147 break;
1148 case '1':
1149 if (s + 2 < ThisTokEnd && s[2] == '6') { // i16 suffix
1150 Bits = 16;
1151 ToSkip = 3;
1152 } else if (s + 3 < ThisTokEnd && s[2] == '2' &&
1153 s[3] == '8') { // i128 suffix
1154 Bits = 128;
1155 ToSkip = 4;
1156 }
1157 break;
1158 case '3':
1159 if (s + 2 < ThisTokEnd && s[2] == '2') { // i32 suffix
1160 Bits = 32;
1161 ToSkip = 3;
1162 }
1163 break;
1164 case '6':
1165 if (s + 2 < ThisTokEnd && s[2] == '4') { // i64 suffix
1166 Bits = 64;
1167 ToSkip = 3;
1168 }
1169 break;
1170 default:
1171 break;
1172 }
1173 if (Bits) {
1174 if (HasSize)
1175 break;
1176 HasSize = true;
1177 MicrosoftInteger = Bits;
1178 s += ToSkip;
1179 assert(s <= ThisTokEnd && "didn't maximally munch?");
1180 break;
1181 }
1182 }
1183 [[fallthrough]];
1184 case 'j':
1185 case 'J':
1186 if (isImaginary) break; // Cannot be repeated.
1187 isImaginary = true;
1188 continue; // Success.
1189 case '_':
1190 if (isFPConstant)
1191 break; // Invalid for floats
1192 if (HasSize)
1193 break;
1194 // There is currently no way to reach this with DoubleUnderscore set.
1195 // If new double underscope literals are added handle it here as above.
1196 assert(!DoubleUnderscore && "unhandled double underscore case");
1197 if (LangOpts.CPlusPlus && s + 2 < ThisTokEnd &&
1198 s[1] == '_') { // s + 2 < ThisTokEnd to ensure some character exists
1199 // after __
1200 DoubleUnderscore = true;
1201 s += 2; // Skip both '_'
1202 if (s + 1 < ThisTokEnd &&
1203 (*s == 'u' || *s == 'U')) { // Ensure some character after 'u'/'U'
1204 isUnsigned = true;
1205 ++s;
1206 }
1207 if (s + 1 < ThisTokEnd &&
1208 ((*s == 'w' && *(++s) == 'b') || (*s == 'W' && *(++s) == 'B'))) {
1209 isBitInt = true;
1210 HasSize = true;
1211 continue;
1212 }
1213 }
1214 break;
1215 case 'w':
1216 case 'W':
1217 if (isFPConstant)
1218 break; // Invalid for floats.
1219 if (HasSize)
1220 break; // Invalid if we already have a size for the literal.
1221
1222 // wb and WB are allowed, but a mixture of cases like Wb or wB is not. We
1223 // explicitly do not support the suffix in C++ as an extension because a
1224 // library-based UDL that resolves to a library type may be more
1225 // appropriate there. The same rules apply for __wb/__WB.
1226 if ((!LangOpts.CPlusPlus || DoubleUnderscore) && s + 1 < ThisTokEnd &&
1227 ((s[0] == 'w' && s[1] == 'b') || (s[0] == 'W' && s[1] == 'B'))) {
1228 isBitInt = true;
1229 HasSize = true;
1230 ++s; // Skip both characters (2nd char skipped on continue).
1231 continue; // Success.
1232 }
1233 }
1234 // If we reached here, there was an error or a ud-suffix.
1235 break;
1236 }
1237
1238 // "i", "if", and "il" are user-defined suffixes in C++1y.
1239 if (s != ThisTokEnd || isImaginary) {
1240 // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
1241 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
1242 if (isValidUDSuffix(LangOpts, Suffix: UDSuffixBuf)) {
1243 if (!isImaginary) {
1244 // Any suffix pieces we might have parsed are actually part of the
1245 // ud-suffix.
1246 isLong = false;
1247 isUnsigned = false;
1248 isLongLong = false;
1249 isSizeT = false;
1250 isFloat = false;
1251 isFloat16 = false;
1252 isHalf = false;
1253 isImaginary = false;
1254 isBitInt = false;
1255 MicrosoftInteger = 0;
1256 saw_fixed_point_suffix = false;
1257 isFract = false;
1258 isAccum = false;
1259 }
1260
1261 saw_ud_suffix = true;
1262 return;
1263 }
1264
1265 if (s != ThisTokEnd) {
1266 // Report an error if there are any.
1267 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1268 TokStart: TokLoc, Characters: SuffixBegin - ThisTokBegin, SM, LangOpts),
1269 DiagID: diag::err_invalid_suffix_constant)
1270 << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
1271 << (isFixedPointConstant ? 2 : isFPConstant);
1272 hadError = true;
1273 }
1274 }
1275
1276 if (!hadError && saw_fixed_point_suffix) {
1277 assert(isFract || isAccum);
1278 }
1279}
1280
1281/// ParseDecimalOrOctalCommon - This method is called for decimal or octal
1282/// numbers. It issues an error for illegal digits, and handles floating point
1283/// parsing. If it detects a floating point number, the radix is set to 10.
1284void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
1285 assert((radix == 8 || radix == 10) && "Unexpected radix");
1286
1287 // If we have a hex digit other than 'e' (which denotes a FP exponent) then
1288 // the code is using an incorrect base.
1289 if (isHexDigit(c: *s) && *s != 'e' && *s != 'E' &&
1290 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1291 Diags.Report(
1292 Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts),
1293 DiagID: diag::err_invalid_digit)
1294 << StringRef(s, 1) << (radix == 8 ? 1 : 0);
1295 hadError = true;
1296 return;
1297 }
1298
1299 if (*s == '.') {
1300 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1301 s++;
1302 radix = 10;
1303 saw_period = true;
1304 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1305 s = SkipDigits(ptr: s); // Skip suffix.
1306 }
1307 if (*s == 'e' || *s == 'E') { // exponent
1308 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1309 const char *Exponent = s;
1310 s++;
1311 radix = 10;
1312 saw_exponent = true;
1313 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1314 const char *first_non_digit = SkipDigits(ptr: s);
1315 if (containsDigits(Start: s, End: first_non_digit)) {
1316 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1317 s = first_non_digit;
1318 } else {
1319 if (!hadError) {
1320 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1321 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1322 DiagID: diag::err_exponent_has_no_digits);
1323 hadError = true;
1324 }
1325 return;
1326 }
1327 }
1328}
1329
1330/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1331/// suffixes as ud-suffixes, because the diagnostic experience is better if we
1332/// treat it as an invalid suffix.
1333bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1334 StringRef Suffix) {
1335 if (!LangOpts.CPlusPlus11 || Suffix.empty())
1336 return false;
1337
1338 // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
1339 // Suffixes starting with '__' (double underscore) are for use by
1340 // the implementation.
1341 if (Suffix.starts_with(Prefix: "_") && !Suffix.starts_with(Prefix: "__"))
1342 return true;
1343
1344 // In C++11, there are no library suffixes.
1345 if (!LangOpts.CPlusPlus14)
1346 return false;
1347
1348 // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
1349 // Per tweaked N3660, "il", "i", and "if" are also used in the library.
1350 // In C++2a "d" and "y" are used in the library.
1351 return llvm::StringSwitch<bool>(Suffix)
1352 .Cases(CaseStrings: {"h", "min", "s"}, Value: true)
1353 .Cases(CaseStrings: {"ms", "us", "ns"}, Value: true)
1354 .Cases(CaseStrings: {"il", "i", "if"}, Value: true)
1355 .Cases(CaseStrings: {"d", "y"}, Value: LangOpts.CPlusPlus20)
1356 .Default(Value: false);
1357}
1358
1359void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
1360 const char *Pos,
1361 CheckSeparatorKind IsAfterDigits) {
1362 if (IsAfterDigits == CSK_AfterDigits) {
1363 if (Pos == ThisTokBegin)
1364 return;
1365 --Pos;
1366 } else if (Pos == ThisTokEnd)
1367 return;
1368
1369 if (isDigitSeparator(C: *Pos)) {
1370 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: Pos - ThisTokBegin, SM,
1371 LangOpts),
1372 DiagID: diag::err_digit_separator_not_between_digits)
1373 << IsAfterDigits;
1374 hadError = true;
1375 }
1376}
1377
1378/// ParseNumberStartingWithZero - This method is called when the first character
1379/// of the number is found to be a zero. This means it is either an octal
1380/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
1381/// a floating point number (01239.123e4). Eat the prefix, determining the
1382/// radix etc.
1383void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
1384 assert(s[0] == '0' && "Invalid method call");
1385 s++;
1386
1387 int c1 = s[0];
1388
1389 // Handle a hex number like 0x1234.
1390 if ((c1 == 'x' || c1 == 'X') && (isHexDigit(c: s[1]) || s[1] == '.')) {
1391 s++;
1392 assert(s < ThisTokEnd && "didn't maximally munch?");
1393 radix = 16;
1394 DigitsBegin = s;
1395 s = SkipHexDigits(ptr: s);
1396 bool HasSignificandDigits = containsDigits(Start: DigitsBegin, End: s);
1397 if (s == ThisTokEnd) {
1398 // Done.
1399 } else if (*s == '.') {
1400 s++;
1401 saw_period = true;
1402 const char *floatDigitsBegin = s;
1403 s = SkipHexDigits(ptr: s);
1404 if (containsDigits(Start: floatDigitsBegin, End: s))
1405 HasSignificandDigits = true;
1406 if (HasSignificandDigits)
1407 checkSeparator(TokLoc, Pos: floatDigitsBegin, IsAfterDigits: CSK_BeforeDigits);
1408 }
1409
1410 if (!HasSignificandDigits) {
1411 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1412 LangOpts),
1413 DiagID: diag::err_hex_constant_requires)
1414 << LangOpts.CPlusPlus << 1;
1415 hadError = true;
1416 return;
1417 }
1418
1419 // A binary exponent can appear with or with a '.'. If dotted, the
1420 // binary exponent is required.
1421 if (*s == 'p' || *s == 'P') {
1422 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1423 const char *Exponent = s;
1424 s++;
1425 saw_exponent = true;
1426 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1427 const char *first_non_digit = SkipDigits(ptr: s);
1428 if (!containsDigits(Start: s, End: first_non_digit)) {
1429 if (!hadError) {
1430 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(
1431 TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1432 DiagID: diag::err_exponent_has_no_digits);
1433 hadError = true;
1434 }
1435 return;
1436 }
1437 checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1438 s = first_non_digit;
1439
1440 if (!LangOpts.HexFloats)
1441 Diags.Report(Loc: TokLoc, DiagID: LangOpts.CPlusPlus
1442 ? diag::compat_pre_cxx17_hex_literal
1443 : diag::ext_hex_constant_invalid);
1444 else if (LangOpts.CPlusPlus17)
1445 Diags.Report(Loc: TokLoc, DiagID: diag::compat_cxx17_hex_literal);
1446 } else if (saw_period) {
1447 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1448 LangOpts),
1449 DiagID: diag::err_hex_constant_requires)
1450 << LangOpts.CPlusPlus << 0;
1451 hadError = true;
1452 }
1453 return;
1454 }
1455
1456 // Handle simple binary numbers 0b01010
1457 if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
1458 // 0b101010 is a C++14 and C23 extension.
1459 unsigned DiagId;
1460 if (LangOpts.CPlusPlus14)
1461 DiagId = diag::warn_cxx11_compat_binary_literal;
1462 else if (LangOpts.C23)
1463 DiagId = diag::warn_c23_compat_binary_literal;
1464 else if (LangOpts.CPlusPlus)
1465 DiagId = diag::ext_binary_literal_cxx14;
1466 else
1467 DiagId = diag::ext_binary_literal;
1468 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1469 ++s;
1470 assert(s < ThisTokEnd && "didn't maximally munch?");
1471 radix = 2;
1472 DigitsBegin = s;
1473 s = SkipBinaryDigits(ptr: s);
1474 if (s == ThisTokEnd) {
1475 // Done.
1476 } else if (isHexDigit(c: *s) &&
1477 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1478 Diags.Report(Loc: Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1479 LangOpts),
1480 DiagID: diag::err_invalid_digit)
1481 << StringRef(s, 1) << 2;
1482 hadError = true;
1483 }
1484 // Other suffixes will be diagnosed by the caller.
1485 return;
1486 }
1487
1488 // Parse a potential octal literal prefix.
1489 bool IsSingleZero = false;
1490 if ((c1 == 'O' || c1 == 'o') && (s[1] >= '0' && s[1] <= '7')) {
1491 unsigned DiagId;
1492 if (LangOpts.C2y)
1493 DiagId = diag::warn_c2y_compat_octal_literal;
1494 else if (LangOpts.CPlusPlus)
1495 DiagId = diag::ext_cpp_octal_literal;
1496 else
1497 DiagId = diag::ext_octal_literal;
1498 Diags.Report(Loc: TokLoc, DiagID: DiagId);
1499
1500 ++s;
1501 DigitsBegin = s;
1502 radix = 8;
1503 s = SkipOctalDigits(ptr: s);
1504 if (s == ThisTokEnd) {
1505 // Done
1506 } else if ((isHexDigit(c: *s) && *s != 'e' && *s != 'E' && *s != '.') &&
1507 !isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1508 auto InvalidDigitLoc = Lexer::AdvanceToTokenCharacter(
1509 TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts);
1510 Diags.Report(Loc: InvalidDigitLoc, DiagID: diag::err_invalid_digit)
1511 << StringRef(s, 1) << 1;
1512 hadError = true;
1513 }
1514 // Other suffixes will be diagnosed by the caller.
1515 return;
1516 }
1517
1518 llvm::scope_exit _([&] {
1519 // If we still have an octal value but we did not see an octal prefix,
1520 // diagnose as being an obsolescent feature starting in C2y.
1521 if (radix == 8 && LangOpts.C2y && !hadError && !IsSingleZero)
1522 Diags.Report(Loc: TokLoc, DiagID: diag::warn_unprefixed_octal_deprecated);
1523 });
1524
1525 // For now, the radix is set to 8. If we discover that we have a
1526 // floating point constant, the radix will change to 10. Octal floating
1527 // point constants are not permitted (only decimal and hexadecimal).
1528 radix = 8;
1529 const char *PossibleNewDigitStart = s;
1530 s = SkipOctalDigits(ptr: s);
1531 // When the value is 0 followed by a suffix (like 0wb), we want to leave 0
1532 // as the start of the digits. So if skipping octal digits does not skip
1533 // anything, we leave the digit start where it was.
1534 if (s != PossibleNewDigitStart)
1535 DigitsBegin = PossibleNewDigitStart;
1536 else
1537 IsSingleZero = (s == ThisTokBegin + 1);
1538
1539 if (s == ThisTokEnd)
1540 return; // Done, simple octal number like 01234
1541
1542 // If we have some other non-octal digit that *is* a decimal digit, see if
1543 // this is part of a floating point number like 094.123 or 09e1.
1544 if (isDigit(c: *s)) {
1545 const char *EndDecimal = SkipDigits(ptr: s);
1546 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
1547 s = EndDecimal;
1548 radix = 10;
1549 }
1550 }
1551
1552 ParseDecimalOrOctalCommon(TokLoc);
1553}
1554
1555static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
1556 switch (Radix) {
1557 case 2:
1558 return NumDigits <= 64;
1559 case 8:
1560 return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
1561 case 10:
1562 return NumDigits <= 19; // floor(log10(2^64))
1563 case 16:
1564 return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
1565 default:
1566 llvm_unreachable("impossible Radix");
1567 }
1568}
1569
1570/// GetIntegerValue - Convert this numeric literal value to an APInt that
1571/// matches Val's input width. If there is an overflow, set Val to the low bits
1572/// of the result and return true. Otherwise, return false.
1573bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1574 // Fast path: Compute a conservative bound on the maximum number of
1575 // bits per digit in this radix. If we can't possibly overflow a
1576 // uint64 based on that bound then do the simple conversion to
1577 // integer. This avoids the expensive overflow checking below, and
1578 // handles the common cases that matter (small decimal integers and
1579 // hex/octal values which don't overflow).
1580 const unsigned NumDigits = SuffixBegin - DigitsBegin;
1581 if (alwaysFitsInto64Bits(Radix: radix, NumDigits)) {
1582 uint64_t N = 0;
1583 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
1584 if (!isDigitSeparator(C: *Ptr))
1585 N = N * radix + llvm::hexDigitValue(C: *Ptr);
1586
1587 // This will truncate the value to Val's input width. Simply check
1588 // for overflow by comparing.
1589 Val = N;
1590 return Val.getZExtValue() != N;
1591 }
1592
1593 Val = 0;
1594 const char *Ptr = DigitsBegin;
1595
1596 llvm::APInt RadixVal(Val.getBitWidth(), radix);
1597 llvm::APInt CharVal(Val.getBitWidth(), 0);
1598 llvm::APInt OldVal = Val;
1599
1600 bool OverflowOccurred = false;
1601 while (Ptr < SuffixBegin) {
1602 if (isDigitSeparator(C: *Ptr)) {
1603 ++Ptr;
1604 continue;
1605 }
1606
1607 unsigned C = llvm::hexDigitValue(C: *Ptr++);
1608
1609 // If this letter is out of bound for this radix, reject it.
1610 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1611
1612 CharVal = C;
1613
1614 // Add the digit to the value in the appropriate radix. If adding in digits
1615 // made the value smaller, then this overflowed.
1616 OldVal = Val;
1617
1618 // Multiply by radix, did overflow occur on the multiply?
1619 Val *= RadixVal;
1620 OverflowOccurred |= Val.udiv(RHS: RadixVal) != OldVal;
1621
1622 // Add value, did overflow occur on the value?
1623 // (a + b) ult b <=> overflow
1624 Val += CharVal;
1625 OverflowOccurred |= Val.ult(RHS: CharVal);
1626 }
1627 return OverflowOccurred;
1628}
1629
1630llvm::APFloat::opStatus
1631NumericLiteralParser::GetFloatValue(llvm::APFloat &Result,
1632 llvm::RoundingMode RM) {
1633 using llvm::APFloat;
1634
1635 unsigned n = std::min(a: SuffixBegin - ThisTokBegin, b: ThisTokEnd - ThisTokBegin);
1636
1637 llvm::SmallString<16> Buffer;
1638 StringRef Str(ThisTokBegin, n);
1639 if (Str.contains(C: '\'')) {
1640 Buffer.reserve(N: n);
1641 std::remove_copy_if(first: Str.begin(), last: Str.end(), result: std::back_inserter(x&: Buffer),
1642 pred: &isDigitSeparator);
1643 Str = Buffer;
1644 }
1645
1646 auto StatusOrErr = Result.convertFromString(Str, RM);
1647 assert(StatusOrErr && "Invalid floating point representation");
1648 return !errorToBool(Err: StatusOrErr.takeError()) ? *StatusOrErr
1649 : APFloat::opInvalidOp;
1650}
1651
1652static inline bool IsExponentPart(char c, bool isHex) {
1653 if (isHex)
1654 return c == 'p' || c == 'P';
1655 return c == 'e' || c == 'E';
1656}
1657
1658bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1659 assert(radix == 16 || radix == 10);
1660
1661 // Find how many digits are needed to store the whole literal.
1662 unsigned NumDigits = SuffixBegin - DigitsBegin;
1663 if (saw_period) --NumDigits;
1664
1665 // Initial scan of the exponent if it exists
1666 bool ExpOverflowOccurred = false;
1667 bool NegativeExponent = false;
1668 const char *ExponentBegin;
1669 uint64_t Exponent = 0;
1670 int64_t BaseShift = 0;
1671 if (saw_exponent) {
1672 const char *Ptr = DigitsBegin;
1673
1674 while (!IsExponentPart(c: *Ptr, isHex: radix == 16))
1675 ++Ptr;
1676 ExponentBegin = Ptr;
1677 ++Ptr;
1678 NegativeExponent = *Ptr == '-';
1679 if (NegativeExponent) ++Ptr;
1680
1681 unsigned NumExpDigits = SuffixBegin - Ptr;
1682 if (alwaysFitsInto64Bits(Radix: radix, NumDigits: NumExpDigits)) {
1683 llvm::StringRef ExpStr(Ptr, NumExpDigits);
1684 llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1685 Exponent = ExpInt.getZExtValue();
1686 } else {
1687 ExpOverflowOccurred = true;
1688 }
1689
1690 if (NegativeExponent) BaseShift -= Exponent;
1691 else BaseShift += Exponent;
1692 }
1693
1694 // Number of bits needed for decimal literal is
1695 // ceil(NumDigits * log2(10)) Integral part
1696 // + Scale Fractional part
1697 // + ceil(Exponent * log2(10)) Exponent
1698 // --------------------------------------------------
1699 // ceil((NumDigits + Exponent) * log2(10)) + Scale
1700 //
1701 // But for simplicity in handling integers, we can round up log2(10) to 4,
1702 // making:
1703 // 4 * (NumDigits + Exponent) + Scale
1704 //
1705 // Number of digits needed for hexadecimal literal is
1706 // 4 * NumDigits Integral part
1707 // + Scale Fractional part
1708 // + Exponent Exponent
1709 // --------------------------------------------------
1710 // (4 * NumDigits) + Scale + Exponent
1711 uint64_t NumBitsNeeded;
1712 if (radix == 10)
1713 NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1714 else
1715 NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1716
1717 if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1718 ExpOverflowOccurred = true;
1719 llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1720
1721 bool FoundDecimal = false;
1722
1723 int64_t FractBaseShift = 0;
1724 const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1725 for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1726 if (*Ptr == '.') {
1727 FoundDecimal = true;
1728 continue;
1729 }
1730
1731 // Normal reading of an integer
1732 unsigned C = llvm::hexDigitValue(C: *Ptr);
1733 assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1734
1735 Val *= radix;
1736 Val += C;
1737
1738 if (FoundDecimal)
1739 // Keep track of how much we will need to adjust this value by from the
1740 // number of digits past the radix point.
1741 --FractBaseShift;
1742 }
1743
1744 // For a radix of 16, we will be multiplying by 2 instead of 16.
1745 if (radix == 16) FractBaseShift *= 4;
1746 BaseShift += FractBaseShift;
1747
1748 Val <<= Scale;
1749
1750 uint64_t Base = (radix == 16) ? 2 : 10;
1751 if (BaseShift > 0) {
1752 for (int64_t i = 0; i < BaseShift; ++i) {
1753 Val *= Base;
1754 }
1755 } else if (BaseShift < 0) {
1756 for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i)
1757 Val = Val.udiv(RHS: Base);
1758 }
1759
1760 bool IntOverflowOccurred = false;
1761 auto MaxVal = llvm::APInt::getMaxValue(numBits: StoreVal.getBitWidth());
1762 if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1763 IntOverflowOccurred |= Val.ugt(RHS: MaxVal.zext(width: Val.getBitWidth()));
1764 StoreVal = Val.trunc(width: StoreVal.getBitWidth());
1765 } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1766 IntOverflowOccurred |= Val.zext(width: MaxVal.getBitWidth()).ugt(RHS: MaxVal);
1767 StoreVal = Val.zext(width: StoreVal.getBitWidth());
1768 } else {
1769 StoreVal = std::move(Val);
1770 }
1771
1772 return IntOverflowOccurred || ExpOverflowOccurred;
1773}
1774
1775/// \verbatim
1776/// user-defined-character-literal: [C++11 lex.ext]
1777/// character-literal ud-suffix
1778/// ud-suffix:
1779/// identifier
1780/// character-literal: [C++11 lex.ccon]
1781/// ' c-char-sequence '
1782/// u' c-char-sequence '
1783/// U' c-char-sequence '
1784/// L' c-char-sequence '
1785/// u8' c-char-sequence ' [C++1z lex.ccon]
1786/// c-char-sequence:
1787/// c-char
1788/// c-char-sequence c-char
1789/// c-char:
1790/// any member of the source character set except the single-quote ',
1791/// backslash \, or new-line character
1792/// escape-sequence
1793/// universal-character-name
1794/// escape-sequence:
1795/// simple-escape-sequence
1796/// octal-escape-sequence
1797/// hexadecimal-escape-sequence
1798/// simple-escape-sequence:
1799/// one of \' \" \? \\ \a \b \f \n \r \t \v
1800/// octal-escape-sequence:
1801/// \ octal-digit
1802/// \ octal-digit octal-digit
1803/// \ octal-digit octal-digit octal-digit
1804/// hexadecimal-escape-sequence:
1805/// \x hexadecimal-digit
1806/// hexadecimal-escape-sequence hexadecimal-digit
1807/// universal-character-name: [C++11 lex.charset]
1808/// \u hex-quad
1809/// \U hex-quad hex-quad
1810/// hex-quad:
1811/// hex-digit hex-digit hex-digit hex-digit
1812/// \endverbatim
1813///
1814CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1815 SourceLocation Loc, Preprocessor &PP,
1816 tok::TokenKind kind) {
1817 // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1818 HadError = false;
1819
1820 Kind = kind;
1821
1822 const char *TokBegin = begin;
1823
1824 // Skip over wide character determinant.
1825 if (Kind != tok::char_constant)
1826 ++begin;
1827 if (Kind == tok::utf8_char_constant)
1828 ++begin;
1829
1830 // Skip over the entry quote.
1831 if (begin[0] != '\'') {
1832 PP.Diag(Loc, DiagID: diag::err_lexing_char);
1833 HadError = true;
1834 return;
1835 }
1836
1837 ++begin;
1838
1839 // Remove an optional ud-suffix.
1840 if (end[-1] != '\'') {
1841 const char *UDSuffixEnd = end;
1842 do {
1843 --end;
1844 } while (end[-1] != '\'');
1845 // FIXME: Don't bother with this if !tok.hasUCN().
1846 expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(end, UDSuffixEnd - end));
1847 UDSuffixOffset = end - TokBegin;
1848 }
1849
1850 // Trim the ending quote.
1851 assert(end != begin && "Invalid token lexed");
1852 --end;
1853
1854 // FIXME: The "Value" is an uint64_t so we can handle char literals of
1855 // up to 64-bits.
1856 // FIXME: This extensively assumes that 'char' is 8-bits.
1857 assert(PP.getTargetInfo().getCharWidth() == 8 &&
1858 "Assumes char is 8 bits");
1859 assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1860 (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1861 "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1862 assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1863 "Assumes sizeof(wchar) on target is <= 64");
1864
1865 SmallVector<uint32_t, 4> codepoint_buffer;
1866 codepoint_buffer.resize(N: end - begin);
1867 uint32_t *buffer_begin = &codepoint_buffer.front();
1868 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1869
1870 const TextEncoding &TE = PP.getTextEncoding();
1871 llvm::TextEncodingConverter *Converter = nullptr;
1872 if (isOrdinary())
1873 Converter = TE.getConverter(Action: CA_ToLiteralEncoding);
1874
1875 // Unicode escapes representing characters that cannot be correctly
1876 // represented in a single code unit are disallowed in character literals
1877 // by this implementation.
1878 uint32_t largest_character_for_kind;
1879 if (tok::wide_char_constant == Kind) {
1880 largest_character_for_kind =
1881 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1882 } else if (tok::utf8_char_constant == Kind) {
1883 largest_character_for_kind = 0x7F;
1884 } else if (tok::utf16_char_constant == Kind) {
1885 largest_character_for_kind = 0xFFFF;
1886 } else if (tok::utf32_char_constant == Kind) {
1887 largest_character_for_kind = 0x10FFFF;
1888 } else {
1889 largest_character_for_kind = (Converter == nullptr) ? 0x7Fu : 0xFFu;
1890 }
1891
1892 while (begin != end) {
1893 // Is this a span of non-escape characters?
1894 if (begin[0] != '\\') {
1895 char const *start = begin;
1896 do {
1897 ++begin;
1898 } while (begin != end && *begin != '\\');
1899
1900 char const *tmp_in_start = start;
1901 uint32_t *tmp_out_start = buffer_begin;
1902 std::string UTF8String(start, begin);
1903 llvm::ConversionResult res =
1904 llvm::ConvertUTF8toUTF32(sourceStart: reinterpret_cast<llvm::UTF8 const **>(&start),
1905 sourceEnd: reinterpret_cast<llvm::UTF8 const *>(begin),
1906 targetStart: &buffer_begin, targetEnd: buffer_end, flags: llvm::strictConversion);
1907 if (res != llvm::conversionOK) {
1908 // If we see bad encoding for unprefixed character literals, warn and
1909 // simply copy the byte values, for compatibility with gcc and
1910 // older versions of clang.
1911 bool NoErrorOnBadEncoding = isOrdinary();
1912 unsigned Msg = diag::err_bad_character_encoding;
1913 if (NoErrorOnBadEncoding)
1914 Msg = diag::warn_bad_character_encoding;
1915 PP.Diag(Loc, DiagID: Msg);
1916 if (NoErrorOnBadEncoding) {
1917 start = tmp_in_start;
1918 buffer_begin = tmp_out_start;
1919 for (; start != begin; ++start, ++buffer_begin)
1920 *buffer_begin = static_cast<uint8_t>(*start);
1921 } else {
1922 HadError = true;
1923 }
1924 } else {
1925 uint32_t *validation_ptr = tmp_out_start;
1926 for (; validation_ptr < buffer_begin; ++validation_ptr) {
1927 if (*validation_ptr > largest_character_for_kind) {
1928 HadError = true;
1929 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1930 }
1931 }
1932
1933 // Convert to execution character set if needed
1934 if (!HadError && Converter) {
1935 assert(isOrdinary() && "Only ordinary characters are supported");
1936 SmallString<4> Converted;
1937 auto ErrorOrChar = Converter->convert(Source: UTF8String, Result&: Converted);
1938 if (!ErrorOrChar) {
1939 for (int i = 0; tmp_out_start < buffer_begin;
1940 ++tmp_out_start, ++i) {
1941 *tmp_out_start = Converted[i];
1942 }
1943 } else {
1944 HadError = true;
1945 PP.Diag(Loc, DiagID: diag::err_exec_charset_conversion_failed)
1946 << ErrorOrChar.message();
1947 }
1948 }
1949 }
1950
1951 continue;
1952 }
1953 // Is this a Universal Character Name escape?
1954 if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') {
1955 if (Converter == nullptr) {
1956 unsigned short UcnLen = 0;
1957 if (!ProcessUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, UcnVal&: *buffer_begin, UcnLen,
1958 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1959 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(), in_char_string_literal: true)) {
1960 HadError = true;
1961 } else if (*buffer_begin > largest_character_for_kind) {
1962 HadError = true;
1963 PP.Diag(Loc, DiagID: diag::err_character_too_large);
1964 }
1965 } else {
1966 char Cp[5];
1967 char *ResultPtr = Cp;
1968 EncodeUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, ResultBuf&: ResultPtr, HadError,
1969 Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1970 /*CharByteWidth=*/1u, Diags: &PP.getDiagnostics(),
1971 Features: PP.getLangOpts(), Converter);
1972 if (!HadError)
1973 *buffer_begin = *Cp;
1974 }
1975 ++buffer_begin;
1976 continue;
1977 }
1978 unsigned CharWidth = getCharWidth(kind: Kind, Target: PP.getTargetInfo());
1979 uint64_t result =
1980 ProcessCharEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, HadError,
1981 Loc: FullSourceLoc(Loc, PP.getSourceManager()), CharWidth,
1982 Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(),
1983 EvalMethod: StringLiteralEvalMethod::Evaluated, Converter: nullptr);
1984 *buffer_begin++ = result;
1985 }
1986
1987 unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1988
1989 if (NumCharsSoFar > 1) {
1990 if (isOrdinary() && NumCharsSoFar == 4)
1991 PP.Diag(Loc, DiagID: diag::warn_four_char_character_literal);
1992 else if (isOrdinary())
1993 PP.Diag(Loc, DiagID: diag::warn_multichar_character_literal);
1994 else {
1995 PP.Diag(Loc, DiagID: diag::err_multichar_character_literal) << (isWide() ? 0 : 1);
1996 HadError = true;
1997 }
1998 IsMultiChar = true;
1999 } else {
2000 IsMultiChar = false;
2001 }
2002
2003 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
2004
2005 // Narrow character literals act as though their value is concatenated
2006 // in this implementation, but warn on overflow.
2007 bool multi_char_too_long = false;
2008 if (isOrdinary() && isMultiChar()) {
2009 LitVal = 0;
2010 for (size_t i = 0; i < NumCharsSoFar; ++i) {
2011 // check for enough leading zeros to shift into
2012 multi_char_too_long |= (LitVal.countl_zero() < 8);
2013 LitVal <<= 8;
2014 LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
2015 }
2016 } else if (NumCharsSoFar > 0) {
2017 // otherwise just take the last character
2018 LitVal = buffer_begin[-1];
2019 }
2020
2021 if (!HadError && multi_char_too_long) {
2022 PP.Diag(Loc, DiagID: diag::warn_char_constant_too_large);
2023 }
2024
2025 // Transfer the value from APInt to uint64_t
2026 Value = LitVal.getZExtValue();
2027
2028 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
2029 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple
2030 // character constants are not sign extended in the this implementation:
2031 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
2032 if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) &&
2033 PP.getLangOpts().CharIsSigned)
2034 Value = (signed char)Value;
2035}
2036
2037/// \verbatim
2038/// string-literal: [C++0x lex.string]
2039/// encoding-prefix " [s-char-sequence] "
2040/// encoding-prefix R raw-string
2041/// encoding-prefix:
2042/// u8
2043/// u
2044/// U
2045/// L
2046/// s-char-sequence:
2047/// s-char
2048/// s-char-sequence s-char
2049/// s-char:
2050/// any member of the source character set except the double-quote ",
2051/// backslash \, or new-line character
2052/// escape-sequence
2053/// universal-character-name
2054/// raw-string:
2055/// " d-char-sequence ( r-char-sequence ) d-char-sequence "
2056/// r-char-sequence:
2057/// r-char
2058/// r-char-sequence r-char
2059/// r-char:
2060/// any member of the source character set, except a right parenthesis )
2061/// followed by the initial d-char-sequence (which may be empty)
2062/// followed by a double quote ".
2063/// d-char-sequence:
2064/// d-char
2065/// d-char-sequence d-char
2066/// d-char:
2067/// any member of the basic source character set except:
2068/// space, the left parenthesis (, the right parenthesis ),
2069/// the backslash \, and the control characters representing horizontal
2070/// tab, vertical tab, form feed, and newline.
2071/// escape-sequence: [C++0x lex.ccon]
2072/// simple-escape-sequence
2073/// octal-escape-sequence
2074/// hexadecimal-escape-sequence
2075/// simple-escape-sequence:
2076/// one of \' \" \? \\ \a \b \f \n \r \t \v
2077/// octal-escape-sequence:
2078/// \ octal-digit
2079/// \ octal-digit octal-digit
2080/// \ octal-digit octal-digit octal-digit
2081/// hexadecimal-escape-sequence:
2082/// \x hexadecimal-digit
2083/// hexadecimal-escape-sequence hexadecimal-digit
2084/// universal-character-name:
2085/// \u hex-quad
2086/// \U hex-quad hex-quad
2087/// hex-quad:
2088/// hex-digit hex-digit hex-digit hex-digit
2089/// \endverbatim
2090///
2091StringLiteralParser::StringLiteralParser(ArrayRef<Token> StringToks,
2092 Preprocessor &PP,
2093 StringLiteralEvalMethod EvalMethod,
2094 ConversionAction Action)
2095 : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
2096 Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()),
2097 TE(&PP.getTextEncoding()), MaxTokenLength(0), SizeBound(0),
2098 CharByteWidth(0), Kind(tok::unknown), ResultPtr(ResultBuf.data()),
2099 EvalMethod(EvalMethod), hadError(false), Pascal(false) {
2100 init(StringToks, Action);
2101}
2102
2103void StringLiteralParser::init(ArrayRef<Token> StringToks,
2104 ConversionAction Action) {
2105 // The literal token may have come from an invalid source location (e.g. due
2106 // to a PCH error), in which case the token length will be 0.
2107 if (StringToks.empty() || StringToks[0].getLength() < 2)
2108 return DiagnoseLexingError(Loc: SourceLocation());
2109
2110 // Scan all of the string portions, remember the max individual token length,
2111 // computing a bound on the concatenated string length, and see whether any
2112 // piece is a wide-string. If any of the string portions is a wide-string
2113 // literal, the result is a wide-string literal [C99 6.4.5p4].
2114 assert(!StringToks.empty() && "expected at least one token");
2115 MaxTokenLength = StringToks[0].getLength();
2116 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
2117 SizeBound = StringToks[0].getLength() - 2; // -2 for "".
2118 hadError = false;
2119
2120 // Determines the kind of string from the prefix
2121 Kind = tok::string_literal;
2122
2123 /// (C99 5.1.1.2p1). The common case is only one string fragment.
2124 for (const Token &Tok : StringToks) {
2125 if (Tok.getLength() < 2)
2126 return DiagnoseLexingError(Loc: Tok.getLocation());
2127
2128 // The string could be shorter than this if it needs cleaning, but this is a
2129 // reasonable bound, which is all we need.
2130 assert(Tok.getLength() >= 2 && "literal token is invalid!");
2131 SizeBound += Tok.getLength() - 2; // -2 for "".
2132
2133 // Remember maximum string piece length.
2134 if (Tok.getLength() > MaxTokenLength)
2135 MaxTokenLength = Tok.getLength();
2136
2137 // Remember if we see any wide or utf-8/16/32 strings.
2138 // Also check for illegal concatenations.
2139 if (isUnevaluated() && Tok.getKind() != tok::string_literal) {
2140 if (Diags) {
2141 SourceLocation PrefixEndLoc = Lexer::AdvanceToTokenCharacter(
2142 TokStart: Tok.getLocation(), Characters: getEncodingPrefixLen(kind: Tok.getKind()), SM,
2143 LangOpts: Features);
2144 CharSourceRange Range =
2145 CharSourceRange::getCharRange(R: {Tok.getLocation(), PrefixEndLoc});
2146 StringRef Prefix(SM.getCharacterData(SL: Tok.getLocation()),
2147 getEncodingPrefixLen(kind: Tok.getKind()));
2148 Diags->Report(Loc: Tok.getLocation(),
2149 DiagID: Features.CPlusPlus26
2150 ? diag::err_unevaluated_string_prefix
2151 : diag::warn_unevaluated_string_prefix)
2152 << Prefix << Features.CPlusPlus << FixItHint::CreateRemoval(RemoveRange: Range);
2153 }
2154 if (Features.CPlusPlus26)
2155 hadError = true;
2156 } else if (Tok.isNot(K: Kind) && Tok.isNot(K: tok::string_literal)) {
2157 if (isOrdinary()) {
2158 Kind = Tok.getKind();
2159 } else {
2160 if (Diags)
2161 Diags->Report(Loc: Tok.getLocation(), DiagID: diag::err_unsupported_string_concat);
2162 hadError = true;
2163 }
2164 }
2165 }
2166
2167 // Include space for the null terminator.
2168 ++SizeBound;
2169
2170 // TODO: K&R warning: "traditional C rejects string constant concatenation"
2171
2172 // Get the width in bytes of char/wchar_t/char16_t/char32_t
2173 CharByteWidth = getCharWidth(kind: Kind, Target);
2174 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
2175 CharByteWidth /= 8;
2176
2177 // The output buffer size needs to be large enough to hold wide characters.
2178 // This is a worst-case assumption which basically corresponds to L"" "long".
2179 SizeBound *= CharByteWidth;
2180
2181 // Size the temporary buffer to hold the result string data.
2182 ResultBuf.resize(N: SizeBound);
2183
2184 // Likewise, but for each string piece.
2185 SmallString<512> TokenBuf;
2186 TokenBuf.resize(N: MaxTokenLength);
2187
2188 // Loop over all the strings, getting their spelling, and expanding them to
2189 // wide strings as appropriate.
2190 ResultPtr = &ResultBuf[0]; // Next byte to fill in.
2191
2192 Pascal = false;
2193
2194 SourceLocation UDSuffixTokLoc;
2195
2196 llvm::TextEncodingConverter *Converter = nullptr;
2197 if (isOrdinary() && TE)
2198 Converter = TE->getConverter(Action);
2199
2200 for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
2201 const char *ThisTokBuf = &TokenBuf[0];
2202 // Get the spelling of the token, which eliminates trigraphs, etc. We know
2203 // that ThisTokBuf points to a buffer that is big enough for the whole token
2204 // and 'spelled' tokens can only shrink.
2205 bool StringInvalid = false;
2206 unsigned ThisTokLen =
2207 Lexer::getSpelling(Tok: StringToks[i], Buffer&: ThisTokBuf, SourceMgr: SM, LangOpts: Features,
2208 Invalid: &StringInvalid);
2209 if (StringInvalid)
2210 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2211
2212 const char *ThisTokBegin = ThisTokBuf;
2213 const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
2214
2215 // Remove an optional ud-suffix.
2216 if (ThisTokEnd[-1] != '"') {
2217 const char *UDSuffixEnd = ThisTokEnd;
2218 do {
2219 --ThisTokEnd;
2220 } while (ThisTokEnd[-1] != '"');
2221
2222 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
2223
2224 if (UDSuffixBuf.empty()) {
2225 if (StringToks[i].hasUCN())
2226 expandUCNs(Buf&: UDSuffixBuf, Input: UDSuffix);
2227 else
2228 UDSuffixBuf.assign(RHS: UDSuffix);
2229 UDSuffixToken = i;
2230 UDSuffixOffset = ThisTokEnd - ThisTokBuf;
2231 UDSuffixTokLoc = StringToks[i].getLocation();
2232 } else {
2233 SmallString<32> ExpandedUDSuffix;
2234 if (StringToks[i].hasUCN()) {
2235 expandUCNs(Buf&: ExpandedUDSuffix, Input: UDSuffix);
2236 UDSuffix = ExpandedUDSuffix;
2237 }
2238
2239 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
2240 // result of a concatenation involving at least one user-defined-string-
2241 // literal, all the participating user-defined-string-literals shall
2242 // have the same ud-suffix.
2243 bool UnevaluatedStringHasUDL = isUnevaluated() && !UDSuffix.empty();
2244 if (UDSuffixBuf != UDSuffix || UnevaluatedStringHasUDL) {
2245 if (Diags) {
2246 SourceLocation TokLoc = StringToks[i].getLocation();
2247 if (UnevaluatedStringHasUDL) {
2248 Diags->Report(Loc: TokLoc, DiagID: diag::err_unevaluated_string_udl)
2249 << SourceRange(TokLoc, TokLoc);
2250 } else {
2251 Diags->Report(Loc: TokLoc, DiagID: diag::err_string_concat_mixed_suffix)
2252 << UDSuffixBuf << UDSuffix
2253 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc);
2254 }
2255 }
2256 hadError = true;
2257 }
2258 }
2259 }
2260
2261 // Strip the end quote.
2262 --ThisTokEnd;
2263
2264 // TODO: Input character set mapping support.
2265
2266 // Skip marker for wide or unicode strings.
2267 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
2268 ++ThisTokBuf;
2269 // Skip 8 of u8 marker for utf8 strings.
2270 if (ThisTokBuf[0] == '8')
2271 ++ThisTokBuf;
2272 }
2273
2274 // Check for raw string
2275 if (ThisTokBuf[0] == 'R') {
2276 if (ThisTokBuf[1] != '"') {
2277 // The file may have come from PCH and then changed after loading the
2278 // PCH; Fail gracefully.
2279 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2280 }
2281 ThisTokBuf += 2; // skip R"
2282
2283 // C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16
2284 // characters.
2285 constexpr unsigned MaxRawStrDelimLen = 16;
2286
2287 const char *Prefix = ThisTokBuf;
2288 while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen &&
2289 ThisTokBuf[0] != '(')
2290 ++ThisTokBuf;
2291 if (ThisTokBuf[0] != '(')
2292 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2293 ++ThisTokBuf; // skip '('
2294
2295 // Remove same number of characters from the end
2296 ThisTokEnd -= ThisTokBuf - Prefix;
2297 if (ThisTokEnd < ThisTokBuf)
2298 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2299
2300 // C++14 [lex.string]p4: A source-file new-line in a raw string literal
2301 // results in a new-line in the resulting execution string-literal.
2302 StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
2303 while (!RemainingTokenSpan.empty()) {
2304 // Split the string literal on \r\n boundaries.
2305 size_t CRLFPos = RemainingTokenSpan.find(Str: "\r\n");
2306 StringRef BeforeCRLF = RemainingTokenSpan.substr(Start: 0, N: CRLFPos);
2307 StringRef AfterCRLF = RemainingTokenSpan.substr(Start: CRLFPos);
2308
2309 // Copy everything before the \r\n sequence into the string literal.
2310 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin, Fragment: BeforeCRLF,
2311 Converter))
2312 hadError = true;
2313
2314 // Point into the \n inside the \r\n sequence and operate on the
2315 // remaining portion of the literal.
2316 RemainingTokenSpan = AfterCRLF.substr(Start: 1);
2317 }
2318 } else {
2319 if (ThisTokBuf[0] != '"') {
2320 // The file may have come from PCH and then changed after loading the
2321 // PCH; Fail gracefully.
2322 return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2323 }
2324 ++ThisTokBuf; // skip "
2325
2326 // Check if this is a pascal string
2327 if (!isUnevaluated() && Features.PascalStrings &&
2328 ThisTokBuf + 1 != ThisTokEnd && ThisTokBuf[0] == '\\' &&
2329 ThisTokBuf[1] == 'p') {
2330
2331 // If the \p sequence is found in the first token, we have a pascal string
2332 // Otherwise, if we already have a pascal string, ignore the first \p
2333 if (i == 0) {
2334 ++ThisTokBuf;
2335 Pascal = true;
2336 } else if (Pascal)
2337 ThisTokBuf += 2;
2338 }
2339
2340 while (ThisTokBuf != ThisTokEnd) {
2341 // Is this a span of non-escape characters?
2342 if (ThisTokBuf[0] != '\\') {
2343 const char *InStart = ThisTokBuf;
2344 do {
2345 ++ThisTokBuf;
2346 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
2347
2348 // Copy the character span over.
2349 if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin,
2350 Fragment: StringRef(InStart, ThisTokBuf - InStart),
2351 Converter))
2352 hadError = true;
2353 continue;
2354 }
2355 // Is this a Universal Character Name escape?
2356 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' ||
2357 ThisTokBuf[1] == 'N') {
2358 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, ResultBuf&: ResultPtr,
2359 HadError&: hadError,
2360 Loc: FullSourceLoc(StringToks[i].getLocation(), SM),
2361 CharByteWidth, Diags, Features, Converter);
2362 continue;
2363 }
2364 // Otherwise, this is a non-UCN escape character. Process it.
2365 unsigned ResultChar = ProcessCharEscape(
2366 ThisTokBegin, ThisTokBuf, ThisTokEnd, HadError&: hadError,
2367 Loc: FullSourceLoc(StringToks[i].getLocation(), SM), CharWidth: CharByteWidth * 8,
2368 Diags, Features, EvalMethod, Converter);
2369
2370 if (CharByteWidth == 4) {
2371 // FIXME: Make the type of the result buffer correct instead of
2372 // using reinterpret_cast.
2373 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
2374 *ResultWidePtr = ResultChar;
2375 ResultPtr += 4;
2376 } else if (CharByteWidth == 2) {
2377 // FIXME: Make the type of the result buffer correct instead of
2378 // using reinterpret_cast.
2379 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
2380 *ResultWidePtr = ResultChar & 0xFFFF;
2381 ResultPtr += 2;
2382 } else {
2383 assert(CharByteWidth == 1 && "Unexpected char width");
2384 *ResultPtr++ = ResultChar & 0xFF;
2385 }
2386 }
2387 }
2388 }
2389
2390 assert((!Pascal || !isUnevaluated()) &&
2391 "Pascal string in unevaluated context");
2392 if (Pascal) {
2393 if (CharByteWidth == 4) {
2394 // FIXME: Make the type of the result buffer correct instead of
2395 // using reinterpret_cast.
2396 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
2397 ResultWidePtr[0] = GetNumStringChars() - 1;
2398 } else if (CharByteWidth == 2) {
2399 // FIXME: Make the type of the result buffer correct instead of
2400 // using reinterpret_cast.
2401 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
2402 ResultWidePtr[0] = GetNumStringChars() - 1;
2403 } else {
2404 assert(CharByteWidth == 1 && "Unexpected char width");
2405 ResultBuf[0] = GetNumStringChars() - 1;
2406 }
2407
2408 // Verify that pascal strings aren't too large.
2409 if (GetStringLength() > 256) {
2410 if (Diags)
2411 Diags->Report(Loc: StringToks.front().getLocation(),
2412 DiagID: diag::err_pascal_string_too_long)
2413 << SourceRange(StringToks.front().getLocation(),
2414 StringToks.back().getLocation());
2415 hadError = true;
2416 return;
2417 }
2418 } else if (Diags) {
2419 // Complain if this string literal has too many characters.
2420 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
2421
2422 if (GetNumStringChars() > MaxChars)
2423 Diags->Report(Loc: StringToks.front().getLocation(),
2424 DiagID: diag::ext_string_too_long)
2425 << GetNumStringChars() << MaxChars
2426 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
2427 << SourceRange(StringToks.front().getLocation(),
2428 StringToks.back().getLocation());
2429 }
2430}
2431
2432static const char *resyncUTF8(const char *Err, const char *End) {
2433 if (Err == End)
2434 return End;
2435 End = Err + std::min<unsigned>(a: llvm::getNumBytesForUTF8(firstByte: *Err), b: End-Err);
2436 while (++Err != End && (*Err & 0xC0) == 0x80)
2437 ;
2438 return Err;
2439}
2440
2441/// This function copies from Fragment, which is a sequence of bytes
2442/// within Tok's contents (which begin at TokBegin) into ResultPtr.
2443/// Performs widening for multi-byte characters.
2444bool StringLiteralParser::CopyStringFragment(
2445 const Token &Tok, const char *TokBegin, StringRef Fragment,
2446 llvm::TextEncodingConverter *Converter) {
2447
2448 const llvm::UTF8 *ErrorPtrTmp;
2449 if (ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: Fragment, ResultPtr, ErrorPtr&: ErrorPtrTmp)) {
2450 if (Converter) {
2451 assert(isOrdinary() && "Only ordinary literals are supported");
2452 SmallString<64> CpConv;
2453 char *Cp = ResultPtr - Fragment.size();
2454 auto EC = Converter->convert(Source: Fragment, Result&: CpConv);
2455 if (!EC) {
2456 memcpy(dest: Cp, src: CpConv.data(), n: CpConv.size());
2457 ResultPtr = Cp + CpConv.size();
2458 } else { // there was a conversion error
2459 if (Diags)
2460 Diags->Report(Loc: Tok.getLocation(),
2461 DiagID: diag::err_exec_charset_conversion_failed)
2462 << EC.message();
2463 return true;
2464 }
2465 }
2466 return false;
2467 }
2468
2469 // If we see bad encoding for unprefixed string literals, warn and
2470 // simply copy the byte values, for compatibility with gcc and older
2471 // versions of clang.
2472 bool NoErrorOnBadEncoding = isOrdinary();
2473 if (NoErrorOnBadEncoding) {
2474 memcpy(dest: ResultPtr, src: Fragment.data(), n: Fragment.size());
2475 ResultPtr += Fragment.size();
2476 }
2477
2478 if (Diags) {
2479 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2480
2481 FullSourceLoc SourceLoc(Tok.getLocation(), SM);
2482 const DiagnosticBuilder &Builder =
2483 Diag(Diags, Features, TokLoc: SourceLoc, TokBegin,
2484 TokRangeBegin: ErrorPtr, TokRangeEnd: resyncUTF8(Err: ErrorPtr, End: Fragment.end()),
2485 DiagID: NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
2486 : diag::err_bad_string_encoding);
2487
2488 const char *NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2489 StringRef NextFragment(NextStart, Fragment.end()-NextStart);
2490
2491 // Decode into a dummy buffer.
2492 SmallString<512> Dummy;
2493 Dummy.reserve(N: Fragment.size() * CharByteWidth);
2494 char *Ptr = Dummy.data();
2495
2496 while (!ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: NextFragment, ResultPtr&: Ptr, ErrorPtr&: ErrorPtrTmp)) {
2497 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2498 NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2499 Builder << MakeCharSourceRange(Features, TokLoc: SourceLoc, TokBegin,
2500 TokRangeBegin: ErrorPtr, TokRangeEnd: NextStart);
2501 NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
2502 }
2503 }
2504 return !NoErrorOnBadEncoding;
2505}
2506
2507void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
2508 hadError = true;
2509 if (Diags)
2510 Diags->Report(Loc, DiagID: diag::err_lexing_string);
2511}
2512
2513/// getOffsetOfStringByte - This function returns the offset of the
2514/// specified byte of the string data represented by Token. This handles
2515/// advancing over escape sequences in the string.
2516unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
2517 unsigned ByteNo) const {
2518 // Get the spelling of the token.
2519 SmallString<32> SpellingBuffer;
2520 SpellingBuffer.resize(N: Tok.getLength());
2521
2522 bool StringInvalid = false;
2523 const char *SpellingPtr = &SpellingBuffer[0];
2524 unsigned TokLen = Lexer::getSpelling(Tok, Buffer&: SpellingPtr, SourceMgr: SM, LangOpts: Features,
2525 Invalid: &StringInvalid);
2526 if (StringInvalid)
2527 return 0;
2528
2529 const char *SpellingStart = SpellingPtr;
2530 const char *SpellingEnd = SpellingPtr+TokLen;
2531
2532 // Handle UTF-8 strings just like narrow strings.
2533 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
2534 SpellingPtr += 2;
2535
2536 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
2537 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
2538
2539 // For raw string literals, this is easy.
2540 if (SpellingPtr[0] == 'R') {
2541 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
2542 // Skip 'R"'.
2543 SpellingPtr += 2;
2544 while (*SpellingPtr != '(') {
2545 ++SpellingPtr;
2546 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
2547 }
2548 // Skip '('.
2549 ++SpellingPtr;
2550 return SpellingPtr - SpellingStart + ByteNo;
2551 }
2552
2553 // Skip over the leading quote
2554 assert(SpellingPtr[0] == '"' && "Should be a string literal!");
2555 ++SpellingPtr;
2556
2557 // Skip over bytes until we find the offset we're looking for.
2558 while (ByteNo) {
2559 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
2560
2561 // Step over non-escapes simply.
2562 if (*SpellingPtr != '\\') {
2563 ++SpellingPtr;
2564 --ByteNo;
2565 continue;
2566 }
2567
2568 // Otherwise, this is an escape character. Advance over it.
2569 bool HadError = false;
2570 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' ||
2571 SpellingPtr[1] == 'N') {
2572 const char *EscapePtr = SpellingPtr;
2573 unsigned Len = MeasureUCNEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd,
2574 CharByteWidth: 1, Features, HadError);
2575 if (Len > ByteNo) {
2576 // ByteNo is somewhere within the escape sequence.
2577 SpellingPtr = EscapePtr;
2578 break;
2579 }
2580 ByteNo -= Len;
2581 } else {
2582 ProcessCharEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd, HadError,
2583 Loc: FullSourceLoc(Tok.getLocation(), SM), CharWidth: CharByteWidth * 8,
2584 Diags, Features, EvalMethod: StringLiteralEvalMethod::Evaluated,
2585 /*TextEncoding=*/Converter: nullptr);
2586 --ByteNo;
2587 }
2588 assert(!HadError && "This method isn't valid on erroneous strings");
2589 }
2590
2591 return SpellingPtr-SpellingStart;
2592}
2593
2594/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
2595/// suffixes as ud-suffixes, because the diagnostic experience is better if we
2596/// treat it as an invalid suffix.
2597bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
2598 StringRef Suffix) {
2599 return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
2600 Suffix == "sv";
2601}
2602