1//===- DependencyDirectivesScanner.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the interface for scanning header and source files to get the
11/// minimum necessary preprocessor directives for evaluating includes. It
12/// reduces the source down to #define, #include, #import, @import, and any
13/// conditional preprocessor logic that contains one of those.
14///
15//===----------------------------------------------------------------------===//
16
17#include "clang/Lex/DependencyDirectivesScanner.h"
18#include "clang/Basic/CharInfo.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Lex/LexDiagnostic.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Pragma.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
27#include <optional>
28
29using namespace clang;
30using namespace clang::dependency_directives_scan;
31using namespace llvm;
32
33namespace {
34
35struct DirectiveWithTokens {
36 DirectiveKind Kind;
37 unsigned NumTokens;
38
39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
40 : Kind(Kind), NumTokens(NumTokens) {}
41};
42
43enum class CXX20ModuleDirectiveKind {
44 None,
45 GlobalModuleFragment,
46 NamedModuleDeclaration,
47 ImportDeclaration,
48};
49
50static CXX20ModuleDirectiveKind scanFirstCXX20ModuleDirective(StringRef Source);
51
52/// Does an efficient "scan" of the sources to detect the presence of
53/// preprocessor (or module import) directives and collects the raw lexed tokens
54/// for those directives so that the \p Lexer can "replay" them when the file is
55/// included.
56///
57/// Note that the behavior of the raw lexer is affected by the language mode,
58/// while at this point we want to do a scan and collect tokens once,
59/// irrespective of the language mode that the file will get included in. To
60/// compensate for that the \p Lexer, while "replaying", will adjust a token
61/// where appropriate, when it could affect the preprocessor's state.
62/// For example in a directive like
63///
64/// \code
65/// #if __has_cpp_attribute(clang::fallthrough)
66/// \endcode
67///
68/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
69/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
70/// while in C++ mode.
71struct Scanner {
72 Scanner(StringRef Input,
73 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
74 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
75 : Input(Input), Tokens(Tokens), Diags(Diags),
76 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
77 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
78 Input.end()) {}
79
80 static LangOptions getLangOptsForDepScanning() {
81 LangOptions LangOpts;
82 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
83 LangOpts.ObjC = true;
84 LangOpts.LineComment = true;
85 LangOpts.RawStringLiterals = true;
86 LangOpts.AllowLiteralDigitSeparator = true;
87 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
88 return LangOpts;
89 }
90
91 /// Lex the provided source and emit the directive tokens.
92 ///
93 /// \returns True on error.
94 bool scan(SmallVectorImpl<Directive> &Directives);
95
96 friend CXX20ModuleDirectiveKind
97 scanFirstCXX20ModuleDirective(StringRef Source);
98 friend bool clang::isPreprocessedModuleFile(StringRef Source);
99
100private:
101 /// Lexes next token and advances \p First and the \p Lexer.
102 [[nodiscard]] dependency_directives_scan::Token &
103 lexToken(const char *&First, const char *const End);
104
105 [[nodiscard]] dependency_directives_scan::Token &
106 lexIncludeFilename(const char *&First, const char *const End);
107
108 void skipLine(const char *&First, const char *const End);
109 void skipDirective(StringRef Name, const char *&First, const char *const End);
110
111 /// Returns the spelling of a string literal or identifier after performing
112 /// any processing needed to handle \c clang::Token::NeedsCleaning.
113 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);
114
115 /// Lexes next token and if it is identifier returns its string, otherwise
116 /// it skips the current line and returns \p std::nullopt.
117 ///
118 /// In any case (whatever the token kind) \p First and the \p Lexer will
119 /// advance beyond the token.
120 [[nodiscard]] std::optional<StringRef>
121 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
122
123 /// Used when it is certain that next token is an identifier.
124 [[nodiscard]] StringRef lexIdentifier(const char *&First,
125 const char *const End);
126
127 /// Lexes next token and returns true iff it is an identifier that matches \p
128 /// Id, otherwise it skips the current line and returns false.
129 ///
130 /// In any case (whatever the token kind) \p First and the \p Lexer will
131 /// advance beyond the token.
132 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,
133 const char *&First,
134 const char *const End);
135
136 /// Lexes next token and returns true iff it matches the kind \p K.
137 /// Otherwise it skips the current line and returns false.
138 ///
139 /// In any case (whatever the token kind) \p First and the \p Lexer will
140 /// advance beyond the token.
141 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
142 const char *const End);
143
144 /// Lexes next token and if it is string literal, returns its string.
145 /// Otherwise, it skips the current line and returns \p std::nullopt.
146 ///
147 /// In any case (whatever the token kind) \p First and the \p Lexer will
148 /// advance beyond the token.
149 [[nodiscard]] std::optional<StringRef>
150 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);
151
152 [[nodiscard]] bool scanImpl(const char *First, const char *const End);
153 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
154 [[nodiscard]] bool lexAt(const char *&First, const char *const End);
155 [[nodiscard]] bool lexModule(const char *&First, const char *const End);
156 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
157 const char *const End);
158 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
159 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);
160 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
161 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
162 const char *const End);
163 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,
164 const char *&First,
165 const char *const End);
166 void lexPPDirectiveBody(const char *&First, const char *const End);
167
168 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
169 Tokens.append(RHS: CurDirToks);
170 DirsWithToks.emplace_back(Args&: Kind, Args: CurDirToks.size());
171 CurDirToks.clear();
172 return DirsWithToks.back();
173 }
174 void popDirective() {
175 Tokens.pop_back_n(NumItems: DirsWithToks.pop_back_val().NumTokens);
176 }
177 DirectiveKind topDirective() const {
178 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
179 }
180
181 unsigned getOffsetAt(const char *CurPtr) const {
182 return CurPtr - Input.data();
183 }
184
185 /// Reports a diagnostic if the diagnostic engine is provided. Always returns
186 /// true at the end.
187 bool reportError(const char *CurPtr, unsigned Err);
188
189 bool ScanningPreprocessedModuleFile = false;
190 StringMap<char> SplitIds;
191 StringRef Input;
192 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;
193 DiagnosticsEngine *Diags;
194 SourceLocation InputSourceLoc;
195
196 const char *LastTokenPtr = nullptr;
197 /// Keeps track of the tokens for the currently lexed directive. Once a
198 /// directive is fully lexed and "committed" then the tokens get appended to
199 /// \p Tokens and \p CurDirToks is cleared for the next directive.
200 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;
201 /// The directives that were lexed along with the number of tokens that each
202 /// directive contains. The tokens of all the directives are kept in \p Tokens
203 /// vector, in the same order as the directives order in \p DirsWithToks.
204 SmallVector<DirectiveWithTokens, 64> DirsWithToks;
205 LangOptions LangOpts;
206 Lexer TheLexer;
207};
208
209} // end anonymous namespace
210
211bool Scanner::reportError(const char *CurPtr, unsigned Err) {
212 if (!Diags)
213 return true;
214 assert(CurPtr >= Input.data() && "invalid buffer ptr");
215 Diags->Report(Loc: InputSourceLoc.getLocWithOffset(Offset: getOffsetAt(CurPtr)), DiagID: Err);
216 return true;
217}
218
219static void skipOverSpaces(const char *&First, const char *const End) {
220 while (First != End && isHorizontalWhitespace(c: *First))
221 ++First;
222}
223
224// Move back by one character, skipping escaped newlines (backslash + \n)
225static char previousChar(const char *First, const char *&Current) {
226 assert(Current > First);
227 --Current;
228 while (Current > First && isVerticalWhitespace(c: *Current)) {
229 // Check if the previous character is a backslash
230 if (Current > First && *(Current - 1) == '\\') {
231 // Use Lexer's getEscapedNewLineSize to get the size of the escaped
232 // newline
233 unsigned EscapeSize = Lexer::getEscapedNewLineSize(P: Current);
234 if (EscapeSize > 0) {
235 // Skip back over the entire escaped newline sequence (backslash +
236 // newline)
237 Current -= (1 + EscapeSize);
238 } else {
239 break;
240 }
241 } else {
242 break;
243 }
244 }
245 return *Current;
246}
247
248[[nodiscard]] static bool isRawStringLiteral(const char *First,
249 const char *Current) {
250 assert(First <= Current);
251
252 // Check if we can even back up.
253 if (*Current != '"' || First == Current)
254 return false;
255
256 // Check for an "R".
257 if (previousChar(First, Current) != 'R')
258 return false;
259 if (First == Current ||
260 !isAsciiIdentifierContinue(c: previousChar(First, Current)))
261 return true;
262
263 // Check for a prefix of "u", "U", or "L".
264 if (*Current == 'u' || *Current == 'U' || *Current == 'L')
265 return First == Current ||
266 !isAsciiIdentifierContinue(c: previousChar(First, Current));
267
268 // Check for a prefix of "u8".
269 if (*Current != '8' || First == Current ||
270 previousChar(First, Current) != 'u')
271 return false;
272 return First == Current ||
273 !isAsciiIdentifierContinue(c: previousChar(First, Current));
274}
275
276static void skipRawString(const char *&First, const char *const End) {
277 assert(First[0] == '"');
278
279 const char *Last = ++First;
280 while (Last != End && *Last != '(')
281 ++Last;
282 if (Last == End) {
283 First = Last; // Hit the end... just give up.
284 return;
285 }
286
287 StringRef Terminator(First, Last - First);
288 for (;;) {
289 // Move First to just past the next ")".
290 First = Last;
291 while (First != End && *First != ')')
292 ++First;
293 if (First == End)
294 return;
295 ++First;
296
297 // Look ahead for the terminator sequence.
298 Last = First;
299 while (Last != End && size_t(Last - First) < Terminator.size() &&
300 Terminator[Last - First] == *Last)
301 ++Last;
302
303 // Check if we hit it (or the end of the file).
304 if (Last == End) {
305 First = Last;
306 return;
307 }
308 if (size_t(Last - First) < Terminator.size())
309 continue;
310 if (*Last != '"')
311 continue;
312 First = Last + 1;
313 return;
314 }
315}
316
317// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
318static unsigned isEOL(const char *First, const char *const End) {
319 if (First == End)
320 return 0;
321 if (End - First > 1 && isVerticalWhitespace(c: First[0]) &&
322 isVerticalWhitespace(c: First[1]) && First[0] != First[1])
323 return 2;
324 return !!isVerticalWhitespace(c: First[0]);
325}
326
327static void skipString(const char *&First, const char *const End) {
328 assert(*First == '\'' || *First == '"' || *First == '<');
329 const char Terminator = *First == '<' ? '>' : *First;
330 for (++First; First != End && *First != Terminator; ++First) {
331 // String and character literals don't extend past the end of the line.
332 if (isVerticalWhitespace(c: *First))
333 return;
334 if (*First != '\\')
335 continue;
336 // Skip past backslash to the next character. This ensures that the
337 // character right after it is skipped as well, which matters if it's
338 // the terminator.
339 if (++First == End)
340 return;
341 if (!isWhitespace(c: *First))
342 continue;
343 // Whitespace after the backslash might indicate a line continuation.
344 const char *FirstAfterBackslashPastSpace = First;
345 skipOverSpaces(First&: FirstAfterBackslashPastSpace, End);
346 if (unsigned NLSize = isEOL(First: FirstAfterBackslashPastSpace, End)) {
347 // Advance the character pointer to the next line for the next
348 // iteration.
349 First = FirstAfterBackslashPastSpace + NLSize - 1;
350 }
351 }
352 if (First != End)
353 ++First; // Finish off the string.
354}
355
356// Returns the length of the skipped newline
357static unsigned skipNewline(const char *&First, const char *End) {
358 if (First == End)
359 return 0;
360 assert(isVerticalWhitespace(*First));
361 unsigned Len = isEOL(First, End);
362 assert(Len && "expected newline");
363 First += Len;
364 return Len;
365}
366
367static void skipToNewlineRaw(const char *&First, const char *const End) {
368 for (;;) {
369 if (First == End)
370 return;
371
372 unsigned Len = isEOL(First, End);
373 if (Len)
374 return;
375
376 char LastNonWhitespace = ' ';
377 do {
378 if (!isHorizontalWhitespace(c: *First))
379 LastNonWhitespace = *First;
380 if (++First == End)
381 return;
382 Len = isEOL(First, End);
383 } while (!Len);
384
385 if (LastNonWhitespace != '\\')
386 return;
387
388 First += Len;
389 // Keep skipping lines...
390 }
391}
392
393static void skipLineComment(const char *&First, const char *const End) {
394 assert(First[0] == '/' && First[1] == '/');
395 First += 2;
396 skipToNewlineRaw(First, End);
397}
398
399static void skipBlockComment(const char *&First, const char *const End) {
400 assert(First[0] == '/' && First[1] == '*');
401 if (End - First < 4) {
402 First = End;
403 return;
404 }
405 for (First += 3; First != End; ++First)
406 if (First[-1] == '*' && First[0] == '/') {
407 ++First;
408 return;
409 }
410}
411
412/// \returns True if the current single quotation mark character is a C++14
413/// digit separator.
414static bool isQuoteCppDigitSeparator(const char *const Start,
415 const char *const Cur,
416 const char *const End) {
417 assert(*Cur == '\'' && "expected quotation character");
418 // skipLine called in places where we don't expect a valid number
419 // body before `start` on the same line, so always return false at the start.
420 if (Start == Cur)
421 return false;
422 // The previous character must be a valid PP number character.
423 // Make sure that the L, u, U, u8 prefixes don't get marked as a
424 // separator though.
425 char Prev = *(Cur - 1);
426 if (Prev == 'L' || Prev == 'U' || Prev == 'u')
427 return false;
428 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
429 return false;
430 if (!isPreprocessingNumberBody(c: Prev))
431 return false;
432 // The next character should be a valid identifier body character.
433 return (Cur + 1) < End && isAsciiIdentifierContinue(c: *(Cur + 1));
434}
435
436void Scanner::skipLine(const char *&First, const char *const End) {
437 for (;;) {
438 assert(First <= End);
439 if (First == End)
440 return;
441
442 if (isVerticalWhitespace(c: *First)) {
443 skipNewline(First, End);
444 return;
445 }
446 const char *Start = First;
447 // Use `LastNonWhitespace`to track if a line-continuation has ever been seen
448 // before a new-line character:
449 char LastNonWhitespace = ' ';
450 while (First != End && !isVerticalWhitespace(c: *First)) {
451 // Iterate over strings correctly to avoid comments and newlines.
452 if (*First == '"' ||
453 (*First == '\'' && !isQuoteCppDigitSeparator(Start, Cur: First, End))) {
454 LastTokenPtr = First;
455 if (isRawStringLiteral(First: Start, Current: First))
456 skipRawString(First, End);
457 else
458 skipString(First, End);
459 continue;
460 }
461
462 // Continue on the same line if an EOL is preceded with backslash
463 if (First + 1 < End && *First == '\\') {
464 if (unsigned Len = isEOL(First: First + 1, End)) {
465 First += 1 + Len;
466 continue;
467 }
468 }
469
470 // Iterate over comments correctly.
471 if (*First != '/' || End - First < 2) {
472 LastTokenPtr = First;
473 if (!isWhitespace(c: *First))
474 LastNonWhitespace = *First;
475 ++First;
476 continue;
477 }
478
479 if (First[1] == '/') {
480 // "//...".
481 skipLineComment(First, End);
482 continue;
483 }
484
485 if (First[1] != '*') {
486 LastTokenPtr = First;
487 if (!isWhitespace(c: *First))
488 LastNonWhitespace = *First;
489 ++First;
490 continue;
491 }
492
493 // "/*...*/".
494 skipBlockComment(First, End);
495 }
496 if (First == End)
497 return;
498
499 // Skip over the newline.
500 skipNewline(First, End);
501
502 if (LastNonWhitespace != '\\')
503 break;
504 }
505}
506
507void Scanner::skipDirective(StringRef Name, const char *&First,
508 const char *const End) {
509 if (llvm::StringSwitch<bool>(Name)
510 .Case(S: "warning", Value: true)
511 .Case(S: "error", Value: true)
512 .Default(Value: false))
513 // Do not process quotes or comments.
514 skipToNewlineRaw(First, End);
515 else
516 skipLine(First, End);
517}
518
519static void skipWhitespace(const char *&First, const char *const End) {
520 for (;;) {
521 assert(First <= End);
522 skipOverSpaces(First, End);
523
524 if (End - First < 2)
525 return;
526
527 if (*First == '\\') {
528 const char *Ptr = First + 1;
529 while (Ptr < End && isHorizontalWhitespace(c: *Ptr))
530 ++Ptr;
531 if (Ptr != End && isVerticalWhitespace(c: *Ptr)) {
532 skipNewline(First&: Ptr, End);
533 First = Ptr;
534 continue;
535 }
536 return;
537 }
538
539 // Check for a non-comment character.
540 if (First[0] != '/')
541 return;
542
543 // "// ...".
544 if (First[1] == '/') {
545 skipLineComment(First, End);
546 return;
547 }
548
549 // Cannot be a comment.
550 if (First[1] != '*')
551 return;
552
553 // "/*...*/".
554 skipBlockComment(First, End);
555 }
556}
557
558bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
559 const char *const End) {
560 assert(Kind == DirectiveKind::cxx_export_import_decl ||
561 Kind == DirectiveKind::cxx_export_module_decl ||
562 Kind == DirectiveKind::cxx_import_decl ||
563 Kind == DirectiveKind::cxx_module_decl ||
564 Kind == DirectiveKind::decl_at_import);
565
566 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
567 for (;;) {
568 // Keep a copy of the First char incase it needs to be reset.
569 const char *Previous = First;
570 const dependency_directives_scan::Token &Tok = lexToken(First, End);
571 if ((Tok.is(K: tok::hash) || Tok.is(K: tok::at)) &&
572 (Tok.Flags & clang::Token::StartOfLine)) {
573 CurDirToks.pop_back();
574 First = Previous;
575 return false;
576 }
577 if (Tok.isOneOf(Ks: tok::eof, Ks: tok::eod))
578 return reportError(
579 CurPtr: DirectiveLoc,
580 Err: diag::err_dep_source_scanner_missing_semi_after_at_import);
581 if (Tok.is(K: tok::semi))
582 break;
583 }
584
585 bool IsCXXModules = Kind == DirectiveKind::cxx_export_import_decl ||
586 Kind == DirectiveKind::cxx_export_module_decl ||
587 Kind == DirectiveKind::cxx_import_decl ||
588 Kind == DirectiveKind::cxx_module_decl;
589 if (IsCXXModules) {
590 lexPPDirectiveBody(First, End);
591 pushDirective(Kind);
592 return false;
593 }
594
595 const auto &Tok = lexToken(First, End);
596 pushDirective(Kind);
597 if (Tok.is(K: tok::eof) || Tok.is(K: tok::eod))
598 return false;
599 return reportError(CurPtr: DirectiveLoc,
600 Err: diag::err_dep_source_scanner_unexpected_tokens_at_import);
601}
602
603dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
604 const char *const End) {
605 clang::Token Tok;
606 TheLexer.LexFromRawLexer(Result&: Tok);
607 First = Input.data() + TheLexer.getCurrentBufferOffset();
608 assert(First <= End);
609
610 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
611 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
612 Args: Tok.getFlags());
613 return CurDirToks.back();
614}
615
616dependency_directives_scan::Token &
617Scanner::lexIncludeFilename(const char *&First, const char *const End) {
618 clang::Token Tok;
619 TheLexer.LexIncludeFilename(FilenameTok&: Tok);
620 First = Input.data() + TheLexer.getCurrentBufferOffset();
621 assert(First <= End);
622
623 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
624 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
625 Args: Tok.getFlags());
626 return CurDirToks.back();
627}
628
629void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
630 while (true) {
631 const dependency_directives_scan::Token &Tok = lexToken(First, End);
632 if (Tok.is(K: tok::eod) || Tok.is(K: tok::eof))
633 break;
634 }
635}
636
637StringRef
638Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
639 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
640 if (LLVM_LIKELY(!NeedsCleaning))
641 return Input.slice(Start: Tok.Offset, End: Tok.getEnd());
642
643 SmallString<64> Spelling;
644 Spelling.resize(N: Tok.Length);
645
646 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow
647 // in the Lexer). Currently we cannot see them due to our LangOpts.
648
649 unsigned SpellingLength = 0;
650 const char *BufPtr = Input.begin() + Tok.Offset;
651 const char *AfterIdent = Input.begin() + Tok.getEnd();
652 while (BufPtr < AfterIdent) {
653 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
654 Spelling[SpellingLength++] = Char;
655 BufPtr += Size;
656 }
657
658 return SplitIds.try_emplace(Key: StringRef(Spelling.begin(), SpellingLength), Args: 0)
659 .first->first();
660}
661
662std::optional<StringRef>
663Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
664 const dependency_directives_scan::Token &Tok = lexToken(First, End);
665 if (Tok.isNot(K: tok::raw_identifier)) {
666 if (!Tok.is(K: tok::eod))
667 skipLine(First, End);
668 return std::nullopt;
669 }
670
671 return cleanStringIfNeeded(Tok);
672}
673
674StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
675 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
676 assert(Id && "expected identifier token");
677 return *Id;
678}
679
680bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
681 const char *const End) {
682 if (std::optional<StringRef> FoundId =
683 tryLexIdentifierOrSkipLine(First, End)) {
684 if (*FoundId == Id)
685 return true;
686 skipLine(First, End);
687 }
688 return false;
689}
690
691bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
692 const char *const End) {
693 const dependency_directives_scan::Token &Tok = lexToken(First, End);
694 if (Tok.is(K))
695 return true;
696 skipLine(First, End);
697 return false;
698}
699
700std::optional<StringRef>
701Scanner::tryLexStringLiteralOrSkipLine(const char *&First,
702 const char *const End) {
703 const dependency_directives_scan::Token &Tok = lexToken(First, End);
704 if (!tok::isStringLiteral(K: Tok.Kind)) {
705 if (!Tok.is(K: tok::eod))
706 skipLine(First, End);
707 return std::nullopt;
708 }
709
710 return cleanStringIfNeeded(Tok);
711}
712
713bool Scanner::lexAt(const char *&First, const char *const End) {
714 // Handle "@import".
715
716 // Lex '@'.
717 const dependency_directives_scan::Token &AtTok = lexToken(First, End);
718 assert(AtTok.is(tok::at));
719 (void)AtTok;
720
721 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
722 return false;
723 return lexModuleDirectiveBody(Kind: decl_at_import, First, End);
724}
725
726bool Scanner::lexModule(const char *&First, const char *const End) {
727 StringRef Id = lexIdentifier(First, End);
728 bool Export = false;
729 if (Id == "export") {
730 Export = true;
731 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
732 if (!NextId)
733 return false;
734 Id = *NextId;
735 }
736
737 StringRef Module =
738 ScanningPreprocessedModuleFile ? "__preprocessed_module" : "module";
739 StringRef Import =
740 ScanningPreprocessedModuleFile ? "__preprocessed_import" : "import";
741
742 if (Id != Module && Id != Import) {
743 skipLine(First, End);
744 return false;
745 }
746
747 skipWhitespace(First, End);
748
749 // Ignore this as a module directive if the next character can't be part of
750 // an import.
751
752 switch (*First) {
753 case ':': {
754 // `module :` is never the start of a valid module declaration.
755 if (Id == Module) {
756 skipLine(First, End);
757 return false;
758 }
759 // A module partition starts with exactly one ':'. If we have '::', this is
760 // a scope resolution instead and shouldn't be recognized as a directive
761 // per P1857R3.
762 if (First + 1 != End && First[1] == ':') {
763 skipLine(First, End);
764 return false;
765 }
766 // `import:(type)name` is a valid ObjC method decl, so check one more token.
767 (void)lexToken(First, End);
768 if (!tryLexIdentifierOrSkipLine(First, End))
769 return false;
770 break;
771 }
772 case ';': {
773 // Handle the global module fragment `module;`.
774 if (Id == Module && !Export)
775 break;
776 skipLine(First, End);
777 return false;
778 }
779 case '<':
780 case '"':
781 break;
782 default:
783 if (!isAsciiIdentifierContinue(c: *First)) {
784 skipLine(First, End);
785 return false;
786 }
787 }
788
789 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ false);
790
791 DirectiveKind Kind;
792 if (Id == Module)
793 Kind = Export ? cxx_export_module_decl : cxx_module_decl;
794 else
795 Kind = Export ? cxx_export_import_decl : cxx_import_decl;
796
797 return lexModuleDirectiveBody(Kind, First, End);
798}
799
800bool Scanner::lex_Pragma(const char *&First, const char *const End) {
801 if (!isNextTokenOrSkipLine(K: tok::l_paren, First, End))
802 return false;
803
804 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);
805
806 if (!Str || !isNextTokenOrSkipLine(K: tok::r_paren, First, End))
807 return false;
808
809 SmallString<64> Buffer(*Str);
810 prepare_PragmaString(StrVal&: Buffer);
811
812 // Use a new scanner instance since the tokens will be inside the allocated
813 // string. We should already have captured all the relevant tokens in the
814 // current scanner.
815 SmallVector<dependency_directives_scan::Token> DiscardTokens;
816 const char *Begin = Buffer.c_str();
817 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
818 InputSourceLoc};
819
820 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
821 if (PragmaScanner.lexPragma(First&: Begin, End: Buffer.end()))
822 return true;
823
824 DirectiveKind K = PragmaScanner.topDirective();
825 if (K == pp_none) {
826 skipLine(First, End);
827 return false;
828 }
829
830 assert(Begin == Buffer.end());
831 pushDirective(Kind: K);
832 return false;
833}
834
835bool Scanner::lexPragma(const char *&First, const char *const End) {
836 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
837 if (!FoundId)
838 return false;
839
840 StringRef Id = *FoundId;
841 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
842 .Case(S: "once", Value: pp_pragma_once)
843 .Case(S: "push_macro", Value: pp_pragma_push_macro)
844 .Case(S: "pop_macro", Value: pp_pragma_pop_macro)
845 .Case(S: "include_alias", Value: pp_pragma_include_alias)
846 .Default(Value: pp_none);
847 if (Kind != pp_none) {
848 lexPPDirectiveBody(First, End);
849 pushDirective(Kind);
850 return false;
851 }
852
853 if (Id != "clang") {
854 skipLine(First, End);
855 return false;
856 }
857
858 FoundId = tryLexIdentifierOrSkipLine(First, End);
859 if (!FoundId)
860 return false;
861 Id = *FoundId;
862
863 // #pragma clang system_header
864 if (Id == "system_header") {
865 lexPPDirectiveBody(First, End);
866 pushDirective(Kind: pp_pragma_system_header);
867 return false;
868 }
869
870 if (Id != "module") {
871 skipLine(First, End);
872 return false;
873 }
874
875 // #pragma clang module.
876 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
877 return false;
878
879 // #pragma clang module import.
880 lexPPDirectiveBody(First, End);
881 pushDirective(Kind: pp_pragma_import);
882 return false;
883}
884
885bool Scanner::lexEndif(const char *&First, const char *const End) {
886 // Strip out "#else" if it's empty.
887 if (topDirective() == pp_else)
888 popDirective();
889
890 // If "#ifdef" is empty, strip it and skip the "#endif".
891 //
892 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
893 // we can skip empty `#if` and `#elif` blocks as well after scanning for a
894 // literal __has_include in the condition. Even without that rule we could
895 // drop the tokens if we scan for identifiers in the condition and find none.
896 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
897 popDirective();
898 skipLine(First, End);
899 return false;
900 }
901
902 return lexDefault(Kind: pp_endif, First, End);
903}
904
905bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
906 const char *const End) {
907 lexPPDirectiveBody(First, End);
908 pushDirective(Kind);
909 return false;
910}
911
912static bool isStartOfRelevantLine(char First) {
913 switch (First) {
914 case '#':
915 case '@':
916 case 'i':
917 case 'e':
918 case 'm':
919 case '_':
920 return true;
921 }
922 return false;
923}
924
925static inline bool isStartWithPreprocessedModuleDirective(const char *First,
926 const char *End) {
927 assert(First <= End);
928 if (*First == '_') {
929 StringRef Str(First, End - First);
930 return Str.starts_with(
931 Prefix: tok::getPPKeywordSpelling(Kind: tok::pp___preprocessed_module)) ||
932 Str.starts_with(
933 Prefix: tok::getPPKeywordSpelling(Kind: tok::pp___preprocessed_import));
934 }
935 return false;
936}
937
938bool Scanner::lexPPLine(const char *&First, const char *const End) {
939 assert(First != End);
940
941 skipWhitespace(First, End);
942 assert(First <= End);
943 if (First == End)
944 return false;
945
946 if (!isStartOfRelevantLine(First: *First)) {
947 skipLine(First, End);
948 assert(First <= End);
949 return false;
950 }
951
952 LastTokenPtr = First;
953
954 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ true);
955
956 llvm::scope_exit ScEx1([&]() {
957 /// Clear Scanner's CurDirToks before returning, in case we didn't push a
958 /// new directive.
959 CurDirToks.clear();
960 });
961
962 bool IsPreprocessedModule =
963 isStartWithPreprocessedModuleDirective(First, End);
964 if (*First == '_' && !IsPreprocessedModule) {
965 if (isNextIdentifierOrSkipLine(Id: "_Pragma", First, End))
966 return lex_Pragma(First, End);
967 return false;
968 }
969
970 // Handle preprocessing directives.
971
972 TheLexer.setParsingPreprocessorDirective(true);
973 llvm::scope_exit ScEx2(
974 [&]() { TheLexer.setParsingPreprocessorDirective(false); });
975
976 if (*First == '@')
977 return lexAt(First, End);
978
979 // Handle module directives for C++20 modules.
980 if (*First == 'i' || *First == 'e' || *First == 'm' || IsPreprocessedModule)
981 return lexModule(First, End);
982
983 // Lex '#'.
984 const dependency_directives_scan::Token &HashTok = lexToken(First, End);
985 if (HashTok.is(K: tok::hashhash)) {
986 // A \p tok::hashhash at this location is passed by the preprocessor to the
987 // parser to interpret, like any other token. So for dependency scanning
988 // skip it like a normal token not affecting the preprocessor.
989 skipLine(First, End);
990 assert(First <= End);
991 return false;
992 }
993 assert(HashTok.is(tok::hash));
994 (void)HashTok;
995
996 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
997 if (!FoundId)
998 return false;
999
1000 StringRef Id = *FoundId;
1001
1002 if (Id == "pragma")
1003 return lexPragma(First, End);
1004
1005 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
1006 .Case(S: "include", Value: pp_include)
1007 .Case(S: "__include_macros", Value: pp___include_macros)
1008 .Case(S: "define", Value: pp_define)
1009 .Case(S: "undef", Value: pp_undef)
1010 .Case(S: "import", Value: pp_import)
1011 .Case(S: "include_next", Value: pp_include_next)
1012 .Case(S: "if", Value: pp_if)
1013 .Case(S: "ifdef", Value: pp_ifdef)
1014 .Case(S: "ifndef", Value: pp_ifndef)
1015 .Case(S: "elif", Value: pp_elif)
1016 .Case(S: "elifdef", Value: pp_elifdef)
1017 .Case(S: "elifndef", Value: pp_elifndef)
1018 .Case(S: "else", Value: pp_else)
1019 .Case(S: "endif", Value: pp_endif)
1020 .Default(Value: pp_none);
1021 if (Kind == pp_none) {
1022 skipDirective(Name: Id, First, End);
1023 return false;
1024 }
1025
1026 if (Kind == pp_endif)
1027 return lexEndif(First, End);
1028
1029 switch (Kind) {
1030 case pp_include:
1031 case pp___include_macros:
1032 case pp_include_next:
1033 case pp_import:
1034 // Ignore missing filenames in include or import directives.
1035 if (lexIncludeFilename(First, End).is(K: tok::eod)) {
1036 return false;
1037 }
1038 break;
1039 default:
1040 break;
1041 }
1042
1043 // Everything else.
1044 return lexDefault(Kind, First, End);
1045}
1046
1047static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
1048 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
1049 First[2] == '\xbf')
1050 First += 3;
1051}
1052
1053bool Scanner::scanImpl(const char *First, const char *const End) {
1054 skipUTF8ByteOrderMark(First, End);
1055 while (First != End)
1056 if (lexPPLine(First, End))
1057 return true;
1058 return false;
1059}
1060
1061bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
1062 ScanningPreprocessedModuleFile = clang::isPreprocessedModuleFile(Source: Input);
1063 bool Error = scanImpl(First: Input.begin(), End: Input.end());
1064
1065 if (!Error) {
1066 // Add an EOF on success.
1067 if (LastTokenPtr &&
1068 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
1069 pushDirective(Kind: tokens_present_before_eof);
1070 pushDirective(Kind: pp_eof);
1071 }
1072
1073 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
1074 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
1075 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
1076 Directives.emplace_back(Args: DirWithToks.Kind,
1077 Args: RemainingTokens.take_front(N: DirWithToks.NumTokens));
1078 RemainingTokens = RemainingTokens.drop_front(N: DirWithToks.NumTokens);
1079 }
1080 assert(RemainingTokens.empty());
1081
1082 return Error;
1083}
1084
1085bool clang::scanSourceForDependencyDirectives(
1086 StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
1087 SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
1088 SourceLocation InputSourceLoc) {
1089 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
1090}
1091
1092void clang::printDependencyDirectivesAsSource(
1093 StringRef Source,
1094 ArrayRef<dependency_directives_scan::Directive> Directives,
1095 llvm::raw_ostream &OS) {
1096 // Add a space separator where it is convenient for testing purposes.
1097 auto needsSpaceSeparator =
1098 [](tok::TokenKind Prev,
1099 const dependency_directives_scan::Token &Tok) -> bool {
1100 if (Prev == Tok.Kind)
1101 return !Tok.isOneOf(Ks: tok::l_paren, Ks: tok::r_paren, Ks: tok::l_square,
1102 Ks: tok::r_square);
1103 if (Prev == tok::raw_identifier &&
1104 Tok.isOneOf(Ks: tok::hash, Ks: tok::numeric_constant, Ks: tok::string_literal,
1105 Ks: tok::char_constant, Ks: tok::header_name))
1106 return true;
1107 if (Prev == tok::r_paren &&
1108 Tok.isOneOf(Ks: tok::raw_identifier, Ks: tok::hash, Ks: tok::string_literal,
1109 Ks: tok::char_constant, Ks: tok::unknown))
1110 return true;
1111 if (Prev == tok::comma &&
1112 Tok.isOneOf(Ks: tok::l_paren, Ks: tok::string_literal, Ks: tok::less))
1113 return true;
1114 return false;
1115 };
1116
1117 for (const dependency_directives_scan::Directive &Directive : Directives) {
1118 if (Directive.Kind == tokens_present_before_eof)
1119 OS << "<TokBeforeEOF>";
1120 std::optional<tok::TokenKind> PrevTokenKind;
1121 for (const dependency_directives_scan::Token &Tok : Directive.Tokens) {
1122 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
1123 OS << ' ';
1124 PrevTokenKind = Tok.Kind;
1125 OS << Source.slice(Start: Tok.Offset, End: Tok.getEnd());
1126 }
1127 }
1128}
1129
1130static void skipUntilMaybeCXX20ModuleDirective(const char *&First,
1131 const char *const End) {
1132 assert(First <= End);
1133 while (First != End) {
1134 if (*First == '#') {
1135 ++First;
1136 skipToNewlineRaw(First, End);
1137 }
1138 skipWhitespace(First, End);
1139 if (const auto Len = isEOL(First, End)) {
1140 First += Len;
1141 continue;
1142 }
1143 break;
1144 }
1145}
1146
1147namespace {
1148
1149static CXX20ModuleDirectiveKind
1150scanFirstCXX20ModuleDirective(StringRef Source) {
1151 const char *First = Source.begin();
1152 const char *const End = Source.end();
1153 skipUntilMaybeCXX20ModuleDirective(First, End);
1154 if (First == End)
1155 return CXX20ModuleDirectiveKind::None;
1156
1157 // Check if the next token can even be a module directive before creating a
1158 // full lexer.
1159 if (!(*First == 'i' || *First == 'e' || *First == 'm'))
1160 return CXX20ModuleDirectiveKind::None;
1161
1162 llvm::SmallVector<dependency_directives_scan::Token> Tokens;
1163 Scanner S(StringRef(First, End - First), Tokens, nullptr, SourceLocation());
1164 S.TheLexer.setParsingPreprocessorDirective(true);
1165 if (S.lexModule(First, End) || S.DirsWithToks.empty())
1166 return CXX20ModuleDirectiveKind::None;
1167
1168 assert(S.DirsWithToks.size() == 1);
1169 const DirectiveWithTokens &Directive = S.DirsWithToks.front();
1170 switch (Directive.Kind) {
1171 case dependency_directives_scan::cxx_module_decl:
1172 assert(Directive.NumTokens >= 2);
1173 return Tokens[1].is(K: tok::semi)
1174 ? CXX20ModuleDirectiveKind::GlobalModuleFragment
1175 : CXX20ModuleDirectiveKind::NamedModuleDeclaration;
1176 case dependency_directives_scan::cxx_export_module_decl:
1177 return CXX20ModuleDirectiveKind::NamedModuleDeclaration;
1178 case dependency_directives_scan::cxx_import_decl:
1179 case dependency_directives_scan::cxx_export_import_decl:
1180 return CXX20ModuleDirectiveKind::ImportDeclaration;
1181 default:
1182 llvm_unreachable("unexpected C++20 module directive kind");
1183 }
1184}
1185
1186} // namespace
1187
1188bool clang::scanInputForCXX20ModulesUsage(StringRef Source) {
1189 return scanFirstCXX20ModuleDirective(Source) !=
1190 CXX20ModuleDirectiveKind::None;
1191}
1192
1193ModuleUnitKind clang::scanInputForCXX20ModuleUnit(StringRef Source) {
1194 switch (scanFirstCXX20ModuleDirective(Source)) {
1195 case CXX20ModuleDirectiveKind::GlobalModuleFragment:
1196 return ModuleUnitKind::HasGlobalModuleFragment;
1197 case CXX20ModuleDirectiveKind::NamedModuleDeclaration:
1198 return ModuleUnitKind::NamedModuleWithoutGlobalModuleFragment;
1199 case CXX20ModuleDirectiveKind::None:
1200 case CXX20ModuleDirectiveKind::ImportDeclaration:
1201 return ModuleUnitKind::NotModuleUnit;
1202 }
1203 llvm_unreachable("unexpected C++20 module directive kind");
1204}
1205
1206bool clang::isPreprocessedModuleFile(StringRef Source) {
1207 const char *First = Source.begin();
1208 const char *const End = Source.end();
1209
1210 skipUntilMaybeCXX20ModuleDirective(First, End);
1211 if (First == End)
1212 return false;
1213
1214 llvm::SmallVector<dependency_directives_scan::Token> Tokens;
1215 Scanner S(StringRef(First, End - First), Tokens, nullptr, SourceLocation());
1216 while (First != End) {
1217 if (*First == '#') {
1218 ++First;
1219 skipToNewlineRaw(First, End);
1220 } else if (*First == 'e') {
1221 S.TheLexer.seek(Offset: S.getOffsetAt(CurPtr: First), /*IsAtStartOfLine=*/true);
1222 StringRef Id = S.lexIdentifier(First, End);
1223 if (Id == "export") {
1224 std::optional<StringRef> NextId =
1225 S.tryLexIdentifierOrSkipLine(First, End);
1226 if (!NextId)
1227 return false;
1228 Id = *NextId;
1229 }
1230 if (Id == "__preprocessed_module" || Id == "__preprocessed_import")
1231 return true;
1232 skipToNewlineRaw(First, End);
1233 } else if (isStartWithPreprocessedModuleDirective(First, End))
1234 return true;
1235 else
1236 skipToNewlineRaw(First, End);
1237
1238 skipWhitespace(First, End);
1239 if (const auto Len = isEOL(First, End)) {
1240 First += Len;
1241 continue;
1242 }
1243 break;
1244 }
1245 return false;
1246}
1247