1//===- DependencyDirectivesScanner.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the interface for scanning header and source files to get the
11/// minimum necessary preprocessor directives for evaluating includes. It
12/// reduces the source down to #define, #include, #import, @import, and any
13/// conditional preprocessor logic that contains one of those.
14///
15//===----------------------------------------------------------------------===//
16
17#include "clang/Lex/DependencyDirectivesScanner.h"
18#include "clang/Basic/CharInfo.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Lex/LexDiagnostic.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Pragma.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
27#include <optional>
28
29using namespace clang;
30using namespace clang::dependency_directives_scan;
31using namespace llvm;
32
33namespace {
34
35struct DirectiveWithTokens {
36 DirectiveKind Kind;
37 unsigned NumTokens;
38
39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
40 : Kind(Kind), NumTokens(NumTokens) {}
41};
42
43/// Does an efficient "scan" of the sources to detect the presence of
44/// preprocessor (or module import) directives and collects the raw lexed tokens
45/// for those directives so that the \p Lexer can "replay" them when the file is
46/// included.
47///
48/// Note that the behavior of the raw lexer is affected by the language mode,
49/// while at this point we want to do a scan and collect tokens once,
50/// irrespective of the language mode that the file will get included in. To
51/// compensate for that the \p Lexer, while "replaying", will adjust a token
52/// where appropriate, when it could affect the preprocessor's state.
53/// For example in a directive like
54///
55/// \code
56/// #if __has_cpp_attribute(clang::fallthrough)
57/// \endcode
58///
59/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
60/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
61/// while in C++ mode.
62struct Scanner {
63 Scanner(StringRef Input,
64 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
69 Input.end()) {}
70
71 static LangOptions getLangOptsForDepScanning() {
72 LangOptions LangOpts;
73 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
74 LangOpts.ObjC = true;
75 LangOpts.LineComment = true;
76 LangOpts.RawStringLiterals = true;
77 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
78 return LangOpts;
79 }
80
81 /// Lex the provided source and emit the directive tokens.
82 ///
83 /// \returns True on error.
84 bool scan(SmallVectorImpl<Directive> &Directives);
85
86private:
87 /// Lexes next token and advances \p First and the \p Lexer.
88 [[nodiscard]] dependency_directives_scan::Token &
89 lexToken(const char *&First, const char *const End);
90
91 [[nodiscard]] dependency_directives_scan::Token &
92 lexIncludeFilename(const char *&First, const char *const End);
93
94 void skipLine(const char *&First, const char *const End);
95 void skipDirective(StringRef Name, const char *&First, const char *const End);
96
97 /// Returns the spelling of a string literal or identifier after performing
98 /// any processing needed to handle \c clang::Token::NeedsCleaning.
99 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);
100
101 /// Lexes next token and if it is identifier returns its string, otherwise
102 /// it skips the current line and returns \p std::nullopt.
103 ///
104 /// In any case (whatever the token kind) \p First and the \p Lexer will
105 /// advance beyond the token.
106 [[nodiscard]] std::optional<StringRef>
107 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
108
109 /// Used when it is certain that next token is an identifier.
110 [[nodiscard]] StringRef lexIdentifier(const char *&First,
111 const char *const End);
112
113 /// Lexes next token and returns true iff it is an identifier that matches \p
114 /// Id, otherwise it skips the current line and returns false.
115 ///
116 /// In any case (whatever the token kind) \p First and the \p Lexer will
117 /// advance beyond the token.
118 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,
119 const char *&First,
120 const char *const End);
121
122 /// Lexes next token and returns true iff it matches the kind \p K.
123 /// Otherwise it skips the current line and returns false.
124 ///
125 /// In any case (whatever the token kind) \p First and the \p Lexer will
126 /// advance beyond the token.
127 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
128 const char *const End);
129
130 /// Lexes next token and if it is string literal, returns its string.
131 /// Otherwise, it skips the current line and returns \p std::nullopt.
132 ///
133 /// In any case (whatever the token kind) \p First and the \p Lexer will
134 /// advance beyond the token.
135 [[nodiscard]] std::optional<StringRef>
136 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);
137
138 [[nodiscard]] bool scanImpl(const char *First, const char *const End);
139 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
140 [[nodiscard]] bool lexAt(const char *&First, const char *const End);
141 [[nodiscard]] bool lexModule(const char *&First, const char *const End);
142 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
143 const char *const End);
144 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
145 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);
146 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
147 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
148 const char *const End);
149 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,
150 const char *&First,
151 const char *const End);
152 void lexPPDirectiveBody(const char *&First, const char *const End);
153
154 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
155 Tokens.append(RHS: CurDirToks);
156 DirsWithToks.emplace_back(Args&: Kind, Args: CurDirToks.size());
157 CurDirToks.clear();
158 return DirsWithToks.back();
159 }
160 void popDirective() {
161 Tokens.pop_back_n(NumItems: DirsWithToks.pop_back_val().NumTokens);
162 }
163 DirectiveKind topDirective() const {
164 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
165 }
166
167 unsigned getOffsetAt(const char *CurPtr) const {
168 return CurPtr - Input.data();
169 }
170
171 /// Reports a diagnostic if the diagnostic engine is provided. Always returns
172 /// true at the end.
173 bool reportError(const char *CurPtr, unsigned Err);
174
175 StringMap<char> SplitIds;
176 StringRef Input;
177 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;
178 DiagnosticsEngine *Diags;
179 SourceLocation InputSourceLoc;
180
181 const char *LastTokenPtr = nullptr;
182 /// Keeps track of the tokens for the currently lexed directive. Once a
183 /// directive is fully lexed and "committed" then the tokens get appended to
184 /// \p Tokens and \p CurDirToks is cleared for the next directive.
185 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;
186 /// The directives that were lexed along with the number of tokens that each
187 /// directive contains. The tokens of all the directives are kept in \p Tokens
188 /// vector, in the same order as the directives order in \p DirsWithToks.
189 SmallVector<DirectiveWithTokens, 64> DirsWithToks;
190 LangOptions LangOpts;
191 Lexer TheLexer;
192};
193
194} // end anonymous namespace
195
196bool Scanner::reportError(const char *CurPtr, unsigned Err) {
197 if (!Diags)
198 return true;
199 assert(CurPtr >= Input.data() && "invalid buffer ptr");
200 Diags->Report(Loc: InputSourceLoc.getLocWithOffset(Offset: getOffsetAt(CurPtr)), DiagID: Err);
201 return true;
202}
203
204static void skipOverSpaces(const char *&First, const char *const End) {
205 while (First != End && isHorizontalWhitespace(c: *First))
206 ++First;
207}
208
209[[nodiscard]] static bool isRawStringLiteral(const char *First,
210 const char *Current) {
211 assert(First <= Current);
212
213 // Check if we can even back up.
214 if (*Current != '"' || First == Current)
215 return false;
216
217 // Check for an "R".
218 --Current;
219 if (*Current != 'R')
220 return false;
221 if (First == Current || !isAsciiIdentifierContinue(c: *--Current))
222 return true;
223
224 // Check for a prefix of "u", "U", or "L".
225 if (*Current == 'u' || *Current == 'U' || *Current == 'L')
226 return First == Current || !isAsciiIdentifierContinue(c: *--Current);
227
228 // Check for a prefix of "u8".
229 if (*Current != '8' || First == Current || *Current-- != 'u')
230 return false;
231 return First == Current || !isAsciiIdentifierContinue(c: *--Current);
232}
233
234static void skipRawString(const char *&First, const char *const End) {
235 assert(First[0] == '"');
236 assert(First[-1] == 'R');
237
238 const char *Last = ++First;
239 while (Last != End && *Last != '(')
240 ++Last;
241 if (Last == End) {
242 First = Last; // Hit the end... just give up.
243 return;
244 }
245
246 StringRef Terminator(First, Last - First);
247 for (;;) {
248 // Move First to just past the next ")".
249 First = Last;
250 while (First != End && *First != ')')
251 ++First;
252 if (First == End)
253 return;
254 ++First;
255
256 // Look ahead for the terminator sequence.
257 Last = First;
258 while (Last != End && size_t(Last - First) < Terminator.size() &&
259 Terminator[Last - First] == *Last)
260 ++Last;
261
262 // Check if we hit it (or the end of the file).
263 if (Last == End) {
264 First = Last;
265 return;
266 }
267 if (size_t(Last - First) < Terminator.size())
268 continue;
269 if (*Last != '"')
270 continue;
271 First = Last + 1;
272 return;
273 }
274}
275
276// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
277static unsigned isEOL(const char *First, const char *const End) {
278 if (First == End)
279 return 0;
280 if (End - First > 1 && isVerticalWhitespace(c: First[0]) &&
281 isVerticalWhitespace(c: First[1]) && First[0] != First[1])
282 return 2;
283 return !!isVerticalWhitespace(c: First[0]);
284}
285
286static void skipString(const char *&First, const char *const End) {
287 assert(*First == '\'' || *First == '"' || *First == '<');
288 const char Terminator = *First == '<' ? '>' : *First;
289 for (++First; First != End && *First != Terminator; ++First) {
290 // String and character literals don't extend past the end of the line.
291 if (isVerticalWhitespace(c: *First))
292 return;
293 if (*First != '\\')
294 continue;
295 // Skip past backslash to the next character. This ensures that the
296 // character right after it is skipped as well, which matters if it's
297 // the terminator.
298 if (++First == End)
299 return;
300 if (!isWhitespace(c: *First))
301 continue;
302 // Whitespace after the backslash might indicate a line continuation.
303 const char *FirstAfterBackslashPastSpace = First;
304 skipOverSpaces(First&: FirstAfterBackslashPastSpace, End);
305 if (unsigned NLSize = isEOL(First: FirstAfterBackslashPastSpace, End)) {
306 // Advance the character pointer to the next line for the next
307 // iteration.
308 First = FirstAfterBackslashPastSpace + NLSize - 1;
309 }
310 }
311 if (First != End)
312 ++First; // Finish off the string.
313}
314
315// Returns the length of the skipped newline
316static unsigned skipNewline(const char *&First, const char *End) {
317 if (First == End)
318 return 0;
319 assert(isVerticalWhitespace(*First));
320 unsigned Len = isEOL(First, End);
321 assert(Len && "expected newline");
322 First += Len;
323 return Len;
324}
325
326static bool wasLineContinuation(const char *First, unsigned EOLLen) {
327 return *(First - (int)EOLLen - 1) == '\\';
328}
329
330static void skipToNewlineRaw(const char *&First, const char *const End) {
331 for (;;) {
332 if (First == End)
333 return;
334
335 unsigned Len = isEOL(First, End);
336 if (Len)
337 return;
338
339 do {
340 if (++First == End)
341 return;
342 Len = isEOL(First, End);
343 } while (!Len);
344
345 if (First[-1] != '\\')
346 return;
347
348 First += Len;
349 // Keep skipping lines...
350 }
351}
352
353static void skipLineComment(const char *&First, const char *const End) {
354 assert(First[0] == '/' && First[1] == '/');
355 First += 2;
356 skipToNewlineRaw(First, End);
357}
358
359static void skipBlockComment(const char *&First, const char *const End) {
360 assert(First[0] == '/' && First[1] == '*');
361 if (End - First < 4) {
362 First = End;
363 return;
364 }
365 for (First += 3; First != End; ++First)
366 if (First[-1] == '*' && First[0] == '/') {
367 ++First;
368 return;
369 }
370}
371
372/// \returns True if the current single quotation mark character is a C++14
373/// digit separator.
374static bool isQuoteCppDigitSeparator(const char *const Start,
375 const char *const Cur,
376 const char *const End) {
377 assert(*Cur == '\'' && "expected quotation character");
378 // skipLine called in places where we don't expect a valid number
379 // body before `start` on the same line, so always return false at the start.
380 if (Start == Cur)
381 return false;
382 // The previous character must be a valid PP number character.
383 // Make sure that the L, u, U, u8 prefixes don't get marked as a
384 // separator though.
385 char Prev = *(Cur - 1);
386 if (Prev == 'L' || Prev == 'U' || Prev == 'u')
387 return false;
388 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
389 return false;
390 if (!isPreprocessingNumberBody(c: Prev))
391 return false;
392 // The next character should be a valid identifier body character.
393 return (Cur + 1) < End && isAsciiIdentifierContinue(c: *(Cur + 1));
394}
395
396void Scanner::skipLine(const char *&First, const char *const End) {
397 for (;;) {
398 assert(First <= End);
399 if (First == End)
400 return;
401
402 if (isVerticalWhitespace(c: *First)) {
403 skipNewline(First, End);
404 return;
405 }
406 const char *Start = First;
407 while (First != End && !isVerticalWhitespace(c: *First)) {
408 // Iterate over strings correctly to avoid comments and newlines.
409 if (*First == '"' ||
410 (*First == '\'' && !isQuoteCppDigitSeparator(Start, Cur: First, End))) {
411 LastTokenPtr = First;
412 if (isRawStringLiteral(First: Start, Current: First))
413 skipRawString(First, End);
414 else
415 skipString(First, End);
416 continue;
417 }
418
419 // Iterate over comments correctly.
420 if (*First != '/' || End - First < 2) {
421 LastTokenPtr = First;
422 ++First;
423 continue;
424 }
425
426 if (First[1] == '/') {
427 // "//...".
428 skipLineComment(First, End);
429 continue;
430 }
431
432 if (First[1] != '*') {
433 LastTokenPtr = First;
434 ++First;
435 continue;
436 }
437
438 // "/*...*/".
439 skipBlockComment(First, End);
440 }
441 if (First == End)
442 return;
443
444 // Skip over the newline.
445 unsigned Len = skipNewline(First, End);
446 if (!wasLineContinuation(First, EOLLen: Len)) // Continue past line-continuations.
447 break;
448 }
449}
450
451void Scanner::skipDirective(StringRef Name, const char *&First,
452 const char *const End) {
453 if (llvm::StringSwitch<bool>(Name)
454 .Case(S: "warning", Value: true)
455 .Case(S: "error", Value: true)
456 .Default(Value: false))
457 // Do not process quotes or comments.
458 skipToNewlineRaw(First, End);
459 else
460 skipLine(First, End);
461}
462
463static void skipWhitespace(const char *&First, const char *const End) {
464 for (;;) {
465 assert(First <= End);
466 skipOverSpaces(First, End);
467
468 if (End - First < 2)
469 return;
470
471 if (First[0] == '\\' && isVerticalWhitespace(c: First[1])) {
472 skipNewline(First&: ++First, End);
473 continue;
474 }
475
476 // Check for a non-comment character.
477 if (First[0] != '/')
478 return;
479
480 // "// ...".
481 if (First[1] == '/') {
482 skipLineComment(First, End);
483 return;
484 }
485
486 // Cannot be a comment.
487 if (First[1] != '*')
488 return;
489
490 // "/*...*/".
491 skipBlockComment(First, End);
492 }
493}
494
495bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
496 const char *const End) {
497 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
498 for (;;) {
499 const dependency_directives_scan::Token &Tok = lexToken(First, End);
500 if (Tok.is(K: tok::eof))
501 return reportError(
502 CurPtr: DirectiveLoc,
503 Err: diag::err_dep_source_scanner_missing_semi_after_at_import);
504 if (Tok.is(K: tok::semi))
505 break;
506 }
507 pushDirective(Kind);
508 skipWhitespace(First, End);
509 if (First == End)
510 return false;
511 if (!isVerticalWhitespace(c: *First))
512 return reportError(
513 CurPtr: DirectiveLoc, Err: diag::err_dep_source_scanner_unexpected_tokens_at_import);
514 skipNewline(First, End);
515 return false;
516}
517
518dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
519 const char *const End) {
520 clang::Token Tok;
521 TheLexer.LexFromRawLexer(Result&: Tok);
522 First = Input.data() + TheLexer.getCurrentBufferOffset();
523 assert(First <= End);
524
525 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
526 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
527 Args: Tok.getFlags());
528 return CurDirToks.back();
529}
530
531dependency_directives_scan::Token &
532Scanner::lexIncludeFilename(const char *&First, const char *const End) {
533 clang::Token Tok;
534 TheLexer.LexIncludeFilename(FilenameTok&: Tok);
535 First = Input.data() + TheLexer.getCurrentBufferOffset();
536 assert(First <= End);
537
538 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
539 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
540 Args: Tok.getFlags());
541 return CurDirToks.back();
542}
543
544void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
545 while (true) {
546 const dependency_directives_scan::Token &Tok = lexToken(First, End);
547 if (Tok.is(K: tok::eod) || Tok.is(K: tok::eof))
548 break;
549 }
550}
551
552StringRef
553Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
554 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
555 if (LLVM_LIKELY(!NeedsCleaning))
556 return Input.slice(Start: Tok.Offset, End: Tok.getEnd());
557
558 SmallString<64> Spelling;
559 Spelling.resize(N: Tok.Length);
560
561 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow
562 // in the Lexer). Currently we cannot see them due to our LangOpts.
563
564 unsigned SpellingLength = 0;
565 const char *BufPtr = Input.begin() + Tok.Offset;
566 const char *AfterIdent = Input.begin() + Tok.getEnd();
567 while (BufPtr < AfterIdent) {
568 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
569 Spelling[SpellingLength++] = Char;
570 BufPtr += Size;
571 }
572
573 return SplitIds.try_emplace(Key: StringRef(Spelling.begin(), SpellingLength), Args: 0)
574 .first->first();
575}
576
577std::optional<StringRef>
578Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
579 const dependency_directives_scan::Token &Tok = lexToken(First, End);
580 if (Tok.isNot(K: tok::raw_identifier)) {
581 if (!Tok.is(K: tok::eod))
582 skipLine(First, End);
583 return std::nullopt;
584 }
585
586 return cleanStringIfNeeded(Tok);
587}
588
589StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
590 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
591 assert(Id && "expected identifier token");
592 return *Id;
593}
594
595bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
596 const char *const End) {
597 if (std::optional<StringRef> FoundId =
598 tryLexIdentifierOrSkipLine(First, End)) {
599 if (*FoundId == Id)
600 return true;
601 skipLine(First, End);
602 }
603 return false;
604}
605
606bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
607 const char *const End) {
608 const dependency_directives_scan::Token &Tok = lexToken(First, End);
609 if (Tok.is(K))
610 return true;
611 skipLine(First, End);
612 return false;
613}
614
615std::optional<StringRef>
616Scanner::tryLexStringLiteralOrSkipLine(const char *&First,
617 const char *const End) {
618 const dependency_directives_scan::Token &Tok = lexToken(First, End);
619 if (!tok::isStringLiteral(K: Tok.Kind)) {
620 if (!Tok.is(K: tok::eod))
621 skipLine(First, End);
622 return std::nullopt;
623 }
624
625 return cleanStringIfNeeded(Tok);
626}
627
628bool Scanner::lexAt(const char *&First, const char *const End) {
629 // Handle "@import".
630
631 // Lex '@'.
632 const dependency_directives_scan::Token &AtTok = lexToken(First, End);
633 assert(AtTok.is(tok::at));
634 (void)AtTok;
635
636 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
637 return false;
638 return lexModuleDirectiveBody(Kind: decl_at_import, First, End);
639}
640
641bool Scanner::lexModule(const char *&First, const char *const End) {
642 StringRef Id = lexIdentifier(First, End);
643 bool Export = false;
644 if (Id == "export") {
645 Export = true;
646 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
647 if (!NextId)
648 return false;
649 Id = *NextId;
650 }
651
652 if (Id != "module" && Id != "import") {
653 skipLine(First, End);
654 return false;
655 }
656
657 skipWhitespace(First, End);
658
659 // Ignore this as a module directive if the next character can't be part of
660 // an import.
661
662 switch (*First) {
663 case ':': {
664 // `module :` is never the start of a valid module declaration.
665 if (Id == "module") {
666 skipLine(First, End);
667 return false;
668 }
669 // `import:(type)name` is a valid ObjC method decl, so check one more token.
670 (void)lexToken(First, End);
671 if (!tryLexIdentifierOrSkipLine(First, End))
672 return false;
673 break;
674 }
675 case '<':
676 case '"':
677 break;
678 default:
679 if (!isAsciiIdentifierContinue(c: *First)) {
680 skipLine(First, End);
681 return false;
682 }
683 }
684
685 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ false);
686
687 DirectiveKind Kind;
688 if (Id == "module")
689 Kind = Export ? cxx_export_module_decl : cxx_module_decl;
690 else
691 Kind = Export ? cxx_export_import_decl : cxx_import_decl;
692
693 return lexModuleDirectiveBody(Kind, First, End);
694}
695
696bool Scanner::lex_Pragma(const char *&First, const char *const End) {
697 if (!isNextTokenOrSkipLine(K: tok::l_paren, First, End))
698 return false;
699
700 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);
701
702 if (!Str || !isNextTokenOrSkipLine(K: tok::r_paren, First, End))
703 return false;
704
705 SmallString<64> Buffer(*Str);
706 prepare_PragmaString(StrVal&: Buffer);
707
708 // Use a new scanner instance since the tokens will be inside the allocated
709 // string. We should already have captured all the relevant tokens in the
710 // current scanner.
711 SmallVector<dependency_directives_scan::Token> DiscardTokens;
712 const char *Begin = Buffer.c_str();
713 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
714 InputSourceLoc};
715
716 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
717 if (PragmaScanner.lexPragma(First&: Begin, End: Buffer.end()))
718 return true;
719
720 DirectiveKind K = PragmaScanner.topDirective();
721 if (K == pp_none) {
722 skipLine(First, End);
723 return false;
724 }
725
726 assert(Begin == Buffer.end());
727 pushDirective(Kind: K);
728 return false;
729}
730
731bool Scanner::lexPragma(const char *&First, const char *const End) {
732 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
733 if (!FoundId)
734 return false;
735
736 StringRef Id = *FoundId;
737 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
738 .Case(S: "once", Value: pp_pragma_once)
739 .Case(S: "push_macro", Value: pp_pragma_push_macro)
740 .Case(S: "pop_macro", Value: pp_pragma_pop_macro)
741 .Case(S: "include_alias", Value: pp_pragma_include_alias)
742 .Default(Value: pp_none);
743 if (Kind != pp_none) {
744 lexPPDirectiveBody(First, End);
745 pushDirective(Kind);
746 return false;
747 }
748
749 if (Id != "clang") {
750 skipLine(First, End);
751 return false;
752 }
753
754 FoundId = tryLexIdentifierOrSkipLine(First, End);
755 if (!FoundId)
756 return false;
757 Id = *FoundId;
758
759 // #pragma clang system_header
760 if (Id == "system_header") {
761 lexPPDirectiveBody(First, End);
762 pushDirective(Kind: pp_pragma_system_header);
763 return false;
764 }
765
766 if (Id != "module") {
767 skipLine(First, End);
768 return false;
769 }
770
771 // #pragma clang module.
772 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
773 return false;
774
775 // #pragma clang module import.
776 lexPPDirectiveBody(First, End);
777 pushDirective(Kind: pp_pragma_import);
778 return false;
779}
780
781bool Scanner::lexEndif(const char *&First, const char *const End) {
782 // Strip out "#else" if it's empty.
783 if (topDirective() == pp_else)
784 popDirective();
785
786 // If "#ifdef" is empty, strip it and skip the "#endif".
787 //
788 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
789 // we can skip empty `#if` and `#elif` blocks as well after scanning for a
790 // literal __has_include in the condition. Even without that rule we could
791 // drop the tokens if we scan for identifiers in the condition and find none.
792 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
793 popDirective();
794 skipLine(First, End);
795 return false;
796 }
797
798 return lexDefault(Kind: pp_endif, First, End);
799}
800
801bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
802 const char *const End) {
803 lexPPDirectiveBody(First, End);
804 pushDirective(Kind);
805 return false;
806}
807
808static bool isStartOfRelevantLine(char First) {
809 switch (First) {
810 case '#':
811 case '@':
812 case 'i':
813 case 'e':
814 case 'm':
815 case '_':
816 return true;
817 }
818 return false;
819}
820
821bool Scanner::lexPPLine(const char *&First, const char *const End) {
822 assert(First != End);
823
824 skipWhitespace(First, End);
825 assert(First <= End);
826 if (First == End)
827 return false;
828
829 if (!isStartOfRelevantLine(First: *First)) {
830 skipLine(First, End);
831 assert(First <= End);
832 return false;
833 }
834
835 LastTokenPtr = First;
836
837 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ true);
838
839 auto ScEx1 = make_scope_exit(F: [&]() {
840 /// Clear Scanner's CurDirToks before returning, in case we didn't push a
841 /// new directive.
842 CurDirToks.clear();
843 });
844
845 // Handle "@import".
846 if (*First == '@')
847 return lexAt(First, End);
848
849 if (*First == 'i' || *First == 'e' || *First == 'm')
850 return lexModule(First, End);
851
852 if (*First == '_') {
853 if (isNextIdentifierOrSkipLine(Id: "_Pragma", First, End))
854 return lex_Pragma(First, End);
855 return false;
856 }
857
858 // Handle preprocessing directives.
859
860 TheLexer.setParsingPreprocessorDirective(true);
861 auto ScEx2 = make_scope_exit(
862 F: [&]() { TheLexer.setParsingPreprocessorDirective(false); });
863
864 // Lex '#'.
865 const dependency_directives_scan::Token &HashTok = lexToken(First, End);
866 if (HashTok.is(K: tok::hashhash)) {
867 // A \p tok::hashhash at this location is passed by the preprocessor to the
868 // parser to interpret, like any other token. So for dependency scanning
869 // skip it like a normal token not affecting the preprocessor.
870 skipLine(First, End);
871 assert(First <= End);
872 return false;
873 }
874 assert(HashTok.is(tok::hash));
875 (void)HashTok;
876
877 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
878 if (!FoundId)
879 return false;
880
881 StringRef Id = *FoundId;
882
883 if (Id == "pragma")
884 return lexPragma(First, End);
885
886 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
887 .Case(S: "include", Value: pp_include)
888 .Case(S: "__include_macros", Value: pp___include_macros)
889 .Case(S: "define", Value: pp_define)
890 .Case(S: "undef", Value: pp_undef)
891 .Case(S: "import", Value: pp_import)
892 .Case(S: "include_next", Value: pp_include_next)
893 .Case(S: "if", Value: pp_if)
894 .Case(S: "ifdef", Value: pp_ifdef)
895 .Case(S: "ifndef", Value: pp_ifndef)
896 .Case(S: "elif", Value: pp_elif)
897 .Case(S: "elifdef", Value: pp_elifdef)
898 .Case(S: "elifndef", Value: pp_elifndef)
899 .Case(S: "else", Value: pp_else)
900 .Case(S: "endif", Value: pp_endif)
901 .Default(Value: pp_none);
902 if (Kind == pp_none) {
903 skipDirective(Name: Id, First, End);
904 return false;
905 }
906
907 if (Kind == pp_endif)
908 return lexEndif(First, End);
909
910 switch (Kind) {
911 case pp_include:
912 case pp___include_macros:
913 case pp_include_next:
914 case pp_import:
915 // Ignore missing filenames in include or import directives.
916 if (lexIncludeFilename(First, End).is(K: tok::eod)) {
917 skipDirective(Name: Id, First, End);
918 return true;
919 }
920 break;
921 default:
922 break;
923 }
924
925 // Everything else.
926 return lexDefault(Kind, First, End);
927}
928
929static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
930 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
931 First[2] == '\xbf')
932 First += 3;
933}
934
935bool Scanner::scanImpl(const char *First, const char *const End) {
936 skipUTF8ByteOrderMark(First, End);
937 while (First != End)
938 if (lexPPLine(First, End))
939 return true;
940 return false;
941}
942
943bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
944 bool Error = scanImpl(First: Input.begin(), End: Input.end());
945
946 if (!Error) {
947 // Add an EOF on success.
948 if (LastTokenPtr &&
949 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
950 pushDirective(Kind: tokens_present_before_eof);
951 pushDirective(Kind: pp_eof);
952 }
953
954 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
955 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
956 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
957 Directives.emplace_back(Args: DirWithToks.Kind,
958 Args: RemainingTokens.take_front(N: DirWithToks.NumTokens));
959 RemainingTokens = RemainingTokens.drop_front(N: DirWithToks.NumTokens);
960 }
961 assert(RemainingTokens.empty());
962
963 return Error;
964}
965
966bool clang::scanSourceForDependencyDirectives(
967 StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
968 SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
969 SourceLocation InputSourceLoc) {
970 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
971}
972
973void clang::printDependencyDirectivesAsSource(
974 StringRef Source,
975 ArrayRef<dependency_directives_scan::Directive> Directives,
976 llvm::raw_ostream &OS) {
977 // Add a space separator where it is convenient for testing purposes.
978 auto needsSpaceSeparator =
979 [](tok::TokenKind Prev,
980 const dependency_directives_scan::Token &Tok) -> bool {
981 if (Prev == Tok.Kind)
982 return !Tok.isOneOf(K1: tok::l_paren, Ks: tok::r_paren, Ks: tok::l_square,
983 Ks: tok::r_square);
984 if (Prev == tok::raw_identifier &&
985 Tok.isOneOf(K1: tok::hash, Ks: tok::numeric_constant, Ks: tok::string_literal,
986 Ks: tok::char_constant, Ks: tok::header_name))
987 return true;
988 if (Prev == tok::r_paren &&
989 Tok.isOneOf(K1: tok::raw_identifier, Ks: tok::hash, Ks: tok::string_literal,
990 Ks: tok::char_constant, Ks: tok::unknown))
991 return true;
992 if (Prev == tok::comma &&
993 Tok.isOneOf(K1: tok::l_paren, Ks: tok::string_literal, Ks: tok::less))
994 return true;
995 return false;
996 };
997
998 for (const dependency_directives_scan::Directive &Directive : Directives) {
999 if (Directive.Kind == tokens_present_before_eof)
1000 OS << "<TokBeforeEOF>";
1001 std::optional<tok::TokenKind> PrevTokenKind;
1002 for (const dependency_directives_scan::Token &Tok : Directive.Tokens) {
1003 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
1004 OS << ' ';
1005 PrevTokenKind = Tok.Kind;
1006 OS << Source.slice(Start: Tok.Offset, End: Tok.getEnd());
1007 }
1008 }
1009}
1010