1//===- DependencyDirectivesScanner.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the interface for scanning header and source files to get the
11/// minimum necessary preprocessor directives for evaluating includes. It
12/// reduces the source down to #define, #include, #import, @import, and any
13/// conditional preprocessor logic that contains one of those.
14///
15//===----------------------------------------------------------------------===//
16
17#include "clang/Lex/DependencyDirectivesScanner.h"
18#include "clang/Basic/CharInfo.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Lex/LexDiagnostic.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Pragma.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
27#include <optional>
28
29using namespace clang;
30using namespace clang::dependency_directives_scan;
31using namespace llvm;
32
33namespace {
34
35struct DirectiveWithTokens {
36 DirectiveKind Kind;
37 unsigned NumTokens;
38
39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
40 : Kind(Kind), NumTokens(NumTokens) {}
41};
42
43/// Does an efficient "scan" of the sources to detect the presence of
44/// preprocessor (or module import) directives and collects the raw lexed tokens
45/// for those directives so that the \p Lexer can "replay" them when the file is
46/// included.
47///
48/// Note that the behavior of the raw lexer is affected by the language mode,
49/// while at this point we want to do a scan and collect tokens once,
50/// irrespective of the language mode that the file will get included in. To
51/// compensate for that the \p Lexer, while "replaying", will adjust a token
52/// where appropriate, when it could affect the preprocessor's state.
53/// For example in a directive like
54///
55/// \code
56/// #if __has_cpp_attribute(clang::fallthrough)
57/// \endcode
58///
59/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
60/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
61/// while in C++ mode.
62struct Scanner {
63 Scanner(StringRef Input,
64 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
69 Input.end()) {}
70
71 static LangOptions getLangOptsForDepScanning() {
72 LangOptions LangOpts;
73 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
74 LangOpts.ObjC = true;
75 LangOpts.LineComment = true;
76 LangOpts.RawStringLiterals = true;
77 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
78 return LangOpts;
79 }
80
81 /// Lex the provided source and emit the directive tokens.
82 ///
83 /// \returns True on error.
84 bool scan(SmallVectorImpl<Directive> &Directives);
85
86private:
87 /// Lexes next token and advances \p First and the \p Lexer.
88 [[nodiscard]] dependency_directives_scan::Token &
89 lexToken(const char *&First, const char *const End);
90
91 [[nodiscard]] dependency_directives_scan::Token &
92 lexIncludeFilename(const char *&First, const char *const End);
93
94 void skipLine(const char *&First, const char *const End);
95 void skipDirective(StringRef Name, const char *&First, const char *const End);
96
97 /// Returns the spelling of a string literal or identifier after performing
98 /// any processing needed to handle \c clang::Token::NeedsCleaning.
99 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);
100
101 /// Lexes next token and if it is identifier returns its string, otherwise
102 /// it skips the current line and returns \p std::nullopt.
103 ///
104 /// In any case (whatever the token kind) \p First and the \p Lexer will
105 /// advance beyond the token.
106 [[nodiscard]] std::optional<StringRef>
107 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
108
109 /// Used when it is certain that next token is an identifier.
110 [[nodiscard]] StringRef lexIdentifier(const char *&First,
111 const char *const End);
112
113 /// Lexes next token and returns true iff it is an identifier that matches \p
114 /// Id, otherwise it skips the current line and returns false.
115 ///
116 /// In any case (whatever the token kind) \p First and the \p Lexer will
117 /// advance beyond the token.
118 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,
119 const char *&First,
120 const char *const End);
121
122 /// Lexes next token and returns true iff it matches the kind \p K.
123 /// Otherwise it skips the current line and returns false.
124 ///
125 /// In any case (whatever the token kind) \p First and the \p Lexer will
126 /// advance beyond the token.
127 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
128 const char *const End);
129
130 /// Lexes next token and if it is string literal, returns its string.
131 /// Otherwise, it skips the current line and returns \p std::nullopt.
132 ///
133 /// In any case (whatever the token kind) \p First and the \p Lexer will
134 /// advance beyond the token.
135 [[nodiscard]] std::optional<StringRef>
136 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);
137
138 [[nodiscard]] bool scanImpl(const char *First, const char *const End);
139 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
140 [[nodiscard]] bool lexAt(const char *&First, const char *const End);
141 [[nodiscard]] bool lexModule(const char *&First, const char *const End);
142 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
143 const char *const End);
144 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
145 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);
146 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
147 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
148 const char *const End);
149 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,
150 const char *&First,
151 const char *const End);
152 void lexPPDirectiveBody(const char *&First, const char *const End);
153
154 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
155 Tokens.append(RHS: CurDirToks);
156 DirsWithToks.emplace_back(Args&: Kind, Args: CurDirToks.size());
157 CurDirToks.clear();
158 return DirsWithToks.back();
159 }
160 void popDirective() {
161 Tokens.pop_back_n(NumItems: DirsWithToks.pop_back_val().NumTokens);
162 }
163 DirectiveKind topDirective() const {
164 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
165 }
166
167 unsigned getOffsetAt(const char *CurPtr) const {
168 return CurPtr - Input.data();
169 }
170
171 /// Reports a diagnostic if the diagnostic engine is provided. Always returns
172 /// true at the end.
173 bool reportError(const char *CurPtr, unsigned Err);
174
175 StringMap<char> SplitIds;
176 StringRef Input;
177 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;
178 DiagnosticsEngine *Diags;
179 SourceLocation InputSourceLoc;
180
181 const char *LastTokenPtr = nullptr;
182 /// Keeps track of the tokens for the currently lexed directive. Once a
183 /// directive is fully lexed and "committed" then the tokens get appended to
184 /// \p Tokens and \p CurDirToks is cleared for the next directive.
185 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;
186 /// The directives that were lexed along with the number of tokens that each
187 /// directive contains. The tokens of all the directives are kept in \p Tokens
188 /// vector, in the same order as the directives order in \p DirsWithToks.
189 SmallVector<DirectiveWithTokens, 64> DirsWithToks;
190 LangOptions LangOpts;
191 Lexer TheLexer;
192};
193
194} // end anonymous namespace
195
196bool Scanner::reportError(const char *CurPtr, unsigned Err) {
197 if (!Diags)
198 return true;
199 assert(CurPtr >= Input.data() && "invalid buffer ptr");
200 Diags->Report(Loc: InputSourceLoc.getLocWithOffset(Offset: getOffsetAt(CurPtr)), DiagID: Err);
201 return true;
202}
203
204static void skipOverSpaces(const char *&First, const char *const End) {
205 while (First != End && isHorizontalWhitespace(c: *First))
206 ++First;
207}
208
209// Move back by one character, skipping escaped newlines (backslash + \n)
210static char previousChar(const char *First, const char *&Current) {
211 assert(Current > First);
212 --Current;
213 while (Current > First && isVerticalWhitespace(c: *Current)) {
214 // Check if the previous character is a backslash
215 if (Current > First && *(Current - 1) == '\\') {
216 // Use Lexer's getEscapedNewLineSize to get the size of the escaped
217 // newline
218 unsigned EscapeSize = Lexer::getEscapedNewLineSize(P: Current);
219 if (EscapeSize > 0) {
220 // Skip back over the entire escaped newline sequence (backslash +
221 // newline)
222 Current -= (1 + EscapeSize);
223 } else {
224 break;
225 }
226 } else {
227 break;
228 }
229 }
230 return *Current;
231}
232
233[[nodiscard]] static bool isRawStringLiteral(const char *First,
234 const char *Current) {
235 assert(First <= Current);
236
237 // Check if we can even back up.
238 if (*Current != '"' || First == Current)
239 return false;
240
241 // Check for an "R".
242 if (previousChar(First, Current) != 'R')
243 return false;
244 if (First == Current ||
245 !isAsciiIdentifierContinue(c: previousChar(First, Current)))
246 return true;
247
248 // Check for a prefix of "u", "U", or "L".
249 if (*Current == 'u' || *Current == 'U' || *Current == 'L')
250 return First == Current ||
251 !isAsciiIdentifierContinue(c: previousChar(First, Current));
252
253 // Check for a prefix of "u8".
254 if (*Current != '8' || First == Current ||
255 previousChar(First, Current) != 'u')
256 return false;
257 return First == Current ||
258 !isAsciiIdentifierContinue(c: previousChar(First, Current));
259}
260
261static void skipRawString(const char *&First, const char *const End) {
262 assert(First[0] == '"');
263
264 const char *Last = ++First;
265 while (Last != End && *Last != '(')
266 ++Last;
267 if (Last == End) {
268 First = Last; // Hit the end... just give up.
269 return;
270 }
271
272 StringRef Terminator(First, Last - First);
273 for (;;) {
274 // Move First to just past the next ")".
275 First = Last;
276 while (First != End && *First != ')')
277 ++First;
278 if (First == End)
279 return;
280 ++First;
281
282 // Look ahead for the terminator sequence.
283 Last = First;
284 while (Last != End && size_t(Last - First) < Terminator.size() &&
285 Terminator[Last - First] == *Last)
286 ++Last;
287
288 // Check if we hit it (or the end of the file).
289 if (Last == End) {
290 First = Last;
291 return;
292 }
293 if (size_t(Last - First) < Terminator.size())
294 continue;
295 if (*Last != '"')
296 continue;
297 First = Last + 1;
298 return;
299 }
300}
301
302// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
303static unsigned isEOL(const char *First, const char *const End) {
304 if (First == End)
305 return 0;
306 if (End - First > 1 && isVerticalWhitespace(c: First[0]) &&
307 isVerticalWhitespace(c: First[1]) && First[0] != First[1])
308 return 2;
309 return !!isVerticalWhitespace(c: First[0]);
310}
311
312static void skipString(const char *&First, const char *const End) {
313 assert(*First == '\'' || *First == '"' || *First == '<');
314 const char Terminator = *First == '<' ? '>' : *First;
315 for (++First; First != End && *First != Terminator; ++First) {
316 // String and character literals don't extend past the end of the line.
317 if (isVerticalWhitespace(c: *First))
318 return;
319 if (*First != '\\')
320 continue;
321 // Skip past backslash to the next character. This ensures that the
322 // character right after it is skipped as well, which matters if it's
323 // the terminator.
324 if (++First == End)
325 return;
326 if (!isWhitespace(c: *First))
327 continue;
328 // Whitespace after the backslash might indicate a line continuation.
329 const char *FirstAfterBackslashPastSpace = First;
330 skipOverSpaces(First&: FirstAfterBackslashPastSpace, End);
331 if (unsigned NLSize = isEOL(First: FirstAfterBackslashPastSpace, End)) {
332 // Advance the character pointer to the next line for the next
333 // iteration.
334 First = FirstAfterBackslashPastSpace + NLSize - 1;
335 }
336 }
337 if (First != End)
338 ++First; // Finish off the string.
339}
340
341// Returns the length of the skipped newline
342static unsigned skipNewline(const char *&First, const char *End) {
343 if (First == End)
344 return 0;
345 assert(isVerticalWhitespace(*First));
346 unsigned Len = isEOL(First, End);
347 assert(Len && "expected newline");
348 First += Len;
349 return Len;
350}
351
352static void skipToNewlineRaw(const char *&First, const char *const End) {
353 for (;;) {
354 if (First == End)
355 return;
356
357 unsigned Len = isEOL(First, End);
358 if (Len)
359 return;
360
361 char LastNonWhitespace = ' ';
362 do {
363 if (!isHorizontalWhitespace(c: *First))
364 LastNonWhitespace = *First;
365 if (++First == End)
366 return;
367 Len = isEOL(First, End);
368 } while (!Len);
369
370 if (LastNonWhitespace != '\\')
371 return;
372
373 First += Len;
374 // Keep skipping lines...
375 }
376}
377
378static void skipLineComment(const char *&First, const char *const End) {
379 assert(First[0] == '/' && First[1] == '/');
380 First += 2;
381 skipToNewlineRaw(First, End);
382}
383
384static void skipBlockComment(const char *&First, const char *const End) {
385 assert(First[0] == '/' && First[1] == '*');
386 if (End - First < 4) {
387 First = End;
388 return;
389 }
390 for (First += 3; First != End; ++First)
391 if (First[-1] == '*' && First[0] == '/') {
392 ++First;
393 return;
394 }
395}
396
397/// \returns True if the current single quotation mark character is a C++14
398/// digit separator.
399static bool isQuoteCppDigitSeparator(const char *const Start,
400 const char *const Cur,
401 const char *const End) {
402 assert(*Cur == '\'' && "expected quotation character");
403 // skipLine called in places where we don't expect a valid number
404 // body before `start` on the same line, so always return false at the start.
405 if (Start == Cur)
406 return false;
407 // The previous character must be a valid PP number character.
408 // Make sure that the L, u, U, u8 prefixes don't get marked as a
409 // separator though.
410 char Prev = *(Cur - 1);
411 if (Prev == 'L' || Prev == 'U' || Prev == 'u')
412 return false;
413 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
414 return false;
415 if (!isPreprocessingNumberBody(c: Prev))
416 return false;
417 // The next character should be a valid identifier body character.
418 return (Cur + 1) < End && isAsciiIdentifierContinue(c: *(Cur + 1));
419}
420
421void Scanner::skipLine(const char *&First, const char *const End) {
422 for (;;) {
423 assert(First <= End);
424 if (First == End)
425 return;
426
427 if (isVerticalWhitespace(c: *First)) {
428 skipNewline(First, End);
429 return;
430 }
431 const char *Start = First;
432 // Use `LastNonWhitespace`to track if a line-continuation has ever been seen
433 // before a new-line character:
434 char LastNonWhitespace = ' ';
435 while (First != End && !isVerticalWhitespace(c: *First)) {
436 // Iterate over strings correctly to avoid comments and newlines.
437 if (*First == '"' ||
438 (*First == '\'' && !isQuoteCppDigitSeparator(Start, Cur: First, End))) {
439 LastTokenPtr = First;
440 if (isRawStringLiteral(First: Start, Current: First))
441 skipRawString(First, End);
442 else
443 skipString(First, End);
444 continue;
445 }
446
447 // Continue on the same line if an EOL is preceded with backslash
448 if (First + 1 < End && *First == '\\') {
449 if (unsigned Len = isEOL(First: First + 1, End)) {
450 First += 1 + Len;
451 continue;
452 }
453 }
454
455 // Iterate over comments correctly.
456 if (*First != '/' || End - First < 2) {
457 LastTokenPtr = First;
458 if (!isWhitespace(c: *First))
459 LastNonWhitespace = *First;
460 ++First;
461 continue;
462 }
463
464 if (First[1] == '/') {
465 // "//...".
466 skipLineComment(First, End);
467 continue;
468 }
469
470 if (First[1] != '*') {
471 LastTokenPtr = First;
472 if (!isWhitespace(c: *First))
473 LastNonWhitespace = *First;
474 ++First;
475 continue;
476 }
477
478 // "/*...*/".
479 skipBlockComment(First, End);
480 }
481 if (First == End)
482 return;
483
484 // Skip over the newline.
485 skipNewline(First, End);
486
487 if (LastNonWhitespace != '\\')
488 break;
489 }
490}
491
492void Scanner::skipDirective(StringRef Name, const char *&First,
493 const char *const End) {
494 if (llvm::StringSwitch<bool>(Name)
495 .Case(S: "warning", Value: true)
496 .Case(S: "error", Value: true)
497 .Default(Value: false))
498 // Do not process quotes or comments.
499 skipToNewlineRaw(First, End);
500 else
501 skipLine(First, End);
502}
503
504static void skipWhitespace(const char *&First, const char *const End) {
505 for (;;) {
506 assert(First <= End);
507 skipOverSpaces(First, End);
508
509 if (End - First < 2)
510 return;
511
512 if (*First == '\\') {
513 const char *Ptr = First + 1;
514 while (Ptr < End && isHorizontalWhitespace(c: *Ptr))
515 ++Ptr;
516 if (Ptr != End && isVerticalWhitespace(c: *Ptr)) {
517 skipNewline(First&: Ptr, End);
518 First = Ptr;
519 continue;
520 }
521 return;
522 }
523
524 // Check for a non-comment character.
525 if (First[0] != '/')
526 return;
527
528 // "// ...".
529 if (First[1] == '/') {
530 skipLineComment(First, End);
531 return;
532 }
533
534 // Cannot be a comment.
535 if (First[1] != '*')
536 return;
537
538 // "/*...*/".
539 skipBlockComment(First, End);
540 }
541}
542
543bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
544 const char *const End) {
545 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
546 for (;;) {
547 // Keep a copy of the First char incase it needs to be reset.
548 const char *Previous = First;
549 const dependency_directives_scan::Token &Tok = lexToken(First, End);
550 if ((Tok.is(K: tok::hash) || Tok.is(K: tok::at)) &&
551 (Tok.Flags & clang::Token::StartOfLine)) {
552 CurDirToks.pop_back();
553 First = Previous;
554 return false;
555 }
556 if (Tok.is(K: tok::eof))
557 return reportError(
558 CurPtr: DirectiveLoc,
559 Err: diag::err_dep_source_scanner_missing_semi_after_at_import);
560 if (Tok.is(K: tok::semi))
561 break;
562 }
563 pushDirective(Kind);
564 skipWhitespace(First, End);
565 if (First == End)
566 return false;
567 if (!isVerticalWhitespace(c: *First))
568 return reportError(
569 CurPtr: DirectiveLoc, Err: diag::err_dep_source_scanner_unexpected_tokens_at_import);
570 skipNewline(First, End);
571 return false;
572}
573
574dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
575 const char *const End) {
576 clang::Token Tok;
577 TheLexer.LexFromRawLexer(Result&: Tok);
578 First = Input.data() + TheLexer.getCurrentBufferOffset();
579 assert(First <= End);
580
581 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
582 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
583 Args: Tok.getFlags());
584 return CurDirToks.back();
585}
586
587dependency_directives_scan::Token &
588Scanner::lexIncludeFilename(const char *&First, const char *const End) {
589 clang::Token Tok;
590 TheLexer.LexIncludeFilename(FilenameTok&: Tok);
591 First = Input.data() + TheLexer.getCurrentBufferOffset();
592 assert(First <= End);
593
594 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
595 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
596 Args: Tok.getFlags());
597 return CurDirToks.back();
598}
599
600void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
601 while (true) {
602 const dependency_directives_scan::Token &Tok = lexToken(First, End);
603 if (Tok.is(K: tok::eod) || Tok.is(K: tok::eof))
604 break;
605 }
606}
607
608StringRef
609Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
610 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
611 if (LLVM_LIKELY(!NeedsCleaning))
612 return Input.slice(Start: Tok.Offset, End: Tok.getEnd());
613
614 SmallString<64> Spelling;
615 Spelling.resize(N: Tok.Length);
616
617 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow
618 // in the Lexer). Currently we cannot see them due to our LangOpts.
619
620 unsigned SpellingLength = 0;
621 const char *BufPtr = Input.begin() + Tok.Offset;
622 const char *AfterIdent = Input.begin() + Tok.getEnd();
623 while (BufPtr < AfterIdent) {
624 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
625 Spelling[SpellingLength++] = Char;
626 BufPtr += Size;
627 }
628
629 return SplitIds.try_emplace(Key: StringRef(Spelling.begin(), SpellingLength), Args: 0)
630 .first->first();
631}
632
633std::optional<StringRef>
634Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
635 const dependency_directives_scan::Token &Tok = lexToken(First, End);
636 if (Tok.isNot(K: tok::raw_identifier)) {
637 if (!Tok.is(K: tok::eod))
638 skipLine(First, End);
639 return std::nullopt;
640 }
641
642 return cleanStringIfNeeded(Tok);
643}
644
645StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
646 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
647 assert(Id && "expected identifier token");
648 return *Id;
649}
650
651bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
652 const char *const End) {
653 if (std::optional<StringRef> FoundId =
654 tryLexIdentifierOrSkipLine(First, End)) {
655 if (*FoundId == Id)
656 return true;
657 skipLine(First, End);
658 }
659 return false;
660}
661
662bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
663 const char *const End) {
664 const dependency_directives_scan::Token &Tok = lexToken(First, End);
665 if (Tok.is(K))
666 return true;
667 skipLine(First, End);
668 return false;
669}
670
671std::optional<StringRef>
672Scanner::tryLexStringLiteralOrSkipLine(const char *&First,
673 const char *const End) {
674 const dependency_directives_scan::Token &Tok = lexToken(First, End);
675 if (!tok::isStringLiteral(K: Tok.Kind)) {
676 if (!Tok.is(K: tok::eod))
677 skipLine(First, End);
678 return std::nullopt;
679 }
680
681 return cleanStringIfNeeded(Tok);
682}
683
684bool Scanner::lexAt(const char *&First, const char *const End) {
685 // Handle "@import".
686
687 // Lex '@'.
688 const dependency_directives_scan::Token &AtTok = lexToken(First, End);
689 assert(AtTok.is(tok::at));
690 (void)AtTok;
691
692 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
693 return false;
694 return lexModuleDirectiveBody(Kind: decl_at_import, First, End);
695}
696
697bool Scanner::lexModule(const char *&First, const char *const End) {
698 StringRef Id = lexIdentifier(First, End);
699 bool Export = false;
700 if (Id == "export") {
701 Export = true;
702 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
703 if (!NextId)
704 return false;
705 Id = *NextId;
706 }
707
708 if (Id != "module" && Id != "import") {
709 skipLine(First, End);
710 return false;
711 }
712
713 skipWhitespace(First, End);
714
715 // Ignore this as a module directive if the next character can't be part of
716 // an import.
717
718 switch (*First) {
719 case ':': {
720 // `module :` is never the start of a valid module declaration.
721 if (Id == "module") {
722 skipLine(First, End);
723 return false;
724 }
725 // `import:(type)name` is a valid ObjC method decl, so check one more token.
726 (void)lexToken(First, End);
727 if (!tryLexIdentifierOrSkipLine(First, End))
728 return false;
729 break;
730 }
731 case '<':
732 case '"':
733 break;
734 default:
735 if (!isAsciiIdentifierContinue(c: *First)) {
736 skipLine(First, End);
737 return false;
738 }
739 }
740
741 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ false);
742
743 DirectiveKind Kind;
744 if (Id == "module")
745 Kind = Export ? cxx_export_module_decl : cxx_module_decl;
746 else
747 Kind = Export ? cxx_export_import_decl : cxx_import_decl;
748
749 return lexModuleDirectiveBody(Kind, First, End);
750}
751
752bool Scanner::lex_Pragma(const char *&First, const char *const End) {
753 if (!isNextTokenOrSkipLine(K: tok::l_paren, First, End))
754 return false;
755
756 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);
757
758 if (!Str || !isNextTokenOrSkipLine(K: tok::r_paren, First, End))
759 return false;
760
761 SmallString<64> Buffer(*Str);
762 prepare_PragmaString(StrVal&: Buffer);
763
764 // Use a new scanner instance since the tokens will be inside the allocated
765 // string. We should already have captured all the relevant tokens in the
766 // current scanner.
767 SmallVector<dependency_directives_scan::Token> DiscardTokens;
768 const char *Begin = Buffer.c_str();
769 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
770 InputSourceLoc};
771
772 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
773 if (PragmaScanner.lexPragma(First&: Begin, End: Buffer.end()))
774 return true;
775
776 DirectiveKind K = PragmaScanner.topDirective();
777 if (K == pp_none) {
778 skipLine(First, End);
779 return false;
780 }
781
782 assert(Begin == Buffer.end());
783 pushDirective(Kind: K);
784 return false;
785}
786
787bool Scanner::lexPragma(const char *&First, const char *const End) {
788 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
789 if (!FoundId)
790 return false;
791
792 StringRef Id = *FoundId;
793 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
794 .Case(S: "once", Value: pp_pragma_once)
795 .Case(S: "push_macro", Value: pp_pragma_push_macro)
796 .Case(S: "pop_macro", Value: pp_pragma_pop_macro)
797 .Case(S: "include_alias", Value: pp_pragma_include_alias)
798 .Default(Value: pp_none);
799 if (Kind != pp_none) {
800 lexPPDirectiveBody(First, End);
801 pushDirective(Kind);
802 return false;
803 }
804
805 if (Id != "clang") {
806 skipLine(First, End);
807 return false;
808 }
809
810 FoundId = tryLexIdentifierOrSkipLine(First, End);
811 if (!FoundId)
812 return false;
813 Id = *FoundId;
814
815 // #pragma clang system_header
816 if (Id == "system_header") {
817 lexPPDirectiveBody(First, End);
818 pushDirective(Kind: pp_pragma_system_header);
819 return false;
820 }
821
822 if (Id != "module") {
823 skipLine(First, End);
824 return false;
825 }
826
827 // #pragma clang module.
828 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
829 return false;
830
831 // #pragma clang module import.
832 lexPPDirectiveBody(First, End);
833 pushDirective(Kind: pp_pragma_import);
834 return false;
835}
836
837bool Scanner::lexEndif(const char *&First, const char *const End) {
838 // Strip out "#else" if it's empty.
839 if (topDirective() == pp_else)
840 popDirective();
841
842 // If "#ifdef" is empty, strip it and skip the "#endif".
843 //
844 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
845 // we can skip empty `#if` and `#elif` blocks as well after scanning for a
846 // literal __has_include in the condition. Even without that rule we could
847 // drop the tokens if we scan for identifiers in the condition and find none.
848 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
849 popDirective();
850 skipLine(First, End);
851 return false;
852 }
853
854 return lexDefault(Kind: pp_endif, First, End);
855}
856
857bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
858 const char *const End) {
859 lexPPDirectiveBody(First, End);
860 pushDirective(Kind);
861 return false;
862}
863
864static bool isStartOfRelevantLine(char First) {
865 switch (First) {
866 case '#':
867 case '@':
868 case 'i':
869 case 'e':
870 case 'm':
871 case '_':
872 return true;
873 }
874 return false;
875}
876
877bool Scanner::lexPPLine(const char *&First, const char *const End) {
878 assert(First != End);
879
880 skipWhitespace(First, End);
881 assert(First <= End);
882 if (First == End)
883 return false;
884
885 if (!isStartOfRelevantLine(First: *First)) {
886 skipLine(First, End);
887 assert(First <= End);
888 return false;
889 }
890
891 LastTokenPtr = First;
892
893 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ true);
894
895 auto ScEx1 = make_scope_exit(F: [&]() {
896 /// Clear Scanner's CurDirToks before returning, in case we didn't push a
897 /// new directive.
898 CurDirToks.clear();
899 });
900
901 // Handle "@import".
902 if (*First == '@')
903 return lexAt(First, End);
904
905 // Handle module directives for C++20 modules.
906 if (*First == 'i' || *First == 'e' || *First == 'm')
907 return lexModule(First, End);
908
909 if (*First == '_') {
910 if (isNextIdentifierOrSkipLine(Id: "_Pragma", First, End))
911 return lex_Pragma(First, End);
912 return false;
913 }
914
915 // Handle preprocessing directives.
916
917 TheLexer.setParsingPreprocessorDirective(true);
918 auto ScEx2 = make_scope_exit(
919 F: [&]() { TheLexer.setParsingPreprocessorDirective(false); });
920
921 // Lex '#'.
922 const dependency_directives_scan::Token &HashTok = lexToken(First, End);
923 if (HashTok.is(K: tok::hashhash)) {
924 // A \p tok::hashhash at this location is passed by the preprocessor to the
925 // parser to interpret, like any other token. So for dependency scanning
926 // skip it like a normal token not affecting the preprocessor.
927 skipLine(First, End);
928 assert(First <= End);
929 return false;
930 }
931 assert(HashTok.is(tok::hash));
932 (void)HashTok;
933
934 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
935 if (!FoundId)
936 return false;
937
938 StringRef Id = *FoundId;
939
940 if (Id == "pragma")
941 return lexPragma(First, End);
942
943 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
944 .Case(S: "include", Value: pp_include)
945 .Case(S: "__include_macros", Value: pp___include_macros)
946 .Case(S: "define", Value: pp_define)
947 .Case(S: "undef", Value: pp_undef)
948 .Case(S: "import", Value: pp_import)
949 .Case(S: "include_next", Value: pp_include_next)
950 .Case(S: "if", Value: pp_if)
951 .Case(S: "ifdef", Value: pp_ifdef)
952 .Case(S: "ifndef", Value: pp_ifndef)
953 .Case(S: "elif", Value: pp_elif)
954 .Case(S: "elifdef", Value: pp_elifdef)
955 .Case(S: "elifndef", Value: pp_elifndef)
956 .Case(S: "else", Value: pp_else)
957 .Case(S: "endif", Value: pp_endif)
958 .Default(Value: pp_none);
959 if (Kind == pp_none) {
960 skipDirective(Name: Id, First, End);
961 return false;
962 }
963
964 if (Kind == pp_endif)
965 return lexEndif(First, End);
966
967 switch (Kind) {
968 case pp_include:
969 case pp___include_macros:
970 case pp_include_next:
971 case pp_import:
972 // Ignore missing filenames in include or import directives.
973 if (lexIncludeFilename(First, End).is(K: tok::eod)) {
974 return false;
975 }
976 break;
977 default:
978 break;
979 }
980
981 // Everything else.
982 return lexDefault(Kind, First, End);
983}
984
985static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
986 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
987 First[2] == '\xbf')
988 First += 3;
989}
990
991bool Scanner::scanImpl(const char *First, const char *const End) {
992 skipUTF8ByteOrderMark(First, End);
993 while (First != End)
994 if (lexPPLine(First, End))
995 return true;
996 return false;
997}
998
999bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
1000 bool Error = scanImpl(First: Input.begin(), End: Input.end());
1001
1002 if (!Error) {
1003 // Add an EOF on success.
1004 if (LastTokenPtr &&
1005 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
1006 pushDirective(Kind: tokens_present_before_eof);
1007 pushDirective(Kind: pp_eof);
1008 }
1009
1010 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
1011 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
1012 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
1013 Directives.emplace_back(Args: DirWithToks.Kind,
1014 Args: RemainingTokens.take_front(N: DirWithToks.NumTokens));
1015 RemainingTokens = RemainingTokens.drop_front(N: DirWithToks.NumTokens);
1016 }
1017 assert(RemainingTokens.empty());
1018
1019 return Error;
1020}
1021
1022bool clang::scanSourceForDependencyDirectives(
1023 StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
1024 SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
1025 SourceLocation InputSourceLoc) {
1026 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
1027}
1028
1029void clang::printDependencyDirectivesAsSource(
1030 StringRef Source,
1031 ArrayRef<dependency_directives_scan::Directive> Directives,
1032 llvm::raw_ostream &OS) {
1033 // Add a space separator where it is convenient for testing purposes.
1034 auto needsSpaceSeparator =
1035 [](tok::TokenKind Prev,
1036 const dependency_directives_scan::Token &Tok) -> bool {
1037 if (Prev == Tok.Kind)
1038 return !Tok.isOneOf(K1: tok::l_paren, Ks: tok::r_paren, Ks: tok::l_square,
1039 Ks: tok::r_square);
1040 if (Prev == tok::raw_identifier &&
1041 Tok.isOneOf(K1: tok::hash, Ks: tok::numeric_constant, Ks: tok::string_literal,
1042 Ks: tok::char_constant, Ks: tok::header_name))
1043 return true;
1044 if (Prev == tok::r_paren &&
1045 Tok.isOneOf(K1: tok::raw_identifier, Ks: tok::hash, Ks: tok::string_literal,
1046 Ks: tok::char_constant, Ks: tok::unknown))
1047 return true;
1048 if (Prev == tok::comma &&
1049 Tok.isOneOf(K1: tok::l_paren, Ks: tok::string_literal, Ks: tok::less))
1050 return true;
1051 return false;
1052 };
1053
1054 for (const dependency_directives_scan::Directive &Directive : Directives) {
1055 if (Directive.Kind == tokens_present_before_eof)
1056 OS << "<TokBeforeEOF>";
1057 std::optional<tok::TokenKind> PrevTokenKind;
1058 for (const dependency_directives_scan::Token &Tok : Directive.Tokens) {
1059 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
1060 OS << ' ';
1061 PrevTokenKind = Tok.Kind;
1062 OS << Source.slice(Start: Tok.Offset, End: Tok.getEnd());
1063 }
1064 }
1065}
1066