1//===--- PPDirectives.cpp - Directive Handling for Preprocessor -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Implements # directive processing for the Preprocessor.
11///
12//===----------------------------------------------------------------------===//
13
14#include "clang/Basic/AttributeCommonInfo.h"
15#include "clang/Basic/Attributes.h"
16#include "clang/Basic/CharInfo.h"
17#include "clang/Basic/DirectoryEntry.h"
18#include "clang/Basic/FileManager.h"
19#include "clang/Basic/IdentifierTable.h"
20#include "clang/Basic/LangOptions.h"
21#include "clang/Basic/Module.h"
22#include "clang/Basic/SourceLocation.h"
23#include "clang/Basic/SourceManager.h"
24#include "clang/Basic/TargetInfo.h"
25#include "clang/Basic/TokenKinds.h"
26#include "clang/Lex/CodeCompletionHandler.h"
27#include "clang/Lex/HeaderSearch.h"
28#include "clang/Lex/LexDiagnostic.h"
29#include "clang/Lex/LiteralSupport.h"
30#include "clang/Lex/MacroInfo.h"
31#include "clang/Lex/ModuleLoader.h"
32#include "clang/Lex/ModuleMap.h"
33#include "clang/Lex/PPCallbacks.h"
34#include "clang/Lex/Pragma.h"
35#include "clang/Lex/Preprocessor.h"
36#include "clang/Lex/PreprocessorOptions.h"
37#include "clang/Lex/Token.h"
38#include "clang/Lex/VariadicMacroSupport.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/STLExtras.h"
41#include "llvm/ADT/ScopeExit.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/ADT/StringExtras.h"
44#include "llvm/ADT/StringRef.h"
45#include "llvm/ADT/StringSwitch.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/SaveAndRestore.h"
49#include <algorithm>
50#include <cassert>
51#include <cstddef>
52#include <cstring>
53#include <optional>
54#include <string>
55#include <utility>
56
57using namespace clang;
58
59//===----------------------------------------------------------------------===//
60// Utility Methods for Preprocessor Directive Handling.
61//===----------------------------------------------------------------------===//
62
63MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
64 static_assert(std::is_trivially_destructible_v<MacroInfo>, "");
65 return new (BP) MacroInfo(L);
66}
67
68DefMacroDirective *Preprocessor::AllocateDefMacroDirective(MacroInfo *MI,
69 SourceLocation Loc) {
70 return new (BP) DefMacroDirective(MI, Loc);
71}
72
73UndefMacroDirective *
74Preprocessor::AllocateUndefMacroDirective(SourceLocation UndefLoc) {
75 return new (BP) UndefMacroDirective(UndefLoc);
76}
77
78VisibilityMacroDirective *
79Preprocessor::AllocateVisibilityMacroDirective(SourceLocation Loc,
80 bool isPublic) {
81 return new (BP) VisibilityMacroDirective(Loc, isPublic);
82}
83
84/// Read and discard all tokens remaining on the current line until
85/// the tok::eod token is found.
86SourceRange Preprocessor::DiscardUntilEndOfDirective(
87 Token &Tmp, SmallVectorImpl<Token> *DiscardedToks) {
88 SourceRange Res;
89 auto ReadNextTok = [&]() {
90 LexUnexpandedToken(Result&: Tmp);
91 if (DiscardedToks && Tmp.isNot(K: tok::eod))
92 DiscardedToks->push_back(Elt: Tmp);
93 };
94 ReadNextTok();
95 Res.setBegin(Tmp.getLocation());
96 while (Tmp.isNot(K: tok::eod)) {
97 assert(Tmp.isNot(tok::eof) && "EOF seen while discarding directive tokens");
98 ReadNextTok();
99 }
100 Res.setEnd(Tmp.getLocation());
101 return Res;
102}
103
104/// Enumerates possible cases of #define/#undef a reserved identifier.
105enum MacroDiag {
106 MD_NoWarn, //> Not a reserved identifier
107 MD_KeywordDef, //> Macro hides keyword, enabled by default
108 MD_ReservedMacro, //> #define of #undef reserved id, disabled by default
109 MD_ReservedAttributeIdentifier
110};
111
112/// Enumerates possible %select values for the pp_err_elif_after_else and
113/// pp_err_elif_without_if diagnostics.
114enum PPElifDiag {
115 PED_Elif,
116 PED_Elifdef,
117 PED_Elifndef
118};
119
120static bool isFeatureTestMacro(StringRef MacroName) {
121 // list from:
122 // * https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_macros.html
123 // * https://docs.microsoft.com/en-us/cpp/c-runtime-library/security-features-in-the-crt?view=msvc-160
124 // * man 7 feature_test_macros
125 // The list must be sorted for correct binary search.
126 static constexpr StringRef ReservedMacro[] = {
127 "_ATFILE_SOURCE",
128 "_BSD_SOURCE",
129 "_CRT_NONSTDC_NO_WARNINGS",
130 "_CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES",
131 "_CRT_SECURE_NO_WARNINGS",
132 "_FILE_OFFSET_BITS",
133 "_FORTIFY_SOURCE",
134 "_GLIBCXX_ASSERTIONS",
135 "_GLIBCXX_CONCEPT_CHECKS",
136 "_GLIBCXX_DEBUG",
137 "_GLIBCXX_DEBUG_PEDANTIC",
138 "_GLIBCXX_PARALLEL",
139 "_GLIBCXX_PARALLEL_ASSERTIONS",
140 "_GLIBCXX_SANITIZE_VECTOR",
141 "_GLIBCXX_USE_CXX11_ABI",
142 "_GLIBCXX_USE_DEPRECATED",
143 "_GNU_SOURCE",
144 "_ISOC11_SOURCE",
145 "_ISOC95_SOURCE",
146 "_ISOC99_SOURCE",
147 "_LARGEFILE64_SOURCE",
148 "_POSIX_C_SOURCE",
149 "_REENTRANT",
150 "_SVID_SOURCE",
151 "_THREAD_SAFE",
152 "_XOPEN_SOURCE",
153 "_XOPEN_SOURCE_EXTENDED",
154 "__STDCPP_WANT_MATH_SPEC_FUNCS__",
155 "__STDC_FORMAT_MACROS",
156 };
157 return llvm::binary_search(Range: ReservedMacro, Value&: MacroName);
158}
159
160static bool isLanguageDefinedBuiltin(const SourceManager &SourceMgr,
161 const MacroInfo *MI,
162 const StringRef MacroName) {
163 // If this is a macro with special handling (like __LINE__) then it's language
164 // defined.
165 if (MI->isBuiltinMacro())
166 return true;
167 // Builtin macros are defined in the builtin file
168 if (!SourceMgr.isWrittenInBuiltinFile(Loc: MI->getDefinitionLoc()))
169 return false;
170 // C defines macros starting with __STDC, and C++ defines macros starting with
171 // __STDCPP
172 if (MacroName.starts_with(Prefix: "__STDC"))
173 return true;
174 // C++ defines the __cplusplus macro
175 if (MacroName == "__cplusplus")
176 return true;
177 // C++ defines various feature-test macros starting with __cpp
178 if (MacroName.starts_with(Prefix: "__cpp"))
179 return true;
180 // Anything else isn't language-defined
181 return false;
182}
183
184static bool isReservedCXXAttributeName(Preprocessor &PP, IdentifierInfo *II) {
185 const LangOptions &Lang = PP.getLangOpts();
186 if (Lang.CPlusPlus &&
187 hasAttribute(Syntax: AttributeCommonInfo::AS_CXX11, /* Scope*/ nullptr, Attr: II,
188 Target: PP.getTargetInfo(), LangOpts: Lang, /*CheckPlugins*/ false) > 0) {
189 AttributeCommonInfo::AttrArgsInfo AttrArgsInfo =
190 AttributeCommonInfo::getCXX11AttrArgsInfo(Name: II);
191 if (AttrArgsInfo == AttributeCommonInfo::AttrArgsInfo::Required)
192 return PP.isNextPPTokenOneOf(Ks: tok::l_paren);
193
194 return !PP.isNextPPTokenOneOf(Ks: tok::l_paren) ||
195 AttrArgsInfo == AttributeCommonInfo::AttrArgsInfo::Optional;
196 }
197 return false;
198}
199
200static MacroDiag shouldWarnOnMacroDef(Preprocessor &PP, IdentifierInfo *II) {
201 const LangOptions &Lang = PP.getLangOpts();
202 StringRef Text = II->getName();
203 if (isReservedInAllContexts(Status: II->isReserved(LangOpts: Lang)))
204 return isFeatureTestMacro(MacroName: Text) ? MD_NoWarn : MD_ReservedMacro;
205 if (II->isKeyword(LangOpts: Lang))
206 return MD_KeywordDef;
207 if (Lang.CPlusPlus11 && (Text == "override" || Text == "final"))
208 return MD_KeywordDef;
209 if (isReservedCXXAttributeName(PP, II))
210 return MD_ReservedAttributeIdentifier;
211 return MD_NoWarn;
212}
213
214static MacroDiag shouldWarnOnMacroUndef(Preprocessor &PP, IdentifierInfo *II) {
215 const LangOptions &Lang = PP.getLangOpts();
216 // Do not warn on keyword undef. It is generally harmless and widely used.
217 if (isReservedInAllContexts(Status: II->isReserved(LangOpts: Lang)))
218 return MD_ReservedMacro;
219 if (isReservedCXXAttributeName(PP, II))
220 return MD_ReservedAttributeIdentifier;
221 return MD_NoWarn;
222}
223
224// Return true if we want to issue a diagnostic by default if we
225// encounter this name in a #include with the wrong case. For now,
226// this includes the standard C and C++ headers, Posix headers,
227// and Boost headers. Improper case for these #includes is a
228// potential portability issue.
229static bool warnByDefaultOnWrongCase(StringRef Include) {
230 // If the first component of the path is "boost", treat this like a standard header
231 // for the purposes of diagnostics.
232 if (::llvm::sys::path::begin(path: Include)->equals_insensitive(RHS: "boost"))
233 return true;
234
235 // "condition_variable" is the longest standard header name at 18 characters.
236 // If the include file name is longer than that, it can't be a standard header.
237 static const size_t MaxStdHeaderNameLen = 18u;
238 if (Include.size() > MaxStdHeaderNameLen)
239 return false;
240
241 // Lowercase and normalize the search string.
242 SmallString<32> LowerInclude{Include};
243 for (char &Ch : LowerInclude) {
244 // In the ASCII range?
245 if (static_cast<unsigned char>(Ch) > 0x7f)
246 return false; // Can't be a standard header
247 // ASCII lowercase:
248 if (Ch >= 'A' && Ch <= 'Z')
249 Ch += 'a' - 'A';
250 // Normalize path separators for comparison purposes.
251 else if (::llvm::sys::path::is_separator(value: Ch))
252 Ch = '/';
253 }
254
255 // The standard C/C++ and Posix headers
256 return llvm::StringSwitch<bool>(LowerInclude)
257 // C library headers
258 .Cases(CaseStrings: {"assert.h", "complex.h", "ctype.h", "errno.h", "fenv.h"}, Value: true)
259 .Cases(CaseStrings: {"float.h", "inttypes.h", "iso646.h", "limits.h", "locale.h"},
260 Value: true)
261 .Cases(CaseStrings: {"math.h", "setjmp.h", "signal.h", "stdalign.h", "stdarg.h"}, Value: true)
262 .Cases(CaseStrings: {"stdatomic.h", "stdbool.h", "stdckdint.h", "stdcountof.h"}, Value: true)
263 .Cases(CaseStrings: {"stddef.h", "stdint.h", "stdio.h", "stdlib.h", "stdnoreturn.h"},
264 Value: true)
265 .Cases(CaseStrings: {"string.h", "tgmath.h", "threads.h", "time.h", "uchar.h"}, Value: true)
266 .Cases(CaseStrings: {"wchar.h", "wctype.h"}, Value: true)
267
268 // C++ headers for C library facilities
269 .Cases(CaseStrings: {"cassert", "ccomplex", "cctype", "cerrno", "cfenv"}, Value: true)
270 .Cases(CaseStrings: {"cfloat", "cinttypes", "ciso646", "climits", "clocale"}, Value: true)
271 .Cases(CaseStrings: {"cmath", "csetjmp", "csignal", "cstdalign", "cstdarg"}, Value: true)
272 .Cases(CaseStrings: {"cstdbool", "cstddef", "cstdint", "cstdio", "cstdlib"}, Value: true)
273 .Cases(CaseStrings: {"cstring", "ctgmath", "ctime", "cuchar", "cwchar"}, Value: true)
274 .Case(S: "cwctype", Value: true)
275
276 // C++ library headers
277 .Cases(CaseStrings: {"algorithm", "fstream", "list", "regex", "thread"}, Value: true)
278 .Cases(CaseStrings: {"array", "functional", "locale", "scoped_allocator", "tuple"},
279 Value: true)
280 .Cases(CaseStrings: {"atomic", "future", "map", "set", "type_traits"}, Value: true)
281 .Cases(
282 CaseStrings: {"bitset", "initializer_list", "memory", "shared_mutex", "typeindex"},
283 Value: true)
284 .Cases(CaseStrings: {"chrono", "iomanip", "mutex", "sstream", "typeinfo"}, Value: true)
285 .Cases(CaseStrings: {"codecvt", "ios", "new", "stack", "unordered_map"}, Value: true)
286 .Cases(CaseStrings: {"complex", "iosfwd", "numeric", "stdexcept", "unordered_set"},
287 Value: true)
288 .Cases(
289 CaseStrings: {"condition_variable", "iostream", "ostream", "streambuf", "utility"},
290 Value: true)
291 .Cases(CaseStrings: {"deque", "istream", "queue", "string", "valarray"}, Value: true)
292 .Cases(CaseStrings: {"exception", "iterator", "random", "strstream", "vector"}, Value: true)
293 .Cases(CaseStrings: {"forward_list", "limits", "ratio", "system_error"}, Value: true)
294
295 // POSIX headers (which aren't also C headers)
296 .Cases(CaseStrings: {"aio.h", "arpa/inet.h", "cpio.h", "dirent.h", "dlfcn.h"}, Value: true)
297 .Cases(CaseStrings: {"fcntl.h", "fmtmsg.h", "fnmatch.h", "ftw.h", "glob.h"}, Value: true)
298 .Cases(CaseStrings: {"grp.h", "iconv.h", "langinfo.h", "libgen.h", "monetary.h"}, Value: true)
299 .Cases(CaseStrings: {"mqueue.h", "ndbm.h", "net/if.h", "netdb.h", "netinet/in.h"},
300 Value: true)
301 .Cases(CaseStrings: {"netinet/tcp.h", "nl_types.h", "poll.h", "pthread.h", "pwd.h"},
302 Value: true)
303 .Cases(CaseStrings: {"regex.h", "sched.h", "search.h", "semaphore.h", "spawn.h"}, Value: true)
304 .Cases(CaseStrings: {"strings.h", "stropts.h", "sys/ipc.h", "sys/mman.h", "sys/msg.h"},
305 Value: true)
306 .Cases(CaseStrings: {"sys/resource.h", "sys/select.h", "sys/sem.h", "sys/shm.h",
307 "sys/socket.h"},
308 Value: true)
309 .Cases(CaseStrings: {"sys/stat.h", "sys/statvfs.h", "sys/time.h", "sys/times.h",
310 "sys/types.h"},
311 Value: true)
312 .Cases(
313 CaseStrings: {"sys/uio.h", "sys/un.h", "sys/utsname.h", "sys/wait.h", "syslog.h"},
314 Value: true)
315 .Cases(CaseStrings: {"tar.h", "termios.h", "trace.h", "ulimit.h"}, Value: true)
316 .Cases(CaseStrings: {"unistd.h", "utime.h", "utmpx.h", "wordexp.h"}, Value: true)
317 .Default(Value: false);
318}
319
320/// Find a similar string in `Candidates`.
321///
322/// \param LHS a string for a similar string in `Candidates`
323///
324/// \param Candidates the candidates to find a similar string.
325///
326/// \returns a similar string if exists. If no similar string exists,
327/// returns std::nullopt.
328static std::optional<StringRef> findSimilarStr(StringRef LHS,
329 ArrayRef<StringRef> Candidates) {
330 // We need to check if `Candidates` has the exact case-insensitive string
331 // because the Levenshtein distance match does not care about it.
332 for (StringRef C : Candidates) {
333 if (LHS.equals_insensitive(RHS: C)) {
334 return C;
335 }
336 }
337
338 // Keep going with the Levenshtein distance match.
339 // If the LHS size is less than 3, use the LHS size minus 1 and if not,
340 // use the LHS size divided by 3.
341 size_t Length = LHS.size();
342 size_t MaxDist = Length < 3 ? Length - 1 : Length / 3;
343
344 std::optional<std::pair<StringRef, size_t>> SimilarStr;
345 for (StringRef C : Candidates) {
346 size_t CurDist = LHS.edit_distance(Other: C, AllowReplacements: true);
347 if (CurDist <= MaxDist) {
348 if (!SimilarStr) {
349 // The first similar string found.
350 SimilarStr = {C, CurDist};
351 } else if (CurDist < SimilarStr->second) {
352 // More similar string found.
353 SimilarStr = {C, CurDist};
354 }
355 }
356 }
357
358 if (SimilarStr) {
359 return SimilarStr->first;
360 } else {
361 return std::nullopt;
362 }
363}
364
365bool Preprocessor::CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
366 bool *ShadowFlag) {
367 // Missing macro name?
368 if (MacroNameTok.is(K: tok::eod))
369 return Diag(Tok: MacroNameTok, DiagID: diag::err_pp_missing_macro_name);
370
371 IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
372 if (!II)
373 return Diag(Tok: MacroNameTok, DiagID: diag::err_pp_macro_not_identifier);
374
375 if (II->isCPlusPlusOperatorKeyword()) {
376 // C++ 2.5p2: Alternative tokens behave the same as its primary token
377 // except for their spellings.
378 Diag(Tok: MacroNameTok, DiagID: getLangOpts().MicrosoftExt
379 ? diag::ext_pp_operator_used_as_macro_name
380 : diag::err_pp_operator_used_as_macro_name)
381 << II << MacroNameTok.getKind();
382 // Allow #defining |and| and friends for Microsoft compatibility or
383 // recovery when legacy C headers are included in C++.
384 }
385
386 if ((isDefineUndef != MU_Other) && II->getPPKeywordID() == tok::pp_defined) {
387 // Error if defining "defined": C99 6.10.8/4, C++ [cpp.predefined]p4.
388 return Diag(Tok: MacroNameTok, DiagID: diag::err_defined_macro_name);
389 }
390
391 // If defining/undefining reserved identifier or a keyword, we need to issue
392 // a warning.
393 SourceLocation MacroNameLoc = MacroNameTok.getLocation();
394 if (ShadowFlag)
395 *ShadowFlag = false;
396 // Macro names with reserved identifiers are accepted if built-in or passed
397 // through the command line (the later may be present if -dD was used to
398 // generate the preprocessed file).
399 // NB: isInPredefinedFile() (via getPresumedLoc) is relatively expensive, so
400 // only run it for names that can actually warn.
401 MacroDiag D = MD_NoWarn;
402 if (isDefineUndef == MU_Define) {
403 D = shouldWarnOnMacroDef(PP&: *this, II);
404 } else if (isDefineUndef == MU_Undef)
405 D = shouldWarnOnMacroUndef(PP&: *this, II);
406 if (D != MD_NoWarn && !SourceMgr.isInSystemHeader(Loc: MacroNameLoc) &&
407 !SourceMgr.isInPredefinedFile(Loc: MacroNameLoc)) {
408 if (D == MD_KeywordDef) {
409 // We do not want to warn on some patterns widely used in configuration
410 // scripts. This requires analyzing next tokens, so do not issue warnings
411 // now, only inform caller.
412 if (ShadowFlag)
413 *ShadowFlag = true;
414 }
415 if (D == MD_ReservedMacro)
416 Diag(Tok: MacroNameTok, DiagID: diag::warn_pp_macro_is_reserved_id);
417 if (D == MD_ReservedAttributeIdentifier)
418 Diag(Tok: MacroNameTok, DiagID: diag::warn_pp_macro_is_reserved_attribute_id)
419 << II->getName();
420 }
421
422 // Okay, we got a good identifier.
423 return false;
424}
425
426/// Lex and validate a macro name, which occurs after a
427/// \#define or \#undef.
428///
429/// This sets the token kind to eod and discards the rest of the macro line if
430/// the macro name is invalid.
431///
432/// \param MacroNameTok Token that is expected to be a macro name.
433/// \param isDefineUndef Context in which macro is used.
434/// \param ShadowFlag Points to a flag that is set if macro shadows a keyword.
435void Preprocessor::ReadMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
436 bool *ShadowFlag) {
437 // Read the token, don't allow macro expansion on it.
438 LexUnexpandedToken(Result&: MacroNameTok);
439
440 if (MacroNameTok.is(K: tok::code_completion)) {
441 if (CodeComplete)
442 CodeComplete->CodeCompleteMacroName(IsDefinition: isDefineUndef == MU_Define);
443 setCodeCompletionReached();
444 LexUnexpandedToken(Result&: MacroNameTok);
445 }
446
447 if (!CheckMacroName(MacroNameTok, isDefineUndef, ShadowFlag))
448 return;
449
450 // Invalid macro name, read and discard the rest of the line and set the
451 // token kind to tok::eod if necessary.
452 if (MacroNameTok.isNot(K: tok::eod)) {
453 MacroNameTok.setKind(tok::eod);
454 DiscardUntilEndOfDirective();
455 }
456}
457
458/// Ensure that the next token is a tok::eod token.
459///
460/// If not, emit a diagnostic and consume up until the eod. If EnableMacros is
461/// true, then we consider macros that expand to zero tokens as being ok.
462///
463/// Returns the location of the end of the directive.
464SourceLocation
465Preprocessor::CheckEndOfDirective(StringRef DirType, bool EnableMacros,
466 SmallVectorImpl<Token> *ExtraToks) {
467 Token Tmp;
468 // Avoid use-of-uninitialized-memory for edge case(s) where there is no extra
469 // token to be parsed.
470 Tmp.startToken();
471 auto ReadNextTok = [this, ExtraToks, &Tmp](auto &&LexFn) {
472 std::invoke(LexFn, this, Tmp);
473 if (ExtraToks && Tmp.isNot(K: tok::eod))
474 ExtraToks->push_back(Elt: Tmp);
475 };
476 // Lex unexpanded tokens for most directives: macros might expand to zero
477 // tokens, causing us to miss diagnosing invalid lines. Some directives (like
478 // #line) allow empty macros.
479 if (EnableMacros)
480 ReadNextTok(&Preprocessor::Lex);
481 else
482 ReadNextTok(&Preprocessor::LexUnexpandedToken);
483
484 // There should be no tokens after the directive, but we allow them as an
485 // extension.
486 while (Tmp.is(K: tok::comment)) // Skip comments in -C mode.
487 ReadNextTok(&Preprocessor::LexUnexpandedToken);
488
489 if (Tmp.is(K: tok::eod))
490 return Tmp.getLocation();
491
492 // Add a fixit in GNU/C99/C++ mode. Don't offer a fixit for strict-C89,
493 // or if this is a macro-style preprocessing directive, because it is more
494 // trouble than it is worth to insert /**/ and check that there is no /**/
495 // in the range also.
496 FixItHint Hint;
497 if ((LangOpts.GNUMode || LangOpts.C99 || LangOpts.CPlusPlus) &&
498 !CurTokenLexer)
499 Hint = FixItHint::CreateInsertion(InsertionLoc: Tmp.getLocation(),Code: "//");
500
501 unsigned DiagID = diag::ext_pp_extra_tokens_at_eol;
502 // C++20 import or module directive has no '#' prefix.
503 if (getLangOpts().CPlusPlusModules &&
504 (DirType == "import" || DirType == "module"))
505 DiagID = diag::warn_pp_extra_tokens_at_module_directive_eol;
506
507 Diag(Tok: Tmp, DiagID) << DirType << Hint;
508 return DiscardUntilEndOfDirective(DiscardedToks: ExtraToks).getEnd();
509}
510
511void Preprocessor::SuggestTypoedDirective(const Token &Tok,
512 StringRef Directive) const {
513 // If this is a `.S` file, treat unknown # directives as non-preprocessor
514 // directives.
515 if (getLangOpts().AsmPreprocessor) return;
516
517 // A known directive (e.g. #include or #define inside a skipped conditional
518 // block) is not a typo of a conditional; don't scan it.
519 if (getIdentifierInfo(Name: Directive)->getPPKeywordID() != tok::pp_not_keyword)
520 return;
521
522 // The scan only feeds this diagnostic; skip it when the diagnostic is
523 // disabled at this location (e.g. -w).
524 if (getDiagnostics().isIgnored(DiagID: diag::warn_pp_invalid_directive,
525 Loc: Tok.getLocation()))
526 return;
527
528 static constexpr StringRef AllCandidates[] = {
529 "if", "ifdef", "ifndef", "elif", "else", "endif", "elifdef", "elifndef"};
530 ArrayRef<StringRef> Candidates(AllCandidates);
531 // #elifdef/#elifndef are only suggested in C23/C++23 and later.
532 if (!LangOpts.C23 && !LangOpts.CPlusPlus23)
533 Candidates = Candidates.drop_back(N: 2);
534
535 if (std::optional<StringRef> Sugg = findSimilarStr(LHS: Directive, Candidates)) {
536 // Directive cannot be coming from macro.
537 assert(Tok.getLocation().isFileID());
538 CharSourceRange DirectiveRange = CharSourceRange::getCharRange(
539 B: Tok.getLocation(),
540 E: Tok.getLocation().getLocWithOffset(Offset: Directive.size()));
541 StringRef SuggValue = *Sugg;
542
543 auto Hint = FixItHint::CreateReplacement(RemoveRange: DirectiveRange, Code: SuggValue);
544 Diag(Tok, DiagID: diag::warn_pp_invalid_directive) << 1 << SuggValue << Hint;
545 }
546}
547
548/// SkipExcludedConditionalBlock - We just read a \#if or related directive and
549/// decided that the subsequent tokens are in the \#if'd out portion of the
550/// file. Lex the rest of the file, until we see an \#endif. If
551/// FoundNonSkipPortion is true, then we have already emitted code for part of
552/// this \#if directive, so \#else/\#elif blocks should never be entered.
553/// If ElseOk is true, then \#else directives are ok, if not, then we have
554/// already seen one so a \#else directive is a duplicate. When this returns,
555/// the caller can lex the first valid token.
556void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
557 SourceLocation IfTokenLoc,
558 bool FoundNonSkipPortion,
559 bool FoundElse,
560 SourceLocation ElseLoc) {
561 // In SkippingRangeStateTy we are depending on SkipExcludedConditionalBlock()
562 // not getting called recursively by storing the RecordedSkippedRanges
563 // DenseMap lookup pointer (field SkipRangePtr). SkippingRangeStateTy expects
564 // that RecordedSkippedRanges won't get modified and SkipRangePtr won't be
565 // invalidated. If this changes and there is a need to call
566 // SkipExcludedConditionalBlock() recursively, SkippingRangeStateTy should
567 // change to do a second lookup in endLexPass function instead of reusing the
568 // lookup pointer.
569 assert(!SkippingExcludedConditionalBlock &&
570 "calling SkipExcludedConditionalBlock recursively");
571 llvm::SaveAndRestore SARSkipping(SkippingExcludedConditionalBlock, true);
572
573 ++NumSkipped;
574 assert(!CurTokenLexer && "Conditional PP block cannot appear in a macro!");
575 assert(CurPPLexer && "Conditional PP block must be in a file!");
576 assert(CurLexer && "Conditional PP block but no current lexer set!");
577
578 if (PreambleConditionalStack.reachedEOFWhileSkipping())
579 PreambleConditionalStack.clearSkipInfo();
580 else
581 CurPPLexer->pushConditionalLevel(DirectiveStart: IfTokenLoc, /*isSkipping*/ WasSkipping: false,
582 FoundNonSkip: FoundNonSkipPortion, FoundElse);
583
584 // Enter raw mode to disable identifier lookup (and thus macro expansion),
585 // disabling warnings, etc.
586 CurPPLexer->LexingRawMode = true;
587 Token Tok;
588 SourceLocation endLoc;
589
590 /// Keeps track and caches skipped ranges and also retrieves a prior skipped
591 /// range if the same block is re-visited.
592 struct SkippingRangeStateTy {
593 Preprocessor &PP;
594
595 const char *BeginPtr = nullptr;
596 unsigned *SkipRangePtr = nullptr;
597
598 SkippingRangeStateTy(Preprocessor &PP) : PP(PP) {}
599
600 void beginLexPass() {
601 if (BeginPtr)
602 return; // continue skipping a block.
603
604 // Initiate a skipping block and adjust the lexer if we already skipped it
605 // before.
606 BeginPtr = PP.CurLexer->getBufferLocation();
607 SkipRangePtr = &PP.RecordedSkippedRanges[BeginPtr];
608 if (*SkipRangePtr) {
609 PP.CurLexer->seek(Offset: PP.CurLexer->getCurrentBufferOffset() + *SkipRangePtr,
610 /*IsAtStartOfLine*/ true);
611 }
612 }
613
614 void endLexPass(const char *Hashptr) {
615 if (!BeginPtr) {
616 // Not doing normal lexing.
617 assert(PP.CurLexer->isDependencyDirectivesLexer());
618 return;
619 }
620
621 // Finished skipping a block, record the range if it's first time visited.
622 if (!*SkipRangePtr) {
623 *SkipRangePtr = Hashptr - BeginPtr;
624 }
625 assert(*SkipRangePtr == unsigned(Hashptr - BeginPtr));
626 BeginPtr = nullptr;
627 SkipRangePtr = nullptr;
628 }
629 } SkippingRangeState(*this);
630
631 while (true) {
632 if (CurLexer->isDependencyDirectivesLexer()) {
633 CurLexer->LexDependencyDirectiveTokenWhileSkipping(Result&: Tok);
634 } else {
635 SkippingRangeState.beginLexPass();
636 while (true) {
637 CurLexer->Lex(Result&: Tok);
638
639 if (Tok.is(K: tok::code_completion)) {
640 setCodeCompletionReached();
641 if (CodeComplete)
642 CodeComplete->CodeCompleteInConditionalExclusion();
643 continue;
644 }
645
646 // There is actually no "skipped block" in the above because the module
647 // directive is not a text-line (https://wg21.link/cpp.pre#2) nor
648 // anything else that is allowed in a group
649 // (https://eel.is/c++draft/cpp.pre#nt:group-part).
650 //
651 // A preprocessor diagnostic (effective with -E) that triggers whenever
652 // a module directive is encountered where a control-line or a text-line
653 // is required.
654 if (getLangOpts().CPlusPlusModules && Tok.isAtStartOfLine() &&
655 Tok.is(K: tok::raw_identifier) &&
656 (Tok.getRawIdentifier() == "export" ||
657 Tok.getRawIdentifier() == "module")) {
658 llvm::SaveAndRestore ModuleDirectiveSkipping(LastExportKeyword);
659 LastExportKeyword.startToken();
660 LookUpIdentifierInfo(Identifier&: Tok);
661 IdentifierInfo *II = Tok.getIdentifierInfo();
662
663 if (II->getName()[0] == 'e') { // export
664 HandleModuleContextualKeyword(Result&: Tok);
665 CurLexer->Lex(Result&: Tok);
666 if (Tok.is(K: tok::raw_identifier)) {
667 LookUpIdentifierInfo(Identifier&: Tok);
668 II = Tok.getIdentifierInfo();
669 }
670 }
671
672 if (II->getName()[0] == 'm') { // module
673 // HandleModuleContextualKeyword changes the lexer state, so we need
674 // to save RawLexingMode
675 llvm::SaveAndRestore RestoreLexingRawMode(CurPPLexer->LexingRawMode,
676 false);
677 if (HandleModuleContextualKeyword(Result&: Tok)) {
678 // We just parsed a # character at the start of a line, so we're
679 // in directive mode. Tell the lexer this so any newlines we see
680 // will be converted into an EOD token (this terminates the
681 // macro).
682 CurPPLexer->ParsingPreprocessorDirective = true;
683 SourceLocation StartLoc = Tok.getLocation();
684 SourceLocation End = DiscardUntilEndOfDirective().getEnd();
685 Diag(Loc: StartLoc, DiagID: diag::err_pp_cond_span_module_decl)
686 << SourceRange(StartLoc, End);
687 CurPPLexer->ParsingPreprocessorDirective = false;
688 // Restore comment saving mode.
689 if (CurLexer)
690 CurLexer->resetExtendedTokenMode();
691 continue;
692 }
693 }
694 }
695
696 // If this is the end of the buffer, we have an error.
697 if (Tok.is(K: tok::eof)) {
698 // We don't emit errors for unterminated conditionals here,
699 // Lexer::LexEndOfFile can do that properly.
700 // Just return and let the caller lex after this #include.
701 if (PreambleConditionalStack.isRecording())
702 PreambleConditionalStack.SkipInfo.emplace(args&: HashTokenLoc, args&: IfTokenLoc,
703 args&: FoundNonSkipPortion,
704 args&: FoundElse, args&: ElseLoc);
705 break;
706 }
707
708 // If this token is not a preprocessor directive, just skip it.
709 if (Tok.isNot(K: tok::hash) || !Tok.isAtStartOfLine())
710 continue;
711
712 break;
713 }
714 }
715 if (Tok.is(K: tok::eof))
716 break;
717
718 // We just parsed a # character at the start of a line, so we're in
719 // directive mode. Tell the lexer this so any newlines we see will be
720 // converted into an EOD token (this terminates the macro).
721 CurPPLexer->ParsingPreprocessorDirective = true;
722 if (CurLexer) CurLexer->SetKeepWhitespaceMode(false);
723
724 assert(Tok.is(tok::hash));
725 const char *Hashptr = CurLexer->getBufferLocation() - Tok.getLength();
726 assert(CurLexer->getSourceLocation(Hashptr) == Tok.getLocation());
727
728 // Read the next token, the directive flavor.
729 LexUnexpandedToken(Result&: Tok);
730
731 // If this isn't an identifier directive (e.g. is "# 1\n" or "#\n", or
732 // something bogus), skip it.
733 if (Tok.isNot(K: tok::raw_identifier)) {
734 CurPPLexer->ParsingPreprocessorDirective = false;
735 // Restore comment saving mode.
736 if (CurLexer) CurLexer->resetExtendedTokenMode();
737 continue;
738 }
739
740 // If the first letter isn't i or e, it isn't intesting to us. We know that
741 // this is safe in the face of spelling differences, because there is no way
742 // to spell an i/e in a strange way that is another letter. Skipping this
743 // allows us to avoid looking up the identifier info for #define/#undef and
744 // other common directives.
745 StringRef RI = Tok.getRawIdentifier();
746
747 char FirstChar = RI[0];
748 if (FirstChar >= 'a' && FirstChar <= 'z' &&
749 FirstChar != 'i' && FirstChar != 'e') {
750 CurPPLexer->ParsingPreprocessorDirective = false;
751 // Restore comment saving mode.
752 if (CurLexer) CurLexer->resetExtendedTokenMode();
753 continue;
754 }
755
756 // Get the identifier name without trigraphs or embedded newlines. Note
757 // that we can't use Tok.getIdentifierInfo() because its lookup is disabled
758 // when skipping.
759 char DirectiveBuf[20];
760 StringRef Directive;
761 if (!Tok.needsCleaning() && RI.size() < 20) {
762 Directive = RI;
763 } else {
764 std::string DirectiveStr = getSpelling(Tok);
765 size_t IdLen = DirectiveStr.size();
766 if (IdLen >= 20) {
767 CurPPLexer->ParsingPreprocessorDirective = false;
768 // Restore comment saving mode.
769 if (CurLexer) CurLexer->resetExtendedTokenMode();
770 continue;
771 }
772 memcpy(dest: DirectiveBuf, src: &DirectiveStr[0], n: IdLen);
773 Directive = StringRef(DirectiveBuf, IdLen);
774 }
775
776 if (Directive.starts_with(Prefix: "if")) {
777 StringRef Sub = Directive.substr(Start: 2);
778 if (Sub.empty() || // "if"
779 Sub == "def" || // "ifdef"
780 Sub == "ndef") { // "ifndef"
781 // We know the entire #if/#ifdef/#ifndef block will be skipped, don't
782 // bother parsing the condition.
783 DiscardUntilEndOfDirective();
784 CurPPLexer->pushConditionalLevel(DirectiveStart: Tok.getLocation(), /*wasskipping*/WasSkipping: true,
785 /*foundnonskip*/FoundNonSkip: false,
786 /*foundelse*/FoundElse: false);
787 } else {
788 SuggestTypoedDirective(Tok, Directive);
789 }
790 } else if (Directive[0] == 'e') {
791 StringRef Sub = Directive.substr(Start: 1);
792 if (Sub == "ndif") { // "endif"
793 PPConditionalInfo CondInfo;
794 CondInfo.WasSkipping = true; // Silence bogus warning.
795 bool InCond = CurPPLexer->popConditionalLevel(CI&: CondInfo);
796 (void)InCond; // Silence warning in no-asserts mode.
797 assert(!InCond && "Can't be skipping if not in a conditional!");
798
799 // If we popped the outermost skipping block, we're done skipping!
800 if (!CondInfo.WasSkipping) {
801 SkippingRangeState.endLexPass(Hashptr);
802 // Restore the value of LexingRawMode so that trailing comments
803 // are handled correctly, if we've reached the outermost block.
804 CurPPLexer->LexingRawMode = false;
805 endLoc = CheckEndOfDirective(DirType: "endif");
806 CurPPLexer->LexingRawMode = true;
807 if (Callbacks)
808 Callbacks->Endif(Loc: Tok.getLocation(), IfLoc: CondInfo.IfLoc);
809 break;
810 } else {
811 DiscardUntilEndOfDirective();
812 }
813 } else if (Sub == "lse") { // "else".
814 // #else directive in a skipping conditional. If not in some other
815 // skipping conditional, and if #else hasn't already been seen, enter it
816 // as a non-skipping conditional.
817 PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
818
819 if (!CondInfo.WasSkipping)
820 SkippingRangeState.endLexPass(Hashptr);
821
822 // If this is a #else with a #else before it, report the error.
823 if (CondInfo.FoundElse)
824 Diag(Tok, DiagID: diag::pp_err_else_after_else);
825
826 // Note that we've seen a #else in this conditional.
827 CondInfo.FoundElse = true;
828
829 // If the conditional is at the top level, and the #if block wasn't
830 // entered, enter the #else block now.
831 if (!CondInfo.WasSkipping && !CondInfo.FoundNonSkip) {
832 CondInfo.FoundNonSkip = true;
833 // Restore the value of LexingRawMode so that trailing comments
834 // are handled correctly.
835 CurPPLexer->LexingRawMode = false;
836 endLoc = CheckEndOfDirective(DirType: "else");
837 CurPPLexer->LexingRawMode = true;
838 if (Callbacks)
839 Callbacks->Else(Loc: Tok.getLocation(), IfLoc: CondInfo.IfLoc);
840 break;
841 } else {
842 DiscardUntilEndOfDirective(); // C99 6.10p4.
843 }
844 } else if (Sub == "lif") { // "elif".
845 PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
846
847 if (!CondInfo.WasSkipping)
848 SkippingRangeState.endLexPass(Hashptr);
849
850 // If this is a #elif with a #else before it, report the error.
851 if (CondInfo.FoundElse)
852 Diag(Tok, DiagID: diag::pp_err_elif_after_else) << PED_Elif;
853
854 // If this is in a skipping block or if we're already handled this #if
855 // block, don't bother parsing the condition.
856 if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
857 // FIXME: We should probably do at least some minimal parsing of the
858 // condition to verify that it is well-formed. The current state
859 // allows #elif* directives with completely malformed (or missing)
860 // conditions.
861 DiscardUntilEndOfDirective();
862 } else {
863 // Restore the value of LexingRawMode so that identifiers are
864 // looked up, etc, inside the #elif expression.
865 assert(CurPPLexer->LexingRawMode && "We have to be skipping here!");
866 CurPPLexer->LexingRawMode = false;
867 IdentifierInfo *IfNDefMacro = nullptr;
868 DirectiveEvalResult DER = EvaluateDirectiveExpression(IfNDefMacro);
869 // Stop if Lexer became invalid after hitting code completion token.
870 if (!CurPPLexer)
871 return;
872 const bool CondValue = DER.Conditional;
873 CurPPLexer->LexingRawMode = true;
874 if (Callbacks) {
875 Callbacks->Elif(
876 Loc: Tok.getLocation(), ConditionRange: DER.ExprRange,
877 ConditionValue: (CondValue ? PPCallbacks::CVK_True : PPCallbacks::CVK_False),
878 IfLoc: CondInfo.IfLoc);
879 }
880 // If this condition is true, enter it!
881 if (CondValue) {
882 CondInfo.FoundNonSkip = true;
883 break;
884 }
885 }
886 } else if (Sub == "lifdef" || // "elifdef"
887 Sub == "lifndef") { // "elifndef"
888 bool IsElifDef = Sub == "lifdef";
889 PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
890 Token DirectiveToken = Tok;
891
892 if (!CondInfo.WasSkipping)
893 SkippingRangeState.endLexPass(Hashptr);
894
895 // Warn if using `#elifdef` & `#elifndef` in not C23 & C++23 mode even
896 // if this branch is in a skipping block.
897 unsigned DiagID = LangOpts.CPlusPlus ? diag_compat::cxx23_pp_directive
898 : diag_compat::c23_pp_directive;
899 DiagCompat(Tok, CompatDiagID: DiagID) << (IsElifDef ? PED_Elifdef : PED_Elifndef);
900
901 // If this is a #elif with a #else before it, report the error.
902 if (CondInfo.FoundElse)
903 Diag(Tok, DiagID: diag::pp_err_elif_after_else)
904 << (IsElifDef ? PED_Elifdef : PED_Elifndef);
905
906 // If this is in a skipping block or if we're already handled this #if
907 // block, don't bother parsing the condition.
908 if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
909 // FIXME: We should probably do at least some minimal parsing of the
910 // condition to verify that it is well-formed. The current state
911 // allows #elif* directives with completely malformed (or missing)
912 // conditions.
913 DiscardUntilEndOfDirective();
914 } else {
915 // Restore the value of LexingRawMode so that identifiers are
916 // looked up, etc, inside the #elif[n]def expression.
917 assert(CurPPLexer->LexingRawMode && "We have to be skipping here!");
918 CurPPLexer->LexingRawMode = false;
919 Token MacroNameTok;
920 ReadMacroName(MacroNameTok);
921 CurPPLexer->LexingRawMode = true;
922
923 // If the macro name token is tok::eod, there was an error that was
924 // already reported.
925 if (MacroNameTok.is(K: tok::eod)) {
926 // Skip code until we get to #endif. This helps with recovery by
927 // not emitting an error when the #endif is reached.
928 continue;
929 }
930
931 emitMacroExpansionWarnings(Identifier: MacroNameTok);
932
933 CheckEndOfDirective(DirType: IsElifDef ? "elifdef" : "elifndef");
934
935 IdentifierInfo *MII = MacroNameTok.getIdentifierInfo();
936 auto MD = getMacroDefinition(II: MII);
937 MacroInfo *MI = MD.getMacroInfo();
938
939 if (Callbacks) {
940 if (IsElifDef) {
941 Callbacks->Elifdef(Loc: DirectiveToken.getLocation(), MacroNameTok,
942 MD);
943 } else {
944 Callbacks->Elifndef(Loc: DirectiveToken.getLocation(), MacroNameTok,
945 MD);
946 }
947 }
948 // If this condition is true, enter it!
949 if (static_cast<bool>(MI) == IsElifDef) {
950 CondInfo.FoundNonSkip = true;
951 break;
952 }
953 }
954 } else {
955 SuggestTypoedDirective(Tok, Directive);
956 }
957 } else {
958 SuggestTypoedDirective(Tok, Directive);
959 }
960
961 CurPPLexer->ParsingPreprocessorDirective = false;
962 // Restore comment saving mode.
963 if (CurLexer) CurLexer->resetExtendedTokenMode();
964 }
965
966 // Finally, if we are out of the conditional (saw an #endif or ran off the end
967 // of the file, just stop skipping and return to lexing whatever came after
968 // the #if block.
969 CurPPLexer->LexingRawMode = false;
970
971 // The last skipped range isn't actually skipped yet if it's truncated
972 // by the end of the preamble; we'll resume parsing after the preamble.
973 if (Callbacks && (Tok.isNot(K: tok::eof) || !isRecordingPreamble()))
974 Callbacks->SourceRangeSkipped(
975 Range: SourceRange(HashTokenLoc, endLoc.isValid()
976 ? endLoc
977 : CurPPLexer->getSourceLocation()),
978 EndifLoc: Tok.getLocation());
979}
980
981Module *Preprocessor::getModuleForLocation(SourceLocation Loc,
982 bool AllowTextual) {
983 if (!SourceMgr.isInMainFile(Loc)) {
984 // Try to determine the module of the include directive.
985 // FIXME: Look into directly passing the FileEntry from LookupFile instead.
986 FileID IDOfIncl = SourceMgr.getFileID(SpellingLoc: SourceMgr.getExpansionLoc(Loc));
987 if (auto EntryOfIncl = SourceMgr.getFileEntryRefForID(FID: IDOfIncl)) {
988 // The include comes from an included file.
989 return HeaderInfo.getModuleMap()
990 .findModuleForHeader(File: *EntryOfIncl, AllowTextual)
991 .getModule();
992 }
993 }
994
995 // This is either in the main file or not in a file at all. It belongs
996 // to the current module, if there is one.
997 return getLangOpts().CurrentModule.empty()
998 ? nullptr
999 : HeaderInfo.lookupModule(ModuleName: getLangOpts().CurrentModule, ImportLoc: Loc);
1000}
1001
1002OptionalFileEntryRef
1003Preprocessor::getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
1004 SourceLocation Loc) {
1005 Module *IncM = getModuleForLocation(
1006 Loc: IncLoc, AllowTextual: LangOpts.ModulesValidateTextualHeaderIncludes);
1007
1008 // Walk up through the include stack, looking through textual headers of M
1009 // until we hit a non-textual header that we can #include. (We assume textual
1010 // headers of a module with non-textual headers aren't meant to be used to
1011 // import entities from the module.)
1012 auto &SM = getSourceManager();
1013 while (!Loc.isInvalid() && !SM.isInMainFile(Loc)) {
1014 auto ID = SM.getFileID(SpellingLoc: SM.getExpansionLoc(Loc));
1015 auto FE = SM.getFileEntryRefForID(FID: ID);
1016 if (!FE)
1017 break;
1018
1019 // We want to find all possible modules that might contain this header, so
1020 // search all enclosing directories for module maps and load them.
1021 HeaderInfo.hasModuleMap(Filename: FE->getName(), /*Root*/ nullptr,
1022 IsSystem: SourceMgr.isInSystemHeader(Loc));
1023
1024 bool InPrivateHeader = false;
1025 for (auto Header : HeaderInfo.findAllModulesForHeader(File: *FE)) {
1026 if (!Header.isAccessibleFrom(M: IncM)) {
1027 // It's in a private header; we can't #include it.
1028 // FIXME: If there's a public header in some module that re-exports it,
1029 // then we could suggest including that, but it's not clear that's the
1030 // expected way to make this entity visible.
1031 InPrivateHeader = true;
1032 continue;
1033 }
1034
1035 // Don't suggest explicitly excluded headers.
1036 if (Header.getRole() == ModuleMap::ExcludedHeader)
1037 continue;
1038
1039 // We'll suggest including textual headers below if they're
1040 // include-guarded.
1041 if (Header.getRole() & ModuleMap::TextualHeader)
1042 continue;
1043
1044 // If we have a module import syntax, we shouldn't include a header to
1045 // make a particular module visible. Let the caller know they should
1046 // suggest an import instead.
1047 if (getLangOpts().ObjC || getLangOpts().CPlusPlusModules)
1048 return std::nullopt;
1049
1050 // If this is an accessible, non-textual header of M's top-level module
1051 // that transitively includes the given location and makes the
1052 // corresponding module visible, this is the thing to #include.
1053 return *FE;
1054 }
1055
1056 // FIXME: If we're bailing out due to a private header, we shouldn't suggest
1057 // an import either.
1058 if (InPrivateHeader)
1059 return std::nullopt;
1060
1061 // If the header is includable and has an include guard, assume the
1062 // intended way to expose its contents is by #include, not by importing a
1063 // module that transitively includes it.
1064 if (getHeaderSearchInfo().isFileMultipleIncludeGuarded(File: *FE))
1065 return *FE;
1066
1067 Loc = SM.getIncludeLoc(FID: ID);
1068 }
1069
1070 return std::nullopt;
1071}
1072
1073OptionalFileEntryRef Preprocessor::LookupFile(
1074 SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
1075 ConstSearchDirIterator FromDir, const FileEntry *FromFile,
1076 ConstSearchDirIterator *CurDirArg, SmallVectorImpl<char> *SearchPath,
1077 SmallVectorImpl<char> *RelativePath,
1078 ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
1079 bool *IsFrameworkFound, bool SkipCache, bool OpenFile, bool CacheFailures) {
1080 ConstSearchDirIterator CurDirLocal = nullptr;
1081 ConstSearchDirIterator &CurDir = CurDirArg ? *CurDirArg : CurDirLocal;
1082
1083 Module *RequestingModule = getModuleForLocation(
1084 Loc: FilenameLoc, AllowTextual: LangOpts.ModulesValidateTextualHeaderIncludes);
1085
1086 // If the header lookup mechanism may be relative to the current inclusion
1087 // stack, record the parent #includes.
1088 SmallVector<std::pair<OptionalFileEntryRef, DirectoryEntryRef>, 16> Includers;
1089 bool BuildSystemModule = false;
1090 if (!FromDir && !FromFile) {
1091 FileID FID = getCurrentFileLexer()->getFileID();
1092 OptionalFileEntryRef FileEnt = SourceMgr.getFileEntryRefForID(FID);
1093
1094 // If there is no file entry associated with this file, it must be the
1095 // predefines buffer or the module includes buffer. Any other file is not
1096 // lexed with a normal lexer, so it won't be scanned for preprocessor
1097 // directives.
1098 //
1099 // If we have the predefines buffer, resolve #include references (which come
1100 // from the -include command line argument) from the current working
1101 // directory instead of relative to the main file.
1102 //
1103 // If we have the module includes buffer, resolve #include references (which
1104 // come from header declarations in the module map) relative to the module
1105 // map file.
1106 if (!FileEnt) {
1107 if (FID == SourceMgr.getMainFileID() && MainFileDir) {
1108 auto IncludeDir =
1109 HeaderInfo.getModuleMap().shouldImportRelativeToBuiltinIncludeDir(
1110 FileName: Filename, Module: getCurrentModule())
1111 ? HeaderInfo.getModuleMap().getBuiltinDir()
1112 : MainFileDir;
1113 Includers.push_back(Elt: std::make_pair(x: std::nullopt, y&: *IncludeDir));
1114 BuildSystemModule = getCurrentModule()->IsSystem;
1115 } else if ((FileEnt = SourceMgr.getFileEntryRefForID(
1116 FID: SourceMgr.getMainFileID()))) {
1117 auto CWD = FileMgr.getOptionalDirectoryRef(DirName: ".");
1118 Includers.push_back(Elt: std::make_pair(x&: *FileEnt, y&: *CWD));
1119 }
1120 } else {
1121 Includers.push_back(Elt: std::make_pair(x&: *FileEnt, y: FileEnt->getDir()));
1122 }
1123
1124 // MSVC searches the current include stack from top to bottom for
1125 // headers included by quoted include directives.
1126 // See: http://msdn.microsoft.com/en-us/library/36k2cdd4.aspx
1127 if (LangOpts.MSVCCompat && !isAngled) {
1128 for (IncludeStackInfo &ISEntry : llvm::reverse(C&: IncludeMacroStack)) {
1129 if (IsFileLexer(I: ISEntry))
1130 if ((FileEnt = ISEntry.ThePPLexer->getFileEntry()))
1131 Includers.push_back(Elt: std::make_pair(x&: *FileEnt, y: FileEnt->getDir()));
1132 }
1133 }
1134 }
1135
1136 CurDir = CurDirLookup;
1137
1138 if (FromFile) {
1139 // We're supposed to start looking from after a particular file. Search
1140 // the include path until we find that file or run out of files.
1141 ConstSearchDirIterator TmpCurDir = CurDir;
1142 ConstSearchDirIterator TmpFromDir = nullptr;
1143 while (OptionalFileEntryRef FE = HeaderInfo.LookupFile(
1144 Filename, IncludeLoc: FilenameLoc, isAngled, FromDir: TmpFromDir, CurDir: &TmpCurDir,
1145 Includers, SearchPath, RelativePath, RequestingModule,
1146 SuggestedModule, /*IsMapped=*/nullptr,
1147 /*IsFrameworkFound=*/nullptr, SkipCache)) {
1148 // Keep looking as if this file did a #include_next.
1149 TmpFromDir = TmpCurDir;
1150 ++TmpFromDir;
1151 if (&FE->getFileEntry() == FromFile) {
1152 // Found it.
1153 FromDir = TmpFromDir;
1154 CurDir = TmpCurDir;
1155 break;
1156 }
1157 }
1158 }
1159
1160 // Do a standard file entry lookup.
1161 OptionalFileEntryRef FE = HeaderInfo.LookupFile(
1162 Filename, IncludeLoc: FilenameLoc, isAngled, FromDir, CurDir: &CurDir, Includers, SearchPath,
1163 RelativePath, RequestingModule, SuggestedModule, IsMapped,
1164 IsFrameworkFound, SkipCache, BuildSystemModule, OpenFile, CacheFailures);
1165 if (FE)
1166 return FE;
1167
1168 OptionalFileEntryRef CurFileEnt;
1169 // Otherwise, see if this is a subframework header. If so, this is relative
1170 // to one of the headers on the #include stack. Walk the list of the current
1171 // headers on the #include stack and pass them to HeaderInfo.
1172 if (IsFileLexer()) {
1173 if ((CurFileEnt = CurPPLexer->getFileEntry())) {
1174 if (OptionalFileEntryRef FE = HeaderInfo.LookupSubframeworkHeader(
1175 Filename, ContextFileEnt: *CurFileEnt, SearchPath, RelativePath, RequestingModule,
1176 SuggestedModule)) {
1177 return FE;
1178 }
1179 }
1180 }
1181
1182 for (IncludeStackInfo &ISEntry : llvm::reverse(C&: IncludeMacroStack)) {
1183 if (IsFileLexer(I: ISEntry)) {
1184 if ((CurFileEnt = ISEntry.ThePPLexer->getFileEntry())) {
1185 if (OptionalFileEntryRef FE = HeaderInfo.LookupSubframeworkHeader(
1186 Filename, ContextFileEnt: *CurFileEnt, SearchPath, RelativePath,
1187 RequestingModule, SuggestedModule)) {
1188 return FE;
1189 }
1190 }
1191 }
1192 }
1193
1194 // Otherwise, we really couldn't find the file.
1195 return std::nullopt;
1196}
1197
1198OptionalFileEntryRef Preprocessor::LookupEmbedFile(StringRef Filename,
1199 bool isAngled,
1200 bool OpenFile) {
1201 FileManager &FM = this->getFileManager();
1202 if (llvm::sys::path::is_absolute(path: Filename)) {
1203 // lookup path or immediately fail
1204 return FM.getOptionalFileRef(Filename, OpenFile, /*CacheFailure=*/true,
1205 /*IsText=*/false);
1206 }
1207
1208 auto SeparateComponents = [](SmallVectorImpl<char> &LookupPath,
1209 StringRef StartingFrom, StringRef FileName,
1210 bool RemoveInitialFileComponentFromLookupPath) {
1211 llvm::sys::path::native(path: StartingFrom, result&: LookupPath);
1212 if (RemoveInitialFileComponentFromLookupPath)
1213 llvm::sys::path::remove_filename(path&: LookupPath);
1214 if (!LookupPath.empty() &&
1215 !llvm::sys::path::is_separator(value: LookupPath.back())) {
1216 LookupPath.push_back(Elt: llvm::sys::path::get_separator().front());
1217 }
1218 LookupPath.append(in_start: FileName.begin(), in_end: FileName.end());
1219 };
1220
1221 // Otherwise, it's search time!
1222 SmallString<512> LookupPath;
1223 // Non-angled lookup
1224 if (!isAngled) {
1225 OptionalFileEntryRef LookupFromFile = getCurrentFileLexer()->getFileEntry();
1226 if (LookupFromFile) {
1227 // Use file-based lookup.
1228 SmallString<1024> TmpDir;
1229 TmpDir = LookupFromFile->getDir().getName();
1230 llvm::sys::path::append(path&: TmpDir, a: Filename);
1231 if (!TmpDir.empty()) {
1232 OptionalFileEntryRef ShouldBeEntry = FM.getOptionalFileRef(
1233 Filename: TmpDir, OpenFile, /*CacheFailure=*/true, /*IsText=*/false);
1234 if (ShouldBeEntry)
1235 return ShouldBeEntry;
1236 }
1237 }
1238
1239 // Otherwise, do working directory lookup.
1240 LookupPath.clear();
1241 auto MaybeWorkingDirEntry = FM.getOptionalDirectoryRef(DirName: ".");
1242 if (MaybeWorkingDirEntry) {
1243 DirectoryEntryRef WorkingDirEntry = *MaybeWorkingDirEntry;
1244 StringRef WorkingDir = WorkingDirEntry.getName();
1245 if (!WorkingDir.empty()) {
1246 SeparateComponents(LookupPath, WorkingDir, Filename, false);
1247 OptionalFileEntryRef ShouldBeEntry = FM.getOptionalFileRef(
1248 Filename: LookupPath, OpenFile, /*CacheFailure=*/true, /*IsText=*/false);
1249 if (ShouldBeEntry)
1250 return ShouldBeEntry;
1251 }
1252 }
1253 }
1254
1255 for (const auto &Entry : PPOpts.EmbedEntries) {
1256 LookupPath.clear();
1257 SeparateComponents(LookupPath, Entry, Filename, false);
1258 OptionalFileEntryRef ShouldBeEntry = FM.getOptionalFileRef(
1259 Filename: LookupPath, OpenFile, /*CacheFailure=*/true, /*IsText=*/false);
1260 if (ShouldBeEntry)
1261 return ShouldBeEntry;
1262 }
1263 return std::nullopt;
1264}
1265
1266//===----------------------------------------------------------------------===//
1267// Preprocessor Directive Handling.
1268//===----------------------------------------------------------------------===//
1269
1270class Preprocessor::ResetMacroExpansionHelper {
1271public:
1272 ResetMacroExpansionHelper(Preprocessor *pp)
1273 : PP(pp), save(pp->DisableMacroExpansion) {
1274 if (pp->MacroExpansionInDirectivesOverride)
1275 pp->DisableMacroExpansion = false;
1276 }
1277
1278 ~ResetMacroExpansionHelper() {
1279 PP->DisableMacroExpansion = save;
1280 }
1281
1282private:
1283 Preprocessor *PP;
1284 bool save;
1285};
1286
1287/// Process a directive while looking for the through header or a #pragma
1288/// hdrstop. The following directives are handled:
1289/// #include (to check if it is the through header)
1290/// #define (to warn about macros that don't match the PCH)
1291/// #pragma (to check for pragma hdrstop).
1292/// All other directives are completely discarded.
1293void Preprocessor::HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1294 SourceLocation HashLoc) {
1295 if (const IdentifierInfo *II = Result.getIdentifierInfo()) {
1296 if (II->getPPKeywordID() == tok::pp_define) {
1297 return HandleDefineDirective(Tok&: Result,
1298 /*ImmediatelyAfterHeaderGuard=*/false);
1299 }
1300 if (SkippingUntilPCHThroughHeader &&
1301 II->getPPKeywordID() == tok::pp_include) {
1302 return HandleIncludeDirective(HashLoc, Tok&: Result);
1303 }
1304 if (SkippingUntilPragmaHdrStop && II->getPPKeywordID() == tok::pp_pragma) {
1305 Lex(Result);
1306 auto *II = Result.getIdentifierInfo();
1307 if (II && II->getName() == "hdrstop")
1308 return HandlePragmaHdrstop(Tok&: Result);
1309 }
1310 }
1311 DiscardUntilEndOfDirective();
1312}
1313
1314/// HandleDirective - This callback is invoked when the lexer sees a # token
1315/// at the start of a line. This consumes the directive, modifies the
1316/// lexer/preprocessor state, and advances the lexer(s) so that the next token
1317/// read is the correct one.
1318void Preprocessor::HandleDirective(Token &Result) {
1319 // FIXME: Traditional: # with whitespace before it not recognized by K&R?
1320
1321 // We just parsed a # or @ character at the start of a line, so we're in
1322 // directive mode. Tell the lexer this so any newlines we see will be
1323 // converted into an EOD token (which terminates the directive).
1324 CurPPLexer->ParsingPreprocessorDirective = true;
1325 if (CurLexer) CurLexer->SetKeepWhitespaceMode(false);
1326
1327 bool ImmediatelyAfterTopLevelIfndef =
1328 CurPPLexer->MIOpt.getImmediatelyAfterTopLevelIfndef();
1329 CurPPLexer->MIOpt.resetImmediatelyAfterTopLevelIfndef();
1330
1331 ++NumDirectives;
1332
1333 // We are about to read a token. For the multiple-include optimization FA to
1334 // work, we have to remember if we had read any tokens *before* this
1335 // pp-directive.
1336 bool ReadAnyTokensBeforeDirective =CurPPLexer->MIOpt.getHasReadAnyTokensVal();
1337
1338 // Save the directive-introducing token ('#', '@', or import/module in C++20)
1339 // in case we need to return it later.
1340 Token Introducer = Result;
1341
1342 // Read the next token, the directive flavor. This isn't expanded due to
1343 // C99 6.10.3p8.
1344 if (Introducer.isOneOf(Ks: tok::hash, Ks: tok::at))
1345 LexUnexpandedToken(Result);
1346
1347 // C99 6.10.3p11: Is this preprocessor directive in macro invocation? e.g.:
1348 // #define A(x) #x
1349 // A(abc
1350 // #warning blah
1351 // def)
1352 // If so, the user is relying on undefined behavior, emit a diagnostic. Do
1353 // not support this for #include-like directives, since that can result in
1354 // terrible diagnostics, and does not work in GCC.
1355 if (InMacroArgs) {
1356 if (IdentifierInfo *II = Result.getIdentifierInfo()) {
1357 switch (II->getPPKeywordID()) {
1358 case tok::pp_include:
1359 case tok::pp_import:
1360 case tok::pp_include_next:
1361 case tok::pp___include_macros:
1362 case tok::pp_pragma:
1363 case tok::pp_embed:
1364 case tok::pp_module:
1365 case tok::pp___preprocessed_module:
1366 case tok::pp___preprocessed_import:
1367 Diag(Tok: Result, DiagID: diag::err_embedded_directive)
1368 << Introducer.is(K: tok::hash) << II->getName();
1369 Diag(Tok: *ArgMacro, DiagID: diag::note_macro_expansion_here)
1370 << ArgMacro->getIdentifierInfo();
1371 DiscardUntilEndOfDirective();
1372 return;
1373 default:
1374 break;
1375 }
1376 }
1377 Diag(Tok: Result, DiagID: diag::ext_embedded_directive);
1378 }
1379
1380 // Temporarily enable macro expansion if set so
1381 // and reset to previous state when returning from this function.
1382 ResetMacroExpansionHelper helper(this);
1383
1384 if (SkippingUntilPCHThroughHeader || SkippingUntilPragmaHdrStop)
1385 return HandleSkippedDirectiveWhileUsingPCH(Result,
1386 HashLoc: Introducer.getLocation());
1387
1388 switch (Result.getKind()) {
1389 case tok::eod:
1390 // Ignore the null directive with regards to the multiple-include
1391 // optimization, i.e. allow the null directive to appear outside of the
1392 // include guard and still enable the multiple-include optimization.
1393 CurPPLexer->MIOpt.SetReadToken(ReadAnyTokensBeforeDirective);
1394 return; // null directive.
1395 case tok::code_completion:
1396 setCodeCompletionReached();
1397 if (CodeComplete)
1398 CodeComplete->CodeCompleteDirective(
1399 InConditional: CurPPLexer->getConditionalStackDepth() > 0);
1400 return;
1401 case tok::numeric_constant: // # 7 GNU line marker directive.
1402 // In a .S file "# 4" may be a comment so don't treat it as a preprocessor
1403 // directive. However do permit it in the predefines file, as we use line
1404 // markers to mark the builtin macros as being in a system header.
1405 if (getLangOpts().AsmPreprocessor &&
1406 SourceMgr.getFileID(SpellingLoc: Introducer.getLocation()) != getPredefinesFileID())
1407 break;
1408 return HandleDigitDirective(Tok&: Result);
1409 default:
1410 IdentifierInfo *II = Result.getIdentifierInfo();
1411 if (!II) break; // Not an identifier.
1412
1413 // Ask what the preprocessor keyword ID is.
1414 switch (II->getPPKeywordID()) {
1415 default: break;
1416 // C99 6.10.1 - Conditional Inclusion.
1417 case tok::pp_if:
1418 return HandleIfDirective(IfToken&: Result, HashToken: Introducer,
1419 ReadAnyTokensBeforeDirective);
1420 case tok::pp_ifdef:
1421 return HandleIfdefDirective(Result, HashToken: Introducer, isIfndef: false,
1422 ReadAnyTokensBeforeDirective: true /*not valid for miopt*/);
1423 case tok::pp_ifndef:
1424 return HandleIfdefDirective(Result, HashToken: Introducer, isIfndef: true,
1425 ReadAnyTokensBeforeDirective);
1426 case tok::pp_elif:
1427 case tok::pp_elifdef:
1428 case tok::pp_elifndef:
1429 return HandleElifFamilyDirective(ElifToken&: Result, HashToken: Introducer,
1430 Kind: II->getPPKeywordID());
1431
1432 case tok::pp_else:
1433 return HandleElseDirective(Result, HashToken: Introducer);
1434 case tok::pp_endif:
1435 return HandleEndifDirective(EndifToken&: Result);
1436
1437 // C99 6.10.2 - Source File Inclusion.
1438 case tok::pp_include:
1439 // Handle #include.
1440 return HandleIncludeDirective(HashLoc: Introducer.getLocation(), Tok&: Result);
1441 case tok::pp___include_macros:
1442 // Handle -imacros.
1443 return HandleIncludeMacrosDirective(HashLoc: Introducer.getLocation(), Tok&: Result);
1444
1445 // C99 6.10.3 - Macro Replacement.
1446 case tok::pp_define:
1447 return HandleDefineDirective(Tok&: Result, ImmediatelyAfterHeaderGuard: ImmediatelyAfterTopLevelIfndef);
1448 case tok::pp_undef:
1449 return HandleUndefDirective();
1450
1451 // C99 6.10.4 - Line Control.
1452 case tok::pp_line:
1453 return HandleLineDirective();
1454
1455 // C99 6.10.5 - Error Directive.
1456 case tok::pp_error:
1457 return HandleUserDiagnosticDirective(Tok&: Result, isWarning: false);
1458
1459 // C99 6.10.6 - Pragma Directive.
1460 case tok::pp_pragma:
1461 return HandlePragmaDirective(Introducer: {.Kind: PIK_HashPragma, .Loc: Introducer.getLocation()});
1462 case tok::pp_module:
1463 case tok::pp___preprocessed_module:
1464 if (Introducer.isModuleContextualKeyword())
1465 return HandleCXXModuleDirective(Module: Result);
1466 break;
1467 case tok::pp___preprocessed_import:
1468 return HandleCXXImportDirective(Import: Result);
1469 case tok::pp_import:
1470 switch (Introducer.getKind()) {
1471 case tok::hash:
1472 return HandleImportDirective(HashLoc: Introducer.getLocation(), Tok&: Result);
1473 case tok::at:
1474 return HandleObjCImportDirective(AtTok&: Introducer, ImportTok&: Result);
1475 case tok::kw_import:
1476 return HandleCXXImportDirective(Import: Result);
1477 default:
1478 llvm_unreachable("not a valid import directive");
1479 }
1480
1481 // GNU Extensions.
1482 case tok::pp_include_next:
1483 return HandleIncludeNextDirective(HashLoc: Introducer.getLocation(), Tok&: Result);
1484
1485 case tok::pp_warning:
1486 if (LangOpts.CPlusPlus)
1487 Diag(Tok: Result, DiagID: LangOpts.CPlusPlus23
1488 ? diag::warn_cxx23_compat_warning_directive
1489 : diag::ext_pp_warning_directive)
1490 << /*C++23*/ 1;
1491 else
1492 Diag(Tok: Result, DiagID: LangOpts.C23 ? diag::warn_c23_compat_warning_directive
1493 : diag::ext_pp_warning_directive)
1494 << /*C23*/ 0;
1495
1496 return HandleUserDiagnosticDirective(Tok&: Result, isWarning: true);
1497 case tok::pp_ident:
1498 return HandleIdentSCCSDirective(Tok&: Result);
1499 case tok::pp_sccs:
1500 return HandleIdentSCCSDirective(Tok&: Result);
1501 case tok::pp_embed:
1502 return HandleEmbedDirective(HashLoc: Introducer.getLocation(), Tok&: Result);
1503 case tok::pp_assert:
1504 //isExtension = true; // FIXME: implement #assert
1505 break;
1506 case tok::pp_unassert:
1507 //isExtension = true; // FIXME: implement #unassert
1508 break;
1509
1510 case tok::pp___public_macro:
1511 if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility)
1512 return HandleMacroPublicDirective(Tok&: Result);
1513 break;
1514
1515 case tok::pp___private_macro:
1516 if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility)
1517 return HandleMacroPrivateDirective();
1518 break;
1519 }
1520 break;
1521 }
1522
1523 // If this is a .S file, treat unknown # directives as non-preprocessor
1524 // directives. This is important because # may be a comment or introduce
1525 // various pseudo-ops. Just return the # token and push back the following
1526 // token to be lexed next time.
1527 if (getLangOpts().AsmPreprocessor) {
1528 auto Toks = std::make_unique<Token[]>(num: 2);
1529 // Return the # and the token after it.
1530 Toks[0] = Introducer;
1531 Toks[1] = Result;
1532
1533 // If the second token is a hashhash token, then we need to translate it to
1534 // unknown so the token lexer doesn't try to perform token pasting.
1535 if (Result.is(K: tok::hashhash))
1536 Toks[1].setKind(tok::unknown);
1537
1538 // Enter this token stream so that we re-lex the tokens. Make sure to
1539 // enable macro expansion, in case the token after the # is an identifier
1540 // that is expanded.
1541 EnterTokenStream(Toks: std::move(Toks), NumToks: 2, DisableMacroExpansion: false, /*IsReinject*/false);
1542 return;
1543 }
1544
1545 // If we reached here, the preprocessing token is not valid!
1546 // Start suggesting if a similar directive found.
1547 Diag(Tok: Result, DiagID: diag::err_pp_invalid_directive) << 0;
1548
1549 // Read the rest of the PP line.
1550 DiscardUntilEndOfDirective();
1551
1552 // Okay, we're done parsing the directive.
1553}
1554
1555/// GetLineValue - Convert a numeric token into an unsigned value, emitting
1556/// Diagnostic DiagID if it is invalid, and returning the value in Val.
1557static bool GetLineValue(Token &DigitTok, unsigned &Val,
1558 unsigned DiagID, Preprocessor &PP,
1559 bool IsGNULineDirective=false) {
1560 if (DigitTok.isNot(K: tok::numeric_constant)) {
1561 PP.Diag(Tok: DigitTok, DiagID);
1562
1563 if (DigitTok.isNot(K: tok::eod))
1564 PP.DiscardUntilEndOfDirective();
1565 return true;
1566 }
1567
1568 SmallString<64> IntegerBuffer;
1569 IntegerBuffer.resize(N: DigitTok.getLength());
1570 const char *DigitTokBegin = &IntegerBuffer[0];
1571 bool Invalid = false;
1572 unsigned ActualLength = PP.getSpelling(Tok: DigitTok, Buffer&: DigitTokBegin, Invalid: &Invalid);
1573 if (Invalid)
1574 return true;
1575
1576 // Verify that we have a simple digit-sequence, and compute the value. This
1577 // is always a simple digit string computed in decimal, so we do this manually
1578 // here.
1579 Val = 0;
1580 for (unsigned i = 0; i != ActualLength; ++i) {
1581 // C++1y [lex.fcon]p1:
1582 // Optional separating single quotes in a digit-sequence are ignored
1583 if (DigitTokBegin[i] == '\'')
1584 continue;
1585
1586 if (!isDigit(c: DigitTokBegin[i])) {
1587 PP.Diag(Loc: PP.AdvanceToTokenCharacter(TokStart: DigitTok.getLocation(), Char: i),
1588 DiagID: diag::err_pp_line_digit_sequence) << IsGNULineDirective;
1589 PP.DiscardUntilEndOfDirective();
1590 return true;
1591 }
1592
1593 unsigned NextVal = Val*10+(DigitTokBegin[i]-'0');
1594 if (NextVal < Val) { // overflow.
1595 PP.Diag(Tok: DigitTok, DiagID);
1596 PP.DiscardUntilEndOfDirective();
1597 return true;
1598 }
1599 Val = NextVal;
1600 }
1601
1602 if (DigitTokBegin[0] == '0' && Val)
1603 PP.Diag(Loc: DigitTok.getLocation(), DiagID: diag::warn_pp_line_decimal)
1604 << IsGNULineDirective;
1605
1606 return false;
1607}
1608
1609/// Handle a \#line directive: C99 6.10.4.
1610///
1611/// The two acceptable forms are:
1612/// \verbatim
1613/// # line digit-sequence
1614/// # line digit-sequence "s-char-sequence"
1615/// \endverbatim
1616void Preprocessor::HandleLineDirective() {
1617 // Read the line # and string argument. Per C99 6.10.4p5, these tokens are
1618 // expanded.
1619 Token DigitTok;
1620 Lex(Result&: DigitTok);
1621
1622 // Validate the number and convert it to an unsigned.
1623 unsigned LineNo;
1624 if (GetLineValue(DigitTok, Val&: LineNo, DiagID: diag::err_pp_line_requires_integer,PP&: *this))
1625 return;
1626
1627 if (LineNo == 0)
1628 Diag(Tok: DigitTok, DiagID: diag::ext_pp_line_zero);
1629
1630 // Enforce C99 6.10.4p3: "The digit sequence shall not specify ... a
1631 // number greater than 2147483647". C90 requires that the line # be <= 32767.
1632 unsigned LineLimit = 32768U;
1633 if (LangOpts.C99 || LangOpts.CPlusPlus11)
1634 LineLimit = 2147483648U;
1635 if (LineNo >= LineLimit)
1636 Diag(Tok: DigitTok, DiagID: diag::ext_pp_line_too_big) << LineLimit;
1637 else if (LangOpts.CPlusPlus11 && LineNo >= 32768U)
1638 Diag(Tok: DigitTok, DiagID: diag::warn_cxx98_compat_pp_line_too_big);
1639
1640 int FilenameID = -1;
1641 Token StrTok;
1642 LexHeaderName(Result&: StrTok);
1643
1644 // If the StrTok is "eod", then it wasn't present. Otherwise, it must be a
1645 // string followed by eod.
1646 if (StrTok.is(K: tok::eod))
1647 ; // ok
1648 else if (StrTok.isNot(K: tok::header_name)) {
1649 Diag(Tok: StrTok, DiagID: diag::err_pp_line_invalid_filename);
1650 DiscardUntilEndOfDirective();
1651 return;
1652 } else {
1653 SmallString<128> FilenameBuffer;
1654 StringRef Filename = getSpelling(Tok: StrTok, Buffer&: FilenameBuffer);
1655 GetLineDirectiveFilenameSpelling(Loc: StrTok.getLocation(), Buffer&: Filename);
1656 FilenameID = SourceMgr.getLineTableFilenameID(Str: Filename);
1657
1658 // Verify that there is nothing after the string, other than EOD. Because
1659 // of C99 6.10.4p5, macros that expand to empty tokens are ok.
1660 CheckEndOfDirective(DirType: "line", EnableMacros: true);
1661 }
1662
1663 // Take the file kind of the file containing the #line directive. #line
1664 // directives are often used for generated sources from the same codebase, so
1665 // the new file should generally be classified the same way as the current
1666 // file. This is visible in GCC's pre-processed output, which rewrites #line
1667 // to GNU line markers.
1668 SrcMgr::CharacteristicKind FileKind =
1669 SourceMgr.getFileCharacteristic(Loc: DigitTok.getLocation());
1670
1671 SourceMgr.AddLineNote(Loc: DigitTok.getLocation(), LineNo, FilenameID, IsFileEntry: false,
1672 IsFileExit: false, FileKind);
1673
1674 if (Callbacks)
1675 Callbacks->FileChanged(Loc: CurPPLexer->getSourceLocation(),
1676 Reason: PPCallbacks::RenameFile, FileType: FileKind);
1677}
1678
1679/// ReadLineMarkerFlags - Parse and validate any flags at the end of a GNU line
1680/// marker directive.
1681static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
1682 SrcMgr::CharacteristicKind &FileKind,
1683 Preprocessor &PP) {
1684 unsigned FlagVal;
1685 Token FlagTok;
1686 PP.Lex(Result&: FlagTok);
1687 if (FlagTok.is(K: tok::eod)) return false;
1688 if (GetLineValue(DigitTok&: FlagTok, Val&: FlagVal, DiagID: diag::err_pp_linemarker_invalid_flag, PP))
1689 return true;
1690
1691 if (FlagVal == 1) {
1692 IsFileEntry = true;
1693
1694 PP.Lex(Result&: FlagTok);
1695 if (FlagTok.is(K: tok::eod)) return false;
1696 if (GetLineValue(DigitTok&: FlagTok, Val&: FlagVal, DiagID: diag::err_pp_linemarker_invalid_flag,PP))
1697 return true;
1698 } else if (FlagVal == 2) {
1699 IsFileExit = true;
1700
1701 SourceManager &SM = PP.getSourceManager();
1702 // If we are leaving the current presumed file, check to make sure the
1703 // presumed include stack isn't empty!
1704 FileID CurFileID =
1705 SM.getDecomposedExpansionLoc(Loc: FlagTok.getLocation()).first;
1706 PresumedLoc PLoc = SM.getPresumedLoc(Loc: FlagTok.getLocation());
1707 if (PLoc.isInvalid())
1708 return true;
1709
1710 // If there is no include loc (main file) or if the include loc is in a
1711 // different physical file, then we aren't in a "1" line marker flag region.
1712 SourceLocation IncLoc = PLoc.getIncludeLoc();
1713 if (IncLoc.isInvalid() ||
1714 SM.getDecomposedExpansionLoc(Loc: IncLoc).first != CurFileID) {
1715 PP.Diag(Tok: FlagTok, DiagID: diag::err_pp_linemarker_invalid_pop);
1716 PP.DiscardUntilEndOfDirective();
1717 return true;
1718 }
1719
1720 PP.Lex(Result&: FlagTok);
1721 if (FlagTok.is(K: tok::eod)) return false;
1722 if (GetLineValue(DigitTok&: FlagTok, Val&: FlagVal, DiagID: diag::err_pp_linemarker_invalid_flag,PP))
1723 return true;
1724 }
1725
1726 // We must have 3 if there are still flags.
1727 if (FlagVal != 3) {
1728 PP.Diag(Tok: FlagTok, DiagID: diag::err_pp_linemarker_invalid_flag);
1729 PP.DiscardUntilEndOfDirective();
1730 return true;
1731 }
1732
1733 FileKind = SrcMgr::C_System;
1734
1735 PP.Lex(Result&: FlagTok);
1736 if (FlagTok.is(K: tok::eod)) return false;
1737 if (GetLineValue(DigitTok&: FlagTok, Val&: FlagVal, DiagID: diag::err_pp_linemarker_invalid_flag, PP))
1738 return true;
1739
1740 // We must have 4 if there is yet another flag.
1741 if (FlagVal != 4) {
1742 PP.Diag(Tok: FlagTok, DiagID: diag::err_pp_linemarker_invalid_flag);
1743 PP.DiscardUntilEndOfDirective();
1744 return true;
1745 }
1746
1747 FileKind = SrcMgr::C_ExternCSystem;
1748
1749 PP.Lex(Result&: FlagTok);
1750 if (FlagTok.is(K: tok::eod)) return false;
1751
1752 // There are no more valid flags here.
1753 PP.Diag(Tok: FlagTok, DiagID: diag::err_pp_linemarker_invalid_flag);
1754 PP.DiscardUntilEndOfDirective();
1755 return true;
1756}
1757
1758/// HandleDigitDirective - Handle a GNU line marker directive, whose syntax is
1759/// one of the following forms:
1760///
1761/// # 42
1762/// # 42 "file" ('1' | '2')?
1763/// # 42 "file" ('1' | '2')? '3' '4'?
1764///
1765void Preprocessor::HandleDigitDirective(Token &DigitTok) {
1766 // Validate the number and convert it to an unsigned. GNU does not have a
1767 // line # limit other than it fit in 32-bits.
1768 unsigned LineNo;
1769 if (GetLineValue(DigitTok, Val&: LineNo, DiagID: diag::err_pp_linemarker_requires_integer,
1770 PP&: *this, IsGNULineDirective: true))
1771 return;
1772
1773 Token StrTok;
1774 LexHeaderName(Result&: StrTok);
1775
1776 bool IsFileEntry = false, IsFileExit = false;
1777 int FilenameID = -1;
1778 SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User;
1779
1780 // If the StrTok is "eod", then it wasn't present. Otherwise, it must be a
1781 // string followed by eod.
1782 if (StrTok.is(K: tok::eod)) {
1783 Diag(Tok: StrTok, DiagID: diag::ext_pp_gnu_line_directive);
1784 // Treat this like "#line NN", which doesn't change file characteristics.
1785 FileKind = SourceMgr.getFileCharacteristic(Loc: DigitTok.getLocation());
1786 } else if (StrTok.isNot(K: tok::header_name)) {
1787 Diag(Tok: StrTok, DiagID: diag::err_pp_linemarker_invalid_filename);
1788 DiscardUntilEndOfDirective();
1789 return;
1790 } else {
1791 SmallString<128> FilenameBuffer;
1792 StringRef Filename = getSpelling(Tok: StrTok, Buffer&: FilenameBuffer);
1793 GetLineDirectiveFilenameSpelling(Loc: StrTok.getLocation(), Buffer&: Filename);
1794 // If a filename was present, read any flags that are present.
1795 if (ReadLineMarkerFlags(IsFileEntry, IsFileExit, FileKind, PP&: *this))
1796 return;
1797 if (!SourceMgr.isInPredefinedFile(Loc: DigitTok.getLocation()))
1798 Diag(Tok: StrTok, DiagID: diag::ext_pp_gnu_line_directive);
1799
1800 // Exiting to an empty string means pop to the including file, so leave
1801 // FilenameID as -1 in that case.
1802 if (!(IsFileExit && Filename.empty()))
1803 FilenameID = SourceMgr.getLineTableFilenameID(Str: Filename);
1804 }
1805
1806 // Create a line note with this information.
1807 SourceMgr.AddLineNote(Loc: DigitTok.getLocation(), LineNo, FilenameID, IsFileEntry,
1808 IsFileExit, FileKind);
1809
1810 // If the preprocessor has callbacks installed, notify them of the #line
1811 // change. This is used so that the line marker comes out in -E mode for
1812 // example.
1813 if (Callbacks) {
1814 PPCallbacks::FileChangeReason Reason = PPCallbacks::RenameFile;
1815 if (IsFileEntry)
1816 Reason = PPCallbacks::EnterFile;
1817 else if (IsFileExit)
1818 Reason = PPCallbacks::ExitFile;
1819
1820 Callbacks->FileChanged(Loc: CurPPLexer->getSourceLocation(), Reason, FileType: FileKind);
1821 }
1822}
1823
1824/// HandleUserDiagnosticDirective - Handle a #warning or #error directive.
1825///
1826void Preprocessor::HandleUserDiagnosticDirective(Token &Tok,
1827 bool isWarning) {
1828 // Read the rest of the line raw. We do this because we don't want macros
1829 // to be expanded and we don't require that the tokens be valid preprocessing
1830 // tokens. For example, this is allowed: "#warning ` 'foo". GCC does
1831 // collapse multiple consecutive white space between tokens, but this isn't
1832 // specified by the standard.
1833 SmallString<128> Message;
1834 CurLexer->ReadToEndOfLine(Result: &Message);
1835
1836 // Find the first non-whitespace character, so that we can make the
1837 // diagnostic more succinct.
1838 StringRef Msg = Message.str().ltrim(Char: ' ');
1839
1840 if (isWarning)
1841 Diag(Tok, DiagID: diag::pp_hash_warning) << Msg;
1842 else
1843 Diag(Tok, DiagID: diag::err_pp_hash_error) << Msg;
1844}
1845
1846/// HandleIdentSCCSDirective - Handle a #ident/#sccs directive.
1847///
1848void Preprocessor::HandleIdentSCCSDirective(Token &Tok) {
1849 // Yes, this directive is an extension.
1850 Diag(Tok, DiagID: diag::ext_pp_ident_directive);
1851
1852 // Read the string argument.
1853 Token StrTok;
1854 Lex(Result&: StrTok);
1855
1856 // If the token kind isn't a string, it's a malformed directive.
1857 if (StrTok.isNot(K: tok::string_literal) &&
1858 StrTok.isNot(K: tok::wide_string_literal)) {
1859 Diag(Tok: StrTok, DiagID: diag::err_pp_malformed_ident);
1860 if (StrTok.isNot(K: tok::eod))
1861 DiscardUntilEndOfDirective();
1862 return;
1863 }
1864
1865 if (StrTok.hasUDSuffix()) {
1866 Diag(Tok: StrTok, DiagID: diag::err_invalid_string_udl);
1867 DiscardUntilEndOfDirective();
1868 return;
1869 }
1870
1871 // Verify that there is nothing after the string, other than EOD.
1872 CheckEndOfDirective(DirType: "ident");
1873
1874 if (Callbacks) {
1875 bool Invalid = false;
1876 std::string Str = getSpelling(Tok: StrTok, Invalid: &Invalid);
1877 if (!Invalid)
1878 Callbacks->Ident(Loc: Tok.getLocation(), str: Str);
1879 }
1880}
1881
1882/// Handle a #public directive.
1883void Preprocessor::HandleMacroPublicDirective(Token &Tok) {
1884 Token MacroNameTok;
1885 ReadMacroName(MacroNameTok, isDefineUndef: MU_Undef);
1886
1887 // Error reading macro name? If so, diagnostic already issued.
1888 if (MacroNameTok.is(K: tok::eod))
1889 return;
1890
1891 // Check to see if this is the last token on the #__public_macro line.
1892 CheckEndOfDirective(DirType: "__public_macro");
1893
1894 IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
1895 // Okay, we finally have a valid identifier to undef.
1896 MacroDirective *MD = getLocalMacroDirective(II);
1897
1898 // If the macro is not defined, this is an error.
1899 if (!MD) {
1900 Diag(Tok: MacroNameTok, DiagID: diag::err_pp_visibility_non_macro) << II;
1901 return;
1902 }
1903
1904 // Note that this macro has now been exported.
1905 appendMacroDirective(II, MD: AllocateVisibilityMacroDirective(
1906 Loc: MacroNameTok.getLocation(), /*isPublic=*/true));
1907}
1908
1909/// Handle a #private directive.
1910void Preprocessor::HandleMacroPrivateDirective() {
1911 Token MacroNameTok;
1912 ReadMacroName(MacroNameTok, isDefineUndef: MU_Undef);
1913
1914 // Error reading macro name? If so, diagnostic already issued.
1915 if (MacroNameTok.is(K: tok::eod))
1916 return;
1917
1918 // Check to see if this is the last token on the #__private_macro line.
1919 CheckEndOfDirective(DirType: "__private_macro");
1920
1921 IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
1922 // Okay, we finally have a valid identifier to undef.
1923 MacroDirective *MD = getLocalMacroDirective(II);
1924
1925 // If the macro is not defined, this is an error.
1926 if (!MD) {
1927 Diag(Tok: MacroNameTok, DiagID: diag::err_pp_visibility_non_macro) << II;
1928 return;
1929 }
1930
1931 // Note that this macro has now been marked private.
1932 appendMacroDirective(II, MD: AllocateVisibilityMacroDirective(
1933 Loc: MacroNameTok.getLocation(), /*isPublic=*/false));
1934}
1935
1936//===----------------------------------------------------------------------===//
1937// Preprocessor Include Directive Handling.
1938//===----------------------------------------------------------------------===//
1939
1940/// GetIncludeFilenameSpelling - Turn the specified lexer token into a fully
1941/// checked and spelled filename, e.g. as an operand of \#include. This returns
1942/// true if the input filename was in <>'s or false if it were in ""'s. The
1943/// caller is expected to provide a buffer that is large enough to hold the
1944/// spelling of the filename, but is also expected to handle the case when
1945/// this method decides to use a different buffer.
1946bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
1947 StringRef &Buffer) {
1948 // Get the text form of the filename.
1949 assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
1950
1951 // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and
1952 // C++20 [lex.header]/2:
1953 //
1954 // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then
1955 // in C: behavior is undefined
1956 // in C++: program is conditionally-supported with implementation-defined
1957 // semantics
1958
1959 // Make sure the filename is <x> or "x".
1960 bool isAngled;
1961 if (Buffer[0] == '<') {
1962 if (Buffer.back() != '>') {
1963 Diag(Loc, DiagID: diag::err_pp_expects_filename);
1964 Buffer = StringRef();
1965 return true;
1966 }
1967 isAngled = true;
1968 } else if (Buffer[0] == '"') {
1969 if (Buffer.back() != '"') {
1970 Diag(Loc, DiagID: diag::err_pp_expects_filename);
1971 Buffer = StringRef();
1972 return true;
1973 }
1974 isAngled = false;
1975 } else {
1976 Diag(Loc, DiagID: diag::err_pp_expects_filename);
1977 Buffer = StringRef();
1978 return true;
1979 }
1980
1981 // Diagnose #include "" as invalid.
1982 if (Buffer.size() <= 2) {
1983 Diag(Loc, DiagID: diag::err_pp_empty_filename);
1984 Buffer = StringRef();
1985 return true;
1986 }
1987
1988 // Skip the brackets.
1989 Buffer = Buffer.substr(Start: 1, N: Buffer.size()-2);
1990 return isAngled;
1991}
1992
1993void Preprocessor::GetLineDirectiveFilenameSpelling(SourceLocation Loc,
1994 StringRef &Buffer) {
1995 // Get the text form of the filename.
1996 assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
1997 if (Buffer.size() < 2 || Buffer.front() != '"' || Buffer.back() != '"') {
1998 Diag(Loc, DiagID: diag::err_pp_line_invalid_filename);
1999 Buffer = StringRef();
2000 return;
2001 }
2002 Buffer = Buffer.substr(Start: 1, N: Buffer.size() - 2);
2003}
2004
2005/// Push a token onto the token stream containing an annotation.
2006void Preprocessor::EnterAnnotationToken(SourceRange Range,
2007 tok::TokenKind Kind,
2008 void *AnnotationVal) {
2009 // FIXME: Produce this as the current token directly, rather than
2010 // allocating a new token for it.
2011 auto Tok = std::make_unique<Token[]>(num: 1);
2012 Tok[0] = Token::createAnnotation(Kind, Range, Value: AnnotationVal);
2013 EnterTokenStream(Toks: std::move(Tok), NumToks: 1, DisableMacroExpansion: true, /*IsReinject*/ false);
2014}
2015
2016/// Produce a diagnostic informing the user that a #include or similar
2017/// was implicitly treated as a module import.
2018static void diagnoseAutoModuleImport(Preprocessor &PP, SourceLocation HashLoc,
2019 Token &IncludeTok,
2020 ArrayRef<IdentifierLoc> Path,
2021 SourceLocation PathEnd) {
2022 SmallString<128> PathString;
2023 for (size_t I = 0, N = Path.size(); I != N; ++I) {
2024 if (I)
2025 PathString += '.';
2026 PathString += Path[I].getIdentifierInfo()->getName();
2027 }
2028
2029 int IncludeKind = 0;
2030 switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
2031 case tok::pp_include:
2032 IncludeKind = 0;
2033 break;
2034
2035 case tok::pp_import:
2036 IncludeKind = 1;
2037 break;
2038
2039 case tok::pp_include_next:
2040 IncludeKind = 2;
2041 break;
2042
2043 case tok::pp___include_macros:
2044 IncludeKind = 3;
2045 break;
2046
2047 default:
2048 llvm_unreachable("unknown include directive kind");
2049 }
2050
2051 PP.Diag(Loc: HashLoc, DiagID: diag::remark_pp_include_directive_modular_translation)
2052 << IncludeKind << PathString;
2053}
2054
2055// Given a vector of path components and a string containing the real
2056// path to the file, build a properly-cased replacement in the vector,
2057// and return true if the replacement should be suggested.
2058static bool trySimplifyPath(SmallVectorImpl<StringRef> &Components,
2059 StringRef RealPathName,
2060 llvm::sys::path::Style Separator) {
2061 auto RealPathComponentIter = llvm::sys::path::rbegin(path: RealPathName);
2062 auto RealPathComponentEnd = llvm::sys::path::rend(path: RealPathName);
2063 int Cnt = 0;
2064 bool SuggestReplacement = false;
2065
2066 auto IsSep = [Separator](StringRef Component) {
2067 return Component.size() == 1 &&
2068 llvm::sys::path::is_separator(value: Component[0], style: Separator);
2069 };
2070
2071 // Below is a best-effort to handle ".." in paths. It is admittedly
2072 // not 100% correct in the presence of symlinks.
2073 for (auto &Component : llvm::reverse(C&: Components)) {
2074 if ("." == Component) {
2075 } else if (".." == Component) {
2076 ++Cnt;
2077 } else if (Cnt) {
2078 --Cnt;
2079 } else if (RealPathComponentIter != RealPathComponentEnd) {
2080 if (!IsSep(Component) && !IsSep(*RealPathComponentIter) &&
2081 Component != *RealPathComponentIter) {
2082 // If these non-separator path components differ by more than just case,
2083 // then we may be looking at symlinked paths. Bail on this diagnostic to
2084 // avoid noisy false positives.
2085 SuggestReplacement =
2086 RealPathComponentIter->equals_insensitive(RHS: Component);
2087 if (!SuggestReplacement)
2088 break;
2089 Component = *RealPathComponentIter;
2090 }
2091 ++RealPathComponentIter;
2092 }
2093 }
2094 return SuggestReplacement;
2095}
2096
2097bool Preprocessor::checkModuleIsAvailable(const LangOptions &LangOpts,
2098 const TargetInfo &TargetInfo,
2099 const Module &M,
2100 DiagnosticsEngine &Diags) {
2101 Module::Requirement Requirement;
2102 Module::UnresolvedHeaderDirective MissingHeader;
2103 Module *ShadowingModule = nullptr;
2104 if (M.isAvailable(LangOpts, Target: TargetInfo, Req&: Requirement, MissingHeader,
2105 ShadowingModule))
2106 return false;
2107
2108 if (MissingHeader.FileNameLoc.isValid()) {
2109 Diags.Report(Loc: MissingHeader.FileNameLoc, DiagID: diag::err_module_header_missing)
2110 << MissingHeader.IsUmbrella << MissingHeader.FileName;
2111 } else if (ShadowingModule) {
2112 Diags.Report(Loc: M.DefinitionLoc, DiagID: diag::err_module_shadowed) << M.Name;
2113 Diags.Report(Loc: ShadowingModule->DefinitionLoc,
2114 DiagID: diag::note_previous_definition);
2115 } else {
2116 // FIXME: Track the location at which the requirement was specified, and
2117 // use it here.
2118 Diags.Report(Loc: M.DefinitionLoc, DiagID: diag::err_module_unavailable)
2119 << M.getFullModuleName() << Requirement.RequiredState
2120 << Requirement.FeatureName;
2121 }
2122 return true;
2123}
2124
2125std::pair<ConstSearchDirIterator, const FileEntry *>
2126Preprocessor::getIncludeNextStart(const Token &IncludeNextTok) const {
2127 // #include_next is like #include, except that we start searching after
2128 // the current found directory. If we can't do this, issue a
2129 // diagnostic.
2130 ConstSearchDirIterator Lookup = CurDirLookup;
2131 const FileEntry *LookupFromFile = nullptr;
2132
2133 if (isInPrimaryFile() && LangOpts.IsHeaderFile) {
2134 // If the main file is a header, then it's either for PCH/AST generation,
2135 // or libclang opened it. Either way, handle it as a normal include below
2136 // and do not complain about include_next.
2137 } else if (isInPrimaryFile()) {
2138 Lookup = nullptr;
2139 Diag(Tok: IncludeNextTok, DiagID: diag::pp_include_next_in_primary);
2140 } else if (CurLexerSubmodule) {
2141 // Start looking up in the directory *after* the one in which the current
2142 // file would be found, if any.
2143 assert(CurPPLexer && "#include_next directive in macro?");
2144 if (auto FE = CurPPLexer->getFileEntry())
2145 LookupFromFile = *FE;
2146 Lookup = nullptr;
2147 } else if (!Lookup) {
2148 // The current file was not found by walking the include path. Either it
2149 // is the primary file (handled above), or it was found by absolute path,
2150 // or it was found relative to such a file.
2151 // FIXME: Track enough information so we know which case we're in.
2152 Diag(Tok: IncludeNextTok, DiagID: diag::pp_include_next_absolute_path);
2153 } else {
2154 // Start looking up in the next directory.
2155 ++Lookup;
2156 }
2157
2158 return {Lookup, LookupFromFile};
2159}
2160
2161/// HandleIncludeDirective - The "\#include" tokens have just been read, read
2162/// the file to be included from the lexer, then include it! This is a common
2163/// routine with functionality shared between \#include, \#include_next and
2164/// \#import. LookupFrom is set when this is a \#include_next directive, it
2165/// specifies the file to start searching from.
2166void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
2167 Token &IncludeTok,
2168 ConstSearchDirIterator LookupFrom,
2169 const FileEntry *LookupFromFile) {
2170 Token FilenameTok;
2171 if (LexHeaderName(Result&: FilenameTok))
2172 return;
2173
2174 if (FilenameTok.isNot(K: tok::header_name)) {
2175 if (FilenameTok.is(K: tok::identifier) &&
2176 (PPOpts.SingleFileParseMode || PPOpts.SingleModuleParseMode)) {
2177 // If we saw #include IDENTIFIER and lexing didn't turn in into a header
2178 // name, it was undefined. In 'single-{file,module}-parse' mode, just skip
2179 // the directive without emitting diagnostics - the identifier might be
2180 // normally defined in previously-skipped include directive.
2181 DiscardUntilEndOfDirective();
2182 return;
2183 }
2184
2185 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_pp_expects_filename);
2186 if (FilenameTok.isNot(K: tok::eod))
2187 DiscardUntilEndOfDirective();
2188 return;
2189 }
2190
2191 // Verify that there is nothing after the filename, other than EOD. Note
2192 // that we allow macros that expand to nothing after the filename, because
2193 // this falls into the category of "#include pp-tokens new-line" specified
2194 // in C99 6.10.2p4.
2195 SourceLocation EndLoc =
2196 CheckEndOfDirective(DirType: IncludeTok.getIdentifierInfo()->getNameStart(), EnableMacros: true);
2197
2198 auto Action = HandleHeaderIncludeOrImport(HashLoc, IncludeTok, FilenameTok,
2199 EndLoc, LookupFrom, LookupFromFile);
2200 switch (Action.Kind) {
2201 case ImportAction::None:
2202 case ImportAction::SkippedModuleImport:
2203 break;
2204 case ImportAction::ModuleBegin:
2205 EnterAnnotationToken(Range: SourceRange(HashLoc, EndLoc),
2206 Kind: tok::annot_module_begin, AnnotationVal: Action.ModuleForHeader);
2207 break;
2208 case ImportAction::HeaderUnitImport:
2209 EnterAnnotationToken(Range: SourceRange(HashLoc, EndLoc), Kind: tok::annot_header_unit,
2210 AnnotationVal: Action.ModuleForHeader);
2211 break;
2212 case ImportAction::ModuleImport:
2213 EnterAnnotationToken(Range: SourceRange(HashLoc, EndLoc),
2214 Kind: tok::annot_module_include, AnnotationVal: Action.ModuleForHeader);
2215 break;
2216 case ImportAction::Failure:
2217 assert(TheModuleLoader.HadFatalFailure &&
2218 "This should be an early exit only to a fatal error");
2219 TheModuleLoader.HadFatalFailure = true;
2220 IncludeTok.setKind(tok::eof);
2221 CurLexer->cutOffLexing();
2222 return;
2223 }
2224}
2225
2226OptionalFileEntryRef Preprocessor::LookupHeaderIncludeOrImport(
2227 ConstSearchDirIterator *CurDir, StringRef &Filename,
2228 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2229 const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2230 bool &IsMapped, ConstSearchDirIterator LookupFrom,
2231 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2232 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2233 ModuleMap::KnownHeader &SuggestedModule, bool isAngled) {
2234 auto DiagnoseHeaderInclusion = [&](FileEntryRef FE) {
2235 if (LangOpts.AsmPreprocessor)
2236 return;
2237
2238 Module *RequestingModule = getModuleForLocation(
2239 Loc: FilenameLoc, AllowTextual: LangOpts.ModulesValidateTextualHeaderIncludes);
2240 bool RequestingModuleIsModuleInterface =
2241 !SourceMgr.isInMainFile(Loc: FilenameLoc);
2242
2243 HeaderInfo.getModuleMap().diagnoseHeaderInclusion(
2244 RequestingModule, RequestingModuleIsModuleInterface, FilenameLoc,
2245 Filename, File: FE);
2246 };
2247
2248 OptionalFileEntryRef File = LookupFile(
2249 FilenameLoc, Filename: LookupFilename, isAngled, FromDir: LookupFrom, FromFile: LookupFromFile, CurDirArg: CurDir,
2250 SearchPath: Callbacks ? &SearchPath : nullptr, RelativePath: Callbacks ? &RelativePath : nullptr,
2251 SuggestedModule: &SuggestedModule, IsMapped: &IsMapped, IsFrameworkFound: &IsFrameworkFound);
2252 if (File) {
2253 DiagnoseHeaderInclusion(*File);
2254 return File;
2255 }
2256
2257 // Give the clients a chance to silently skip this include.
2258 if (Callbacks && Callbacks->FileNotFound(FileName: Filename))
2259 return std::nullopt;
2260
2261 if (SuppressIncludeNotFoundError)
2262 return std::nullopt;
2263
2264 // If the file could not be located and it was included via angle
2265 // brackets, we can attempt a lookup as though it were a quoted path to
2266 // provide the user with a possible fixit.
2267 if (isAngled) {
2268 OptionalFileEntryRef File = LookupFile(
2269 FilenameLoc, Filename: LookupFilename, isAngled: false, FromDir: LookupFrom, FromFile: LookupFromFile, CurDirArg: CurDir,
2270 SearchPath: Callbacks ? &SearchPath : nullptr, RelativePath: Callbacks ? &RelativePath : nullptr,
2271 SuggestedModule: &SuggestedModule, IsMapped: &IsMapped,
2272 /*IsFrameworkFound=*/nullptr);
2273 if (File) {
2274 DiagnoseHeaderInclusion(*File);
2275 Diag(Tok: FilenameTok, DiagID: diag::err_pp_file_not_found_angled_include_not_fatal)
2276 << Filename << IsImportDecl
2277 << FixItHint::CreateReplacement(RemoveRange: FilenameRange,
2278 Code: "\"" + Filename.str() + "\"");
2279 return File;
2280 }
2281 }
2282
2283 // Check for likely typos due to leading or trailing non-isAlphanumeric
2284 // characters
2285 StringRef OriginalFilename = Filename;
2286 if (LangOpts.SpellChecking) {
2287 // A heuristic to correct a typo file name by removing leading and
2288 // trailing non-isAlphanumeric characters.
2289 auto CorrectTypoFilename = [](llvm::StringRef Filename) {
2290 Filename = Filename.drop_until(F: isAlphanumeric);
2291 while (!Filename.empty() && !isAlphanumeric(c: Filename.back())) {
2292 Filename = Filename.drop_back();
2293 }
2294 return Filename;
2295 };
2296 StringRef TypoCorrectionName = CorrectTypoFilename(Filename);
2297 StringRef TypoCorrectionLookupName = CorrectTypoFilename(LookupFilename);
2298
2299 OptionalFileEntryRef File = LookupFile(
2300 FilenameLoc, Filename: TypoCorrectionLookupName, isAngled, FromDir: LookupFrom,
2301 FromFile: LookupFromFile, CurDirArg: CurDir, SearchPath: Callbacks ? &SearchPath : nullptr,
2302 RelativePath: Callbacks ? &RelativePath : nullptr, SuggestedModule: &SuggestedModule, IsMapped: &IsMapped,
2303 /*IsFrameworkFound=*/nullptr);
2304 if (File) {
2305 DiagnoseHeaderInclusion(*File);
2306 auto Hint =
2307 isAngled ? FixItHint::CreateReplacement(
2308 RemoveRange: FilenameRange, Code: "<" + TypoCorrectionName.str() + ">")
2309 : FixItHint::CreateReplacement(
2310 RemoveRange: FilenameRange, Code: "\"" + TypoCorrectionName.str() + "\"");
2311 Diag(Tok: FilenameTok, DiagID: diag::err_pp_file_not_found_typo_not_fatal)
2312 << OriginalFilename << TypoCorrectionName << Hint;
2313 // We found the file, so set the Filename to the name after typo
2314 // correction.
2315 Filename = TypoCorrectionName;
2316 LookupFilename = TypoCorrectionLookupName;
2317 return File;
2318 }
2319 }
2320
2321 // If the file is still not found, just go with the vanilla diagnostic
2322 assert(!File && "expected missing file");
2323 Diag(Tok: FilenameTok, DiagID: diag::err_pp_file_not_found)
2324 << OriginalFilename << FilenameRange;
2325 if (IsFrameworkFound) {
2326 size_t SlashPos = OriginalFilename.find(C: '/');
2327 assert(SlashPos != StringRef::npos &&
2328 "Include with framework name should have '/' in the filename");
2329 StringRef FrameworkName = OriginalFilename.substr(Start: 0, N: SlashPos);
2330 FrameworkCacheEntry &CacheEntry =
2331 HeaderInfo.LookupFrameworkCache(FWName: FrameworkName);
2332 assert(CacheEntry.Directory && "Found framework should be in cache");
2333 Diag(Tok: FilenameTok, DiagID: diag::note_pp_framework_without_header)
2334 << OriginalFilename.substr(Start: SlashPos + 1) << FrameworkName
2335 << CacheEntry.Directory->getName();
2336 }
2337
2338 return std::nullopt;
2339}
2340
2341/// Handle either a #include-like directive or an import declaration that names
2342/// a header file.
2343///
2344/// \param HashLoc The location of the '#' token for an include, or
2345/// SourceLocation() for an import declaration.
2346/// \param IncludeTok The include / include_next / import token.
2347/// \param FilenameTok The header-name token.
2348/// \param EndLoc The location at which any imported macros become visible.
2349/// \param LookupFrom For #include_next, the starting directory for the
2350/// directory lookup.
2351/// \param LookupFromFile For #include_next, the starting file for the directory
2352/// lookup.
2353Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
2354 SourceLocation HashLoc, Token &IncludeTok, Token &FilenameTok,
2355 SourceLocation EndLoc, ConstSearchDirIterator LookupFrom,
2356 const FileEntry *LookupFromFile) {
2357 SmallString<128> FilenameBuffer;
2358 StringRef Filename = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
2359 SourceLocation CharEnd = FilenameTok.getEndLoc();
2360
2361 CharSourceRange FilenameRange
2362 = CharSourceRange::getCharRange(B: FilenameTok.getLocation(), E: CharEnd);
2363 StringRef OriginalFilename = Filename;
2364 bool isAngled =
2365 GetIncludeFilenameSpelling(Loc: FilenameTok.getLocation(), Buffer&: Filename);
2366
2367 // If GetIncludeFilenameSpelling set the start ptr to null, there was an
2368 // error.
2369 if (Filename.empty())
2370 return {ImportAction::None};
2371 if (Filename.ends_with(Suffix: ' ') || Filename.ends_with(Suffix: '.')) {
2372 unsigned Selection = Filename.ends_with(Suffix: '.') ? 1 : 0;
2373 Diag(Tok: FilenameTok, DiagID: diag::pp_nonportable_path_trailing)
2374 << Filename << Selection;
2375 }
2376
2377 bool IsImportDecl = HashLoc.isInvalid();
2378 SourceLocation StartLoc = IsImportDecl ? IncludeTok.getLocation() : HashLoc;
2379
2380 // Complain about attempts to #include files in an audit pragma.
2381 if (PragmaARCCFCodeAuditedInfo.getLoc().isValid()) {
2382 Diag(Loc: StartLoc, DiagID: diag::err_pp_include_in_arc_cf_code_audited) << IsImportDecl;
2383 Diag(Loc: PragmaARCCFCodeAuditedInfo.getLoc(), DiagID: diag::note_pragma_entered_here);
2384
2385 // Immediately leave the pragma.
2386 PragmaARCCFCodeAuditedInfo = IdentifierLoc();
2387 }
2388
2389 // Complain about attempts to #include files in an assume-nonnull pragma.
2390 if (PragmaAssumeNonNullLoc.isValid()) {
2391 Diag(Loc: StartLoc, DiagID: diag::err_pp_include_in_assume_nonnull) << IsImportDecl;
2392 Diag(Loc: PragmaAssumeNonNullLoc, DiagID: diag::note_pragma_entered_here);
2393
2394 // Immediately leave the pragma.
2395 PragmaAssumeNonNullLoc = SourceLocation();
2396 }
2397
2398 if (HeaderInfo.HasIncludeAliasMap()) {
2399 // Map the filename with the brackets still attached. If the name doesn't
2400 // map to anything, fall back on the filename we've already gotten the
2401 // spelling for.
2402 StringRef NewName = HeaderInfo.MapHeaderToIncludeAlias(Source: OriginalFilename);
2403 if (!NewName.empty())
2404 Filename = NewName;
2405 }
2406
2407 // Search include directories.
2408 bool IsMapped = false;
2409 bool IsFrameworkFound = false;
2410 ConstSearchDirIterator CurDir = nullptr;
2411 SmallString<1024> SearchPath;
2412 SmallString<1024> RelativePath;
2413 // We get the raw path only if we have 'Callbacks' to which we later pass
2414 // the path.
2415 ModuleMap::KnownHeader SuggestedModule;
2416 SourceLocation FilenameLoc = FilenameTok.getLocation();
2417 StringRef LookupFilename = Filename;
2418
2419 // Normalize slashes when compiling with -fms-extensions on non-Windows. This
2420 // is unnecessary on Windows since the filesystem there handles backslashes.
2421 SmallString<128> NormalizedPath;
2422 llvm::sys::path::Style BackslashStyle = llvm::sys::path::Style::native;
2423 if (is_style_posix(S: BackslashStyle) && LangOpts.MicrosoftExt) {
2424 NormalizedPath = Filename.str();
2425 llvm::sys::path::native(path&: NormalizedPath);
2426 LookupFilename = NormalizedPath;
2427 BackslashStyle = llvm::sys::path::Style::windows;
2428 }
2429
2430 OptionalFileEntryRef File = LookupHeaderIncludeOrImport(
2431 CurDir: &CurDir, Filename, FilenameLoc, FilenameRange, FilenameTok,
2432 IsFrameworkFound, IsImportDecl, IsMapped, LookupFrom, LookupFromFile,
2433 LookupFilename, RelativePath, SearchPath, SuggestedModule, isAngled);
2434
2435 if (usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) {
2436 if (File && isPCHThroughHeader(FE: &File->getFileEntry()))
2437 SkippingUntilPCHThroughHeader = false;
2438 return {ImportAction::None};
2439 }
2440
2441 // Should we enter the source file? Set to Skip if either the source file is
2442 // known to have no effect beyond its effect on module visibility -- that is,
2443 // if it's got an include guard that is already defined, set to Import if it
2444 // is a modular header we've already built and should import.
2445
2446 // For C++20 Modules
2447 // [cpp.include]/7 If the header identified by the header-name denotes an
2448 // importable header, it is implementation-defined whether the #include
2449 // preprocessing directive is instead replaced by an import directive.
2450 // For this implementation, the translation is permitted when we are parsing
2451 // the Global Module Fragment, and not otherwise (the cases where it would be
2452 // valid to replace an include with an import are highly constrained once in
2453 // named module purview; this choice avoids considerable complexity in
2454 // determining valid cases).
2455
2456 enum { Enter, Import, Skip, IncludeLimitReached } Action = Enter;
2457 bool DependencyScanModuleImport = false;
2458
2459 if (PPOpts.SingleFileParseMode)
2460 Action = IncludeLimitReached;
2461
2462 // If we've reached the max allowed include depth, it is usually due to an
2463 // include cycle. Don't enter already processed files again as it can lead to
2464 // reaching the max allowed include depth again.
2465 if (Action == Enter && HasReachedMaxIncludeDepth && File &&
2466 alreadyIncluded(File: *File))
2467 Action = IncludeLimitReached;
2468
2469 // FIXME: We do not have a good way to disambiguate C++ clang modules from
2470 // C++ standard modules (other than use/non-use of Header Units).
2471
2472 Module *ModuleToImport = SuggestedModule.getModule();
2473
2474 bool MaybeTranslateInclude = Action == Enter && File && ModuleToImport &&
2475 !ModuleToImport->isForBuilding(LangOpts: getLangOpts());
2476
2477 // Maybe a usable Header Unit
2478 bool UsableHeaderUnit = false;
2479 if (getLangOpts().CPlusPlusModules && ModuleToImport &&
2480 ModuleToImport->isHeaderUnit()) {
2481 if (TrackGMFState.inGMF() || IsImportDecl)
2482 UsableHeaderUnit = true;
2483 else if (!IsImportDecl) {
2484 // This is a Header Unit that we do not include-translate
2485 ModuleToImport = nullptr;
2486 }
2487 }
2488 // Maybe a usable clang header module.
2489 bool UsableClangHeaderModule =
2490 (getLangOpts().CPlusPlusModules || getLangOpts().Modules) &&
2491 ModuleToImport && !ModuleToImport->isHeaderUnit();
2492
2493 // Determine whether we should try to import the module for this #include, if
2494 // there is one. Don't do so if precompiled module support is disabled or we
2495 // are processing this module textually (because we're building the module).
2496 if (MaybeTranslateInclude && (UsableHeaderUnit || UsableClangHeaderModule)) {
2497 // If this include corresponds to a module but that module is
2498 // unavailable, diagnose the situation and bail out.
2499 // FIXME: Remove this; loadModule does the same check (but produces
2500 // slightly worse diagnostics).
2501 if (checkModuleIsAvailable(LangOpts: getLangOpts(), TargetInfo: getTargetInfo(), M: *ModuleToImport,
2502 Diags&: getDiagnostics())) {
2503 Diag(Loc: FilenameTok.getLocation(),
2504 DiagID: diag::note_implicit_top_level_module_import_here)
2505 << ModuleToImport->getTopLevelModuleName();
2506 return {ImportAction::None};
2507 }
2508
2509 // Compute the module access path corresponding to this module.
2510 // FIXME: Should we have a second loadModule() overload to avoid this
2511 // extra lookup step?
2512 SmallVector<IdentifierLoc, 2> Path;
2513 for (Module *Mod = ModuleToImport; Mod; Mod = Mod->Parent)
2514 Path.emplace_back(Args: FilenameTok.getLocation(),
2515 Args: getIdentifierInfo(Name: Mod->Name));
2516 std::reverse(first: Path.begin(), last: Path.end());
2517
2518 // Warn that we're replacing the include/import with a module import.
2519 if (!IsImportDecl)
2520 diagnoseAutoModuleImport(PP&: *this, HashLoc: StartLoc, IncludeTok, Path, PathEnd: CharEnd);
2521
2522 if (PPOpts.DependencyScanningModuleMapImports &&
2523 ModuleToImport->Kind == Module::ModuleMapModule) {
2524 // Dependency scanning only needs the module name. Avoid requiring the
2525 // module file, which may not have been built yet.
2526 Action = Skip;
2527 DependencyScanModuleImport = true;
2528 } else {
2529 // Load the module to import its macros. We'll make the declarations
2530 // visible when the parser gets here.
2531 // FIXME: Pass ModuleToImport in here rather than converting it to a path
2532 // and making the module loader convert it back again.
2533 ModuleLoadResult Imported = TheModuleLoader.loadModule(
2534 ImportLoc: IncludeTok.getLocation(), Path, Visibility: Module::Hidden,
2535 /*IsInclusionDirective=*/true);
2536 assert((Imported == nullptr || Imported == ModuleToImport) &&
2537 "the imported module is different than the suggested one");
2538
2539 if (Imported) {
2540 Action = Import;
2541 } else if (Imported.isMissingExpected()) {
2542 markClangModuleAsAffecting(
2543 M: static_cast<Module *>(Imported)->getTopLevelModule());
2544 // We failed to find a submodule that we assumed would exist (because it
2545 // was in the directory of an umbrella header, for instance), but no
2546 // actual module containing it exists (because the umbrella header is
2547 // incomplete). Treat this as a textual inclusion.
2548 ModuleToImport = nullptr;
2549 UsableClangHeaderModule = false;
2550 } else if (Imported.isConfigMismatch()) {
2551 // On a configuration mismatch, enter the header textually. We still
2552 // know that it's part of the corresponding module.
2553 } else {
2554 // We hit an error processing the import. Bail out.
2555 if (hadModuleLoaderFatalFailure()) {
2556 // With a fatal failure in the module loader, we abort parsing.
2557 Token &Result = IncludeTok;
2558 assert(CurLexer && "#include but no current lexer set!");
2559 Result.startToken();
2560 CurLexer->FormTokenWithChars(Result, TokEnd: CurLexer->BufferEnd, Kind: tok::eof);
2561 CurLexer->cutOffLexing();
2562 }
2563 return {ImportAction::None};
2564 }
2565 }
2566 }
2567
2568 // The #included file will be considered to be a system header if either it is
2569 // in a system include directory, or if the #includer is a system include
2570 // header.
2571 SrcMgr::CharacteristicKind FileCharacter =
2572 SourceMgr.getFileCharacteristic(Loc: FilenameTok.getLocation());
2573 if (File)
2574 FileCharacter = std::max(a: HeaderInfo.getFileDirFlavor(File: *File), b: FileCharacter);
2575
2576 // If this is a '#import' or an import-declaration, don't re-enter the file.
2577 //
2578 // FIXME: If we have a suggested module for a '#include', and we've already
2579 // visited this file, don't bother entering it again. We know it has no
2580 // further effect.
2581 bool EnterOnce =
2582 IsImportDecl ||
2583 IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import;
2584
2585 bool IsFirstIncludeOfFile = false;
2586
2587 // Ask HeaderInfo if we should enter this #include file. If not, #including
2588 // this file will have no effect.
2589 if (Action == Enter && File &&
2590 !HeaderInfo.ShouldEnterIncludeFile(PP&: *this, File: *File, isImport: EnterOnce,
2591 ModulesEnabled: getLangOpts().Modules, M: ModuleToImport,
2592 IsFirstIncludeOfFile)) {
2593 // C++ standard modules:
2594 // If we are not in the GMF, then we textually include only
2595 // clang modules:
2596 // Even if we've already preprocessed this header once and know that we
2597 // don't need to see its contents again, we still need to import it if it's
2598 // modular because we might not have imported it from this submodule before.
2599 //
2600 // FIXME: We don't do this when compiling a PCH because the AST
2601 // serialization layer can't cope with it. This means we get local
2602 // submodule visibility semantics wrong in that case.
2603 if (UsableHeaderUnit && !getLangOpts().CompilingPCH)
2604 Action = TrackGMFState.inGMF() ? Import : Skip;
2605 else
2606 Action = (UsableClangHeaderModule && !getLangOpts().CompilingPCH) ? Import
2607 : Skip;
2608 }
2609
2610 // Check for circular inclusion of the main file.
2611 // We can't generate a consistent preamble with regard to the conditional
2612 // stack if the main file is included again as due to the preamble bounds
2613 // some directives (e.g. #endif of a header guard) will never be seen.
2614 // Since this will lead to confusing errors, avoid the inclusion.
2615 if (Action == Enter && File && PreambleConditionalStack.isRecording() &&
2616 SourceMgr.isMainFile(SourceFile: File->getFileEntry())) {
2617 Diag(Loc: FilenameTok.getLocation(),
2618 DiagID: diag::err_pp_including_mainfile_in_preamble);
2619 return {ImportAction::None};
2620 }
2621
2622 if (Callbacks && !IsImportDecl) {
2623 // Notify the callback object that we've seen an inclusion directive.
2624 // FIXME: Use a different callback for a pp-import?
2625 Callbacks->InclusionDirective(
2626 HashLoc, IncludeTok, FileName: LookupFilename, IsAngled: isAngled, FilenameRange, File,
2627 SearchPath, RelativePath, SuggestedModule: SuggestedModule.getModule(),
2628 ModuleImported: Action == Import || DependencyScanModuleImport, FileType: FileCharacter);
2629 if (Action == Skip && File)
2630 Callbacks->FileSkipped(SkippedFile: *File, FilenameTok, FileType: FileCharacter);
2631 }
2632
2633 if (!File)
2634 return {ImportAction::None};
2635
2636 // If this is a C++20 pp-import declaration, diagnose if we didn't find any
2637 // module corresponding to the named header.
2638 if (IsImportDecl && !ModuleToImport) {
2639 Diag(Tok: FilenameTok, DiagID: diag::err_header_import_not_header_unit)
2640 << OriginalFilename << File->getName();
2641 return {ImportAction::None};
2642 }
2643
2644 // Issue a diagnostic if the name of the file on disk has a different case
2645 // than the one we're about to open.
2646 const bool CheckIncludePathPortability =
2647 !IsMapped && !File->getFileEntry().tryGetRealPathName().empty();
2648
2649 if (CheckIncludePathPortability) {
2650 StringRef Name = LookupFilename;
2651 StringRef NameWithoriginalSlashes = Filename;
2652#if defined(_WIN32)
2653 // Skip UNC prefix if present. (tryGetRealPathName() always
2654 // returns a path with the prefix skipped.)
2655 bool NameWasUNC = Name.consume_front("\\\\?\\");
2656 NameWithoriginalSlashes.consume_front("\\\\?\\");
2657#endif
2658 StringRef RealPathName = File->getFileEntry().tryGetRealPathName();
2659 SmallVector<StringRef, 16> Components(llvm::sys::path::begin(path: Name),
2660 llvm::sys::path::end(path: Name));
2661#if defined(_WIN32)
2662 // -Wnonportable-include-path is designed to diagnose includes using
2663 // case even on systems with a case-insensitive file system.
2664 // On Windows, RealPathName always starts with an upper-case drive
2665 // letter for absolute paths, but Name might start with either
2666 // case depending on if `cd c:\foo` or `cd C:\foo` was used in the shell.
2667 // ("foo" will always have on-disk case, no matter which case was
2668 // used in the cd command). To not emit this warning solely for
2669 // the drive letter, whose case is dependent on if `cd` is used
2670 // with upper- or lower-case drive letters, always consider the
2671 // given drive letter case as correct for the purpose of this warning.
2672 SmallString<128> FixedDriveRealPath;
2673 if (llvm::sys::path::is_absolute(Name) &&
2674 llvm::sys::path::is_absolute(RealPathName) &&
2675 toLowercase(Name[0]) == toLowercase(RealPathName[0]) &&
2676 isLowercase(Name[0]) != isLowercase(RealPathName[0])) {
2677 assert(Components.size() >= 3 && "should have drive, backslash, name");
2678 assert(Components[0].size() == 2 && "should start with drive");
2679 assert(Components[0][1] == ':' && "should have colon");
2680 FixedDriveRealPath = (Name.substr(0, 1) + RealPathName.substr(1)).str();
2681 RealPathName = FixedDriveRealPath;
2682 }
2683#endif
2684
2685 if (trySimplifyPath(Components, RealPathName, Separator: BackslashStyle)) {
2686 SmallString<128> Path;
2687 Path.reserve(N: Name.size()+2);
2688 Path.push_back(Elt: isAngled ? '<' : '"');
2689
2690 const auto IsSep = [BackslashStyle](char c) {
2691 return llvm::sys::path::is_separator(value: c, style: BackslashStyle);
2692 };
2693
2694 for (auto Component : Components) {
2695 // On POSIX, Components will contain a single '/' as first element
2696 // exactly if Name is an absolute path.
2697 // On Windows, it will contain "C:" followed by '\' for absolute paths.
2698 // The drive letter is optional for absolute paths on Windows, but
2699 // clang currently cannot process absolute paths in #include lines that
2700 // don't have a drive.
2701 // If the first entry in Components is a directory separator,
2702 // then the code at the bottom of this loop that keeps the original
2703 // directory separator style copies it. If the second entry is
2704 // a directory separator (the C:\ case), then that separator already
2705 // got copied when the C: was processed and we want to skip that entry.
2706 if (!(Component.size() == 1 && IsSep(Component[0])))
2707 Path.append(RHS: Component);
2708 else if (Path.size() != 1)
2709 continue;
2710
2711 // Append the separator(s) the user used, or the close quote
2712 if (Path.size() > NameWithoriginalSlashes.size()) {
2713 Path.push_back(Elt: isAngled ? '>' : '"');
2714 continue;
2715 }
2716 assert(IsSep(NameWithoriginalSlashes[Path.size()-1]));
2717 do
2718 Path.push_back(Elt: NameWithoriginalSlashes[Path.size()-1]);
2719 while (Path.size() <= NameWithoriginalSlashes.size() &&
2720 IsSep(NameWithoriginalSlashes[Path.size()-1]));
2721 }
2722
2723#if defined(_WIN32)
2724 // Restore UNC prefix if it was there.
2725 if (NameWasUNC)
2726 Path = (Path.substr(0, 1) + "\\\\?\\" + Path.substr(1)).str();
2727#endif
2728
2729 // For user files and known standard headers, issue a diagnostic.
2730 // For other system headers, don't. They can be controlled separately.
2731 auto DiagId =
2732 (FileCharacter == SrcMgr::C_User || warnByDefaultOnWrongCase(Include: Name))
2733 ? diag::pp_nonportable_path
2734 : diag::pp_nonportable_system_path;
2735 Diag(Tok: FilenameTok, DiagID: DiagId) << Path <<
2736 FixItHint::CreateReplacement(RemoveRange: FilenameRange, Code: Path);
2737 }
2738
2739 bool SuppressBackslashDiag =
2740 // The diagnostic logic is expensive, so only run it if it's enabled...
2741 Diags->isIgnored(DiagID: diag::pp_nonportable_path_separator, Loc: FilenameLoc) ||
2742 // ...and try to only trigger on paths that appear in source.
2743 FilenameLoc.isMacroID() ||
2744 SourceMgr.isWrittenInBuiltinFile(Loc: FilenameLoc) ||
2745 SourceMgr.isWrittenInModuleIncludes(Loc: FilenameLoc);
2746 if (!SuppressBackslashDiag && OriginalFilename.contains(C: '\\')) {
2747 std::string SuggestedPath = OriginalFilename.str();
2748 llvm::replace(Range&: SuggestedPath, OldValue: '\\', NewValue: '/');
2749 Diag(Tok: FilenameTok, DiagID: diag::pp_nonportable_path_separator)
2750 << Name << FixItHint::CreateReplacement(RemoveRange: FilenameRange, Code: SuggestedPath);
2751 }
2752 }
2753
2754 switch (Action) {
2755 case Skip:
2756 // If we don't need to enter the file, stop now.
2757 if (ModuleToImport)
2758 return {ImportAction::SkippedModuleImport, ModuleToImport};
2759 return {ImportAction::None};
2760
2761 case IncludeLimitReached:
2762 // If we reached our include limit and don't want to enter any more files,
2763 // don't go any further.
2764 return {ImportAction::None};
2765
2766 case Import: {
2767 // If this is a module import, make it visible if needed.
2768 assert(ModuleToImport && "no module to import");
2769
2770 makeModuleVisible(M: ModuleToImport, Loc: EndLoc);
2771
2772 if (IncludeTok.getIdentifierInfo()->getPPKeywordID() ==
2773 tok::pp___include_macros)
2774 return {ImportAction::None};
2775
2776 return {ImportAction::ModuleImport, ModuleToImport};
2777 }
2778
2779 case Enter:
2780 break;
2781 }
2782
2783 // Check that we don't have infinite #include recursion.
2784 if (IncludeMacroStack.size() == MaxAllowedIncludeStackDepth-1) {
2785 Diag(Tok: FilenameTok, DiagID: diag::err_pp_include_too_deep);
2786 HasReachedMaxIncludeDepth = true;
2787 return {ImportAction::None};
2788 }
2789
2790 if (isAngled && isInNamedModule())
2791 Diag(Tok: FilenameTok, DiagID: diag::warn_pp_include_angled_in_module_purview)
2792 << getNamedModuleName();
2793
2794 // Look up the file, create a File ID for it.
2795 SourceLocation IncludePos = FilenameTok.getLocation();
2796 // If the filename string was the result of macro expansions, set the include
2797 // position on the file where it will be included and after the expansions.
2798 if (IncludePos.isMacroID())
2799 IncludePos = SourceMgr.getExpansionRange(Loc: IncludePos).getEnd();
2800 FileID FID = SourceMgr.createFileID(SourceFile: *File, IncludePos, FileCharacter);
2801 if (!FID.isValid()) {
2802 TheModuleLoader.HadFatalFailure = true;
2803 return ImportAction::Failure;
2804 }
2805
2806 // If all is good, enter the new file!
2807 if (EnterSourceFile(FID, Dir: CurDir, Loc: FilenameTok.getLocation(),
2808 IsFirstIncludeOfFile))
2809 return {ImportAction::None};
2810
2811 // Determine if we're switching to building a new submodule, and which one.
2812 // This does not apply for C++20 modules header units.
2813 if (ModuleToImport && !ModuleToImport->isHeaderUnit()) {
2814 if (ModuleToImport->getTopLevelModule()->ShadowingModule) {
2815 // We are building a submodule that belongs to a shadowed module. This
2816 // means we find header files in the shadowed module.
2817 Diag(Loc: ModuleToImport->DefinitionLoc,
2818 DiagID: diag::err_module_build_shadowed_submodule)
2819 << ModuleToImport->getFullModuleName();
2820 Diag(Loc: ModuleToImport->getTopLevelModule()->ShadowingModule->DefinitionLoc,
2821 DiagID: diag::note_previous_definition);
2822 return {ImportAction::None};
2823 }
2824 // When building a pch, -fmodule-name tells the compiler to textually
2825 // include headers in the specified module. We are not building the
2826 // specified module.
2827 //
2828 // FIXME: This is the wrong way to handle this. We should produce a PCH
2829 // that behaves the same as the header would behave in a compilation using
2830 // that PCH, which means we should enter the submodule. We need to teach
2831 // the AST serialization layer to deal with the resulting AST.
2832 if (getLangOpts().CompilingPCH &&
2833 ModuleToImport->isForBuilding(LangOpts: getLangOpts()))
2834 return {ImportAction::None};
2835
2836 assert(!CurLexerSubmodule && "should not have marked this as a module yet");
2837 CurLexerSubmodule = ModuleToImport;
2838
2839 // Let the macro handling code know that any future macros are within
2840 // the new submodule.
2841 EnterSubmodule(M: ModuleToImport, ImportLoc: EndLoc, /*ForPragma*/ false);
2842
2843 // Let the parser know that any future declarations are within the new
2844 // submodule.
2845 // FIXME: There's no point doing this if we're handling a #__include_macros
2846 // directive.
2847 return {ImportAction::ModuleBegin, ModuleToImport};
2848 }
2849
2850 assert(!IsImportDecl && "failed to diagnose missing module for import decl");
2851 return {ImportAction::None};
2852}
2853
2854/// HandleIncludeNextDirective - Implements \#include_next.
2855///
2856void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
2857 Token &IncludeNextTok) {
2858 Diag(Tok: IncludeNextTok, DiagID: diag::ext_pp_include_next_directive);
2859
2860 ConstSearchDirIterator Lookup = nullptr;
2861 const FileEntry *LookupFromFile;
2862 std::tie(args&: Lookup, args&: LookupFromFile) = getIncludeNextStart(IncludeNextTok);
2863
2864 return HandleIncludeDirective(HashLoc, IncludeTok&: IncludeNextTok, LookupFrom: Lookup,
2865 LookupFromFile);
2866}
2867
2868/// HandleMicrosoftImportDirective - Implements \#import for Microsoft Mode
2869void Preprocessor::HandleMicrosoftImportDirective(Token &Tok) {
2870 // The Microsoft #import directive takes a type library and generates header
2871 // files from it, and includes those. This is beyond the scope of what clang
2872 // does, so we ignore it and error out. However, #import can optionally have
2873 // trailing attributes that span multiple lines. We're going to eat those
2874 // so we can continue processing from there.
2875 Diag(Tok, DiagID: diag::err_pp_import_directive_ms );
2876
2877 // Read tokens until we get to the end of the directive. Note that the
2878 // directive can be split over multiple lines using the backslash character.
2879 DiscardUntilEndOfDirective();
2880}
2881
2882/// HandleImportDirective - Implements \#import.
2883///
2884void Preprocessor::HandleImportDirective(SourceLocation HashLoc,
2885 Token &ImportTok) {
2886 if (!LangOpts.ObjC) { // #import is standard for ObjC.
2887 if (LangOpts.MSVCCompat)
2888 return HandleMicrosoftImportDirective(Tok&: ImportTok);
2889 Diag(Tok: ImportTok, DiagID: diag::ext_pp_import_directive);
2890 }
2891 return HandleIncludeDirective(HashLoc, IncludeTok&: ImportTok);
2892}
2893
2894/// HandleIncludeMacrosDirective - The -imacros command line option turns into a
2895/// pseudo directive in the predefines buffer. This handles it by sucking all
2896/// tokens through the preprocessor and discarding them (only keeping the side
2897/// effects on the preprocessor).
2898void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
2899 Token &IncludeMacrosTok) {
2900 // This directive should only occur in the predefines buffer or the internal
2901 // buffer used to enter deferred implicit inputs in a GMF. If not, emit an
2902 // error and reject it.
2903 SourceLocation Loc = IncludeMacrosTok.getLocation();
2904 FileID FID = SourceMgr.getFileID(SpellingLoc: Loc);
2905 if (FID != getPredefinesFileID() && FID != DeferredGMFInputsFileID) {
2906 Diag(Loc: IncludeMacrosTok.getLocation(),
2907 DiagID: diag::pp_include_macros_out_of_predefines);
2908 DiscardUntilEndOfDirective();
2909 return;
2910 }
2911
2912 // Treat this as a normal #include for checking purposes. If this is
2913 // successful, it will push a new lexer onto the include stack.
2914 HandleIncludeDirective(HashLoc, IncludeTok&: IncludeMacrosTok);
2915
2916 Token TmpTok;
2917 do {
2918 Lex(Result&: TmpTok);
2919 assert(TmpTok.isNot(tok::eof) && "Didn't find end of -imacros!");
2920 } while (TmpTok.isNot(K: tok::hashhash));
2921}
2922
2923//===----------------------------------------------------------------------===//
2924// Preprocessor Macro Directive Handling.
2925//===----------------------------------------------------------------------===//
2926
2927/// ReadMacroParameterList - The ( starting a parameter list of a macro
2928/// definition has just been read. Lex the rest of the parameters and the
2929/// closing ), updating MI with what we learn. Return true if an error occurs
2930/// parsing the param list.
2931bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) {
2932 SmallVector<IdentifierInfo*, 32> Parameters;
2933
2934 while (true) {
2935 LexUnexpandedNonComment(Result&: Tok);
2936 switch (Tok.getKind()) {
2937 case tok::r_paren:
2938 // Found the end of the parameter list.
2939 if (Parameters.empty()) // #define FOO()
2940 return false;
2941 // Otherwise we have #define FOO(A,)
2942 Diag(Tok, DiagID: diag::err_pp_expected_ident_in_arg_list);
2943 return true;
2944 case tok::ellipsis: // #define X(... -> C99 varargs
2945 if (!LangOpts.C99)
2946 Diag(Tok, DiagID: LangOpts.CPlusPlus11 ?
2947 diag::warn_cxx98_compat_variadic_macro :
2948 diag::ext_variadic_macro);
2949
2950 // OpenCL v1.2 s6.9.e: variadic macros are not supported.
2951 if (LangOpts.OpenCL && !LangOpts.OpenCLCPlusPlus) {
2952 Diag(Tok, DiagID: diag::ext_pp_opencl_variadic_macros);
2953 }
2954
2955 // Lex the token after the identifier.
2956 LexUnexpandedNonComment(Result&: Tok);
2957 if (Tok.isNot(K: tok::r_paren)) {
2958 Diag(Tok, DiagID: diag::err_pp_missing_rparen_in_macro_def);
2959 return true;
2960 }
2961 // Add the __VA_ARGS__ identifier as a parameter.
2962 Parameters.push_back(Elt: Ident__VA_ARGS__);
2963 MI->setIsC99Varargs();
2964 MI->setParameterList(List: Parameters, PPAllocator&: BP);
2965 return false;
2966 case tok::eod: // #define X(
2967 Diag(Tok, DiagID: diag::err_pp_missing_rparen_in_macro_def);
2968 return true;
2969 default:
2970 // Handle keywords and identifiers here to accept things like
2971 // #define Foo(for) for.
2972 IdentifierInfo *II = Tok.getIdentifierInfo();
2973 if (!II) {
2974 // #define X(1
2975 Diag(Tok, DiagID: diag::err_pp_invalid_tok_in_arg_list);
2976 return true;
2977 }
2978
2979 // If this is already used as a parameter, it is used multiple times (e.g.
2980 // #define X(A,A.
2981 if (llvm::is_contained(Range&: Parameters, Element: II)) { // C99 6.10.3p6
2982 Diag(Tok, DiagID: diag::err_pp_duplicate_name_in_arg_list) << II;
2983 return true;
2984 }
2985
2986 // Add the parameter to the macro info.
2987 Parameters.push_back(Elt: II);
2988
2989 // Lex the token after the identifier.
2990 LexUnexpandedNonComment(Result&: Tok);
2991
2992 switch (Tok.getKind()) {
2993 default: // #define X(A B
2994 Diag(Tok, DiagID: diag::err_pp_expected_comma_in_arg_list);
2995 return true;
2996 case tok::r_paren: // #define X(A)
2997 MI->setParameterList(List: Parameters, PPAllocator&: BP);
2998 return false;
2999 case tok::comma: // #define X(A,
3000 break;
3001 case tok::ellipsis: // #define X(A... -> GCC extension
3002 // Diagnose extension.
3003 Diag(Tok, DiagID: diag::ext_named_variadic_macro);
3004
3005 // Lex the token after the identifier.
3006 LexUnexpandedNonComment(Result&: Tok);
3007 if (Tok.isNot(K: tok::r_paren)) {
3008 Diag(Tok, DiagID: diag::err_pp_missing_rparen_in_macro_def);
3009 return true;
3010 }
3011
3012 MI->setIsGNUVarargs();
3013 MI->setParameterList(List: Parameters, PPAllocator&: BP);
3014 return false;
3015 }
3016 }
3017 }
3018}
3019
3020static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI,
3021 const LangOptions &LOptions) {
3022 if (MI->getNumTokens() == 1) {
3023 const Token &Value = MI->getReplacementToken(Tok: 0);
3024
3025 // Macro that is identity, like '#define inline inline' is a valid pattern.
3026 if (MacroName.getKind() == Value.getKind())
3027 return true;
3028
3029 // Macro that maps a keyword to the same keyword decorated with leading/
3030 // trailing underscores is a valid pattern:
3031 // #define inline __inline
3032 // #define inline __inline__
3033 // #define inline _inline (in MS compatibility mode)
3034 StringRef MacroText = MacroName.getIdentifierInfo()->getName();
3035 if (IdentifierInfo *II = Value.getIdentifierInfo()) {
3036 if (!II->isKeyword(LangOpts: LOptions))
3037 return false;
3038 StringRef ValueText = II->getName();
3039 StringRef TrimmedValue = ValueText;
3040 if (!ValueText.starts_with(Prefix: "__")) {
3041 if (ValueText.starts_with(Prefix: "_"))
3042 TrimmedValue = TrimmedValue.drop_front(N: 1);
3043 else
3044 return false;
3045 } else {
3046 TrimmedValue = TrimmedValue.drop_front(N: 2);
3047 if (TrimmedValue.ends_with(Suffix: "__"))
3048 TrimmedValue = TrimmedValue.drop_back(N: 2);
3049 }
3050 return TrimmedValue == MacroText;
3051 } else {
3052 return false;
3053 }
3054 }
3055
3056 // #define inline
3057 return MacroName.isOneOf(Ks: tok::kw_extern, Ks: tok::kw_inline, Ks: tok::kw_static,
3058 Ks: tok::kw_const) &&
3059 MI->getNumTokens() == 0;
3060}
3061
3062// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
3063// entire line) of the macro's tokens and adds them to MacroInfo, and while
3064// doing so performs certain validity checks including (but not limited to):
3065// - # (stringization) is followed by a macro parameter
3066//
3067// Returns a nullptr if an invalid sequence of tokens is encountered or returns
3068// a pointer to a MacroInfo object.
3069
3070MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
3071 const Token &MacroNameTok, const bool ImmediatelyAfterHeaderGuard) {
3072
3073 Token LastTok = MacroNameTok;
3074 // Create the new macro.
3075 MacroInfo *const MI = AllocateMacroInfo(L: MacroNameTok.getLocation());
3076
3077 Token Tok;
3078 LexUnexpandedToken(Result&: Tok);
3079
3080 // Ensure we consume the rest of the macro body if errors occur.
3081 llvm::scope_exit _([&]() {
3082 // The flag indicates if we are still waiting for 'eod'.
3083 if (CurLexer->ParsingPreprocessorDirective)
3084 DiscardUntilEndOfDirective();
3085 });
3086
3087 // Used to un-poison and then re-poison identifiers of the __VA_ARGS__ ilk
3088 // within their appropriate context.
3089 VariadicMacroScopeGuard VariadicMacroScopeGuard(*this);
3090
3091 // If this is a function-like macro definition, parse the argument list,
3092 // marking each of the identifiers as being used as macro arguments. Also,
3093 // check other constraints on the first token of the macro body.
3094 if (Tok.is(K: tok::eod)) {
3095 if (ImmediatelyAfterHeaderGuard) {
3096 // Save this macro information since it may part of a header guard.
3097 CurPPLexer->MIOpt.SetDefinedMacro(M: MacroNameTok.getIdentifierInfo(),
3098 Loc: MacroNameTok.getLocation());
3099 }
3100 // If there is no body to this macro, we have no special handling here.
3101 } else if (Tok.hasLeadingSpace()) {
3102 // This is a normal token with leading space. Clear the leading space
3103 // marker on the first token to get proper expansion.
3104 Tok.clearFlag(Flag: Token::LeadingSpace);
3105 } else if (Tok.is(K: tok::l_paren)) {
3106 // This is a function-like macro definition. Read the argument list.
3107 MI->setIsFunctionLike();
3108 if (ReadMacroParameterList(MI, Tok&: LastTok))
3109 return nullptr;
3110
3111 // If this is a definition of an ISO C/C++ variadic function-like macro (not
3112 // using the GNU named varargs extension) inform our variadic scope guard
3113 // which un-poisons and re-poisons certain identifiers (e.g. __VA_ARGS__)
3114 // allowed only within the definition of a variadic macro.
3115
3116 if (MI->isC99Varargs()) {
3117 VariadicMacroScopeGuard.enterScope();
3118 }
3119
3120 // Read the first token after the arg list for down below.
3121 LexUnexpandedToken(Result&: Tok);
3122 } else if (LangOpts.C99 || LangOpts.CPlusPlus11) {
3123 // C99 requires whitespace between the macro definition and the body. Emit
3124 // a diagnostic for something like "#define X+".
3125 Diag(Tok, DiagID: diag::ext_c99_whitespace_required_after_macro_name);
3126 } else {
3127 // C90 6.8 TC1 says: "In the definition of an object-like macro, if the
3128 // first character of a replacement list is not a character required by
3129 // subclause 5.2.1, then there shall be white-space separation between the
3130 // identifier and the replacement list.". 5.2.1 lists this set:
3131 // "A-Za-z0-9!"#%&'()*+,_./:;<=>?[\]^_{|}~" as well as whitespace, which
3132 // is irrelevant here.
3133 bool isInvalid = false;
3134 if (Tok.is(K: tok::at)) // @ is not in the list above.
3135 isInvalid = true;
3136 else if (Tok.is(K: tok::unknown)) {
3137 // If we have an unknown token, it is something strange like "`". Since
3138 // all of valid characters would have lexed into a single character
3139 // token of some sort, we know this is not a valid case.
3140 isInvalid = true;
3141 }
3142 if (isInvalid)
3143 Diag(Tok, DiagID: diag::ext_missing_whitespace_after_macro_name);
3144 else
3145 Diag(Tok, DiagID: diag::warn_missing_whitespace_after_macro_name);
3146 }
3147
3148 if (!Tok.is(K: tok::eod))
3149 LastTok = Tok;
3150
3151 SmallVector<Token, 16> Tokens;
3152
3153 // Read the rest of the macro body.
3154 if (MI->isObjectLike()) {
3155 // Object-like macros are very simple, just read their body.
3156 while (Tok.isNot(K: tok::eod)) {
3157 LastTok = Tok;
3158 Tokens.push_back(Elt: Tok);
3159 // Get the next token of the macro.
3160 LexUnexpandedToken(Result&: Tok);
3161 }
3162 } else {
3163 // Otherwise, read the body of a function-like macro. While we are at it,
3164 // check C99 6.10.3.2p1: ensure that # operators are followed by macro
3165 // parameters in function-like macro expansions.
3166
3167 VAOptDefinitionContext VAOCtx(*this);
3168
3169 while (Tok.isNot(K: tok::eod)) {
3170 LastTok = Tok;
3171
3172 if (!Tok.isOneOf(Ks: tok::hash, Ks: tok::hashat, Ks: tok::hashhash)) {
3173 Tokens.push_back(Elt: Tok);
3174
3175 if (VAOCtx.isVAOptToken(T: Tok)) {
3176 // If we're already within a VAOPT, emit an error.
3177 if (VAOCtx.isInVAOpt()) {
3178 Diag(Tok, DiagID: diag::err_pp_vaopt_nested_use);
3179 return nullptr;
3180 }
3181 // Ensure VAOPT is followed by a '(' .
3182 LexUnexpandedToken(Result&: Tok);
3183 if (Tok.isNot(K: tok::l_paren)) {
3184 Diag(Tok, DiagID: diag::err_pp_missing_lparen_in_vaopt_use);
3185 return nullptr;
3186 }
3187 Tokens.push_back(Elt: Tok);
3188 VAOCtx.sawVAOptFollowedByOpeningParens(LParenLoc: Tok.getLocation());
3189 LexUnexpandedToken(Result&: Tok);
3190 if (Tok.is(K: tok::hashhash)) {
3191 Diag(Tok, DiagID: diag::err_vaopt_paste_at_start);
3192 return nullptr;
3193 }
3194 continue;
3195 } else if (VAOCtx.isInVAOpt()) {
3196 if (Tok.is(K: tok::r_paren)) {
3197 if (VAOCtx.sawClosingParen()) {
3198 assert(Tokens.size() >= 3 &&
3199 "Must have seen at least __VA_OPT__( "
3200 "and a subsequent tok::r_paren");
3201 if (Tokens[Tokens.size() - 2].is(K: tok::hashhash)) {
3202 Diag(Tok, DiagID: diag::err_vaopt_paste_at_end);
3203 return nullptr;
3204 }
3205 }
3206 } else if (Tok.is(K: tok::l_paren)) {
3207 VAOCtx.sawOpeningParen(LParenLoc: Tok.getLocation());
3208 }
3209 }
3210 // Get the next token of the macro.
3211 LexUnexpandedToken(Result&: Tok);
3212 continue;
3213 }
3214
3215 // If we're in -traditional mode, then we should ignore stringification
3216 // and token pasting. Mark the tokens as unknown so as not to confuse
3217 // things.
3218 if (getLangOpts().TraditionalCPP) {
3219 Tok.setKind(tok::unknown);
3220 Tokens.push_back(Elt: Tok);
3221
3222 // Get the next token of the macro.
3223 LexUnexpandedToken(Result&: Tok);
3224 continue;
3225 }
3226
3227 if (Tok.is(K: tok::hashhash)) {
3228 // If we see token pasting, check if it looks like the gcc comma
3229 // pasting extension. We'll use this information to suppress
3230 // diagnostics later on.
3231
3232 // Get the next token of the macro.
3233 LexUnexpandedToken(Result&: Tok);
3234
3235 if (Tok.is(K: tok::eod)) {
3236 Tokens.push_back(Elt: LastTok);
3237 break;
3238 }
3239
3240 if (!Tokens.empty() && Tok.getIdentifierInfo() == Ident__VA_ARGS__ &&
3241 Tokens[Tokens.size() - 1].is(K: tok::comma))
3242 MI->setHasCommaPasting();
3243
3244 // Things look ok, add the '##' token to the macro.
3245 Tokens.push_back(Elt: LastTok);
3246 continue;
3247 }
3248
3249 // Our Token is a stringization operator.
3250 // Get the next token of the macro.
3251 LexUnexpandedToken(Result&: Tok);
3252
3253 // Check for a valid macro arg identifier or __VA_OPT__.
3254 if (!VAOCtx.isVAOptToken(T: Tok) &&
3255 (Tok.getIdentifierInfo() == nullptr ||
3256 MI->getParameterNum(Arg: Tok.getIdentifierInfo()) == -1)) {
3257
3258 // If this is assembler-with-cpp mode, we accept random gibberish after
3259 // the '#' because '#' is often a comment character. However, change
3260 // the kind of the token to tok::unknown so that the preprocessor isn't
3261 // confused.
3262 if (getLangOpts().AsmPreprocessor && Tok.isNot(K: tok::eod)) {
3263 LastTok.setKind(tok::unknown);
3264 Tokens.push_back(Elt: LastTok);
3265 continue;
3266 } else {
3267 Diag(Tok, DiagID: diag::err_pp_stringize_not_parameter)
3268 << LastTok.is(K: tok::hashat);
3269 return nullptr;
3270 }
3271 }
3272
3273 // Things look ok, add the '#' and param name tokens to the macro.
3274 Tokens.push_back(Elt: LastTok);
3275
3276 // If the token following '#' is VAOPT, let the next iteration handle it
3277 // and check it for correctness, otherwise add the token and prime the
3278 // loop with the next one.
3279 if (!VAOCtx.isVAOptToken(T: Tok)) {
3280 Tokens.push_back(Elt: Tok);
3281 LastTok = Tok;
3282
3283 // Get the next token of the macro.
3284 LexUnexpandedToken(Result&: Tok);
3285 }
3286 }
3287 if (VAOCtx.isInVAOpt()) {
3288 assert(Tok.is(tok::eod) && "Must be at End Of preprocessing Directive");
3289 Diag(Tok, DiagID: diag::err_pp_expected_after)
3290 << LastTok.getKind() << tok::r_paren;
3291 Diag(Loc: VAOCtx.getUnmatchedOpeningParenLoc(), DiagID: diag::note_matching) << tok::l_paren;
3292 return nullptr;
3293 }
3294 }
3295 MI->setDefinitionEndLoc(LastTok.getLocation());
3296
3297 MI->setTokens(Tokens, PPAllocator&: BP);
3298 return MI;
3299}
3300
3301static bool isObjCProtectedMacro(const IdentifierInfo *II) {
3302 return II->isStr(Str: "__strong") || II->isStr(Str: "__weak") ||
3303 II->isStr(Str: "__unsafe_unretained") || II->isStr(Str: "__autoreleasing");
3304}
3305
3306/// HandleDefineDirective - Implements \#define. This consumes the entire macro
3307/// line then lets the caller lex the next real token.
3308void Preprocessor::HandleDefineDirective(
3309 Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) {
3310 ++NumDefined;
3311
3312 Token MacroNameTok;
3313 bool MacroShadowsKeyword;
3314 ReadMacroName(MacroNameTok, isDefineUndef: MU_Define, ShadowFlag: &MacroShadowsKeyword);
3315
3316 // Error reading macro name? If so, diagnostic already issued.
3317 if (MacroNameTok.is(K: tok::eod))
3318 return;
3319
3320 IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
3321 // Issue a final pragma warning if we're defining a macro that was has been
3322 // undefined and is being redefined.
3323 if (!II->hasMacroDefinition() && II->hadMacroDefinition() && II->isFinal())
3324 emitFinalMacroWarning(Identifier: MacroNameTok, /*IsUndef=*/false);
3325
3326 // If we are supposed to keep comments in #defines, reenable comment saving
3327 // mode.
3328 if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments);
3329
3330 MacroInfo *const MI = ReadOptionalMacroParameterListAndBody(
3331 MacroNameTok, ImmediatelyAfterHeaderGuard);
3332
3333 if (!MI) return;
3334
3335 if (MacroShadowsKeyword &&
3336 !isConfigurationPattern(MacroName&: MacroNameTok, MI, LOptions: getLangOpts())) {
3337 Diag(Tok: MacroNameTok, DiagID: diag::warn_pp_macro_hides_keyword);
3338 }
3339 // Check that there is no paste (##) operator at the beginning or end of the
3340 // replacement list.
3341 unsigned NumTokens = MI->getNumTokens();
3342 if (NumTokens != 0) {
3343 if (MI->getReplacementToken(Tok: 0).is(K: tok::hashhash)) {
3344 Diag(Tok: MI->getReplacementToken(Tok: 0), DiagID: diag::err_paste_at_start);
3345 return;
3346 }
3347 if (MI->getReplacementToken(Tok: NumTokens-1).is(K: tok::hashhash)) {
3348 Diag(Tok: MI->getReplacementToken(Tok: NumTokens-1), DiagID: diag::err_paste_at_end);
3349 return;
3350 }
3351 }
3352
3353 // When skipping just warn about macros that do not match.
3354 if (SkippingUntilPCHThroughHeader) {
3355 const MacroInfo *OtherMI = getMacroInfo(II: MacroNameTok.getIdentifierInfo());
3356 if (!OtherMI || !MI->isIdenticalTo(Other: *OtherMI, PP&: *this,
3357 /*Syntactic=*/Syntactically: LangOpts.MicrosoftExt))
3358 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::warn_pp_macro_def_mismatch_with_pch)
3359 << MacroNameTok.getIdentifierInfo();
3360 // Issue the diagnostic but allow the change if msvc extensions are enabled
3361 if (!LangOpts.MicrosoftExt)
3362 return;
3363 }
3364
3365 // Finally, if this identifier already had a macro defined for it, verify that
3366 // the macro bodies are identical, and issue diagnostics if they are not.
3367 if (const MacroInfo *OtherMI=getMacroInfo(II: MacroNameTok.getIdentifierInfo())) {
3368 // Final macros are hard-mode: they always warn. Even if the bodies are
3369 // identical. Even if they are in system headers. Even if they are things we
3370 // would silently allow in the past.
3371 if (MacroNameTok.getIdentifierInfo()->isFinal())
3372 emitFinalMacroWarning(Identifier: MacroNameTok, /*IsUndef=*/false);
3373
3374 // In Objective-C, ignore attempts to directly redefine the builtin
3375 // definitions of the ownership qualifiers. It's still possible to
3376 // #undef them.
3377 if (getLangOpts().ObjC &&
3378 SourceMgr.getFileID(SpellingLoc: OtherMI->getDefinitionLoc()) ==
3379 getPredefinesFileID() &&
3380 isObjCProtectedMacro(II: MacroNameTok.getIdentifierInfo())) {
3381 // Warn if it changes the tokens.
3382 if ((!getDiagnostics().getSuppressSystemWarnings() ||
3383 !SourceMgr.isInSystemHeader(Loc: DefineTok.getLocation())) &&
3384 !MI->isIdenticalTo(Other: *OtherMI, PP&: *this,
3385 /*Syntactic=*/Syntactically: LangOpts.MicrosoftExt)) {
3386 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::warn_pp_objc_macro_redef_ignored);
3387 }
3388 assert(!OtherMI->isWarnIfUnused());
3389 return;
3390 }
3391
3392 // It is very common for system headers to have tons of macro redefinitions
3393 // and for warnings to be disabled in system headers. If this is the case,
3394 // then don't bother calling MacroInfo::isIdenticalTo.
3395 if (!getDiagnostics().getSuppressSystemWarnings() ||
3396 !SourceMgr.isInSystemHeader(Loc: DefineTok.getLocation())) {
3397
3398 if (!OtherMI->isUsed() && OtherMI->isWarnIfUnused())
3399 Diag(Loc: OtherMI->getDefinitionLoc(), DiagID: diag::pp_macro_not_used);
3400
3401 // Warn if defining "__LINE__" and other builtins, per C99 6.10.8/4 and
3402 // C++ [cpp.predefined]p4, but allow it as an extension.
3403 if (isLanguageDefinedBuiltin(SourceMgr, MI: OtherMI, MacroName: II->getName()))
3404 Diag(Tok: MacroNameTok, DiagID: diag::ext_pp_redef_builtin_macro);
3405 // Macros must be identical. This means all tokens and whitespace
3406 // separation must be the same. C99 6.10.3p2.
3407 else if (!OtherMI->isAllowRedefinitionsWithoutWarning() &&
3408 !MI->isIdenticalTo(Other: *OtherMI, PP&: *this, /*Syntactic=*/Syntactically: LangOpts.MicrosoftExt)) {
3409 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::ext_pp_macro_redef)
3410 << MacroNameTok.getIdentifierInfo();
3411 Diag(Loc: OtherMI->getDefinitionLoc(), DiagID: diag::note_previous_definition);
3412 }
3413 }
3414 if (OtherMI->isWarnIfUnused())
3415 WarnUnusedMacroLocs.erase(V: OtherMI->getDefinitionLoc());
3416 }
3417
3418 DefMacroDirective *MD =
3419 appendDefMacroDirective(II: MacroNameTok.getIdentifierInfo(), MI);
3420
3421 assert(!MI->isUsed());
3422 // If we need warning for not using the macro, add its location in the
3423 // warn-because-unused-macro set. If it gets used it will be removed from set.
3424 if (getSourceManager().isInMainFile(Loc: MI->getDefinitionLoc()) &&
3425 !Diags->isIgnored(DiagID: diag::pp_macro_not_used, Loc: MI->getDefinitionLoc()) &&
3426 !MacroExpansionInDirectivesOverride &&
3427 getSourceManager().getFileID(SpellingLoc: MI->getDefinitionLoc()) !=
3428 getPredefinesFileID()) {
3429 MI->setIsWarnIfUnused(true);
3430 WarnUnusedMacroLocs.insert(V: MI->getDefinitionLoc());
3431 }
3432
3433 // If the callbacks want to know, tell them about the macro definition.
3434 if (Callbacks)
3435 Callbacks->MacroDefined(MacroNameTok, MD);
3436}
3437
3438/// HandleUndefDirective - Implements \#undef.
3439///
3440void Preprocessor::HandleUndefDirective() {
3441 ++NumUndefined;
3442
3443 Token MacroNameTok;
3444 ReadMacroName(MacroNameTok, isDefineUndef: MU_Undef);
3445
3446 // Error reading macro name? If so, diagnostic already issued.
3447 if (MacroNameTok.is(K: tok::eod))
3448 return;
3449
3450 // Check to see if this is the last token on the #undef line.
3451 CheckEndOfDirective(DirType: "undef");
3452
3453 // Okay, we have a valid identifier to undef.
3454 auto *II = MacroNameTok.getIdentifierInfo();
3455 auto MD = getMacroDefinition(II);
3456 UndefMacroDirective *Undef = nullptr;
3457
3458 if (II->isFinal())
3459 emitFinalMacroWarning(Identifier: MacroNameTok, /*IsUndef=*/true);
3460
3461 // If the macro is not defined, this is a noop undef.
3462 if (const MacroInfo *MI = MD.getMacroInfo()) {
3463 if (!MI->isUsed() && MI->isWarnIfUnused())
3464 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::pp_macro_not_used);
3465
3466 // Warn if undefining "__LINE__" and other builtins, per C99 6.10.8/4 and
3467 // C++ [cpp.predefined]p4, but allow it as an extension.
3468 if (isLanguageDefinedBuiltin(SourceMgr, MI, MacroName: II->getName()))
3469 Diag(Tok: MacroNameTok, DiagID: diag::ext_pp_undef_builtin_macro);
3470
3471 if (MI->isWarnIfUnused())
3472 WarnUnusedMacroLocs.erase(V: MI->getDefinitionLoc());
3473
3474 Undef = AllocateUndefMacroDirective(UndefLoc: MacroNameTok.getLocation());
3475 }
3476
3477 // If the callbacks want to know, tell them about the macro #undef.
3478 // Note: no matter if the macro was defined or not.
3479 if (Callbacks)
3480 Callbacks->MacroUndefined(MacroNameTok, MD, Undef);
3481
3482 if (Undef)
3483 appendMacroDirective(II, MD: Undef);
3484}
3485
3486//===----------------------------------------------------------------------===//
3487// Preprocessor Conditional Directive Handling.
3488//===----------------------------------------------------------------------===//
3489
3490/// HandleIfdefDirective - Implements the \#ifdef/\#ifndef directive. isIfndef
3491/// is true when this is a \#ifndef directive. ReadAnyTokensBeforeDirective is
3492/// true if any tokens have been returned or pp-directives activated before this
3493/// \#ifndef has been lexed.
3494///
3495void Preprocessor::HandleIfdefDirective(Token &Result,
3496 const Token &HashToken,
3497 bool isIfndef,
3498 bool ReadAnyTokensBeforeDirective) {
3499 ++NumIf;
3500 Token DirectiveTok = Result;
3501
3502 Token MacroNameTok;
3503 ReadMacroName(MacroNameTok);
3504
3505 // Error reading macro name? If so, diagnostic already issued.
3506 if (MacroNameTok.is(K: tok::eod)) {
3507 // Skip code until we get to #endif. This helps with recovery by not
3508 // emitting an error when the #endif is reached.
3509 SkipExcludedConditionalBlock(HashTokenLoc: HashToken.getLocation(),
3510 IfTokenLoc: DirectiveTok.getLocation(),
3511 /*Foundnonskip*/ FoundNonSkipPortion: false, /*FoundElse*/ false);
3512 return;
3513 }
3514
3515 emitMacroExpansionWarnings(Identifier: MacroNameTok, /*IsIfnDef=*/true);
3516
3517 // Check to see if this is the last token on the #if[n]def line.
3518 CheckEndOfDirective(DirType: isIfndef ? "ifndef" : "ifdef");
3519
3520 IdentifierInfo *MII = MacroNameTok.getIdentifierInfo();
3521 auto MD = getMacroDefinition(II: MII);
3522 MacroInfo *MI = MD.getMacroInfo();
3523
3524 if (CurPPLexer->getConditionalStackDepth() == 0) {
3525 // If the start of a top-level #ifdef and if the macro is not defined,
3526 // inform MIOpt that this might be the start of a proper include guard.
3527 // Otherwise it is some other form of unknown conditional which we can't
3528 // handle.
3529 if (!ReadAnyTokensBeforeDirective && !MI) {
3530 assert(isIfndef && "#ifdef shouldn't reach here");
3531 CurPPLexer->MIOpt.EnterTopLevelIfndef(M: MII, Loc: MacroNameTok.getLocation());
3532 } else
3533 CurPPLexer->MIOpt.EnterTopLevelConditional();
3534 }
3535
3536 // If there is a macro, process it.
3537 if (MI) // Mark it used.
3538 markMacroAsUsed(MI);
3539
3540 if (Callbacks) {
3541 if (isIfndef)
3542 Callbacks->Ifndef(Loc: DirectiveTok.getLocation(), MacroNameTok, MD);
3543 else
3544 Callbacks->Ifdef(Loc: DirectiveTok.getLocation(), MacroNameTok, MD);
3545 }
3546
3547 bool RetainExcludedCB = PPOpts.RetainExcludedConditionalBlocks &&
3548 getSourceManager().isInMainFile(Loc: DirectiveTok.getLocation());
3549
3550 // Should we include the stuff contained by this directive?
3551 if (PPOpts.SingleFileParseMode && !MI) {
3552 // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3553 // the directive blocks.
3554 CurPPLexer->pushConditionalLevel(DirectiveStart: DirectiveTok.getLocation(),
3555 /*wasskip*/WasSkipping: false, /*foundnonskip*/FoundNonSkip: false,
3556 /*foundelse*/FoundElse: false);
3557 } else if (PPOpts.SingleModuleParseMode && !MI) {
3558 // In 'single-module-parse mode' undefined identifiers trigger skipping of
3559 // all the directive blocks. We lie here and set FoundNonSkipPortion so that
3560 // even any \#else blocks get skipped.
3561 SkipExcludedConditionalBlock(
3562 HashTokenLoc: HashToken.getLocation(), IfTokenLoc: DirectiveTok.getLocation(),
3563 /*FoundNonSkipPortion=*/true, /*FoundElse=*/false);
3564 } else if (!MI == isIfndef || RetainExcludedCB) {
3565 // Yes, remember that we are inside a conditional, then lex the next token.
3566 CurPPLexer->pushConditionalLevel(DirectiveStart: DirectiveTok.getLocation(),
3567 /*wasskip*/WasSkipping: false, /*foundnonskip*/FoundNonSkip: true,
3568 /*foundelse*/FoundElse: false);
3569 } else {
3570 // No, skip the contents of this block.
3571 SkipExcludedConditionalBlock(HashTokenLoc: HashToken.getLocation(),
3572 IfTokenLoc: DirectiveTok.getLocation(),
3573 /*Foundnonskip*/ FoundNonSkipPortion: false,
3574 /*FoundElse*/ false);
3575 }
3576}
3577
3578/// HandleIfDirective - Implements the \#if directive.
3579///
3580void Preprocessor::HandleIfDirective(Token &IfToken,
3581 const Token &HashToken,
3582 bool ReadAnyTokensBeforeDirective) {
3583 ++NumIf;
3584
3585 // Parse and evaluate the conditional expression.
3586 IdentifierInfo *IfNDefMacro = nullptr;
3587 const DirectiveEvalResult DER = EvaluateDirectiveExpression(IfNDefMacro);
3588 const bool ConditionalTrue = DER.Conditional;
3589 // Lexer might become invalid if we hit code completion point while evaluating
3590 // expression.
3591 if (!CurPPLexer)
3592 return;
3593
3594 // If this condition is equivalent to #ifndef X, and if this is the first
3595 // directive seen, handle it for the multiple-include optimization.
3596 if (CurPPLexer->getConditionalStackDepth() == 0) {
3597 if (!ReadAnyTokensBeforeDirective && IfNDefMacro && ConditionalTrue)
3598 // FIXME: Pass in the location of the macro name, not the 'if' token.
3599 CurPPLexer->MIOpt.EnterTopLevelIfndef(M: IfNDefMacro, Loc: IfToken.getLocation());
3600 else
3601 CurPPLexer->MIOpt.EnterTopLevelConditional();
3602 }
3603
3604 if (Callbacks)
3605 Callbacks->If(
3606 Loc: IfToken.getLocation(), ConditionRange: DER.ExprRange,
3607 ConditionValue: (ConditionalTrue ? PPCallbacks::CVK_True : PPCallbacks::CVK_False));
3608
3609 bool RetainExcludedCB = PPOpts.RetainExcludedConditionalBlocks &&
3610 getSourceManager().isInMainFile(Loc: IfToken.getLocation());
3611
3612 // Should we include the stuff contained by this directive?
3613 if (PPOpts.SingleFileParseMode && DER.IncludedUndefinedIds) {
3614 // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3615 // the directive blocks.
3616 CurPPLexer->pushConditionalLevel(DirectiveStart: IfToken.getLocation(), /*wasskip*/WasSkipping: false,
3617 /*foundnonskip*/FoundNonSkip: false, /*foundelse*/FoundElse: false);
3618 } else if (PPOpts.SingleModuleParseMode && DER.IncludedUndefinedIds) {
3619 // In 'single-module-parse mode' undefined identifiers trigger skipping of
3620 // all the directive blocks. We lie here and set FoundNonSkipPortion so that
3621 // even any \#else blocks get skipped.
3622 SkipExcludedConditionalBlock(HashTokenLoc: HashToken.getLocation(), IfTokenLoc: IfToken.getLocation(),
3623 /*FoundNonSkipPortion=*/true,
3624 /*FoundElse=*/false);
3625 } else if (ConditionalTrue || RetainExcludedCB) {
3626 // Yes, remember that we are inside a conditional, then lex the next token.
3627 CurPPLexer->pushConditionalLevel(DirectiveStart: IfToken.getLocation(), /*wasskip*/WasSkipping: false,
3628 /*foundnonskip*/FoundNonSkip: true, /*foundelse*/FoundElse: false);
3629 } else {
3630 // No, skip the contents of this block.
3631 SkipExcludedConditionalBlock(HashTokenLoc: HashToken.getLocation(), IfTokenLoc: IfToken.getLocation(),
3632 /*Foundnonskip*/ FoundNonSkipPortion: false,
3633 /*FoundElse*/ false);
3634 }
3635}
3636
3637/// HandleEndifDirective - Implements the \#endif directive.
3638///
3639void Preprocessor::HandleEndifDirective(Token &EndifToken) {
3640 ++NumEndif;
3641
3642 // Check that this is the whole directive.
3643 CheckEndOfDirective(DirType: "endif");
3644
3645 PPConditionalInfo CondInfo;
3646 if (CurPPLexer->popConditionalLevel(CI&: CondInfo)) {
3647 // No conditionals on the stack: this is an #endif without an #if.
3648 Diag(Tok: EndifToken, DiagID: diag::err_pp_endif_without_if);
3649 return;
3650 }
3651
3652 // If this the end of a top-level #endif, inform MIOpt.
3653 if (CurPPLexer->getConditionalStackDepth() == 0)
3654 CurPPLexer->MIOpt.ExitTopLevelConditional();
3655
3656 assert(!CondInfo.WasSkipping && !CurPPLexer->LexingRawMode &&
3657 "This code should only be reachable in the non-skipping case!");
3658
3659 if (Callbacks)
3660 Callbacks->Endif(Loc: EndifToken.getLocation(), IfLoc: CondInfo.IfLoc);
3661}
3662
3663/// HandleElseDirective - Implements the \#else directive.
3664///
3665void Preprocessor::HandleElseDirective(Token &Result, const Token &HashToken) {
3666 ++NumElse;
3667
3668 // #else directive in a non-skipping conditional... start skipping.
3669 CheckEndOfDirective(DirType: "else");
3670
3671 PPConditionalInfo CI;
3672 if (CurPPLexer->popConditionalLevel(CI)) {
3673 Diag(Tok: Result, DiagID: diag::pp_err_else_without_if);
3674 return;
3675 }
3676
3677 // If this is a top-level #else, inform the MIOpt.
3678 if (CurPPLexer->getConditionalStackDepth() == 0)
3679 CurPPLexer->MIOpt.EnterTopLevelConditional();
3680
3681 // If this is a #else with a #else before it, report the error.
3682 if (CI.FoundElse) Diag(Tok: Result, DiagID: diag::pp_err_else_after_else);
3683
3684 if (Callbacks)
3685 Callbacks->Else(Loc: Result.getLocation(), IfLoc: CI.IfLoc);
3686
3687 bool RetainExcludedCB = PPOpts.RetainExcludedConditionalBlocks &&
3688 getSourceManager().isInMainFile(Loc: Result.getLocation());
3689
3690 if ((PPOpts.SingleFileParseMode && !CI.FoundNonSkip) || RetainExcludedCB) {
3691 // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3692 // the directive blocks.
3693 CurPPLexer->pushConditionalLevel(DirectiveStart: CI.IfLoc, /*wasskip*/WasSkipping: false,
3694 /*foundnonskip*/FoundNonSkip: false, /*foundelse*/FoundElse: true);
3695 return;
3696 }
3697
3698 // Finally, skip the rest of the contents of this block.
3699 SkipExcludedConditionalBlock(HashTokenLoc: HashToken.getLocation(), IfTokenLoc: CI.IfLoc,
3700 /*Foundnonskip*/ FoundNonSkipPortion: true,
3701 /*FoundElse*/ true, ElseLoc: Result.getLocation());
3702}
3703
3704/// Implements the \#elif, \#elifdef, and \#elifndef directives.
3705void Preprocessor::HandleElifFamilyDirective(Token &ElifToken,
3706 const Token &HashToken,
3707 tok::PPKeywordKind Kind) {
3708 PPElifDiag DirKind = Kind == tok::pp_elif ? PED_Elif
3709 : Kind == tok::pp_elifdef ? PED_Elifdef
3710 : PED_Elifndef;
3711 ++NumElse;
3712
3713 // Warn if using `#elifdef` & `#elifndef` in not C23 & C++23 mode.
3714 switch (DirKind) {
3715 case PED_Elifdef:
3716 case PED_Elifndef:
3717 DiagCompat(Tok: ElifToken, CompatDiagID: LangOpts.CPlusPlus ? diag_compat::cxx23_pp_directive
3718 : diag_compat::c23_pp_directive)
3719 << DirKind;
3720 break;
3721 default:
3722 break;
3723 }
3724
3725 // #elif directive in a non-skipping conditional... start skipping.
3726 // We don't care what the condition is, because we will always skip it (since
3727 // the block immediately before it was included).
3728 SourceRange ConditionRange = DiscardUntilEndOfDirective();
3729
3730 PPConditionalInfo CI;
3731 if (CurPPLexer->popConditionalLevel(CI)) {
3732 Diag(Tok: ElifToken, DiagID: diag::pp_err_elif_without_if) << DirKind;
3733 return;
3734 }
3735
3736 // If this is a top-level #elif, inform the MIOpt.
3737 if (CurPPLexer->getConditionalStackDepth() == 0)
3738 CurPPLexer->MIOpt.EnterTopLevelConditional();
3739
3740 // If this is a #elif with a #else before it, report the error.
3741 if (CI.FoundElse)
3742 Diag(Tok: ElifToken, DiagID: diag::pp_err_elif_after_else) << DirKind;
3743
3744 if (Callbacks) {
3745 switch (Kind) {
3746 case tok::pp_elif:
3747 Callbacks->Elif(Loc: ElifToken.getLocation(), ConditionRange,
3748 ConditionValue: PPCallbacks::CVK_NotEvaluated, IfLoc: CI.IfLoc);
3749 break;
3750 case tok::pp_elifdef:
3751 Callbacks->Elifdef(Loc: ElifToken.getLocation(), ConditionRange, IfLoc: CI.IfLoc);
3752 break;
3753 case tok::pp_elifndef:
3754 Callbacks->Elifndef(Loc: ElifToken.getLocation(), ConditionRange, IfLoc: CI.IfLoc);
3755 break;
3756 default:
3757 assert(false && "unexpected directive kind");
3758 break;
3759 }
3760 }
3761
3762 bool RetainExcludedCB = PPOpts.RetainExcludedConditionalBlocks &&
3763 getSourceManager().isInMainFile(Loc: ElifToken.getLocation());
3764
3765 if ((PPOpts.SingleFileParseMode && !CI.FoundNonSkip) || RetainExcludedCB) {
3766 // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3767 // the directive blocks.
3768 CurPPLexer->pushConditionalLevel(DirectiveStart: ElifToken.getLocation(), /*wasskip*/WasSkipping: false,
3769 /*foundnonskip*/FoundNonSkip: false, /*foundelse*/FoundElse: false);
3770 return;
3771 }
3772
3773 // Finally, skip the rest of the contents of this block.
3774 SkipExcludedConditionalBlock(
3775 HashTokenLoc: HashToken.getLocation(), IfTokenLoc: CI.IfLoc, /*Foundnonskip*/ FoundNonSkipPortion: true,
3776 /*FoundElse*/ CI.FoundElse, ElseLoc: ElifToken.getLocation());
3777}
3778
3779std::optional<LexEmbedParametersResult>
3780Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
3781 LexEmbedParametersResult Result{};
3782 tok::TokenKind EndTokenKind = ForHasEmbed ? tok::r_paren : tok::eod;
3783
3784 auto DiagMismatchedBracesAndSkipToEOD =
3785 [&](tok::TokenKind Expected,
3786 std::pair<tok::TokenKind, SourceLocation> Matches) {
3787 Diag(Tok: CurTok, DiagID: diag::err_expected) << Expected;
3788 Diag(Loc: Matches.second, DiagID: diag::note_matching) << Matches.first;
3789 if (CurTok.isNot(K: tok::eod))
3790 DiscardUntilEndOfDirective(Tmp&: CurTok);
3791 };
3792
3793 auto ExpectOrDiagAndSkipToEOD = [&](tok::TokenKind Kind) {
3794 if (CurTok.isNot(K: Kind)) {
3795 Diag(Tok: CurTok, DiagID: diag::err_expected) << Kind;
3796 if (CurTok.isNot(K: tok::eod))
3797 DiscardUntilEndOfDirective(Tmp&: CurTok);
3798 return false;
3799 }
3800 return true;
3801 };
3802
3803 // C23 6.10:
3804 // pp-parameter-name:
3805 // pp-standard-parameter
3806 // pp-prefixed-parameter
3807 //
3808 // pp-standard-parameter:
3809 // identifier
3810 //
3811 // pp-prefixed-parameter:
3812 // identifier :: identifier
3813 auto LexPPParameterName = [&]() -> std::optional<std::string> {
3814 // We expect the current token to be an identifier; if it's not, things
3815 // have gone wrong.
3816 if (!ExpectOrDiagAndSkipToEOD(tok::identifier))
3817 return std::nullopt;
3818
3819 const IdentifierInfo *Prefix = CurTok.getIdentifierInfo();
3820
3821 // Lex another token; it is either a :: or we're done with the parameter
3822 // name.
3823 LexNonComment(Result&: CurTok);
3824 if (CurTok.is(K: tok::coloncolon)) {
3825 // We found a ::, so lex another identifier token.
3826 LexNonComment(Result&: CurTok);
3827 if (!ExpectOrDiagAndSkipToEOD(tok::identifier))
3828 return std::nullopt;
3829
3830 const IdentifierInfo *Suffix = CurTok.getIdentifierInfo();
3831
3832 // Lex another token so we're past the name.
3833 LexNonComment(Result&: CurTok);
3834 return (llvm::Twine(Prefix->getName()) + "::" + Suffix->getName()).str();
3835 }
3836 return Prefix->getName().str();
3837 };
3838
3839 // C23 6.10p5: In all aspects, a preprocessor standard parameter specified by
3840 // this document as an identifier pp_param and an identifier of the form
3841 // __pp_param__ shall behave the same when used as a preprocessor parameter,
3842 // except for the spelling.
3843 auto NormalizeParameterName = [](StringRef Name) {
3844 if (Name.size() > 4 && Name.starts_with(Prefix: "__") && Name.ends_with(Suffix: "__"))
3845 return Name.substr(Start: 2, N: Name.size() - 4);
3846 return Name;
3847 };
3848
3849 auto LexParenthesizedIntegerExpr = [&]() -> std::optional<size_t> {
3850 // we have a limit parameter and its internals are processed using
3851 // evaluation rules from #if.
3852 if (!ExpectOrDiagAndSkipToEOD(tok::l_paren))
3853 return std::nullopt;
3854
3855 // We do not consume the ( because EvaluateDirectiveExpression will lex
3856 // the next token for us.
3857 IdentifierInfo *ParameterIfNDef = nullptr;
3858 bool EvaluatedDefined;
3859 DirectiveEvalResult LimitEvalResult = EvaluateDirectiveExpression(
3860 IfNDefMacro&: ParameterIfNDef, Tok&: CurTok, EvaluatedDefined, /*CheckForEOD=*/CheckForEoD: false);
3861
3862 if (!LimitEvalResult.Value) {
3863 // If there was an error evaluating the directive expression, we expect
3864 // to be at the end of directive token.
3865 assert(CurTok.is(tok::eod) && "expect to be at the end of directive");
3866 return std::nullopt;
3867 }
3868
3869 if (!ExpectOrDiagAndSkipToEOD(tok::r_paren))
3870 return std::nullopt;
3871
3872 // Eat the ).
3873 LexNonComment(Result&: CurTok);
3874
3875 // C23 6.10.3.2p2: The token defined shall not appear within the constant
3876 // expression.
3877 if (EvaluatedDefined) {
3878 Diag(Tok: CurTok, DiagID: diag::err_defined_in_pp_embed);
3879 return std::nullopt;
3880 }
3881
3882 if (LimitEvalResult.Value) {
3883 const llvm::APSInt &Result = *LimitEvalResult.Value;
3884 if (Result.isNegative()) {
3885 Diag(Tok: CurTok, DiagID: diag::err_requires_positive_value)
3886 << toString(I: Result, Radix: 10) << /*positive*/ 0;
3887 if (CurTok.isNot(K: EndTokenKind))
3888 DiscardUntilEndOfDirective(Tmp&: CurTok);
3889 return std::nullopt;
3890 }
3891 return Result.getLimitedValue();
3892 }
3893 return std::nullopt;
3894 };
3895
3896 auto GetMatchingCloseBracket = [](tok::TokenKind Kind) {
3897 switch (Kind) {
3898 case tok::l_paren:
3899 return tok::r_paren;
3900 case tok::l_brace:
3901 return tok::r_brace;
3902 case tok::l_square:
3903 return tok::r_square;
3904 default:
3905 llvm_unreachable("should not get here");
3906 }
3907 };
3908
3909 auto LexParenthesizedBalancedTokenSoup =
3910 [&](llvm::SmallVectorImpl<Token> &Tokens) {
3911 std::vector<std::pair<tok::TokenKind, SourceLocation>> BracketStack;
3912
3913 // We expect the current token to be a left paren.
3914 if (!ExpectOrDiagAndSkipToEOD(tok::l_paren))
3915 return false;
3916 LexNonComment(Result&: CurTok); // Eat the (
3917
3918 bool WaitingForInnerCloseParen = false;
3919 while (CurTok.isNot(K: tok::eod) &&
3920 (WaitingForInnerCloseParen || CurTok.isNot(K: tok::r_paren))) {
3921 switch (CurTok.getKind()) {
3922 default: // Shutting up diagnostics about not fully-covered switch.
3923 break;
3924 case tok::l_paren:
3925 WaitingForInnerCloseParen = true;
3926 [[fallthrough]];
3927 case tok::l_brace:
3928 case tok::l_square:
3929 BracketStack.push_back(x: {CurTok.getKind(), CurTok.getLocation()});
3930 break;
3931 case tok::r_paren:
3932 WaitingForInnerCloseParen = false;
3933 [[fallthrough]];
3934 case tok::r_brace:
3935 case tok::r_square: {
3936 if (BracketStack.empty()) {
3937 ExpectOrDiagAndSkipToEOD(tok::r_paren);
3938 return false;
3939 }
3940 tok::TokenKind Matching =
3941 GetMatchingCloseBracket(BracketStack.back().first);
3942 if (CurTok.getKind() != Matching) {
3943 DiagMismatchedBracesAndSkipToEOD(Matching, BracketStack.back());
3944 return false;
3945 }
3946 BracketStack.pop_back();
3947 } break;
3948 }
3949 Tokens.push_back(Elt: CurTok);
3950 LexNonComment(Result&: CurTok);
3951 }
3952
3953 // When we're done, we want to eat the closing paren.
3954 if (!ExpectOrDiagAndSkipToEOD(tok::r_paren))
3955 return false;
3956
3957 LexNonComment(Result&: CurTok); // Eat the )
3958 return true;
3959 };
3960
3961 LexNonComment(Result&: CurTok); // Prime the pump.
3962 while (!CurTok.isOneOf(Ks: EndTokenKind, Ks: tok::eod)) {
3963 SourceLocation ParamStartLoc = CurTok.getLocation();
3964 std::optional<std::string> ParamName = LexPPParameterName();
3965 if (!ParamName)
3966 return std::nullopt;
3967 StringRef Parameter = NormalizeParameterName(*ParamName);
3968
3969 // Lex the parameters (dependent on the parameter type we want!).
3970 //
3971 // C23 6.10.3.Xp1: The X standard embed parameter may appear zero times or
3972 // one time in the embed parameter sequence.
3973 if (Parameter == "limit") {
3974 if (Result.MaybeLimitParam)
3975 Diag(Tok: CurTok, DiagID: diag::err_pp_embed_dup_params) << Parameter;
3976
3977 std::optional<size_t> Limit = LexParenthesizedIntegerExpr();
3978 if (!Limit)
3979 return std::nullopt;
3980 Result.MaybeLimitParam =
3981 PPEmbedParameterLimit{*Limit, {ParamStartLoc, CurTok.getLocation()}};
3982 } else if (Parameter == "clang::offset") {
3983 if (Result.MaybeOffsetParam)
3984 Diag(Tok: CurTok, DiagID: diag::err_pp_embed_dup_params) << Parameter;
3985
3986 std::optional<size_t> Offset = LexParenthesizedIntegerExpr();
3987 if (!Offset)
3988 return std::nullopt;
3989 Result.MaybeOffsetParam = PPEmbedParameterOffset{
3990 *Offset, {ParamStartLoc, CurTok.getLocation()}};
3991 } else if (Parameter == "prefix") {
3992 if (Result.MaybePrefixParam)
3993 Diag(Tok: CurTok, DiagID: diag::err_pp_embed_dup_params) << Parameter;
3994
3995 SmallVector<Token, 4> Soup;
3996 if (!LexParenthesizedBalancedTokenSoup(Soup))
3997 return std::nullopt;
3998 Result.MaybePrefixParam = PPEmbedParameterPrefix{
3999 std::move(Soup), {ParamStartLoc, CurTok.getLocation()}};
4000 } else if (Parameter == "suffix") {
4001 if (Result.MaybeSuffixParam)
4002 Diag(Tok: CurTok, DiagID: diag::err_pp_embed_dup_params) << Parameter;
4003
4004 SmallVector<Token, 4> Soup;
4005 if (!LexParenthesizedBalancedTokenSoup(Soup))
4006 return std::nullopt;
4007 Result.MaybeSuffixParam = PPEmbedParameterSuffix{
4008 std::move(Soup), {ParamStartLoc, CurTok.getLocation()}};
4009 } else if (Parameter == "if_empty") {
4010 if (Result.MaybeIfEmptyParam)
4011 Diag(Tok: CurTok, DiagID: diag::err_pp_embed_dup_params) << Parameter;
4012
4013 SmallVector<Token, 4> Soup;
4014 if (!LexParenthesizedBalancedTokenSoup(Soup))
4015 return std::nullopt;
4016 Result.MaybeIfEmptyParam = PPEmbedParameterIfEmpty{
4017 std::move(Soup), {ParamStartLoc, CurTok.getLocation()}};
4018 } else {
4019 ++Result.UnrecognizedParams;
4020
4021 // If there's a left paren, we need to parse a balanced token sequence
4022 // and just eat those tokens.
4023 if (CurTok.is(K: tok::l_paren)) {
4024 SmallVector<Token, 4> Soup;
4025 if (!LexParenthesizedBalancedTokenSoup(Soup))
4026 return std::nullopt;
4027 }
4028 if (!ForHasEmbed) {
4029 Diag(Loc: ParamStartLoc, DiagID: diag::err_pp_unknown_parameter) << 1 << Parameter;
4030 if (CurTok.isNot(K: EndTokenKind))
4031 DiscardUntilEndOfDirective(Tmp&: CurTok);
4032 return std::nullopt;
4033 }
4034 }
4035 }
4036 return Result;
4037}
4038
4039void Preprocessor::HandleEmbedDirectiveImpl(
4040 SourceLocation HashLoc, const LexEmbedParametersResult &Params,
4041 StringRef BinaryContents, StringRef FileName) {
4042 if (BinaryContents.empty()) {
4043 // If we have no binary contents, the only thing we need to emit are the
4044 // if_empty tokens, if any.
4045 // FIXME: this loses AST fidelity; nothing in the compiler will see that
4046 // these tokens came from #embed. We have to hack around this when printing
4047 // preprocessed output. The same is true for prefix and suffix tokens.
4048 if (Params.MaybeIfEmptyParam) {
4049 ArrayRef<Token> Toks = Params.MaybeIfEmptyParam->Tokens;
4050 size_t TokCount = Toks.size();
4051 auto NewToks = std::make_unique<Token[]>(num: TokCount);
4052 llvm::copy(Range&: Toks, Out: NewToks.get());
4053 EnterTokenStream(Toks: std::move(NewToks), NumToks: TokCount, DisableMacroExpansion: true, IsReinject: true);
4054 }
4055 return;
4056 }
4057
4058 size_t NumPrefixToks = Params.PrefixTokenCount(),
4059 NumSuffixToks = Params.SuffixTokenCount();
4060 size_t TotalNumToks = 1 + NumPrefixToks + NumSuffixToks;
4061 size_t CurIdx = 0;
4062 auto Toks = std::make_unique<Token[]>(num: TotalNumToks);
4063
4064 // Add the prefix tokens, if any.
4065 if (Params.MaybePrefixParam) {
4066 llvm::copy(Range: Params.MaybePrefixParam->Tokens, Out: &Toks[CurIdx]);
4067 CurIdx += NumPrefixToks;
4068 }
4069
4070 EmbedAnnotationData *Data = new (BP) EmbedAnnotationData;
4071 Data->BinaryData = BinaryContents;
4072 Data->FileName = FileName;
4073
4074 Toks[CurIdx++] = Token::createAnnotation(Kind: tok::annot_embed, Range: HashLoc, Value: Data);
4075
4076 // Now add the suffix tokens, if any.
4077 if (Params.MaybeSuffixParam) {
4078 llvm::copy(Range: Params.MaybeSuffixParam->Tokens, Out: &Toks[CurIdx]);
4079 CurIdx += NumSuffixToks;
4080 }
4081
4082 assert(CurIdx == TotalNumToks && "Calculated the incorrect number of tokens");
4083 EnterTokenStream(Toks: std::move(Toks), NumToks: TotalNumToks, DisableMacroExpansion: true, IsReinject: true);
4084}
4085
4086void Preprocessor::HandleEmbedDirective(SourceLocation HashLoc,
4087 Token &EmbedTok) {
4088 // Give the usual extension/compatibility warnings.
4089 if (LangOpts.C23)
4090 Diag(Tok: EmbedTok, DiagID: diag::warn_compat_pp_embed_directive);
4091 else
4092 Diag(Tok: EmbedTok, DiagID: diag::ext_pp_embed_directive)
4093 << (LangOpts.CPlusPlus ? /*Clang*/ 1 : /*C23*/ 0);
4094
4095 // Parse the filename header
4096 Token FilenameTok;
4097 if (LexHeaderName(Result&: FilenameTok))
4098 return;
4099
4100 if (FilenameTok.isNot(K: tok::header_name)) {
4101 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_pp_expects_filename);
4102 if (FilenameTok.isNot(K: tok::eod))
4103 DiscardUntilEndOfDirective();
4104 return;
4105 }
4106
4107 // Parse the optional sequence of
4108 // directive-parameters:
4109 // identifier parameter-name-list[opt] directive-argument-list[opt]
4110 // directive-argument-list:
4111 // '(' balanced-token-sequence ')'
4112 // parameter-name-list:
4113 // '::' identifier parameter-name-list[opt]
4114 Token CurTok;
4115 std::optional<LexEmbedParametersResult> Params =
4116 LexEmbedParameters(CurTok, /*ForHasEmbed=*/false);
4117
4118 assert((Params || CurTok.is(tok::eod)) &&
4119 "expected success or to be at the end of the directive");
4120 if (!Params)
4121 return;
4122
4123 // Now, splat the data out!
4124 SmallString<128> FilenameBuffer;
4125 StringRef Filename = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
4126 StringRef OriginalFilename = Filename;
4127 bool isAngled =
4128 GetIncludeFilenameSpelling(Loc: FilenameTok.getLocation(), Buffer&: Filename);
4129
4130 // If GetIncludeFilenameSpelling set the start ptr to null, there was an
4131 // error.
4132 if (Filename.empty())
4133 return;
4134
4135 OptionalFileEntryRef MaybeFileRef =
4136 this->LookupEmbedFile(Filename, isAngled, /*OpenFile=*/true);
4137 if (!MaybeFileRef) {
4138 // could not find file
4139 if (Callbacks && Callbacks->EmbedFileNotFound(FileName: Filename)) {
4140 return;
4141 }
4142 Diag(Tok: FilenameTok, DiagID: diag::err_pp_file_not_found) << Filename;
4143 return;
4144 }
4145
4146 if (MaybeFileRef->isDeviceFile()) {
4147 Diag(Tok: FilenameTok, DiagID: diag::err_pp_embed_device_file) << Filename;
4148 return;
4149 }
4150
4151 std::optional<llvm::MemoryBufferRef> MaybeFile =
4152 getSourceManager().getMemoryBufferForFileOrNone(File: *MaybeFileRef);
4153 if (!MaybeFile) {
4154 // could not find file
4155 Diag(Tok: FilenameTok, DiagID: diag::err_cannot_open_file)
4156 << Filename << "a buffer to the contents could not be created";
4157 return;
4158 }
4159 StringRef BinaryContents = MaybeFile->getBuffer();
4160
4161 // The order is important between 'offset' and 'limit'; we want to offset
4162 // first and then limit second; otherwise we may reduce the notional resource
4163 // size to something too small to offset into.
4164 if (Params->MaybeOffsetParam) {
4165 // FIXME: just like with the limit() and if_empty() parameters, this loses
4166 // source fidelity in the AST; it has no idea that there was an offset
4167 // involved.
4168 // offsets all the way to the end of the file make for an empty file.
4169 BinaryContents = BinaryContents.substr(Start: Params->MaybeOffsetParam->Offset);
4170 }
4171
4172 if (Params->MaybeLimitParam) {
4173 // FIXME: just like with the clang::offset() and if_empty() parameters,
4174 // this loses source fidelity in the AST; it has no idea there was a limit
4175 // involved.
4176 BinaryContents = BinaryContents.substr(Start: 0, N: Params->MaybeLimitParam->Limit);
4177 }
4178
4179 if (Callbacks)
4180 Callbacks->EmbedDirective(HashLoc, FileName: Filename, IsAngled: isAngled, File: MaybeFileRef,
4181 Params: *Params);
4182 // getSpelling() may return a buffer from the token itself or it may use the
4183 // SmallString buffer we provided. getSpelling() may also return a string that
4184 // is actually longer than FilenameTok.getLength(), so we first pass a
4185 // locally created buffer to getSpelling() to get the string of real length
4186 // and then we allocate a long living buffer because the buffer we used
4187 // previously will only live till the end of this function and we need
4188 // filename info to live longer.
4189 void *Mem = BP.Allocate(Size: OriginalFilename.size(), Alignment: alignof(char *));
4190 memcpy(dest: Mem, src: OriginalFilename.data(), n: OriginalFilename.size());
4191 StringRef FilenameToGo =
4192 StringRef(static_cast<char *>(Mem), OriginalFilename.size());
4193 HandleEmbedDirectiveImpl(HashLoc, Params: *Params, BinaryContents, FileName: FilenameToGo);
4194}
4195
4196/// HandleCXXImportDirective - Handle the C++ modules import directives
4197///
4198/// pp-import:
4199/// export[opt] import header-name pp-tokens[opt] ; new-line
4200/// export[opt] import header-name-tokens pp-tokens[opt] ; new-line
4201/// export[opt] import pp-tokens ; new-line
4202///
4203/// The header importing are replaced by annot_header_unit token, and the
4204/// lexed module name are replaced by annot_module_name token.
4205void Preprocessor::HandleCXXImportDirective(Token ImportTok) {
4206 assert(getLangOpts().CPlusPlusModules && ImportTok.is(tok::kw_import));
4207 llvm::SaveAndRestore<bool> SaveImportingCXXModules(
4208 this->ImportingCXXNamedModules, true);
4209
4210 Token Tok;
4211 if (LexHeaderName(Result&: Tok)) {
4212 if (Tok.isNot(K: tok::eod))
4213 CheckEndOfDirective(DirType: ImportTok.getIdentifierInfo()->getName());
4214 return;
4215 }
4216
4217 SourceLocation UseLoc = ImportTok.getLocation();
4218 SmallVector<Token, 4> DirToks{ImportTok};
4219 SmallVector<IdentifierLoc, 2> Path;
4220 bool ImportingHeader = false;
4221 bool IsPartition = false;
4222
4223 switch (Tok.getKind()) {
4224 case tok::header_name:
4225 ImportingHeader = true;
4226 DirToks.push_back(Elt: Tok);
4227 Lex(Result&: DirToks.emplace_back());
4228 break;
4229 case tok::colon:
4230 IsPartition = true;
4231 DirToks.push_back(Elt: Tok);
4232 UseLoc = Tok.getLocation();
4233 Lex(Result&: Tok);
4234 [[fallthrough]];
4235 case tok::code_completion:
4236 case tok::identifier: {
4237 if (HandleModuleName(DirType: ImportTok.getIdentifierInfo()->getName(), UseLoc, Tok,
4238 Path, DirToks, /*AllowMacroExpansion=*/true,
4239 IsPartition))
4240 return;
4241
4242 std::string FlatName;
4243 bool IsValid =
4244 (IsPartition && ModuleDeclState.isNamedModule()) || !IsPartition;
4245 if (Callbacks && IsValid) {
4246 if (IsPartition && ModuleDeclState.isNamedModule()) {
4247 FlatName += ModuleDeclState.getPrimaryName();
4248 FlatName += ":";
4249 }
4250
4251 FlatName += ModuleLoader::getFlatNameFromPath(Path);
4252 SourceLocation StartLoc = IsPartition ? UseLoc : Path[0].getLoc();
4253 IdentifierLoc FlatNameLoc(StartLoc, getIdentifierInfo(Name: FlatName));
4254
4255 // We don't/shouldn't load the standard c++20 modules when preprocessing.
4256 // so the imported module is nullptr.
4257 Callbacks->moduleImport(ImportLoc: ImportTok.getLocation(),
4258 Path: ModuleIdPath(FlatNameLoc),
4259 /*Imported=*/nullptr);
4260 }
4261 break;
4262 }
4263 default:
4264 DirToks.push_back(Elt: Tok);
4265 break;
4266 }
4267
4268 // Consume the pp-import-suffix and expand any macros in it now, if we're not
4269 // at the semicolon already.
4270 if (!DirToks.back().isOneOf(Ks: tok::semi, Ks: tok::eod))
4271 CollectPPImportSuffix(Toks&: DirToks);
4272
4273 if (DirToks.back().isNot(K: tok::eod))
4274 CheckEndOfDirective(DirType: ImportTok.getIdentifierInfo()->getName());
4275 else
4276 DirToks.pop_back();
4277
4278 // This is not a pp-import after all.
4279 if (DirToks.back().isNot(K: tok::semi)) {
4280 EnterModuleSuffixTokenStream(Toks: DirToks);
4281 return;
4282 }
4283
4284 if (ImportingHeader) {
4285 // C++2a [cpp.module]p1:
4286 // The ';' preprocessing-token terminating a pp-import shall not have
4287 // been produced by macro replacement.
4288 SourceLocation SemiLoc = DirToks.back().getLocation();
4289 if (SemiLoc.isMacroID())
4290 Diag(Loc: SemiLoc, DiagID: diag::err_header_import_semi_in_macro);
4291
4292 auto Action = HandleHeaderIncludeOrImport(
4293 /*HashLoc*/ SourceLocation(), IncludeTok&: ImportTok, FilenameTok&: Tok, EndLoc: SemiLoc);
4294 switch (Action.Kind) {
4295 case ImportAction::None:
4296 break;
4297
4298 case ImportAction::ModuleBegin:
4299 // Let the parser know we're textually entering the module.
4300 DirToks.push_back(Elt: Token::createAnnotation(
4301 Kind: tok::annot_module_begin, Range: SemiLoc, Value: Action.ModuleForHeader));
4302 [[fallthrough]];
4303
4304 case ImportAction::ModuleImport:
4305 case ImportAction::HeaderUnitImport:
4306 case ImportAction::SkippedModuleImport:
4307 // We chose to import (or textually enter) the file. Convert the
4308 // header-name token into a header unit annotation token.
4309 DirToks[1].setKind(tok::annot_header_unit);
4310 DirToks[1].setAnnotationEndLoc(DirToks[0].getLocation());
4311 DirToks[1].setAnnotationValue(Action.ModuleForHeader);
4312 // FIXME: Call the moduleImport callback?
4313 break;
4314 case ImportAction::Failure:
4315 assert(TheModuleLoader.HadFatalFailure &&
4316 "This should be an early exit only to a fatal error");
4317 CurLexer->cutOffLexing();
4318 return;
4319 }
4320 }
4321
4322 EnterModuleSuffixTokenStream(Toks: DirToks);
4323}
4324
4325/// HandleCXXModuleDirective - Handle C++ module declaration directives.
4326///
4327/// pp-module:
4328/// export[opt] module pp-tokens[opt] ; new-line
4329///
4330/// pp-module-name:
4331/// pp-module-name-qualifier[opt] identifier
4332/// pp-module-partition:
4333/// : pp-module-name-qualifier[opt] identifier
4334/// pp-module-name-qualifier:
4335/// identifier .
4336/// pp-module-name-qualifier identifier .
4337///
4338/// global-module-fragment:
4339/// module-keyword ; declaration-seq[opt]
4340///
4341/// private-module-fragment:
4342/// module-keyword : private ; declaration-seq[opt]
4343///
4344/// The lexed module name are replaced by annot_module_name token.
4345void Preprocessor::HandleCXXModuleDirective(Token ModuleTok) {
4346 assert(getLangOpts().CPlusPlusModules && ModuleTok.is(tok::kw_module));
4347 SourceLocation StartLoc = ModuleTok.getLocation();
4348
4349 Token Tok;
4350 SourceLocation UseLoc = ModuleTok.getLocation();
4351 SmallVector<Token, 4> DirToks{ModuleTok};
4352 SmallVector<IdentifierLoc, 2> Path, Partition;
4353 LexUnexpandedToken(Result&: Tok);
4354
4355 switch (Tok.getKind()) {
4356 // Global Module Fragment.
4357 case tok::semi:
4358 DirToks.push_back(Elt: Tok);
4359 break;
4360 case tok::colon:
4361 DirToks.push_back(Elt: Tok);
4362 LexUnexpandedToken(Result&: Tok);
4363 if (Tok.isNot(K: tok::kw_private)) {
4364 if (Tok.isNot(K: tok::eod))
4365 CheckEndOfDirective(DirType: ModuleTok.getIdentifierInfo()->getName(),
4366 /*EnableMacros=*/false, ExtraToks: &DirToks);
4367 EnterModuleSuffixTokenStream(Toks: DirToks);
4368 return;
4369 }
4370 DirToks.push_back(Elt: Tok);
4371 break;
4372 case tok::identifier: {
4373 if (HandleModuleName(DirType: ModuleTok.getIdentifierInfo()->getName(), UseLoc, Tok,
4374 Path, DirToks, /*AllowMacroExpansion=*/false,
4375 /*IsPartition=*/false))
4376 return;
4377
4378 // C++20 [cpp.module]p
4379 // The pp-tokens, if any, of a pp-module shall be of the form:
4380 // pp-module-name pp-module-partition[opt] pp-tokens[opt]
4381 if (Tok.is(K: tok::colon)) {
4382 LexUnexpandedToken(Result&: Tok);
4383 if (HandleModuleName(DirType: ModuleTok.getIdentifierInfo()->getName(), UseLoc,
4384 Tok, Path&: Partition, DirToks,
4385 /*AllowMacroExpansion=*/false, /*IsPartition=*/true))
4386 return;
4387 }
4388
4389 // If the current token is a macro definition, put it back to token stream
4390 // and expand any macros in it later.
4391 //
4392 // export module M ATTR(some_attr); // -D'ATTR(x)=[[x]]'
4393 //
4394 // Current token is `ATTR`.
4395 if (Tok.is(K: tok::identifier) &&
4396 getMacroDefinition(II: Tok.getIdentifierInfo())) {
4397 std::unique_ptr<Token[]> TokCopy = std::make_unique<Token[]>(num: 1);
4398 TokCopy[0] = Tok;
4399 EnterTokenStream(Toks: std::move(TokCopy), /*NumToks=*/1,
4400 /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
4401 Lex(Result&: Tok);
4402 DirToks.back() = Tok;
4403 }
4404 break;
4405 }
4406 default:
4407 DirToks.push_back(Elt: Tok);
4408 break;
4409 }
4410
4411 if (!DirToks.back().isOneOf(Ks: tok::semi, Ks: tok::eod)) {
4412 // Consume the pp-import-suffix and expand any macros in it now. We'll add
4413 // it back into the token stream later.
4414 CollectPPImportSuffix(Toks&: DirToks);
4415 }
4416
4417 SourceLocation End =
4418 DirToks.back().isNot(K: tok::eod)
4419 ? CheckEndOfDirective(DirType: ModuleTok.getIdentifierInfo()->getName(),
4420 /*EnableMacros=*/false, ExtraToks: &DirToks)
4421
4422 : DirToks.pop_back_val().getLocation();
4423
4424 bool IsGMFIntroducer = DirToks.size() == 2 && DirToks[0].is(K: tok::kw_module) &&
4425 DirToks[1].is(K: tok::semi);
4426 bool IsSynthesizedGMF = IsGMFIntroducer && HasSynthesizedGMF &&
4427 CurPPLexer->getFileID() == getPredefinesFileID();
4428
4429 if (!IncludeMacroStack.empty() && !IsSynthesizedGMF) {
4430 Diag(Loc: StartLoc, DiagID: diag::err_pp_module_decl_in_header)
4431 << SourceRange(StartLoc, End);
4432 }
4433
4434 if (CurPPLexer->getConditionalStackDepth() != 0) {
4435 Diag(Loc: StartLoc, DiagID: diag::err_pp_cond_span_module_decl)
4436 << SourceRange(StartLoc, End);
4437 }
4438
4439 // For the global-module-fragment introducer (`module;`), enter any implicit
4440 // macro, PCH, and regular include files that were deferred to the GMF now,
4441 // before re-entering the `module;` token stream. Because the include stack is
4442 // LIFO, the `module;` tokens are consumed first and the included files are
4443 // then lexed inside the fragment (ahead of the rest of the main file).
4444 if (IsGMFIntroducer)
4445 EnterDeferredGMFInputs(IncludeLoc: End);
4446
4447 EnterModuleSuffixTokenStream(Toks: DirToks);
4448}
4449
4450/// Lex a token following the 'import' contextual keyword.
4451///
4452/// pp-import:
4453/// [ObjC] @ import module-name ;
4454///
4455/// module-name:
4456/// module-name-qualifier[opt] identifier
4457///
4458/// module-name-qualifier
4459/// module-name-qualifier[opt] identifier .
4460///
4461/// We respond to a pp-import by importing macros from the named module.
4462void Preprocessor::HandleObjCImportDirective(Token &AtTok, Token &ImportTok) {
4463 assert(getLangOpts().ObjC && AtTok.is(tok::at) &&
4464 ImportTok.isObjCAtKeyword(tok::objc_import));
4465 ImportTok.setKind(tok::kw_import);
4466 SmallVector<Token, 32> DirToks{AtTok, ImportTok};
4467 SmallVector<IdentifierLoc, 3> Path;
4468 SourceLocation UseLoc = ImportTok.getLocation();
4469 ModuleImportLoc = ImportTok.getLocation();
4470 Token Tok;
4471 Lex(Result&: Tok);
4472 if (HandleModuleName(DirType: ImportTok.getIdentifierInfo()->getName(), UseLoc, Tok,
4473 Path, DirToks,
4474 /*AllowMacroExpansion=*/true,
4475 /*IsPartition=*/false))
4476 return;
4477
4478 // Consume the pp-import-suffix and expand any macros in it now, if we're not
4479 // at the semicolon already.
4480 if (!DirToks.back().isOneOf(Ks: tok::semi, Ks: tok::eod))
4481 CollectPPImportSuffix(Toks&: DirToks);
4482
4483 SourceLocation End =
4484 DirToks.back().isNot(K: tok::eod)
4485 ? CheckEndOfDirective(DirType: ImportTok.getIdentifierInfo()->getName(),
4486 /*EnableMacros=*/false, ExtraToks: &DirToks)
4487
4488 : DirToks.pop_back_val().getLocation();
4489
4490 Module *Imported = nullptr;
4491 if (getLangOpts().Modules) {
4492 Imported = TheModuleLoader.loadModule(ImportLoc: ModuleImportLoc, Path, Visibility: Module::Hidden,
4493 /*IsInclusionDirective=*/false);
4494 if (Imported)
4495 makeModuleVisible(M: Imported, Loc: End);
4496 }
4497
4498 if (Callbacks)
4499 Callbacks->moduleImport(ImportLoc: ModuleImportLoc, Path, Imported);
4500
4501 EnterModuleSuffixTokenStream(Toks: DirToks);
4502}
4503