1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Preprocessor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/Basic/FileManager.h"
30#include "clang/Basic/IdentifierTable.h"
31#include "clang/Basic/LLVM.h"
32#include "clang/Basic/LangOptions.h"
33#include "clang/Basic/Module.h"
34#include "clang/Basic/SourceLocation.h"
35#include "clang/Basic/SourceManager.h"
36#include "clang/Basic/TargetInfo.h"
37#include "clang/Lex/CodeCompletionHandler.h"
38#include "clang/Lex/DependencyDirectivesScanner.h"
39#include "clang/Lex/ExternalPreprocessorSource.h"
40#include "clang/Lex/HeaderSearch.h"
41#include "clang/Lex/LexDiagnostic.h"
42#include "clang/Lex/Lexer.h"
43#include "clang/Lex/LiteralSupport.h"
44#include "clang/Lex/MacroArgs.h"
45#include "clang/Lex/MacroInfo.h"
46#include "clang/Lex/ModuleLoader.h"
47#include "clang/Lex/NoTrivialPPDirectiveTracer.h"
48#include "clang/Lex/Pragma.h"
49#include "clang/Lex/PreprocessingRecord.h"
50#include "clang/Lex/PreprocessorLexer.h"
51#include "clang/Lex/PreprocessorOptions.h"
52#include "clang/Lex/ScratchBuffer.h"
53#include "clang/Lex/Token.h"
54#include "clang/Lex/TokenLexer.h"
55#include "clang/Support/Compiler.h"
56#include "llvm/ADT/APInt.h"
57#include "llvm/ADT/ArrayRef.h"
58#include "llvm/ADT/DenseMap.h"
59#include "llvm/ADT/STLExtras.h"
60#include "llvm/ADT/ScopeExit.h"
61#include "llvm/ADT/SmallVector.h"
62#include "llvm/ADT/StringRef.h"
63#include "llvm/Support/Capacity.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/FormatVariadic.h"
66#include "llvm/Support/MemoryBuffer.h"
67#include "llvm/Support/MemoryBufferRef.h"
68#include "llvm/Support/SaveAndRestore.h"
69#include "llvm/Support/raw_ostream.h"
70#include <algorithm>
71#include <cassert>
72#include <memory>
73#include <optional>
74#include <string>
75#include <utility>
76#include <vector>
77
78using namespace clang;
79
80/// Minimum distance between two check points, in tokens.
81static constexpr unsigned CheckPointStepSize = 1024;
82
83LLVM_INSTANTIATE_REGISTRY_EX(CLANG_ABI_EXPORT, PragmaHandlerRegistry)
84
85ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
86
87Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
88 DiagnosticsEngine &diags, const LangOptions &opts,
89 SourceManager &SM, HeaderSearch &Headers,
90 ModuleLoader &TheModuleLoader,
91 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
92 TranslationUnitKind TUKind)
93 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
94 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
95 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
96 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
97 // As the language options may have not been loaded yet (when
98 // deserializing an ASTUnit), adding keywords to the identifier table is
99 // deferred to Preprocessor::Initialize().
100 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
101 TUKind(TUKind), SkipMainFilePreamble(0, true),
102 CurSubmoduleState(&NullSubmoduleState) {
103 OwnsHeaderSearch = OwnsHeaders;
104
105 // Only record check points if we might highlight diagnostic snippets.
106 RecordCheckPoints = getDiagnostics().getShowColors();
107
108 // Default to discarding comments.
109 KeepComments = false;
110 KeepMacroComments = false;
111 SuppressIncludeNotFoundError = false;
112
113 // Macro expansion is enabled.
114 DisableMacroExpansion = false;
115 MacroExpansionInDirectivesOverride = false;
116 InMacroArgs = false;
117 ArgMacro = nullptr;
118 InMacroArgPreExpansion = false;
119 NumCachedTokenLexers = 0;
120 PragmasEnabled = true;
121 ParsingIfOrElifDirective = false;
122 PreprocessedOutput = false;
123
124 // We haven't read anything from the external source.
125 ReadMacrosFromExternalSource = false;
126
127 LastExportKeyword.startToken();
128
129 BuiltinInfo = std::make_unique<Builtin::Context>();
130
131 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
132 // a macro. They get unpoisoned where it is allowed.
133 (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned();
134 SetPoisonReason(II: Ident__VA_ARGS__,DiagID: diag::ext_pp_bad_vaargs_use);
135 (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned();
136 SetPoisonReason(II: Ident__VA_OPT__,DiagID: diag::ext_pp_bad_vaopt_use);
137
138 // Initialize the pragma handlers.
139 RegisterBuiltinPragmas();
140
141 // Initialize builtin macros like __LINE__ and friends.
142 RegisterBuiltinMacros();
143
144 if(LangOpts.Borland) {
145 Ident__exception_info = getIdentifierInfo(Name: "_exception_info");
146 Ident___exception_info = getIdentifierInfo(Name: "__exception_info");
147 Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation");
148 Ident__exception_code = getIdentifierInfo(Name: "_exception_code");
149 Ident___exception_code = getIdentifierInfo(Name: "__exception_code");
150 Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode");
151 Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination");
152 Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination");
153 Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination");
154 } else {
155 Ident__exception_info = Ident__exception_code = nullptr;
156 Ident__abnormal_termination = Ident___exception_info = nullptr;
157 Ident___exception_code = Ident___abnormal_termination = nullptr;
158 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
159 Ident_AbnormalTermination = nullptr;
160 }
161
162 Ident__GLIBCXX__ = getIdentifierInfo(Name: "__GLIBCXX__");
163
164 // Default incremental processing to -fincremental-extensions, clients can
165 // override with `enableIncrementalProcessing` if desired.
166 IncrementalProcessing = LangOpts.IncrementalExtensions;
167
168 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
169 if (usingPCHWithPragmaHdrStop())
170 SkippingUntilPragmaHdrStop = true;
171
172 // If using a PCH with a through header, start skipping tokens.
173 if (!this->PPOpts.PCHThroughHeader.empty() &&
174 !this->PPOpts.ImplicitPCHInclude.empty())
175 SkippingUntilPCHThroughHeader = true;
176
177 if (this->PPOpts.GeneratePreamble)
178 PreambleConditionalStack.startRecording();
179
180 MaxTokens = LangOpts.MaxTokens;
181}
182
183Preprocessor::~Preprocessor() {
184 assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!");
185
186 IncludeMacroStack.clear();
187
188 // Free any cached macro expanders.
189 // This populates MacroArgCache, so all TokenLexers need to be destroyed
190 // before the code below that frees up the MacroArgCache list.
191 std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr);
192 CurTokenLexer.reset();
193
194 // Free any cached MacroArgs.
195 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
196 ArgList = ArgList->deallocate();
197
198 // Delete the header search info, if we own it.
199 if (OwnsHeaderSearch)
200 delete &HeaderInfo;
201}
202
203void Preprocessor::Initialize(const TargetInfo &Target,
204 const TargetInfo *AuxTarget) {
205 assert((!this->Target || this->Target == &Target) &&
206 "Invalid override of target information");
207 this->Target = &Target;
208
209 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
210 "Invalid override of aux target information.");
211 this->AuxTarget = AuxTarget;
212
213 // Initialize information about built-ins.
214 BuiltinInfo->InitializeTarget(Target, AuxTarget);
215 HeaderInfo.setTarget(Target);
216
217 // Populate the identifier table with info about keywords for the current language.
218 Identifiers.AddKeywords(LangOpts);
219
220 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
221 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
222
223 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
224 // Use setting from TargetInfo.
225 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod());
226 else
227 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
228 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod());
229}
230
231void Preprocessor::InitializeForModelFile() {
232 NumEnteredSourceFiles = 0;
233
234 // Reset pragmas
235 PragmaHandlersBackup = std::move(PragmaHandlers);
236 PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef());
237 RegisterBuiltinPragmas();
238
239 // Reset PredefinesFileID
240 PredefinesFileID = FileID();
241}
242
243void Preprocessor::FinalizeForModelFile() {
244 NumEnteredSourceFiles = 1;
245
246 PragmaHandlers = std::move(PragmaHandlersBackup);
247}
248
249void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
250 std::string TokenStr;
251 llvm::raw_string_ostream OS(TokenStr);
252
253 // The alignment of 16 is chosen to comfortably fit most identifiers.
254 OS << llvm::formatv(Fmt: "{0,-16} ", Vals: tok::getTokenName(Kind: Tok.getKind()));
255
256 // Annotation tokens are just markers that don't have a spelling -- they
257 // indicate where something expanded.
258 if (!Tok.isAnnotation()) {
259 OS << "'";
260 // Escape string to prevent token spelling from spanning multiple lines.
261 OS.write_escaped(Str: getSpelling(Tok));
262 OS << "'";
263 }
264
265 // The alignment of 48 (32 characters for the spelling + the 16 for
266 // the identifier name) fits most variable names, keywords and annotations.
267 llvm::errs() << llvm::formatv(Fmt: "{0,-48} ", Vals&: OS.str());
268
269 if (!DumpFlags) return;
270
271 auto Loc = Tok.getLocation();
272 llvm::errs() << "Loc=<";
273 DumpLocation(Loc);
274 llvm::errs() << ">";
275
276 // If the token points directly to a file location (i.e. not a macro
277 // expansion), then add additional padding so that trailing markers
278 // align, provided the line/column numbers are reasonably sized.
279 //
280 // Otherwise, if it's a macro expansion, don't bother with alignment,
281 // as the line will include multiple locations and be very long.
282 //
283 // NOTE: To keep this stateless, it doesn't account for filename
284 // length, so when a header starts markers will be temporarily misaligned.
285 if (Loc.isFileID()) {
286 PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
287
288 if (!PLoc.isInvalid()) {
289 int LineWidth = llvm::utostr(X: PLoc.getLine()).size();
290 int ColumnWidth = llvm::utostr(X: PLoc.getColumn()).size();
291
292 // Reserve space for lines up to 9999 and columns up to 99,
293 // which is 4 + 2 = 6 characters in total.
294 const int ReservedSpace = 6;
295
296 int LeftSpace = ReservedSpace - LineWidth - ColumnWidth;
297 int Padding = std::max<int>(a: 0, b: LeftSpace);
298
299 llvm::errs().indent(NumSpaces: Padding);
300 }
301 }
302
303 if (Tok.isAtStartOfLine())
304 llvm::errs() << " [StartOfLine]";
305 if (Tok.hasLeadingSpace())
306 llvm::errs() << " [LeadingSpace]";
307 if (Tok.isExpandDisabled())
308 llvm::errs() << " [ExpandDisabled]";
309 if (Tok.needsCleaning()) {
310 const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation());
311 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) << "']";
312 }
313}
314
315void Preprocessor::DumpLocation(SourceLocation Loc) const {
316 Loc.print(OS&: llvm::errs(), SM: SourceMgr);
317}
318
319void Preprocessor::DumpMacro(const MacroInfo &MI) const {
320 llvm::errs() << "MACRO: ";
321 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
322 DumpToken(Tok: MI.getReplacementToken(Tok: i));
323 llvm::errs() << " ";
324 }
325 llvm::errs() << "\n";
326}
327
328void Preprocessor::PrintStats() {
329 llvm::errs() << "\n*** Preprocessor Stats:\n";
330 llvm::errs() << NumDirectives << " directives found:\n";
331 llvm::errs() << " " << NumDefined << " #define.\n";
332 llvm::errs() << " " << NumUndefined << " #undef.\n";
333 llvm::errs() << " #include/#include_next/#import:\n";
334 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
335 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
336 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
337 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
338 llvm::errs() << " " << NumEndif << " #endif.\n";
339 llvm::errs() << " " << NumPragma << " #pragma.\n";
340 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
341
342 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
343 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
344 << NumFastMacroExpanded << " on the fast path.\n";
345 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
346 << " token paste (##) operations performed, "
347 << NumFastTokenPaste << " on the fast path.\n";
348
349 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
350
351 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
352 llvm::errs() << "\n Macro Expanded Tokens: "
353 << llvm::capacity_in_bytes(X: MacroExpandedTokens);
354 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
355 // FIXME: List information for all submodules.
356 llvm::errs() << "\n Macros: "
357 << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros);
358 llvm::errs() << "\n #pragma push_macro Info: "
359 << llvm::capacity_in_bytes(X: PragmaPushMacroInfo);
360 llvm::errs() << "\n Poison Reasons: "
361 << llvm::capacity_in_bytes(X: PoisonReasons);
362 llvm::errs() << "\n Comment Handlers: "
363 << llvm::capacity_in_bytes(x: CommentHandlers) << "\n";
364}
365
366llvm::iterator_range<Preprocessor::macro_iterator>
367Preprocessor::macros(bool IncludeExternalMacros) const {
368 if (IncludeExternalMacros && ExternalSource &&
369 !ReadMacrosFromExternalSource) {
370 ReadMacrosFromExternalSource = true;
371 ExternalSource->ReadDefinedMacros();
372 }
373 // Make sure we cover all macros in visible modules.
374 for (const ModuleMacro &Macro : ModuleMacros)
375 CurSubmoduleState->Macros.try_emplace(Key: Macro.II);
376
377 return CurSubmoduleState->Macros;
378}
379
380size_t Preprocessor::getTotalMemory() const {
381 return BP.getTotalMemory()
382 + llvm::capacity_in_bytes(X: MacroExpandedTokens)
383 + Predefines.capacity() /* Predefines buffer. */
384 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
385 // and ModuleMacros.
386 + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros)
387 + llvm::capacity_in_bytes(X: PragmaPushMacroInfo)
388 + llvm::capacity_in_bytes(X: PoisonReasons)
389 + llvm::capacity_in_bytes(x: CommentHandlers);
390}
391
392/// Compares macro tokens with a specified token value sequence.
393static bool MacroDefinitionEquals(const MacroInfo *MI,
394 ArrayRef<TokenValue> Tokens) {
395 return Tokens.size() == MI->getNumTokens() &&
396 std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin());
397}
398
399StringRef Preprocessor::getLastMacroWithSpelling(
400 SourceLocation Loc,
401 ArrayRef<TokenValue> Tokens) const {
402 SourceLocation BestLocation;
403 StringRef BestSpelling;
404 for (const auto &M : macros()) {
405 const MacroDirective::DefInfo Def =
406 M.second.findDirectiveAtLoc(Loc, SourceMgr);
407 if (!Def || !Def.getMacroInfo())
408 continue;
409 if (!Def.getMacroInfo()->isObjectLike())
410 continue;
411 if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens))
412 continue;
413 SourceLocation Location = Def.getLocation();
414 // Choose the macro defined latest.
415 if (BestLocation.isInvalid() ||
416 (Location.isValid() &&
417 SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) {
418 BestLocation = Location;
419 BestSpelling = M.first->getName();
420 }
421 }
422 return BestSpelling;
423}
424
425void Preprocessor::recomputeCurLexerKind() {
426 if (InCachingLexMode())
427 CurLexerCallback = CLK_CachingLexer;
428 else if (CurLexer)
429 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
430 ? CLK_DependencyDirectivesLexer
431 : CLK_Lexer;
432 else if (CurTokenLexer)
433 CurLexerCallback = CLK_TokenLexer;
434 else
435 CurLexerCallback = CLK_Lexer;
436}
437
438bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
439 unsigned CompleteLine,
440 unsigned CompleteColumn) {
441 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
442 assert(!CodeCompletionFile && "Already set");
443
444 // Load the actual file's contents.
445 std::optional<llvm::MemoryBufferRef> Buffer =
446 SourceMgr.getMemoryBufferForFileOrNone(File);
447 if (!Buffer)
448 return true;
449
450 // Find the byte position of the truncation point.
451 const char *Position = Buffer->getBufferStart();
452 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
453 for (; *Position; ++Position) {
454 if (*Position != '\r' && *Position != '\n')
455 continue;
456
457 // Eat \r\n or \n\r as a single line.
458 if ((Position[1] == '\r' || Position[1] == '\n') &&
459 Position[0] != Position[1])
460 ++Position;
461 ++Position;
462 break;
463 }
464 }
465
466 Position += CompleteColumn - 1;
467
468 // If pointing inside the preamble, adjust the position at the beginning of
469 // the file after the preamble.
470 if (SkipMainFilePreamble.first &&
471 SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) {
472 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
473 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
474 }
475
476 if (Position > Buffer->getBufferEnd())
477 Position = Buffer->getBufferEnd();
478
479 CodeCompletionFile = File;
480 CodeCompletionOffset = Position - Buffer->getBufferStart();
481
482 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
483 Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier());
484 char *NewBuf = NewBuffer->getBufferStart();
485 char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf);
486 *NewPos = '\0';
487 std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1);
488 SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer));
489
490 return false;
491}
492
493void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
494 bool IsAngled) {
495 setCodeCompletionReached();
496 if (CodeComplete)
497 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
498}
499
500void Preprocessor::CodeCompleteNaturalLanguage() {
501 setCodeCompletionReached();
502 if (CodeComplete)
503 CodeComplete->CodeCompleteNaturalLanguage();
504}
505
506/// getSpelling - This method is used to get the spelling of a token into a
507/// SmallVector. Note that the returned StringRef may not point to the
508/// supplied buffer if a copy can be avoided.
509StringRef Preprocessor::getSpelling(const Token &Tok,
510 SmallVectorImpl<char> &Buffer,
511 bool *Invalid) const {
512 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
513 if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) {
514 // Try the fast path.
515 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
516 return II->getName();
517 }
518
519 // Resize the buffer if we need to copy into it.
520 if (Tok.needsCleaning())
521 Buffer.resize(N: Tok.getLength());
522
523 const char *Ptr = Buffer.data();
524 unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid);
525 return StringRef(Ptr, Len);
526}
527
528/// CreateString - Plop the specified string into a scratch buffer and return a
529/// location for it. If specified, the source location provides a source
530/// location for the token.
531void Preprocessor::CreateString(StringRef Str, Token &Tok,
532 SourceLocation ExpansionLocStart,
533 SourceLocation ExpansionLocEnd) {
534 Tok.setLength(Str.size());
535
536 const char *DestPtr;
537 SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr);
538
539 if (ExpansionLocStart.isValid())
540 Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart,
541 ExpansionLocEnd, Length: Str.size());
542 Tok.setLocation(Loc);
543
544 // If this is a raw identifier or a literal token, set the pointer data.
545 if (Tok.is(K: tok::raw_identifier))
546 Tok.setRawIdentifierData(DestPtr);
547 else if (Tok.isLiteral())
548 Tok.setLiteralData(DestPtr);
549}
550
551SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
552 auto &SM = getSourceManager();
553 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
554 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc);
555 bool Invalid = false;
556 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
557 if (Invalid)
558 return SourceLocation();
559
560 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
561 const char *DestPtr;
562 SourceLocation Spelling =
563 ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr);
564 return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length));
565}
566
567Module *Preprocessor::getCurrentModule() {
568 if (!getLangOpts().isCompilingModule())
569 return nullptr;
570
571 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule);
572}
573
574Module *Preprocessor::getCurrentModuleImplementation() {
575 if (!getLangOpts().isCompilingModuleImplementation())
576 return nullptr;
577
578 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName);
579}
580
581//===----------------------------------------------------------------------===//
582// Preprocessor Initialization Methods
583//===----------------------------------------------------------------------===//
584
585/// EnterMainSourceFile - Enter the specified FileID as the main source file,
586/// which implicitly adds the builtin defines etc.
587void Preprocessor::EnterMainSourceFile() {
588 // We do not allow the preprocessor to reenter the main file. Doing so will
589 // cause FileID's to accumulate information from both runs (e.g. #line
590 // information) and predefined macros aren't guaranteed to be set properly.
591 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
592 FileID MainFileID = SourceMgr.getMainFileID();
593
594 // Whether and how the main file starts a C++20 module unit. Implicit inputs
595 // are placed in its existing global module fragment, or in a synthesized one
596 // for a named module without a GMF.
597 ModuleUnitKind MainFileModuleUnitKind = ModuleUnitKind::NotModuleUnit;
598
599 // If MainFileID is loaded it means we loaded an AST file, no need to enter
600 // a main file.
601 if (!SourceMgr.isLoadedFileID(FID: MainFileID)) {
602 // Enter the main file source buffer.
603 EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
604
605 // If we've been asked to skip bytes in the main file (e.g., as part of a
606 // precompiled preamble), do so now.
607 if (SkipMainFilePreamble.first > 0)
608 CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first,
609 StartOfLine: SkipMainFilePreamble.second);
610
611 // Tell the header info that the main file was entered. If the file is later
612 // #imported, it won't be re-entered.
613 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID))
614 markIncluded(File: *FE);
615
616 // Record the first PP token in the main file. This is used to generate
617 // better diagnostics for C++ modules.
618 //
619 // // This is a comment.
620 // #define FOO int // note: add 'module;' to the start of the file
621 // ^ FirstPPToken // to introduce a global module fragment.
622 //
623 // export module M; // error: module declaration must occur
624 // // at the start of the translation unit.
625 if (getLangOpts().CPlusPlusModules) {
626 std::optional<StringRef> Input =
627 getSourceManager().getBufferDataOrNone(FID: MainFileID);
628 if (!isPreprocessedModuleFile() && Input)
629 MainFileIsPreprocessedModuleFile =
630 clang::isPreprocessedModuleFile(Source: *Input);
631 if (Input && !MainFileIsPreprocessedModuleFile && hasDeferredGMFInputs())
632 MainFileModuleUnitKind = scanInputForCXX20ModuleUnit(Source: *Input);
633 auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(args&: *this);
634 DirTracer = Tracer.get();
635 addPPCallbacks(C: std::move(Tracer));
636 std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
637 if (FirstPPTok)
638 FirstPPTokenLoc = FirstPPTok->getLocation();
639 }
640 }
641
642 // Preserve the historical placement in the predefines buffer for ordinary
643 // translation units. A module unit opening with `module;` leaves the inputs
644 // deferred until the introducer has been lexed. For a named module without a
645 // GMF, synthesize the introducer before the main file and use the same
646 // deferred-input path.
647 if (hasDeferredGMFInputs()) {
648 if (PredefinesWereReplaced) {
649 // Loading an implicit PCH replaces Predefines with the directives
650 // suggested by ASTReader. For a module unit, those are the only implicit
651 // inputs that still need to be processed in the GMF.
652 if (MainFileModuleUnitKind != ModuleUnitKind::NotModuleUnit) {
653 DeferredGMFInputs = std::move(Predefines);
654 Predefines.clear();
655 }
656 } else if (MainFileModuleUnitKind == ModuleUnitKind::NotModuleUnit) {
657 // Preserve the historical predefines ordering for an ordinary
658 // translation unit.
659 Predefines += DeferredGMFInputs;
660 }
661 if (MainFileModuleUnitKind ==
662 ModuleUnitKind::NamedModuleWithoutGlobalModuleFragment) {
663 Predefines += "# 1 \"<implicit-global-module-fragment>\" 1\nmodule;\n";
664 HasSynthesizedGMF = true;
665 } else if (MainFileModuleUnitKind == ModuleUnitKind::NotModuleUnit) {
666 DeferredGMFInputs.clear();
667 }
668 }
669
670 // Preprocess Predefines to populate the initial preprocessor state.
671 std::unique_ptr<llvm::MemoryBuffer> SB =
672 llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>");
673 assert(SB && "Cannot create predefined source buffer");
674 FileID FID = SourceMgr.createFileID(Buffer: std::move(SB));
675 assert(FID.isValid() && "Could not create FileID for predefines?");
676 setPredefinesFileID(FID);
677
678 // Start parsing the predefines.
679 EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation());
680
681 if (!PPOpts.PCHThroughHeader.empty()) {
682 // Lookup and save the FileID for the through header. If it isn't found
683 // in the search path, it's a fatal error.
684 OptionalFileEntryRef File = LookupFile(
685 FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader,
686 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
687 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
688 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
689 /*IsFrameworkFound=*/nullptr);
690 if (!File) {
691 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_found)
692 << PPOpts.PCHThroughHeader;
693 return;
694 }
695 setPCHThroughHeaderFileID(
696 SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User));
697 }
698
699 // Skip tokens from the Predefines and if needed the main file.
700 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
701 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
702 SkipTokensWhileUsingPCH();
703}
704
705void Preprocessor::EnterDeferredGMFInputs(SourceLocation IncludeLoc) {
706 if (!hasDeferredGMFInputs())
707 return;
708 // Synthesize the implicit input directives and enter them inside the global
709 // module fragment. Attribute the buffer to IncludeLoc so it is ordered within
710 // the translation unit.
711 std::unique_ptr<llvm::MemoryBuffer> MB = llvm::MemoryBuffer::getMemBufferCopy(
712 InputData: DeferredGMFInputs, BufferName: "<gmf-command-line-inputs>");
713 DeferredGMFInputs.clear();
714 DeferredGMFInputsFileID =
715 SourceMgr.createFileID(Buffer: std::move(MB), FileCharacter: SrcMgr::C_User, LoadedID: 0, LoadedOffset: 0, IncludeLoc);
716 EnterSourceFile(FID: DeferredGMFInputsFileID, Dir: nullptr, Loc: IncludeLoc);
717}
718
719void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
720 assert(PCHThroughHeaderFileID.isInvalid() &&
721 "PCHThroughHeaderFileID already set!");
722 PCHThroughHeaderFileID = FID;
723}
724
725bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
726 assert(PCHThroughHeaderFileID.isValid() &&
727 "Invalid PCH through header FileID");
728 return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID);
729}
730
731bool Preprocessor::creatingPCHWithThroughHeader() {
732 return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
733 PCHThroughHeaderFileID.isValid();
734}
735
736bool Preprocessor::usingPCHWithThroughHeader() {
737 return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
738 PCHThroughHeaderFileID.isValid();
739}
740
741bool Preprocessor::creatingPCHWithPragmaHdrStop() {
742 return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop;
743}
744
745bool Preprocessor::usingPCHWithPragmaHdrStop() {
746 return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop;
747}
748
749/// Skip tokens until after the #include of the through header or
750/// until after a #pragma hdrstop is seen. Tokens in the predefines file
751/// and the main file may be skipped. If the end of the predefines file
752/// is reached, skipping continues into the main file. If the end of the
753/// main file is reached, it's a fatal error.
754void Preprocessor::SkipTokensWhileUsingPCH() {
755 bool ReachedMainFileEOF = false;
756 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
757 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
758 Token Tok;
759 while (true) {
760 bool InPredefines =
761 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
762 CurLexerCallback(*this, Tok);
763 if (Tok.is(K: tok::eof) && !InPredefines) {
764 ReachedMainFileEOF = true;
765 break;
766 }
767 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
768 break;
769 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
770 break;
771 }
772 if (ReachedMainFileEOF) {
773 if (UsingPCHThroughHeader)
774 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_seen)
775 << PPOpts.PCHThroughHeader << 1;
776 else if (!PPOpts.PCHWithHdrStopCreate)
777 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_pragma_hdrstop_not_seen);
778 }
779}
780
781void Preprocessor::replayPreambleConditionalStack() {
782 // Restore the conditional stack from the preamble, if there is one.
783 if (PreambleConditionalStack.isReplaying()) {
784 assert(CurPPLexer &&
785 "CurPPLexer is null when calling replayPreambleConditionalStack.");
786 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
787 PreambleConditionalStack.doneReplaying();
788 if (PreambleConditionalStack.reachedEOFWhileSkipping())
789 SkipExcludedConditionalBlock(
790 HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc,
791 IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc,
792 FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
793 FoundElse: PreambleConditionalStack.SkipInfo->FoundElse,
794 ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc);
795 }
796}
797
798void Preprocessor::EndSourceFile() {
799 // Notify the client that we reached the end of the source file.
800 if (Callbacks)
801 Callbacks->EndOfMainFile();
802}
803
804//===----------------------------------------------------------------------===//
805// Lexer Event Handling.
806//===----------------------------------------------------------------------===//
807
808/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
809/// identifier information for the token and install it into the token,
810/// updating the token kind accordingly.
811IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
812 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
813
814 // Look up this token, see if it is a macro, or if it is a language keyword.
815 IdentifierInfo *II;
816 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
817 // No cleaning needed, just use the characters from the lexed buffer.
818 II = getIdentifierInfo(Name: Identifier.getRawIdentifier());
819 } else {
820 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
821 SmallString<64> IdentifierBuffer;
822 StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer);
823
824 if (Identifier.hasUCN()) {
825 SmallString<64> UCNIdentifierBuffer;
826 expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr);
827 II = getIdentifierInfo(Name: UCNIdentifierBuffer);
828 } else {
829 II = getIdentifierInfo(Name: CleanedStr);
830 }
831 }
832
833 // Update the token info (identifier info and appropriate token kind).
834 // FIXME: the raw_identifier may contain leading whitespace which is removed
835 // from the cleaned identifier token. The SourceLocation should be updated to
836 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
837 // line continuation before 'B') is parsed as a single tok::raw_identifier and
838 // is cleaned to tok::identifier "B". After cleaning the token's length is
839 // still 3 and the SourceLocation refers to the location of the backslash.
840 Identifier.setIdentifierInfo(II);
841 Identifier.setKind(II->getTokenID());
842
843 return II;
844}
845
846void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
847 PoisonReasons[II] = DiagID;
848}
849
850void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
851 assert(Ident__exception_code && Ident__exception_info);
852 assert(Ident___exception_code && Ident___exception_info);
853 Ident__exception_code->setIsPoisoned(Poison);
854 Ident___exception_code->setIsPoisoned(Poison);
855 Ident_GetExceptionCode->setIsPoisoned(Poison);
856 Ident__exception_info->setIsPoisoned(Poison);
857 Ident___exception_info->setIsPoisoned(Poison);
858 Ident_GetExceptionInfo->setIsPoisoned(Poison);
859 Ident__abnormal_termination->setIsPoisoned(Poison);
860 Ident___abnormal_termination->setIsPoisoned(Poison);
861 Ident_AbnormalTermination->setIsPoisoned(Poison);
862}
863
864void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
865 assert(Identifier.getIdentifierInfo() &&
866 "Can't handle identifiers without identifier info!");
867 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
868 PoisonReasons.find(Val: Identifier.getIdentifierInfo());
869 if(it == PoisonReasons.end())
870 Diag(Tok: Identifier, DiagID: diag::err_pp_used_poisoned_id);
871 else
872 Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo();
873}
874
875void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const {
876 assert(II.isOutOfDate() && "not out of date");
877 assert(getExternalSource() &&
878 "getExternalSource() should not return nullptr");
879 getExternalSource()->updateOutOfDateIdentifier(II);
880}
881
882/// HandleIdentifier - This callback is invoked when the lexer reads an
883/// identifier. This callback looks up the identifier in the map and/or
884/// potentially macro expands it or turns it into a named token (like 'for').
885///
886/// Note that callers of this method are guarded by checking the
887/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
888/// IdentifierInfo methods that compute these properties will need to change to
889/// match.
890bool Preprocessor::HandleIdentifier(Token &Identifier) {
891 assert(Identifier.getIdentifierInfo() &&
892 "Can't handle identifiers without identifier info!");
893
894 IdentifierInfo &II = *Identifier.getIdentifierInfo();
895
896 // If the information about this identifier is out of date, update it from
897 // the external source.
898 // We have to treat __VA_ARGS__ in a special way, since it gets
899 // serialized with isPoisoned = true, but our preprocessor may have
900 // unpoisoned it if we're defining a C99 macro.
901 if (II.isOutOfDate()) {
902 bool CurrentIsPoisoned = false;
903 const bool IsSpecialVariadicMacro =
904 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
905 if (IsSpecialVariadicMacro)
906 CurrentIsPoisoned = II.isPoisoned();
907
908 updateOutOfDateIdentifier(II);
909 Identifier.setKind(II.getTokenID());
910
911 if (IsSpecialVariadicMacro)
912 II.setIsPoisoned(CurrentIsPoisoned);
913 }
914
915 // If this identifier was poisoned, and if it was not produced from a macro
916 // expansion, emit an error.
917 if (II.isPoisoned() && CurPPLexer) {
918 HandlePoisonedIdentifier(Identifier);
919 }
920
921 // If this is a macro to be expanded, do it.
922 if (const MacroDefinition MD = getMacroDefinition(II: &II)) {
923 const auto *MI = MD.getMacroInfo();
924 assert(MI && "macro definition with no macro info?");
925 if (!DisableMacroExpansion) {
926 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
927 // C99 6.10.3p10: If the preprocessing token immediately after the
928 // macro name isn't a '(', this macro should not be expanded.
929 if (!MI->isFunctionLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
930 return HandleMacroExpandedIdentifier(Identifier, MD);
931 } else {
932 // C99 6.10.3.4p2 says that a disabled macro may never again be
933 // expanded, even if it's in a context where it could be expanded in the
934 // future.
935 Identifier.setFlag(Token::DisableExpand);
936 if (MI->isObjectLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
937 Diag(Tok: Identifier, DiagID: diag::pp_disabled_macro_expansion);
938 }
939 }
940 }
941
942 // If this identifier is a keyword in a newer Standard or proposed Standard,
943 // produce a warning. Don't warn if we're not considering macro expansion,
944 // since this identifier might be the name of a macro.
945 // FIXME: This warning is disabled in cases where it shouldn't be, like
946 // "#define constexpr constexpr", "int constexpr;"
947 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
948 Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts()))
949 << II.getName();
950 // Don't diagnose this keyword again in this translation unit.
951 II.setIsFutureCompatKeyword(false);
952 }
953
954 // If this identifier would be a keyword in C++, diagnose as a compatibility
955 // issue.
956 if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion)
957 Diag(Tok: Identifier, DiagID: diag::warn_pp_identifier_is_cpp_keyword) << &II;
958
959 // If this is an extension token, diagnose its use.
960 // We avoid diagnosing tokens that originate from macro definitions.
961 // FIXME: This warning is disabled in cases where it shouldn't be,
962 // like "#define TY typeof", "TY(1) x".
963 if (II.isExtensionToken() && !DisableMacroExpansion)
964 Diag(Tok: Identifier, DiagID: diag::ext_token_used);
965
966 // Handle module contextual keywords.
967 if (getLangOpts().CPlusPlusModules && CurLexer &&
968 !CurLexer->isLexingRawMode() && !CurLexer->isPragmaLexer() &&
969 !CurLexer->ParsingPreprocessorDirective &&
970 Identifier.isModuleContextualKeyword() &&
971 HandleModuleContextualKeyword(Result&: Identifier)) {
972 HandleDirective(Result&: Identifier);
973 // With a fatal failure in the module loader, we abort parsing.
974 return hadModuleLoaderFatalFailure();
975 }
976
977 return true;
978}
979
980void Preprocessor::Lex(Token &Result) {
981 ++LexLevel;
982
983 // We loop here until a lex function returns a token; this avoids recursion.
984 while (!CurLexerCallback(*this, Result))
985 ;
986
987 if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure)
988 return;
989
990 if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) {
991 // Remember the identifier before code completion token.
992 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
993 setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc());
994 // Set IdenfitierInfo to null to avoid confusing code that handles both
995 // identifiers and completion tokens.
996 Result.setIdentifierInfo(nullptr);
997 }
998
999 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
1000 // if this token is being produced as a result of phase 4 of translation.
1001 // Update TrackGMFState to decide if we are currently in a Global Module
1002 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
1003 // depends on the prevailing StdCXXImportSeq state in two cases.
1004 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
1005 !Result.getFlag(Flag: Token::IsReinjected)) {
1006 switch (Result.getKind()) {
1007 case tok::l_paren: case tok::l_square: case tok::l_brace:
1008 StdCXXImportSeqState.handleOpenBracket();
1009 break;
1010 case tok::r_paren: case tok::r_square:
1011 StdCXXImportSeqState.handleCloseBracket();
1012 break;
1013 case tok::r_brace:
1014 StdCXXImportSeqState.handleCloseBrace();
1015 break;
1016#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
1017// For `#pragma ...` mimic ';'.
1018#include "clang/Basic/TokenKinds.def"
1019#undef PRAGMA_ANNOTATION
1020 // This token is injected to represent the translation of '#include "a.h"'
1021 // into "import a.h;". Mimic the notional ';'.
1022 case tok::annot_module_include:
1023 case tok::semi:
1024 TrackGMFState.handleSemi();
1025 StdCXXImportSeqState.handleSemi();
1026 ModuleDeclState.handleSemi();
1027 break;
1028 case tok::eof:
1029 if (getLangOpts().IncrementalExtensions) {
1030 TrackGMFState.handleSemi();
1031 StdCXXImportSeqState.handleSemi();
1032 ModuleDeclState.handleSemi();
1033 } else {
1034 TrackGMFState.handleMisc();
1035 StdCXXImportSeqState.handleMisc();
1036 ModuleDeclState.handleMisc();
1037 }
1038 break;
1039 case tok::header_name:
1040 case tok::annot_header_unit:
1041 StdCXXImportSeqState.handleHeaderName();
1042 break;
1043 case tok::kw_export:
1044 if (hasSeenNoTrivialPPDirective())
1045 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
1046 TrackGMFState.handleExport();
1047 StdCXXImportSeqState.handleExport();
1048 ModuleDeclState.handleExport();
1049 break;
1050 case tok::colon:
1051 ModuleDeclState.handleColon();
1052 break;
1053 case tok::kw_import:
1054 if (StdCXXImportSeqState.atTopLevel()) {
1055 TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
1056 StdCXXImportSeqState.handleImport();
1057 }
1058 break;
1059 case tok::kw_module:
1060 if (StdCXXImportSeqState.atTopLevel()) {
1061 if (hasSeenNoTrivialPPDirective())
1062 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
1063 TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
1064 ModuleDeclState.handleModule();
1065 }
1066 break;
1067 case tok::annot_module_name:
1068 ModuleDeclState.handleModuleName(
1069 NameLoc: static_cast<ModuleNameLoc *>(Result.getAnnotationValue()));
1070 if (ModuleDeclState.isModuleCandidate())
1071 break;
1072 [[fallthrough]];
1073 default:
1074 TrackGMFState.handleMisc();
1075 StdCXXImportSeqState.handleMisc();
1076 ModuleDeclState.handleMisc();
1077 break;
1078 }
1079 }
1080
1081 if (RecordCheckPoints && CurLexer &&
1082 ++CheckPointCounter == CheckPointStepSize) {
1083 CheckPoints[CurLexer->getFileID()].push_back(Elt: CurLexer->BufferPtr);
1084 CheckPointCounter = 0;
1085 }
1086
1087 if (Result.isNot(K: tok::kw_export))
1088 LastExportKeyword.startToken();
1089
1090 --LexLevel;
1091
1092 // Destroy any lexers that were deferred while we were in nested Lex calls.
1093 // This must happen after decrementing LexLevel but before any other
1094 // processing that might re-enter Lex.
1095 if (LexLevel == 0 && !PendingDestroyLexers.empty())
1096 PendingDestroyLexers.clear();
1097
1098 if ((LexLevel == 0 || PreprocessToken) &&
1099 !Result.getFlag(Flag: Token::IsReinjected)) {
1100 if (LexLevel == 0)
1101 ++TokenCount;
1102 if (OnToken)
1103 OnToken(Result);
1104 }
1105}
1106
1107void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
1108 while (1) {
1109 Token Tok;
1110 Lex(Result&: Tok);
1111 if (Tok.isOneOf(Ks: tok::unknown, Ks: tok::eof, Ks: tok::eod))
1112 break;
1113 if (Tokens != nullptr)
1114 Tokens->push_back(x: Tok);
1115 }
1116}
1117
1118/// Lex a header-name token (including one formed from header-name-tokens if
1119/// \p AllowMacroExpansion is \c true).
1120///
1121/// \param FilenameTok Filled in with the next token. On success, this will
1122/// be either a header_name token. On failure, it will be whatever other
1123/// token was found instead.
1124/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1125/// by macro expansion (concatenating tokens as necessary if the first
1126/// token is a '<').
1127/// \return \c true if we reached EOD or EOF while looking for a > token in
1128/// a concatenated header name and diagnosed it. \c false otherwise.
1129bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1130 // Lex using header-name tokenization rules if tokens are being lexed from
1131 // a file. Just grab a token normally if we're in a macro expansion.
1132 if (CurPPLexer) {
1133 // Avoid nested header-name lexing when macro expansion recurses
1134 // __has_include(__has_include))
1135 if (CurPPLexer->ParsingFilename)
1136 LexUnexpandedToken(Result&: FilenameTok);
1137 else
1138 CurPPLexer->LexIncludeFilename(FilenameTok);
1139 } else {
1140 Lex(Result&: FilenameTok);
1141 }
1142
1143 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1144 // case, glue the tokens together into an angle_string_literal token.
1145 SmallString<128> FilenameBuffer;
1146 if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) {
1147 bool StartOfLine = FilenameTok.isAtStartOfLine();
1148 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1149 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1150
1151 SourceLocation Start = FilenameTok.getLocation();
1152 SourceLocation End;
1153 FilenameBuffer.push_back(Elt: '<');
1154
1155 // Consume tokens until we find a '>'.
1156 // FIXME: A header-name could be formed starting or ending with an
1157 // alternative token. It's not clear whether that's ill-formed in all
1158 // cases.
1159 while (FilenameTok.isNot(K: tok::greater)) {
1160 Lex(Result&: FilenameTok);
1161 if (FilenameTok.isOneOf(Ks: tok::eod, Ks: tok::eof)) {
1162 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_expected) << tok::greater;
1163 Diag(Loc: Start, DiagID: diag::note_matching) << tok::less;
1164 return true;
1165 }
1166
1167 End = FilenameTok.getLocation();
1168
1169 // FIXME: Provide code completion for #includes.
1170 if (FilenameTok.is(K: tok::code_completion)) {
1171 setCodeCompletionReached();
1172 Lex(Result&: FilenameTok);
1173 continue;
1174 }
1175
1176 // Append the spelling of this token to the buffer. If there was a space
1177 // before it, add it now.
1178 if (FilenameTok.hasLeadingSpace())
1179 FilenameBuffer.push_back(Elt: ' ');
1180
1181 // Get the spelling of the token, directly into FilenameBuffer if
1182 // possible.
1183 size_t PreAppendSize = FilenameBuffer.size();
1184 FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength());
1185
1186 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1187 unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr);
1188
1189 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1190 if (BufPtr != &FilenameBuffer[PreAppendSize])
1191 memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen);
1192
1193 // Resize FilenameBuffer to the correct size.
1194 if (FilenameTok.getLength() != ActualLen)
1195 FilenameBuffer.resize(N: PreAppendSize + ActualLen);
1196 }
1197
1198 FilenameTok.startToken();
1199 FilenameTok.setKind(tok::header_name);
1200 FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine);
1201 FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1202 FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro);
1203 CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End);
1204 } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) {
1205 // Convert a string-literal token of the form " h-char-sequence "
1206 // (produced by macro expansion) into a header-name token.
1207 //
1208 // The rules for header-names don't quite match the rules for
1209 // string-literals, but all the places where they differ result in
1210 // undefined behavior, so we can and do treat them the same.
1211 //
1212 // A string-literal with a prefix or suffix is not translated into a
1213 // header-name. This could theoretically be observable via the C++20
1214 // context-sensitive header-name formation rules.
1215 StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
1216 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1217 FilenameTok.setKind(tok::header_name);
1218 }
1219
1220 return false;
1221}
1222
1223std::optional<Token> Preprocessor::peekNextPPToken() const {
1224 // Do some quick tests for rejection cases.
1225 std::optional<Token> Val;
1226 if (CurLexer)
1227 Val = CurLexer->peekNextPPToken();
1228 else
1229 Val = CurTokenLexer->peekNextPPToken();
1230
1231 if (!Val) {
1232 // We have run off the end. If it's a source file we don't
1233 // examine enclosing ones (C99 5.1.1.2p4). Otherwise walk up the
1234 // macro stack.
1235 if (CurPPLexer)
1236 return std::nullopt;
1237 for (const IncludeStackInfo &Entry : llvm::reverse(C: IncludeMacroStack)) {
1238 if (Entry.TheLexer)
1239 Val = Entry.TheLexer->peekNextPPToken();
1240 else
1241 Val = Entry.TheTokenLexer->peekNextPPToken();
1242
1243 if (Val)
1244 break;
1245
1246 // Ran off the end of a source file?
1247 if (Entry.ThePPLexer)
1248 return std::nullopt;
1249 }
1250 }
1251
1252 // Okay, we found the token and return. Otherwise we found the end of the
1253 // translation unit.
1254 return Val;
1255}
1256
1257// We represent the primary and partition names as 'Paths' which are sections
1258// of the hierarchical access path for a clang module. However for C++20
1259// the periods in a name are just another character, and we will need to
1260// flatten them into a string.
1261std::string ModuleLoader::getFlatNameFromPath(ModuleIdPath Path) {
1262 std::string Name;
1263 if (Path.empty())
1264 return Name;
1265
1266 for (auto &Piece : Path) {
1267 assert(Piece.getIdentifierInfo() && Piece.getLoc().isValid());
1268 if (!Name.empty())
1269 Name += ".";
1270 Name += Piece.getIdentifierInfo()->getName();
1271 }
1272 return Name;
1273}
1274
1275ModuleNameLoc *ModuleNameLoc::Create(Preprocessor &PP, ModuleIdPath Path) {
1276 assert(!Path.empty() && "expect at least one identifier in a module name");
1277 void *Mem = PP.getPreprocessorAllocator().Allocate(
1278 Size: totalSizeToAlloc<IdentifierLoc>(Counts: Path.size()), Alignment: alignof(ModuleNameLoc));
1279 return new (Mem) ModuleNameLoc(Path);
1280}
1281
1282bool Preprocessor::LexModuleNameContinue(Token &Tok, SourceLocation UseLoc,
1283 SmallVectorImpl<Token> &Suffix,
1284 SmallVectorImpl<IdentifierLoc> &Path,
1285 bool AllowMacroExpansion,
1286 bool IsPartition) {
1287 auto ConsumeToken = [&]() {
1288 if (AllowMacroExpansion)
1289 Lex(Result&: Tok);
1290 else
1291 LexUnexpandedToken(Result&: Tok);
1292 Suffix.push_back(Elt: Tok);
1293 };
1294
1295 while (true) {
1296 if (Tok.isNot(K: tok::identifier)) {
1297 if (Tok.is(K: tok::code_completion)) {
1298 CurLexer->cutOffLexing();
1299 CodeComplete->CodeCompleteModuleImport(ImportLoc: UseLoc, Path);
1300 return true;
1301 }
1302
1303 Diag(Tok, DiagID: diag::err_pp_module_expected_ident) << Path.empty();
1304 return true;
1305 }
1306
1307 // [cpp.pre]/p2:
1308 // No identifier in the pp-module-name or pp-module-partition shall
1309 // currently be defined as an object-like macro.
1310 if (MacroInfo *MI = getMacroInfo(II: Tok.getIdentifierInfo());
1311 MI && MI->isObjectLike() && getLangOpts().CPlusPlus20 &&
1312 !AllowMacroExpansion) {
1313 Diag(Tok, DiagID: diag::err_pp_module_name_is_macro)
1314 << IsPartition << Tok.getIdentifierInfo();
1315 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::note_macro_here)
1316 << Tok.getIdentifierInfo();
1317 }
1318
1319 // Record this part of the module path.
1320 Path.emplace_back(Args: Tok.getLocation(), Args: Tok.getIdentifierInfo());
1321 ConsumeToken();
1322
1323 if (Tok.isNot(K: tok::period))
1324 return false;
1325
1326 ConsumeToken();
1327 }
1328}
1329
1330bool Preprocessor::HandleModuleName(StringRef DirType, SourceLocation UseLoc,
1331 Token &Tok,
1332 SmallVectorImpl<IdentifierLoc> &Path,
1333 SmallVectorImpl<Token> &DirToks,
1334 bool AllowMacroExpansion,
1335 bool IsPartition) {
1336 bool LeadingSpace = Tok.hasLeadingSpace();
1337 unsigned NumToksInDirective = DirToks.size();
1338 if (LexModuleNameContinue(Tok, UseLoc, Suffix&: DirToks, Path, AllowMacroExpansion,
1339 IsPartition)) {
1340 if (Tok.isNot(K: tok::eod))
1341 CheckEndOfDirective(DirType,
1342 /*EnableMacros=*/false, ExtraToks: &DirToks);
1343 EnterModuleSuffixTokenStream(Toks: DirToks);
1344 return true;
1345 }
1346
1347 // Clean the module-name tokens and replace these tokens with
1348 // annot_module_name.
1349 DirToks.resize(N: NumToksInDirective);
1350 ModuleNameLoc *NameLoc = ModuleNameLoc::Create(PP&: *this, Path);
1351 DirToks.emplace_back();
1352 DirToks.back().setKind(tok::annot_module_name);
1353 DirToks.back().setAnnotationRange(NameLoc->getRange());
1354 DirToks.back().setAnnotationValue(static_cast<void *>(NameLoc));
1355 DirToks.back().setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1356 DirToks.push_back(Elt: Tok);
1357 return false;
1358}
1359
1360/// [cpp.pre]/p2:
1361/// A preprocessing directive consists of a sequence of preprocessing tokens
1362/// that satisfies the following constraints: At the start of translation phase
1363/// 4, the first preprocessing token in the sequence, referred to as a
1364/// directive-introducing token, begins with the first character in the source
1365/// file (optionally after whitespace containing no new-line characters) or
1366/// follows whitespace containing at least one new-line character, and is:
1367/// - a # preprocessing token, or
1368/// - an import preprocessing token immediately followed on the same logical
1369/// source line by a header-name, <, identifier, or : preprocessing token, or
1370/// - a module preprocessing token immediately followed on the same logical
1371/// source line by an identifier, :, or ; preprocessing token, or
1372/// - an export preprocessing token immediately followed on the same logical
1373/// source line by one of the two preceding forms.
1374///
1375///
1376/// At the start of phase 4 an import or module token is treated as starting a
1377/// directive and are converted to their respective keywords iff:
1378/// - After skipping horizontal whitespace are
1379/// - at the start of a logical line, or
1380/// - preceded by an 'export' at the start of the logical line.
1381/// - Are followed by an identifier pp token (before macro expansion), or
1382/// - <, ", or : (but not ::) pp tokens for 'import', or
1383/// - ; for 'module'
1384/// Otherwise the token is treated as an identifier.
1385bool Preprocessor::HandleModuleContextualKeyword(Token &Result) {
1386 if (!getLangOpts().CPlusPlusModules || !Result.isModuleContextualKeyword())
1387 return false;
1388
1389 if (Result.is(K: tok::kw_export)) {
1390 LastExportKeyword = Result;
1391 return false;
1392 }
1393
1394 /// Trait 'module' and 'import' as a identifier when the main file is a
1395 /// preprocessed module file. We only allow '__preprocessed_module' and
1396 /// '__preprocessed_import' in this context.
1397 IdentifierInfo *II = Result.getIdentifierInfo();
1398 if (isPreprocessedModuleFile() &&
1399 (II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_import)) ||
1400 II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_module))))
1401 return false;
1402
1403 if (LastExportKeyword.is(K: tok::kw_export)) {
1404 // The export keyword was not at the start of line, it's not a
1405 // directive-introducing token.
1406 if (!LastExportKeyword.isAtPhysicalStartOfLine())
1407 return false;
1408 // [cpp.pre]/1.4
1409 // export // not a preprocessing directive
1410 // import foo; // preprocessing directive (ill-formed at phase7)
1411 if (Result.isAtPhysicalStartOfLine())
1412 return false;
1413 } else if (!Result.isAtPhysicalStartOfLine())
1414 return false;
1415
1416 assert(CurPPLexer && "CurPPLexer must not be null");
1417
1418 llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective(
1419 CurPPLexer->ParsingPreprocessorDirective, true);
1420
1421 if (II->isModuleKeyword()) {
1422 if (auto NextTok = peekNextPPToken()) {
1423 if (NextTok->is(K: tok::raw_identifier))
1424 LookUpIdentifierInfo(Identifier&: *NextTok);
1425 if (NextTok->isOneOf(Ks: tok::identifier, Ks: tok::colon, Ks: tok::semi)) {
1426 Result.setKind(tok::kw_module);
1427 ModuleDeclLoc = Result.getLocation();
1428 return true;
1429 }
1430 }
1431 return false;
1432 }
1433
1434 if (II->isImportKeyword()) {
1435 llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename,
1436 true);
1437 if (auto NextTok = peekNextPPToken()) {
1438 if (NextTok->is(K: tok::raw_identifier))
1439 LookUpIdentifierInfo(Identifier&: *NextTok);
1440 if (NextTok->isOneOf(Ks: tok::header_name, Ks: tok::identifier, Ks: tok::colon,
1441 Ks: tok::less, Ks: tok::code_completion)) {
1442 Result.setKind(tok::kw_import);
1443 ModuleImportLoc = Result.getLocation();
1444 return true;
1445 }
1446 }
1447 return false;
1448 }
1449
1450 // Ok, it's an identifier.
1451 return false;
1452}
1453
1454bool Preprocessor::CollectPPImportSuffixAndEnterStream(
1455 SmallVectorImpl<Token> &Toks, bool StopUntilEOD) {
1456 CollectPPImportSuffix(Toks);
1457 EnterModuleSuffixTokenStream(Toks);
1458 return false;
1459}
1460
1461/// Collect the tokens of a C++20 pp-import-suffix.
1462void Preprocessor::CollectPPImportSuffix(SmallVectorImpl<Token> &Toks,
1463 bool StopUntilEOD) {
1464 while (true) {
1465 Toks.emplace_back();
1466 Lex(Result&: Toks.back());
1467
1468 switch (Toks.back().getKind()) {
1469 case tok::semi:
1470 if (!StopUntilEOD)
1471 return;
1472 [[fallthrough]];
1473 case tok::eod:
1474 case tok::eof:
1475 return;
1476 default:
1477 break;
1478 }
1479 }
1480}
1481
1482// Allocate a holding buffer for a sequence of tokens and introduce it into
1483// the token stream.
1484void Preprocessor::EnterModuleSuffixTokenStream(ArrayRef<Token> Toks) {
1485 if (Toks.empty())
1486 return;
1487 auto ToksCopy = std::make_unique<Token[]>(num: Toks.size());
1488 std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get());
1489 EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(),
1490 /*DisableMacroExpansion*/ false, /*IsReinject*/ false);
1491 assert(CurTokenLexer && "Must have a TokenLexer");
1492 CurTokenLexer->setLexingCXXModuleDirective();
1493}
1494
1495void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc,
1496 bool IncludeExports) {
1497 CurSubmoduleState->VisibleModules.setVisible(
1498 M, Loc, IncludeExports, Vis: [](Module *) {},
1499 Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1500 // FIXME: Include the path in the diagnostic.
1501 // FIXME: Include the import location for the conflicting module.
1502 Diag(Loc: ModuleImportLoc, DiagID: diag::warn_module_conflict)
1503 << Path[0]->getFullModuleName()
1504 << Conflict->getFullModuleName()
1505 << Message;
1506 });
1507
1508 // Add this module to the imports list of the currently-built submodule.
1509 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1510 BuildingSubmoduleStack.back().M->Imports.push_back(Elt: M);
1511}
1512
1513bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1514 const char *DiagnosticTag,
1515 bool AllowMacroExpansion) {
1516 // We need at least one string literal.
1517 if (Result.isNot(K: tok::string_literal)) {
1518 Diag(Tok: Result, DiagID: diag::err_expected_string_literal)
1519 << /*Source='in...'*/0 << DiagnosticTag;
1520 return false;
1521 }
1522
1523 // Lex string literal tokens, optionally with macro expansion.
1524 SmallVector<Token, 4> StrToks;
1525 do {
1526 StrToks.push_back(Elt: Result);
1527
1528 if (Result.hasUDSuffix())
1529 Diag(Tok: Result, DiagID: diag::err_invalid_string_udl);
1530
1531 if (AllowMacroExpansion)
1532 Lex(Result);
1533 else
1534 LexUnexpandedToken(Result);
1535 } while (Result.is(K: tok::string_literal));
1536
1537 // Concatenate and parse the strings.
1538 StringLiteralParser Literal(StrToks, *this);
1539 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1540
1541 if (Literal.hadError)
1542 return false;
1543
1544 if (Literal.Pascal) {
1545 Diag(Loc: StrToks[0].getLocation(), DiagID: diag::err_expected_string_literal)
1546 << /*Source='in...'*/0 << DiagnosticTag;
1547 return false;
1548 }
1549
1550 String = std::string(Literal.GetString());
1551 return true;
1552}
1553
1554bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1555 assert(Tok.is(tok::numeric_constant));
1556 SmallString<8> IntegerBuffer;
1557 bool NumberInvalid = false;
1558 StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid);
1559 if (NumberInvalid)
1560 return false;
1561 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1562 getLangOpts(), getTargetInfo(),
1563 getDiagnostics());
1564 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1565 return false;
1566 llvm::APInt APVal(64, 0);
1567 if (Literal.GetIntegerValue(Val&: APVal))
1568 return false;
1569 Lex(Result&: Tok);
1570 Value = APVal.getLimitedValue();
1571 return true;
1572}
1573
1574void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1575 assert(Handler && "NULL comment handler");
1576 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1577 "Comment handler already registered");
1578 CommentHandlers.push_back(x: Handler);
1579}
1580
1581void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1582 std::vector<CommentHandler *>::iterator Pos =
1583 llvm::find(Range&: CommentHandlers, Val: Handler);
1584 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1585 CommentHandlers.erase(position: Pos);
1586}
1587
1588bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1589 bool AnyPendingTokens = false;
1590 for (CommentHandler *H : CommentHandlers) {
1591 if (H->HandleComment(PP&: *this, Comment))
1592 AnyPendingTokens = true;
1593 }
1594 if (!AnyPendingTokens || getCommentRetentionState())
1595 return false;
1596 Lex(Result&: result);
1597 return true;
1598}
1599
1600void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1601 const MacroAnnotations &A =
1602 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1603 assert(A.DeprecationInfo &&
1604 "Macro deprecation warning without recorded annotation!");
1605 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1606 if (Info.Message.empty())
1607 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1608 << Identifier.getIdentifierInfo() << 0;
1609 else
1610 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1611 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1612 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 0;
1613}
1614
1615void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1616 const MacroAnnotations &A =
1617 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1618 assert(A.RestrictExpansionInfo &&
1619 "Macro restricted expansion warning without recorded annotation!");
1620 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1621 if (Info.Message.empty())
1622 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1623 << Identifier.getIdentifierInfo() << 0;
1624 else
1625 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1626 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1627 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 1;
1628}
1629
1630void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
1631 unsigned DiagSelection) const {
1632 Diag(Tok: Identifier, DiagID: diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1633}
1634
1635void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1636 bool IsUndef) const {
1637 const MacroAnnotations &A =
1638 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1639 assert(A.FinalAnnotationLoc &&
1640 "Final macro warning without recorded annotation!");
1641
1642 Diag(Tok: Identifier, DiagID: diag::warn_pragma_final_macro)
1643 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1644 Diag(Loc: *A.FinalAnnotationLoc, DiagID: diag::note_pp_macro_annotation) << 2;
1645}
1646
1647bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr,
1648 const SourceLocation &Loc) const {
1649 // The lambda that tests if a `Loc` is in an opt-out region given one opt-out
1650 // region map:
1651 auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map,
1652 const SourceLocation &Loc) -> bool {
1653 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1654 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1655 Range: Map, P: [&SourceMgr,
1656 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1657 return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc);
1658 });
1659
1660 if (FirstRegionEndingAfterLoc != Map.end()) {
1661 // To test if the start location of the found region precedes `Loc`:
1662 return SourceMgr.isBeforeInTranslationUnit(
1663 LHS: FirstRegionEndingAfterLoc->first, RHS: Loc);
1664 }
1665 // If we do not find a region whose end location passes `Loc`, we want to
1666 // check if the current region is still open:
1667 if (!Map.empty() && Map.back().first == Map.back().second)
1668 return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc);
1669 return false;
1670 };
1671
1672 // What the following does:
1673 //
1674 // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`.
1675 // Otherwise, `Loc` is from a loaded AST. We look up the
1676 // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the
1677 // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out
1678 // region w.r.t. the region map. If the region map is absent, it means there
1679 // is no opt-out pragma in that loaded AST.
1680 //
1681 // Opt-out pragmas in the local TU or a loaded AST is not visible to another
1682 // one of them. That means if you put the pragmas around a `#include
1683 // "module.h"`, where module.h is a module, it is not actually suppressing
1684 // warnings in module.h. This is fine because warnings in module.h will be
1685 // reported when module.h is compiled in isolation and nothing in module.h
1686 // will be analyzed ever again. So you will not see warnings from the file
1687 // that imports module.h anyway. And you can't even do the same thing for PCHs
1688 // because they can only be included from the command line.
1689
1690 if (SourceMgr.isLocalSourceLocation(Loc))
1691 return TestInMap(SafeBufferOptOutMap, Loc);
1692
1693 const SafeBufferOptOutRegionsTy *LoadedRegions =
1694 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr);
1695
1696 if (LoadedRegions)
1697 return TestInMap(*LoadedRegions, Loc);
1698 return false;
1699}
1700
1701bool Preprocessor::enterOrExitSafeBufferOptOutRegion(
1702 bool isEnter, const SourceLocation &Loc) {
1703 if (isEnter) {
1704 if (isPPInSafeBufferOptOutRegion())
1705 return true; // invalid enter action
1706 InSafeBufferOptOutRegion = true;
1707 CurrentSafeBufferOptOutStart = Loc;
1708
1709 // To set the start location of a new region:
1710
1711 if (!SafeBufferOptOutMap.empty()) {
1712 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1713 assert(PrevRegion->first != PrevRegion->second &&
1714 "Shall not begin a safe buffer opt-out region before closing the "
1715 "previous one.");
1716 }
1717 // If the start location equals to the end location, we call the region a
1718 // open region or a unclosed region (i.e., end location has not been set
1719 // yet).
1720 SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc);
1721 } else {
1722 if (!isPPInSafeBufferOptOutRegion())
1723 return true; // invalid enter action
1724 InSafeBufferOptOutRegion = false;
1725
1726 // To set the end location of the current open region:
1727
1728 assert(!SafeBufferOptOutMap.empty() &&
1729 "Misordered safe buffer opt-out regions");
1730 auto *CurrRegion = &SafeBufferOptOutMap.back();
1731 assert(CurrRegion->first == CurrRegion->second &&
1732 "Set end location to a closed safe buffer opt-out region");
1733 CurrRegion->second = Loc;
1734 }
1735 return false;
1736}
1737
1738bool Preprocessor::isPPInSafeBufferOptOutRegion() {
1739 return InSafeBufferOptOutRegion;
1740}
1741bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) {
1742 StartLoc = CurrentSafeBufferOptOutStart;
1743 return InSafeBufferOptOutRegion;
1744}
1745
1746SmallVector<SourceLocation, 64>
1747Preprocessor::serializeSafeBufferOptOutMap() const {
1748 assert(!InSafeBufferOptOutRegion &&
1749 "Attempt to serialize safe buffer opt-out regions before file being "
1750 "completely preprocessed");
1751
1752 SmallVector<SourceLocation, 64> SrcSeq;
1753
1754 for (const auto &[begin, end] : SafeBufferOptOutMap) {
1755 SrcSeq.push_back(Elt: begin);
1756 SrcSeq.push_back(Elt: end);
1757 }
1758 // Only `SafeBufferOptOutMap` gets serialized. No need to serialize
1759 // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every
1760 // pch/module in the pch-chain/module-DAG will be loaded one by one in order.
1761 // It means that for each loading pch/module m, it just needs to load m's own
1762 // `SafeBufferOptOutMap`.
1763 return SrcSeq;
1764}
1765
1766bool Preprocessor::setDeserializedSafeBufferOptOutMap(
1767 const SmallVectorImpl<SourceLocation> &SourceLocations) {
1768 if (SourceLocations.size() == 0)
1769 return false;
1770
1771 assert(SourceLocations.size() % 2 == 0 &&
1772 "ill-formed SourceLocation sequence");
1773
1774 auto It = SourceLocations.begin();
1775 SafeBufferOptOutRegionsTy &Regions =
1776 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr);
1777
1778 do {
1779 SourceLocation Begin = *It++;
1780 SourceLocation End = *It++;
1781
1782 Regions.emplace_back(Args&: Begin, Args&: End);
1783 } while (It != SourceLocations.end());
1784 return true;
1785}
1786
1787ModuleLoader::~ModuleLoader() = default;
1788
1789CommentHandler::~CommentHandler() = default;
1790
1791EmptylineHandler::~EmptylineHandler() = default;
1792
1793CodeCompletionHandler::~CodeCompletionHandler() = default;
1794
1795void Preprocessor::createPreprocessingRecord() {
1796 if (Record)
1797 return;
1798
1799 Record = new PreprocessingRecord(getSourceManager());
1800 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record));
1801}
1802
1803void Preprocessor::removePPCallbacks() {
1804 auto IsPreserved = [&](PPCallbacks *C) {
1805 return C == Record || C == DirTracer;
1806 };
1807 SmallVector<PPCallbacks *, 2> Released;
1808 PPCallbacks::releaseIfPreserved(CB&: Callbacks, Pred: IsPreserved, Released);
1809 Callbacks.reset();
1810 for (auto *P : Released)
1811 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(P));
1812}
1813
1814const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
1815 if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) {
1816 const SmallVector<const char *> &FileCheckPoints = It->second;
1817 auto P = llvm::upper_bound(Range: FileCheckPoints, Value&: Start);
1818 if (P == FileCheckPoints.begin())
1819 return nullptr;
1820 return *std::prev(x: P);
1821 }
1822 return nullptr;
1823}
1824
1825bool Preprocessor::hasSeenNoTrivialPPDirective() const {
1826 return DirTracer && DirTracer->hasSeenNoTrivialPPDirective();
1827}
1828
1829bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const {
1830 return SeenNoTrivialPPDirective;
1831}
1832
1833void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
1834 if (InMainFile && !SeenNoTrivialPPDirective)
1835 SeenNoTrivialPPDirective = true;
1836}
1837
1838void NoTrivialPPDirectiveTracer::LexedFileChanged(
1839 FileID FID, LexedFileChangeReason Reason,
1840 SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) {
1841 InMainFile = (FID == PP.getSourceManager().getMainFileID());
1842}
1843
1844void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok,
1845 const MacroDefinition &MD,
1846 SourceRange Range,
1847 const MacroArgs *Args) {
1848 // FIXME: Does only enable builtin macro expansion make sense?
1849 if (!MD.getMacroInfo()->isBuiltinMacro())
1850 setSeenNoTrivialPPDirective();
1851}
1852