1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Preprocessor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/Basic/FileManager.h"
30#include "clang/Basic/IdentifierTable.h"
31#include "clang/Basic/LLVM.h"
32#include "clang/Basic/LangOptions.h"
33#include "clang/Basic/Module.h"
34#include "clang/Basic/SourceLocation.h"
35#include "clang/Basic/SourceManager.h"
36#include "clang/Basic/TargetInfo.h"
37#include "clang/Lex/CodeCompletionHandler.h"
38#include "clang/Lex/DependencyDirectivesScanner.h"
39#include "clang/Lex/ExternalPreprocessorSource.h"
40#include "clang/Lex/HeaderSearch.h"
41#include "clang/Lex/LexDiagnostic.h"
42#include "clang/Lex/Lexer.h"
43#include "clang/Lex/LiteralSupport.h"
44#include "clang/Lex/MacroArgs.h"
45#include "clang/Lex/MacroInfo.h"
46#include "clang/Lex/ModuleLoader.h"
47#include "clang/Lex/NoTrivialPPDirectiveTracer.h"
48#include "clang/Lex/Pragma.h"
49#include "clang/Lex/PreprocessingRecord.h"
50#include "clang/Lex/PreprocessorLexer.h"
51#include "clang/Lex/PreprocessorOptions.h"
52#include "clang/Lex/ScratchBuffer.h"
53#include "clang/Lex/Token.h"
54#include "clang/Lex/TokenLexer.h"
55#include "clang/Support/Compiler.h"
56#include "llvm/ADT/APInt.h"
57#include "llvm/ADT/ArrayRef.h"
58#include "llvm/ADT/DenseMap.h"
59#include "llvm/ADT/STLExtras.h"
60#include "llvm/ADT/ScopeExit.h"
61#include "llvm/ADT/SmallVector.h"
62#include "llvm/ADT/StringRef.h"
63#include "llvm/Support/Capacity.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/FormatVariadic.h"
66#include "llvm/Support/MemoryBuffer.h"
67#include "llvm/Support/MemoryBufferRef.h"
68#include "llvm/Support/SaveAndRestore.h"
69#include "llvm/Support/raw_ostream.h"
70#include <algorithm>
71#include <cassert>
72#include <memory>
73#include <optional>
74#include <string>
75#include <utility>
76#include <vector>
77
78using namespace clang;
79
80/// Minimum distance between two check points, in tokens.
81static constexpr unsigned CheckPointStepSize = 1024;
82
83LLVM_INSTANTIATE_REGISTRY_EX(CLANG_ABI_EXPORT, PragmaHandlerRegistry)
84
85ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
86
87Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
88 DiagnosticsEngine &diags, const LangOptions &opts,
89 SourceManager &SM, HeaderSearch &Headers,
90 ModuleLoader &TheModuleLoader,
91 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
92 TranslationUnitKind TUKind)
93 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
94 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
95 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
96 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
97 // As the language options may have not been loaded yet (when
98 // deserializing an ASTUnit), adding keywords to the identifier table is
99 // deferred to Preprocessor::Initialize().
100 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
101 TUKind(TUKind), SkipMainFilePreamble(0, true),
102 CurSubmoduleState(&NullSubmoduleState) {
103 OwnsHeaderSearch = OwnsHeaders;
104
105 // Only record check points if we might highlight diagnostic snippets.
106 RecordCheckPoints = getDiagnostics().getShowColors();
107
108 // Default to discarding comments.
109 KeepComments = false;
110 KeepMacroComments = false;
111 SuppressIncludeNotFoundError = false;
112
113 // Macro expansion is enabled.
114 DisableMacroExpansion = false;
115 MacroExpansionInDirectivesOverride = false;
116 InMacroArgs = false;
117 ArgMacro = nullptr;
118 InMacroArgPreExpansion = false;
119 NumCachedTokenLexers = 0;
120 PragmasEnabled = true;
121 ParsingIfOrElifDirective = false;
122 PreprocessedOutput = false;
123
124 // We haven't read anything from the external source.
125 ReadMacrosFromExternalSource = false;
126
127 LastExportKeyword.startToken();
128
129 BuiltinInfo = std::make_unique<Builtin::Context>();
130
131 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
132 // a macro. They get unpoisoned where it is allowed.
133 (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned();
134 SetPoisonReason(II: Ident__VA_ARGS__,DiagID: diag::ext_pp_bad_vaargs_use);
135 (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned();
136 SetPoisonReason(II: Ident__VA_OPT__,DiagID: diag::ext_pp_bad_vaopt_use);
137
138 // Initialize the pragma handlers.
139 RegisterBuiltinPragmas();
140
141 // Initialize builtin macros like __LINE__ and friends.
142 RegisterBuiltinMacros();
143
144 if(LangOpts.Borland) {
145 Ident__exception_info = getIdentifierInfo(Name: "_exception_info");
146 Ident___exception_info = getIdentifierInfo(Name: "__exception_info");
147 Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation");
148 Ident__exception_code = getIdentifierInfo(Name: "_exception_code");
149 Ident___exception_code = getIdentifierInfo(Name: "__exception_code");
150 Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode");
151 Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination");
152 Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination");
153 Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination");
154 } else {
155 Ident__exception_info = Ident__exception_code = nullptr;
156 Ident__abnormal_termination = Ident___exception_info = nullptr;
157 Ident___exception_code = Ident___abnormal_termination = nullptr;
158 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
159 Ident_AbnormalTermination = nullptr;
160 }
161
162 Ident__GLIBCXX__ = getIdentifierInfo(Name: "__GLIBCXX__");
163
164 // Default incremental processing to -fincremental-extensions, clients can
165 // override with `enableIncrementalProcessing` if desired.
166 IncrementalProcessing = LangOpts.IncrementalExtensions;
167
168 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
169 if (usingPCHWithPragmaHdrStop())
170 SkippingUntilPragmaHdrStop = true;
171
172 // If using a PCH with a through header, start skipping tokens.
173 if (!this->PPOpts.PCHThroughHeader.empty() &&
174 !this->PPOpts.ImplicitPCHInclude.empty())
175 SkippingUntilPCHThroughHeader = true;
176
177 if (this->PPOpts.GeneratePreamble)
178 PreambleConditionalStack.startRecording();
179
180 MaxTokens = LangOpts.MaxTokens;
181}
182
183Preprocessor::~Preprocessor() {
184 assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!");
185
186 IncludeMacroStack.clear();
187
188 // Free any cached macro expanders.
189 // This populates MacroArgCache, so all TokenLexers need to be destroyed
190 // before the code below that frees up the MacroArgCache list.
191 std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr);
192 CurTokenLexer.reset();
193
194 // Free any cached MacroArgs.
195 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
196 ArgList = ArgList->deallocate();
197
198 // Delete the header search info, if we own it.
199 if (OwnsHeaderSearch)
200 delete &HeaderInfo;
201}
202
203void Preprocessor::Initialize(const TargetInfo &Target,
204 const TargetInfo *AuxTarget) {
205 assert((!this->Target || this->Target == &Target) &&
206 "Invalid override of target information");
207 this->Target = &Target;
208
209 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
210 "Invalid override of aux target information.");
211 this->AuxTarget = AuxTarget;
212
213 // Initialize information about built-ins.
214 BuiltinInfo->InitializeTarget(Target, AuxTarget);
215 HeaderInfo.setTarget(Target);
216
217 // Populate the identifier table with info about keywords for the current language.
218 Identifiers.AddKeywords(LangOpts);
219
220 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
221 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
222
223 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
224 // Use setting from TargetInfo.
225 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod());
226 else
227 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
228 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod());
229}
230
231void Preprocessor::InitializeForModelFile() {
232 NumEnteredSourceFiles = 0;
233
234 // Reset pragmas
235 PragmaHandlersBackup = std::move(PragmaHandlers);
236 PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef());
237 RegisterBuiltinPragmas();
238
239 // Reset PredefinesFileID
240 PredefinesFileID = FileID();
241}
242
243void Preprocessor::FinalizeForModelFile() {
244 NumEnteredSourceFiles = 1;
245
246 PragmaHandlers = std::move(PragmaHandlersBackup);
247}
248
249void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
250 std::string TokenStr;
251 llvm::raw_string_ostream OS(TokenStr);
252
253 // The alignment of 16 is chosen to comfortably fit most identifiers.
254 OS << llvm::formatv(Fmt: "{0,-16} ", Vals: tok::getTokenName(Kind: Tok.getKind()));
255
256 // Annotation tokens are just markers that don't have a spelling -- they
257 // indicate where something expanded.
258 if (!Tok.isAnnotation()) {
259 OS << "'";
260 // Escape string to prevent token spelling from spanning multiple lines.
261 OS.write_escaped(Str: getSpelling(Tok));
262 OS << "'";
263 }
264
265 // The alignment of 48 (32 characters for the spelling + the 16 for
266 // the identifier name) fits most variable names, keywords and annotations.
267 llvm::errs() << llvm::formatv(Fmt: "{0,-48} ", Vals&: OS.str());
268
269 if (!DumpFlags) return;
270
271 auto Loc = Tok.getLocation();
272 llvm::errs() << "Loc=<";
273 DumpLocation(Loc);
274 llvm::errs() << ">";
275
276 // If the token points directly to a file location (i.e. not a macro
277 // expansion), then add additional padding so that trailing markers
278 // align, provided the line/column numbers are reasonably sized.
279 //
280 // Otherwise, if it's a macro expansion, don't bother with alignment,
281 // as the line will include multiple locations and be very long.
282 //
283 // NOTE: To keep this stateless, it doesn't account for filename
284 // length, so when a header starts markers will be temporarily misaligned.
285 if (Loc.isFileID()) {
286 PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
287
288 if (!PLoc.isInvalid()) {
289 int LineWidth = llvm::utostr(X: PLoc.getLine()).size();
290 int ColumnWidth = llvm::utostr(X: PLoc.getColumn()).size();
291
292 // Reserve space for lines up to 9999 and columns up to 99,
293 // which is 4 + 2 = 6 characters in total.
294 const int ReservedSpace = 6;
295
296 int LeftSpace = ReservedSpace - LineWidth - ColumnWidth;
297 int Padding = std::max<int>(a: 0, b: LeftSpace);
298
299 llvm::errs().indent(NumSpaces: Padding);
300 }
301 }
302
303 if (Tok.isAtStartOfLine())
304 llvm::errs() << " [StartOfLine]";
305 if (Tok.hasLeadingSpace())
306 llvm::errs() << " [LeadingSpace]";
307 if (Tok.isExpandDisabled())
308 llvm::errs() << " [ExpandDisabled]";
309 if (Tok.needsCleaning()) {
310 const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation());
311 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) << "']";
312 }
313}
314
315void Preprocessor::DumpLocation(SourceLocation Loc) const {
316 Loc.print(OS&: llvm::errs(), SM: SourceMgr);
317}
318
319void Preprocessor::DumpMacro(const MacroInfo &MI) const {
320 llvm::errs() << "MACRO: ";
321 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
322 DumpToken(Tok: MI.getReplacementToken(Tok: i));
323 llvm::errs() << " ";
324 }
325 llvm::errs() << "\n";
326}
327
328void Preprocessor::PrintStats() {
329 llvm::errs() << "\n*** Preprocessor Stats:\n";
330 llvm::errs() << NumDirectives << " directives found:\n";
331 llvm::errs() << " " << NumDefined << " #define.\n";
332 llvm::errs() << " " << NumUndefined << " #undef.\n";
333 llvm::errs() << " #include/#include_next/#import:\n";
334 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
335 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
336 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
337 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
338 llvm::errs() << " " << NumEndif << " #endif.\n";
339 llvm::errs() << " " << NumPragma << " #pragma.\n";
340 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
341
342 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
343 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
344 << NumFastMacroExpanded << " on the fast path.\n";
345 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
346 << " token paste (##) operations performed, "
347 << NumFastTokenPaste << " on the fast path.\n";
348
349 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
350
351 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
352 llvm::errs() << "\n Macro Expanded Tokens: "
353 << llvm::capacity_in_bytes(X: MacroExpandedTokens);
354 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
355 // FIXME: List information for all submodules.
356 llvm::errs() << "\n Macros: "
357 << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros);
358 llvm::errs() << "\n #pragma push_macro Info: "
359 << llvm::capacity_in_bytes(X: PragmaPushMacroInfo);
360 llvm::errs() << "\n Poison Reasons: "
361 << llvm::capacity_in_bytes(X: PoisonReasons);
362 llvm::errs() << "\n Comment Handlers: "
363 << llvm::capacity_in_bytes(x: CommentHandlers) << "\n";
364}
365
366llvm::iterator_range<Preprocessor::macro_iterator>
367Preprocessor::macros(bool IncludeExternalMacros) const {
368 if (IncludeExternalMacros && ExternalSource &&
369 !ReadMacrosFromExternalSource) {
370 ReadMacrosFromExternalSource = true;
371 ExternalSource->ReadDefinedMacros();
372 }
373 // Make sure we cover all macros in visible modules.
374 for (const ModuleMacro &Macro : ModuleMacros)
375 CurSubmoduleState->Macros.try_emplace(Key: Macro.II);
376
377 return CurSubmoduleState->Macros;
378}
379
380size_t Preprocessor::getTotalMemory() const {
381 return BP.getTotalMemory()
382 + llvm::capacity_in_bytes(X: MacroExpandedTokens)
383 + Predefines.capacity() /* Predefines buffer. */
384 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
385 // and ModuleMacros.
386 + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros)
387 + llvm::capacity_in_bytes(X: PragmaPushMacroInfo)
388 + llvm::capacity_in_bytes(X: PoisonReasons)
389 + llvm::capacity_in_bytes(x: CommentHandlers);
390}
391
392/// Compares macro tokens with a specified token value sequence.
393static bool MacroDefinitionEquals(const MacroInfo *MI,
394 ArrayRef<TokenValue> Tokens) {
395 return Tokens.size() == MI->getNumTokens() &&
396 std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin());
397}
398
399StringRef Preprocessor::getLastMacroWithSpelling(
400 SourceLocation Loc,
401 ArrayRef<TokenValue> Tokens) const {
402 SourceLocation BestLocation;
403 StringRef BestSpelling;
404 for (const auto &M : macros()) {
405 const MacroDirective::DefInfo Def =
406 M.second.findDirectiveAtLoc(Loc, SourceMgr);
407 if (!Def || !Def.getMacroInfo())
408 continue;
409 if (!Def.getMacroInfo()->isObjectLike())
410 continue;
411 if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens))
412 continue;
413 SourceLocation Location = Def.getLocation();
414 // Choose the macro defined latest.
415 if (BestLocation.isInvalid() ||
416 (Location.isValid() &&
417 SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) {
418 BestLocation = Location;
419 BestSpelling = M.first->getName();
420 }
421 }
422 return BestSpelling;
423}
424
425void Preprocessor::recomputeCurLexerKind() {
426 if (InCachingLexMode())
427 CurLexerCallback = CLK_CachingLexer;
428 else if (CurLexer)
429 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
430 ? CLK_DependencyDirectivesLexer
431 : CLK_Lexer;
432 else if (CurTokenLexer)
433 CurLexerCallback = CLK_TokenLexer;
434 else
435 CurLexerCallback = CLK_Lexer;
436}
437
438bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
439 unsigned CompleteLine,
440 unsigned CompleteColumn) {
441 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
442 assert(!CodeCompletionFile && "Already set");
443
444 // Load the actual file's contents.
445 std::optional<llvm::MemoryBufferRef> Buffer =
446 SourceMgr.getMemoryBufferForFileOrNone(File);
447 if (!Buffer)
448 return true;
449
450 // Find the byte position of the truncation point.
451 const char *Position = Buffer->getBufferStart();
452 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
453 for (; *Position; ++Position) {
454 if (*Position != '\r' && *Position != '\n')
455 continue;
456
457 // Eat \r\n or \n\r as a single line.
458 if ((Position[1] == '\r' || Position[1] == '\n') &&
459 Position[0] != Position[1])
460 ++Position;
461 ++Position;
462 break;
463 }
464 }
465
466 Position += CompleteColumn - 1;
467
468 // If pointing inside the preamble, adjust the position at the beginning of
469 // the file after the preamble.
470 if (SkipMainFilePreamble.first &&
471 SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) {
472 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
473 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
474 }
475
476 if (Position > Buffer->getBufferEnd())
477 Position = Buffer->getBufferEnd();
478
479 CodeCompletionFile = File;
480 CodeCompletionOffset = Position - Buffer->getBufferStart();
481
482 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
483 Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier());
484 char *NewBuf = NewBuffer->getBufferStart();
485 char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf);
486 *NewPos = '\0';
487 std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1);
488 SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer));
489
490 return false;
491}
492
493void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
494 bool IsAngled) {
495 setCodeCompletionReached();
496 if (CodeComplete)
497 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
498}
499
500void Preprocessor::CodeCompleteNaturalLanguage() {
501 setCodeCompletionReached();
502 if (CodeComplete)
503 CodeComplete->CodeCompleteNaturalLanguage();
504}
505
506/// getSpelling - This method is used to get the spelling of a token into a
507/// SmallVector. Note that the returned StringRef may not point to the
508/// supplied buffer if a copy can be avoided.
509StringRef Preprocessor::getSpelling(const Token &Tok,
510 SmallVectorImpl<char> &Buffer,
511 bool *Invalid) const {
512 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
513 if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) {
514 // Try the fast path.
515 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
516 return II->getName();
517 }
518
519 // Resize the buffer if we need to copy into it.
520 if (Tok.needsCleaning())
521 Buffer.resize(N: Tok.getLength());
522
523 const char *Ptr = Buffer.data();
524 unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid);
525 return StringRef(Ptr, Len);
526}
527
528/// CreateString - Plop the specified string into a scratch buffer and return a
529/// location for it. If specified, the source location provides a source
530/// location for the token.
531void Preprocessor::CreateString(StringRef Str, Token &Tok,
532 SourceLocation ExpansionLocStart,
533 SourceLocation ExpansionLocEnd) {
534 Tok.setLength(Str.size());
535
536 const char *DestPtr;
537 SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr);
538
539 if (ExpansionLocStart.isValid())
540 Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart,
541 ExpansionLocEnd, Length: Str.size());
542 Tok.setLocation(Loc);
543
544 // If this is a raw identifier or a literal token, set the pointer data.
545 if (Tok.is(K: tok::raw_identifier))
546 Tok.setRawIdentifierData(DestPtr);
547 else if (Tok.isLiteral())
548 Tok.setLiteralData(DestPtr);
549}
550
551SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
552 auto &SM = getSourceManager();
553 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
554 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc);
555 bool Invalid = false;
556 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
557 if (Invalid)
558 return SourceLocation();
559
560 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
561 const char *DestPtr;
562 SourceLocation Spelling =
563 ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr);
564 return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length));
565}
566
567Module *Preprocessor::getCurrentModule() {
568 if (!getLangOpts().isCompilingModule())
569 return nullptr;
570
571 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule);
572}
573
574Module *Preprocessor::getCurrentModuleImplementation() {
575 if (!getLangOpts().isCompilingModuleImplementation())
576 return nullptr;
577
578 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName);
579}
580
581//===----------------------------------------------------------------------===//
582// Preprocessor Initialization Methods
583//===----------------------------------------------------------------------===//
584
585/// EnterMainSourceFile - Enter the specified FileID as the main source file,
586/// which implicitly adds the builtin defines etc.
587void Preprocessor::EnterMainSourceFile() {
588 // We do not allow the preprocessor to reenter the main file. Doing so will
589 // cause FileID's to accumulate information from both runs (e.g. #line
590 // information) and predefined macros aren't guaranteed to be set properly.
591 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
592 FileID MainFileID = SourceMgr.getMainFileID();
593
594 // Whether and how the main file starts a C++20 module unit. Implicit inputs
595 // are placed in its existing global module fragment, or in a synthesized one
596 // for a named module without a GMF.
597 ModuleUnitKind MainFileModuleUnitKind = ModuleUnitKind::NotModuleUnit;
598
599 // If MainFileID is loaded it means we loaded an AST file, no need to enter
600 // a main file.
601 if (!SourceMgr.isLoadedFileID(FID: MainFileID)) {
602 // Enter the main file source buffer.
603 EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
604
605 // If we've been asked to skip bytes in the main file (e.g., as part of a
606 // precompiled preamble), do so now.
607 if (SkipMainFilePreamble.first > 0)
608 CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first,
609 StartOfLine: SkipMainFilePreamble.second);
610
611 // Tell the header info that the main file was entered. If the file is later
612 // #imported, it won't be re-entered.
613 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID))
614 markIncluded(File: *FE);
615
616 // Record the first PP token in the main file. This is used to generate
617 // better diagnostics for C++ modules.
618 //
619 // // This is a comment.
620 // #define FOO int // note: add 'module;' to the start of the file
621 // ^ FirstPPToken // to introduce a global module fragment.
622 //
623 // export module M; // error: module declaration must occur
624 // // at the start of the translation unit.
625 if (getLangOpts().CPlusPlusModules) {
626 std::optional<StringRef> Input =
627 getSourceManager().getBufferDataOrNone(FID: MainFileID);
628 if (!isPreprocessedModuleFile() && Input)
629 MainFileIsPreprocessedModuleFile =
630 clang::isPreprocessedModuleFile(Source: *Input);
631 if (Input && !MainFileIsPreprocessedModuleFile && hasDeferredGMFInputs())
632 MainFileModuleUnitKind = scanInputForCXX20ModuleUnit(Source: *Input);
633 auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(args&: *this);
634 DirTracer = Tracer.get();
635 addPPCallbacks(C: std::move(Tracer));
636 std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
637 if (FirstPPTok)
638 FirstPPTokenLoc = FirstPPTok->getLocation();
639 }
640 }
641
642 // Preserve the historical placement in the predefines buffer for ordinary
643 // translation units. A module unit opening with `module;` leaves the inputs
644 // deferred until the introducer has been lexed. For a named module without a
645 // GMF, synthesize the introducer before the main file and use the same
646 // deferred-input path.
647 if (hasDeferredGMFInputs()) {
648 if (PredefinesWereReplaced) {
649 // Loading an implicit PCH replaces Predefines with the directives
650 // suggested by ASTReader. For a module unit, those are the only implicit
651 // inputs that still need to be processed in the GMF.
652 if (MainFileModuleUnitKind != ModuleUnitKind::NotModuleUnit) {
653 DeferredGMFInputs = std::move(Predefines);
654 Predefines.clear();
655 }
656 } else if (MainFileModuleUnitKind == ModuleUnitKind::NotModuleUnit) {
657 // Preserve the historical predefines ordering for an ordinary
658 // translation unit.
659 Predefines += DeferredGMFInputs;
660 }
661 if (MainFileModuleUnitKind ==
662 ModuleUnitKind::NamedModuleWithoutGlobalModuleFragment) {
663 Predefines += "# 1 \"<implicit-global-module-fragment>\" 1\nmodule;\n";
664 HasSynthesizedGMF = true;
665 } else if (MainFileModuleUnitKind == ModuleUnitKind::NotModuleUnit) {
666 DeferredGMFInputs.clear();
667 }
668 }
669
670 // Preprocess Predefines to populate the initial preprocessor state.
671 std::unique_ptr<llvm::MemoryBuffer> SB =
672 llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>");
673 assert(SB && "Cannot create predefined source buffer");
674 FileID FID = SourceMgr.createFileID(Buffer: std::move(SB));
675 assert(FID.isValid() && "Could not create FileID for predefines?");
676 setPredefinesFileID(FID);
677
678 // Start parsing the predefines.
679 EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation());
680
681 if (!PPOpts.PCHThroughHeader.empty()) {
682 // Lookup and save the FileID for the through header. If it isn't found
683 // in the search path, it's a fatal error.
684 OptionalFileEntryRef File = LookupFile(
685 FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader,
686 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
687 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
688 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
689 /*IsFrameworkFound=*/nullptr);
690 if (!File) {
691 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_found)
692 << PPOpts.PCHThroughHeader;
693 return;
694 }
695 setPCHThroughHeaderFileID(
696 SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User));
697 }
698
699 // Skip tokens from the Predefines and if needed the main file.
700 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
701 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
702 SkipTokensWhileUsingPCH();
703}
704
705void Preprocessor::EnterDeferredGMFInputs(SourceLocation IncludeLoc) {
706 if (!hasDeferredGMFInputs())
707 return;
708 // Synthesize the implicit input directives and enter them inside the global
709 // module fragment. Attribute the buffer to IncludeLoc so it is ordered within
710 // the translation unit.
711 std::unique_ptr<llvm::MemoryBuffer> MB = llvm::MemoryBuffer::getMemBufferCopy(
712 InputData: DeferredGMFInputs, BufferName: "<gmf-command-line-inputs>");
713 DeferredGMFInputs.clear();
714 DeferredGMFInputsFileID =
715 SourceMgr.createFileID(Buffer: std::move(MB), FileCharacter: SrcMgr::C_User, LoadedID: 0, LoadedOffset: 0, IncludeLoc);
716 EnterSourceFile(FID: DeferredGMFInputsFileID, Dir: nullptr, Loc: IncludeLoc);
717}
718
719void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
720 assert(PCHThroughHeaderFileID.isInvalid() &&
721 "PCHThroughHeaderFileID already set!");
722 PCHThroughHeaderFileID = FID;
723}
724
725bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
726 assert(PCHThroughHeaderFileID.isValid() &&
727 "Invalid PCH through header FileID");
728 return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID);
729}
730
731bool Preprocessor::creatingPCHWithThroughHeader() {
732 return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
733 PCHThroughHeaderFileID.isValid();
734}
735
736bool Preprocessor::usingPCHWithThroughHeader() {
737 return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
738 PCHThroughHeaderFileID.isValid();
739}
740
741bool Preprocessor::creatingPCHWithPragmaHdrStop() {
742 return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop;
743}
744
745bool Preprocessor::usingPCHWithPragmaHdrStop() {
746 return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop;
747}
748
749/// Skip tokens until after the #include of the through header or
750/// until after a #pragma hdrstop is seen. Tokens in the predefines file
751/// and the main file may be skipped. If the end of the predefines file
752/// is reached, skipping continues into the main file. If the end of the
753/// main file is reached, it's a fatal error.
754void Preprocessor::SkipTokensWhileUsingPCH() {
755 bool ReachedMainFileEOF = false;
756 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
757 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
758 Token Tok;
759 while (true) {
760 bool InPredefines =
761 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
762 CurLexerCallback(*this, Tok);
763 if (Tok.is(K: tok::eof) && !InPredefines) {
764 ReachedMainFileEOF = true;
765 break;
766 }
767 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
768 break;
769 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
770 break;
771 }
772 if (ReachedMainFileEOF) {
773 if (UsingPCHThroughHeader)
774 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_seen)
775 << PPOpts.PCHThroughHeader << 1;
776 else if (!PPOpts.PCHWithHdrStopCreate)
777 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_pragma_hdrstop_not_seen);
778 }
779}
780
781void Preprocessor::replayPreambleConditionalStack() {
782 // Restore the conditional stack from the preamble, if there is one.
783 if (PreambleConditionalStack.isReplaying()) {
784 assert(CurPPLexer &&
785 "CurPPLexer is null when calling replayPreambleConditionalStack.");
786 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
787 PreambleConditionalStack.doneReplaying();
788 if (PreambleConditionalStack.reachedEOFWhileSkipping())
789 SkipExcludedConditionalBlock(
790 HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc,
791 IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc,
792 FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
793 FoundElse: PreambleConditionalStack.SkipInfo->FoundElse,
794 ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc);
795 }
796}
797
798void Preprocessor::EndSourceFile() {
799 // Notify the client that we reached the end of the source file.
800 if (Callbacks)
801 Callbacks->EndOfMainFile();
802}
803
804//===----------------------------------------------------------------------===//
805// Lexer Event Handling.
806//===----------------------------------------------------------------------===//
807
808/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
809/// identifier information for the token and install it into the token,
810/// updating the token kind accordingly.
811IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
812 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
813
814 // Look up this token, see if it is a macro, or if it is a language keyword.
815 IdentifierInfo *II;
816 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
817 // No cleaning needed, just use the characters from the lexed buffer.
818 II = getIdentifierInfo(Name: Identifier.getRawIdentifier());
819 } else {
820 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
821 SmallString<64> IdentifierBuffer;
822 StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer);
823
824 if (Identifier.hasUCN()) {
825 SmallString<64> UCNIdentifierBuffer;
826 expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr);
827 II = getIdentifierInfo(Name: UCNIdentifierBuffer);
828 } else {
829 II = getIdentifierInfo(Name: CleanedStr);
830 }
831 }
832
833 // Update the token info (identifier info and appropriate token kind).
834 // FIXME: the raw_identifier may contain leading whitespace which is removed
835 // from the cleaned identifier token. The SourceLocation should be updated to
836 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
837 // line continuation before 'B') is parsed as a single tok::raw_identifier and
838 // is cleaned to tok::identifier "B". After cleaning the token's length is
839 // still 3 and the SourceLocation refers to the location of the backslash.
840 Identifier.setIdentifierInfo(II);
841 Identifier.setKind(II->getTokenID());
842
843 return II;
844}
845
846void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
847 PoisonReasons[II] = DiagID;
848}
849
850void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
851 assert(Ident__exception_code && Ident__exception_info);
852 assert(Ident___exception_code && Ident___exception_info);
853 Ident__exception_code->setIsPoisoned(Poison);
854 Ident___exception_code->setIsPoisoned(Poison);
855 Ident_GetExceptionCode->setIsPoisoned(Poison);
856 Ident__exception_info->setIsPoisoned(Poison);
857 Ident___exception_info->setIsPoisoned(Poison);
858 Ident_GetExceptionInfo->setIsPoisoned(Poison);
859 Ident__abnormal_termination->setIsPoisoned(Poison);
860 Ident___abnormal_termination->setIsPoisoned(Poison);
861 Ident_AbnormalTermination->setIsPoisoned(Poison);
862}
863
864void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
865 assert(Identifier.getIdentifierInfo() &&
866 "Can't handle identifiers without identifier info!");
867 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
868 PoisonReasons.find(Val: Identifier.getIdentifierInfo());
869 if(it == PoisonReasons.end())
870 Diag(Tok: Identifier, DiagID: diag::err_pp_used_poisoned_id);
871 else
872 Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo();
873}
874
875void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const {
876 assert(II.isOutOfDate() && "not out of date");
877 assert(getExternalSource() &&
878 "getExternalSource() should not return nullptr");
879 getExternalSource()->updateOutOfDateIdentifier(II);
880}
881
882/// HandleIdentifier - This callback is invoked when the lexer reads an
883/// identifier. This callback looks up the identifier in the map and/or
884/// potentially macro expands it or turns it into a named token (like 'for').
885///
886/// Note that callers of this method are guarded by checking the
887/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
888/// IdentifierInfo methods that compute these properties will need to change to
889/// match.
890bool Preprocessor::HandleIdentifier(Token &Identifier) {
891 assert(Identifier.getIdentifierInfo() &&
892 "Can't handle identifiers without identifier info!");
893
894 IdentifierInfo &II = *Identifier.getIdentifierInfo();
895
896 // If the information about this identifier is out of date, update it from
897 // the external source.
898 // We have to treat __VA_ARGS__ in a special way, since it gets
899 // serialized with isPoisoned = true, but our preprocessor may have
900 // unpoisoned it if we're defining a C99 macro.
901 if (II.isOutOfDate()) {
902 bool CurrentIsPoisoned = false;
903 const bool IsSpecialVariadicMacro =
904 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
905 if (IsSpecialVariadicMacro)
906 CurrentIsPoisoned = II.isPoisoned();
907
908 updateOutOfDateIdentifier(II);
909 Identifier.setKind(II.getTokenID());
910
911 if (IsSpecialVariadicMacro)
912 II.setIsPoisoned(CurrentIsPoisoned);
913 }
914
915 // If this identifier was poisoned, and if it was not produced from a macro
916 // expansion, emit an error.
917 if (II.isPoisoned() && CurPPLexer) {
918 HandlePoisonedIdentifier(Identifier);
919 }
920
921 // If this is a macro to be expanded, do it.
922 if (const MacroDefinition MD = getMacroDefinition(II: &II)) {
923 const auto *MI = MD.getMacroInfo();
924 assert(MI && "macro definition with no macro info?");
925 if (!DisableMacroExpansion) {
926 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
927 // C99 6.10.3p10: If the preprocessing token immediately after the
928 // macro name isn't a '(', this macro should not be expanded.
929 if (!MI->isFunctionLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
930 return HandleMacroExpandedIdentifier(Identifier, MD);
931 } else {
932 // C99 6.10.3.4p2 says that a disabled macro may never again be
933 // expanded, even if it's in a context where it could be expanded in the
934 // future.
935 Identifier.setFlag(Token::DisableExpand);
936 if (MI->isObjectLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
937 Diag(Tok: Identifier, DiagID: diag::pp_disabled_macro_expansion);
938 }
939 }
940 }
941
942 // If this identifier is a keyword in a newer Standard or proposed Standard,
943 // produce a warning. Don't warn if we're not considering macro expansion,
944 // since this identifier might be the name of a macro.
945 // FIXME: This warning is disabled in cases where it shouldn't be, like
946 // "#define constexpr constexpr", "int constexpr;"
947 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
948 Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts()))
949 << II.getName();
950 // Don't diagnose this keyword again in this translation unit.
951 II.setIsFutureCompatKeyword(false);
952 }
953
954 // If this identifier would be a keyword in C++, diagnose as a compatibility
955 // issue.
956 if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion)
957 Diag(Tok: Identifier, DiagID: diag::warn_pp_identifier_is_cpp_keyword) << &II;
958
959 // If this is an extension token, diagnose its use.
960 // We avoid diagnosing tokens that originate from macro definitions.
961 // FIXME: This warning is disabled in cases where it shouldn't be,
962 // like "#define TY typeof", "TY(1) x".
963 if (II.isExtensionToken() && !DisableMacroExpansion)
964 Diag(Tok: Identifier, DiagID: diag::ext_token_used);
965
966 // Handle module contextual keywords.
967 if (getLangOpts().CPlusPlusModules && CurLexer &&
968 !CurLexer->isLexingRawMode() && !CurLexer->isPragmaLexer() &&
969 !CurLexer->ParsingPreprocessorDirective &&
970 Identifier.isModuleContextualKeyword() &&
971 HandleModuleContextualKeyword(Result&: Identifier)) {
972 HandleDirective(Result&: Identifier);
973 // With a fatal failure in the module loader, we abort parsing.
974 return hadModuleLoaderFatalFailure();
975 }
976
977 return true;
978}
979
980void Preprocessor::Lex(Token &Result) {
981 ++LexLevel;
982
983 // We loop here until a lex function returns a token; this avoids recursion.
984 while (!CurLexerCallback(*this, Result))
985 ;
986
987 if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure)
988 return;
989
990 if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) {
991 // Remember the identifier before code completion token.
992 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
993 setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc());
994 // Set IdenfitierInfo to null to avoid confusing code that handles both
995 // identifiers and completion tokens.
996 Result.setIdentifierInfo(nullptr);
997 }
998
999 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
1000 // if this token is being produced as a result of phase 4 of translation.
1001 // Update TrackGMFState to decide if we are currently in a Global Module
1002 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
1003 // depends on the prevailing StdCXXImportSeq state in two cases.
1004 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
1005 !Result.getFlag(Flag: Token::IsReinjected)) {
1006 switch (Result.getKind()) {
1007 case tok::l_paren: case tok::l_square: case tok::l_brace:
1008 StdCXXImportSeqState.handleOpenBracket();
1009 break;
1010 case tok::r_paren: case tok::r_square:
1011 StdCXXImportSeqState.handleCloseBracket();
1012 break;
1013 case tok::r_brace:
1014 StdCXXImportSeqState.handleCloseBrace();
1015 break;
1016#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
1017// For `#pragma ...` mimic ';'.
1018#include "clang/Basic/TokenKinds.def"
1019#undef PRAGMA_ANNOTATION
1020 // This token is injected to represent the translation of '#include "a.h"'
1021 // into "import a.h;". Mimic the notional ';'.
1022 case tok::annot_module_include:
1023 case tok::annot_repl_input_end:
1024 case tok::semi:
1025 TrackGMFState.handleSemi();
1026 StdCXXImportSeqState.handleSemi();
1027 ModuleDeclState.handleSemi();
1028 break;
1029 case tok::header_name:
1030 case tok::annot_header_unit:
1031 StdCXXImportSeqState.handleHeaderName();
1032 break;
1033 case tok::kw_export:
1034 if (hasSeenNoTrivialPPDirective())
1035 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
1036 TrackGMFState.handleExport();
1037 StdCXXImportSeqState.handleExport();
1038 ModuleDeclState.handleExport();
1039 break;
1040 case tok::colon:
1041 ModuleDeclState.handleColon();
1042 break;
1043 case tok::kw_import:
1044 if (StdCXXImportSeqState.atTopLevel()) {
1045 TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
1046 StdCXXImportSeqState.handleImport();
1047 }
1048 break;
1049 case tok::kw_module:
1050 if (StdCXXImportSeqState.atTopLevel()) {
1051 if (hasSeenNoTrivialPPDirective())
1052 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
1053 TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
1054 ModuleDeclState.handleModule();
1055 }
1056 break;
1057 case tok::annot_module_name:
1058 ModuleDeclState.handleModuleName(
1059 NameLoc: static_cast<ModuleNameLoc *>(Result.getAnnotationValue()));
1060 if (ModuleDeclState.isModuleCandidate())
1061 break;
1062 [[fallthrough]];
1063 default:
1064 TrackGMFState.handleMisc();
1065 StdCXXImportSeqState.handleMisc();
1066 ModuleDeclState.handleMisc();
1067 break;
1068 }
1069 }
1070
1071 if (RecordCheckPoints && CurLexer &&
1072 ++CheckPointCounter == CheckPointStepSize) {
1073 CheckPoints[CurLexer->getFileID()].push_back(Elt: CurLexer->BufferPtr);
1074 CheckPointCounter = 0;
1075 }
1076
1077 if (Result.isNot(K: tok::kw_export))
1078 LastExportKeyword.startToken();
1079
1080 --LexLevel;
1081
1082 // Destroy any lexers that were deferred while we were in nested Lex calls.
1083 // This must happen after decrementing LexLevel but before any other
1084 // processing that might re-enter Lex.
1085 if (LexLevel == 0 && !PendingDestroyLexers.empty())
1086 PendingDestroyLexers.clear();
1087
1088 if ((LexLevel == 0 || PreprocessToken) &&
1089 !Result.getFlag(Flag: Token::IsReinjected)) {
1090 if (LexLevel == 0)
1091 ++TokenCount;
1092 if (OnToken)
1093 OnToken(Result);
1094 }
1095}
1096
1097void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
1098 while (1) {
1099 Token Tok;
1100 Lex(Result&: Tok);
1101 if (Tok.isOneOf(Ks: tok::unknown, Ks: tok::eof, Ks: tok::eod,
1102 Ks: tok::annot_repl_input_end))
1103 break;
1104 if (Tokens != nullptr)
1105 Tokens->push_back(x: Tok);
1106 }
1107}
1108
1109/// Lex a header-name token (including one formed from header-name-tokens if
1110/// \p AllowMacroExpansion is \c true).
1111///
1112/// \param FilenameTok Filled in with the next token. On success, this will
1113/// be either a header_name token. On failure, it will be whatever other
1114/// token was found instead.
1115/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1116/// by macro expansion (concatenating tokens as necessary if the first
1117/// token is a '<').
1118/// \return \c true if we reached EOD or EOF while looking for a > token in
1119/// a concatenated header name and diagnosed it. \c false otherwise.
1120bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1121 // Lex using header-name tokenization rules if tokens are being lexed from
1122 // a file. Just grab a token normally if we're in a macro expansion.
1123 if (CurPPLexer) {
1124 // Avoid nested header-name lexing when macro expansion recurses
1125 // __has_include(__has_include))
1126 if (CurPPLexer->ParsingFilename)
1127 LexUnexpandedToken(Result&: FilenameTok);
1128 else
1129 CurPPLexer->LexIncludeFilename(FilenameTok);
1130 } else {
1131 Lex(Result&: FilenameTok);
1132 }
1133
1134 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1135 // case, glue the tokens together into an angle_string_literal token.
1136 SmallString<128> FilenameBuffer;
1137 if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) {
1138 bool StartOfLine = FilenameTok.isAtStartOfLine();
1139 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1140 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1141
1142 SourceLocation Start = FilenameTok.getLocation();
1143 SourceLocation End;
1144 FilenameBuffer.push_back(Elt: '<');
1145
1146 // Consume tokens until we find a '>'.
1147 // FIXME: A header-name could be formed starting or ending with an
1148 // alternative token. It's not clear whether that's ill-formed in all
1149 // cases.
1150 while (FilenameTok.isNot(K: tok::greater)) {
1151 Lex(Result&: FilenameTok);
1152 if (FilenameTok.isOneOf(Ks: tok::eod, Ks: tok::eof)) {
1153 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_expected) << tok::greater;
1154 Diag(Loc: Start, DiagID: diag::note_matching) << tok::less;
1155 return true;
1156 }
1157
1158 End = FilenameTok.getLocation();
1159
1160 // FIXME: Provide code completion for #includes.
1161 if (FilenameTok.is(K: tok::code_completion)) {
1162 setCodeCompletionReached();
1163 Lex(Result&: FilenameTok);
1164 continue;
1165 }
1166
1167 // Append the spelling of this token to the buffer. If there was a space
1168 // before it, add it now.
1169 if (FilenameTok.hasLeadingSpace())
1170 FilenameBuffer.push_back(Elt: ' ');
1171
1172 // Get the spelling of the token, directly into FilenameBuffer if
1173 // possible.
1174 size_t PreAppendSize = FilenameBuffer.size();
1175 FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength());
1176
1177 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1178 unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr);
1179
1180 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1181 if (BufPtr != &FilenameBuffer[PreAppendSize])
1182 memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen);
1183
1184 // Resize FilenameBuffer to the correct size.
1185 if (FilenameTok.getLength() != ActualLen)
1186 FilenameBuffer.resize(N: PreAppendSize + ActualLen);
1187 }
1188
1189 FilenameTok.startToken();
1190 FilenameTok.setKind(tok::header_name);
1191 FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine);
1192 FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1193 FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro);
1194 CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End);
1195 } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) {
1196 // Convert a string-literal token of the form " h-char-sequence "
1197 // (produced by macro expansion) into a header-name token.
1198 //
1199 // The rules for header-names don't quite match the rules for
1200 // string-literals, but all the places where they differ result in
1201 // undefined behavior, so we can and do treat them the same.
1202 //
1203 // A string-literal with a prefix or suffix is not translated into a
1204 // header-name. This could theoretically be observable via the C++20
1205 // context-sensitive header-name formation rules.
1206 StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
1207 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1208 FilenameTok.setKind(tok::header_name);
1209 }
1210
1211 return false;
1212}
1213
1214std::optional<Token> Preprocessor::peekNextPPToken() const {
1215 // Do some quick tests for rejection cases.
1216 std::optional<Token> Val;
1217 if (CurLexer)
1218 Val = CurLexer->peekNextPPToken();
1219 else
1220 Val = CurTokenLexer->peekNextPPToken();
1221
1222 if (!Val) {
1223 // We have run off the end. If it's a source file we don't
1224 // examine enclosing ones (C99 5.1.1.2p4). Otherwise walk up the
1225 // macro stack.
1226 if (CurPPLexer)
1227 return std::nullopt;
1228 for (const IncludeStackInfo &Entry : llvm::reverse(C: IncludeMacroStack)) {
1229 if (Entry.TheLexer)
1230 Val = Entry.TheLexer->peekNextPPToken();
1231 else
1232 Val = Entry.TheTokenLexer->peekNextPPToken();
1233
1234 if (Val)
1235 break;
1236
1237 // Ran off the end of a source file?
1238 if (Entry.ThePPLexer)
1239 return std::nullopt;
1240 }
1241 }
1242
1243 // Okay, we found the token and return. Otherwise we found the end of the
1244 // translation unit.
1245 return Val;
1246}
1247
1248// We represent the primary and partition names as 'Paths' which are sections
1249// of the hierarchical access path for a clang module. However for C++20
1250// the periods in a name are just another character, and we will need to
1251// flatten them into a string.
1252std::string ModuleLoader::getFlatNameFromPath(ModuleIdPath Path) {
1253 std::string Name;
1254 if (Path.empty())
1255 return Name;
1256
1257 for (auto &Piece : Path) {
1258 assert(Piece.getIdentifierInfo() && Piece.getLoc().isValid());
1259 if (!Name.empty())
1260 Name += ".";
1261 Name += Piece.getIdentifierInfo()->getName();
1262 }
1263 return Name;
1264}
1265
1266ModuleNameLoc *ModuleNameLoc::Create(Preprocessor &PP, ModuleIdPath Path) {
1267 assert(!Path.empty() && "expect at least one identifier in a module name");
1268 void *Mem = PP.getPreprocessorAllocator().Allocate(
1269 Size: totalSizeToAlloc<IdentifierLoc>(Counts: Path.size()), Alignment: alignof(ModuleNameLoc));
1270 return new (Mem) ModuleNameLoc(Path);
1271}
1272
1273bool Preprocessor::LexModuleNameContinue(Token &Tok, SourceLocation UseLoc,
1274 SmallVectorImpl<Token> &Suffix,
1275 SmallVectorImpl<IdentifierLoc> &Path,
1276 bool AllowMacroExpansion,
1277 bool IsPartition) {
1278 auto ConsumeToken = [&]() {
1279 if (AllowMacroExpansion)
1280 Lex(Result&: Tok);
1281 else
1282 LexUnexpandedToken(Result&: Tok);
1283 Suffix.push_back(Elt: Tok);
1284 };
1285
1286 while (true) {
1287 if (Tok.isNot(K: tok::identifier)) {
1288 if (Tok.is(K: tok::code_completion)) {
1289 CurLexer->cutOffLexing();
1290 CodeComplete->CodeCompleteModuleImport(ImportLoc: UseLoc, Path);
1291 return true;
1292 }
1293
1294 Diag(Tok, DiagID: diag::err_pp_module_expected_ident) << Path.empty();
1295 return true;
1296 }
1297
1298 // [cpp.pre]/p2:
1299 // No identifier in the pp-module-name or pp-module-partition shall
1300 // currently be defined as an object-like macro.
1301 if (MacroInfo *MI = getMacroInfo(II: Tok.getIdentifierInfo());
1302 MI && MI->isObjectLike() && getLangOpts().CPlusPlus20 &&
1303 !AllowMacroExpansion) {
1304 Diag(Tok, DiagID: diag::err_pp_module_name_is_macro)
1305 << IsPartition << Tok.getIdentifierInfo();
1306 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::note_macro_here)
1307 << Tok.getIdentifierInfo();
1308 }
1309
1310 // Record this part of the module path.
1311 Path.emplace_back(Args: Tok.getLocation(), Args: Tok.getIdentifierInfo());
1312 ConsumeToken();
1313
1314 if (Tok.isNot(K: tok::period))
1315 return false;
1316
1317 ConsumeToken();
1318 }
1319}
1320
1321bool Preprocessor::HandleModuleName(StringRef DirType, SourceLocation UseLoc,
1322 Token &Tok,
1323 SmallVectorImpl<IdentifierLoc> &Path,
1324 SmallVectorImpl<Token> &DirToks,
1325 bool AllowMacroExpansion,
1326 bool IsPartition) {
1327 bool LeadingSpace = Tok.hasLeadingSpace();
1328 unsigned NumToksInDirective = DirToks.size();
1329 if (LexModuleNameContinue(Tok, UseLoc, Suffix&: DirToks, Path, AllowMacroExpansion,
1330 IsPartition)) {
1331 if (Tok.isNot(K: tok::eod))
1332 CheckEndOfDirective(DirType,
1333 /*EnableMacros=*/false, ExtraToks: &DirToks);
1334 EnterModuleSuffixTokenStream(Toks: DirToks);
1335 return true;
1336 }
1337
1338 // Clean the module-name tokens and replace these tokens with
1339 // annot_module_name.
1340 DirToks.resize(N: NumToksInDirective);
1341 ModuleNameLoc *NameLoc = ModuleNameLoc::Create(PP&: *this, Path);
1342 DirToks.emplace_back();
1343 DirToks.back().setKind(tok::annot_module_name);
1344 DirToks.back().setAnnotationRange(NameLoc->getRange());
1345 DirToks.back().setAnnotationValue(static_cast<void *>(NameLoc));
1346 DirToks.back().setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1347 DirToks.push_back(Elt: Tok);
1348 return false;
1349}
1350
1351/// [cpp.pre]/p2:
1352/// A preprocessing directive consists of a sequence of preprocessing tokens
1353/// that satisfies the following constraints: At the start of translation phase
1354/// 4, the first preprocessing token in the sequence, referred to as a
1355/// directive-introducing token, begins with the first character in the source
1356/// file (optionally after whitespace containing no new-line characters) or
1357/// follows whitespace containing at least one new-line character, and is:
1358/// - a # preprocessing token, or
1359/// - an import preprocessing token immediately followed on the same logical
1360/// source line by a header-name, <, identifier, or : preprocessing token, or
1361/// - a module preprocessing token immediately followed on the same logical
1362/// source line by an identifier, :, or ; preprocessing token, or
1363/// - an export preprocessing token immediately followed on the same logical
1364/// source line by one of the two preceding forms.
1365///
1366///
1367/// At the start of phase 4 an import or module token is treated as starting a
1368/// directive and are converted to their respective keywords iff:
1369/// - After skipping horizontal whitespace are
1370/// - at the start of a logical line, or
1371/// - preceded by an 'export' at the start of the logical line.
1372/// - Are followed by an identifier pp token (before macro expansion), or
1373/// - <, ", or : (but not ::) pp tokens for 'import', or
1374/// - ; for 'module'
1375/// Otherwise the token is treated as an identifier.
1376bool Preprocessor::HandleModuleContextualKeyword(Token &Result) {
1377 if (!getLangOpts().CPlusPlusModules || !Result.isModuleContextualKeyword())
1378 return false;
1379
1380 if (Result.is(K: tok::kw_export)) {
1381 LastExportKeyword = Result;
1382 return false;
1383 }
1384
1385 /// Trait 'module' and 'import' as a identifier when the main file is a
1386 /// preprocessed module file. We only allow '__preprocessed_module' and
1387 /// '__preprocessed_import' in this context.
1388 IdentifierInfo *II = Result.getIdentifierInfo();
1389 if (isPreprocessedModuleFile() &&
1390 (II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_import)) ||
1391 II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_module))))
1392 return false;
1393
1394 if (LastExportKeyword.is(K: tok::kw_export)) {
1395 // The export keyword was not at the start of line, it's not a
1396 // directive-introducing token.
1397 if (!LastExportKeyword.isAtPhysicalStartOfLine())
1398 return false;
1399 // [cpp.pre]/1.4
1400 // export // not a preprocessing directive
1401 // import foo; // preprocessing directive (ill-formed at phase7)
1402 if (Result.isAtPhysicalStartOfLine())
1403 return false;
1404 } else if (!Result.isAtPhysicalStartOfLine())
1405 return false;
1406
1407 assert(CurPPLexer && "CurPPLexer must not be null");
1408
1409 llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective(
1410 CurPPLexer->ParsingPreprocessorDirective, true);
1411
1412 if (II->isModuleKeyword()) {
1413 if (auto NextTok = peekNextPPToken()) {
1414 if (NextTok->is(K: tok::raw_identifier))
1415 LookUpIdentifierInfo(Identifier&: *NextTok);
1416 if (NextTok->isOneOf(Ks: tok::identifier, Ks: tok::colon, Ks: tok::semi)) {
1417 Result.setKind(tok::kw_module);
1418 ModuleDeclLoc = Result.getLocation();
1419 return true;
1420 }
1421 }
1422 return false;
1423 }
1424
1425 if (II->isImportKeyword()) {
1426 llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename,
1427 true);
1428 if (auto NextTok = peekNextPPToken()) {
1429 if (NextTok->is(K: tok::raw_identifier))
1430 LookUpIdentifierInfo(Identifier&: *NextTok);
1431 if (NextTok->isOneOf(Ks: tok::header_name, Ks: tok::identifier, Ks: tok::colon,
1432 Ks: tok::less, Ks: tok::code_completion)) {
1433 Result.setKind(tok::kw_import);
1434 ModuleImportLoc = Result.getLocation();
1435 return true;
1436 }
1437 }
1438 return false;
1439 }
1440
1441 // Ok, it's an identifier.
1442 return false;
1443}
1444
1445bool Preprocessor::CollectPPImportSuffixAndEnterStream(
1446 SmallVectorImpl<Token> &Toks, bool StopUntilEOD) {
1447 CollectPPImportSuffix(Toks);
1448 EnterModuleSuffixTokenStream(Toks);
1449 return false;
1450}
1451
1452/// Collect the tokens of a C++20 pp-import-suffix.
1453void Preprocessor::CollectPPImportSuffix(SmallVectorImpl<Token> &Toks,
1454 bool StopUntilEOD) {
1455 while (true) {
1456 Toks.emplace_back();
1457 Lex(Result&: Toks.back());
1458
1459 switch (Toks.back().getKind()) {
1460 case tok::semi:
1461 if (!StopUntilEOD)
1462 return;
1463 [[fallthrough]];
1464 case tok::eod:
1465 case tok::eof:
1466 return;
1467 default:
1468 break;
1469 }
1470 }
1471}
1472
1473// Allocate a holding buffer for a sequence of tokens and introduce it into
1474// the token stream.
1475void Preprocessor::EnterModuleSuffixTokenStream(ArrayRef<Token> Toks) {
1476 if (Toks.empty())
1477 return;
1478 auto ToksCopy = std::make_unique<Token[]>(num: Toks.size());
1479 std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get());
1480 EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(),
1481 /*DisableMacroExpansion*/ false, /*IsReinject*/ false);
1482 assert(CurTokenLexer && "Must have a TokenLexer");
1483 CurTokenLexer->setLexingCXXModuleDirective();
1484}
1485
1486void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc,
1487 bool IncludeExports) {
1488 CurSubmoduleState->VisibleModules.setVisible(
1489 M, Loc, IncludeExports, Vis: [](Module *) {},
1490 Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1491 // FIXME: Include the path in the diagnostic.
1492 // FIXME: Include the import location for the conflicting module.
1493 Diag(Loc: ModuleImportLoc, DiagID: diag::warn_module_conflict)
1494 << Path[0]->getFullModuleName()
1495 << Conflict->getFullModuleName()
1496 << Message;
1497 });
1498
1499 // Add this module to the imports list of the currently-built submodule.
1500 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1501 BuildingSubmoduleStack.back().M->Imports.push_back(Elt: M);
1502}
1503
1504bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1505 const char *DiagnosticTag,
1506 bool AllowMacroExpansion) {
1507 // We need at least one string literal.
1508 if (Result.isNot(K: tok::string_literal)) {
1509 Diag(Tok: Result, DiagID: diag::err_expected_string_literal)
1510 << /*Source='in...'*/0 << DiagnosticTag;
1511 return false;
1512 }
1513
1514 // Lex string literal tokens, optionally with macro expansion.
1515 SmallVector<Token, 4> StrToks;
1516 do {
1517 StrToks.push_back(Elt: Result);
1518
1519 if (Result.hasUDSuffix())
1520 Diag(Tok: Result, DiagID: diag::err_invalid_string_udl);
1521
1522 if (AllowMacroExpansion)
1523 Lex(Result);
1524 else
1525 LexUnexpandedToken(Result);
1526 } while (Result.is(K: tok::string_literal));
1527
1528 // Concatenate and parse the strings.
1529 StringLiteralParser Literal(StrToks, *this);
1530 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1531
1532 if (Literal.hadError)
1533 return false;
1534
1535 if (Literal.Pascal) {
1536 Diag(Loc: StrToks[0].getLocation(), DiagID: diag::err_expected_string_literal)
1537 << /*Source='in...'*/0 << DiagnosticTag;
1538 return false;
1539 }
1540
1541 String = std::string(Literal.GetString());
1542 return true;
1543}
1544
1545bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1546 assert(Tok.is(tok::numeric_constant));
1547 SmallString<8> IntegerBuffer;
1548 bool NumberInvalid = false;
1549 StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid);
1550 if (NumberInvalid)
1551 return false;
1552 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1553 getLangOpts(), getTargetInfo(),
1554 getDiagnostics());
1555 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1556 return false;
1557 llvm::APInt APVal(64, 0);
1558 if (Literal.GetIntegerValue(Val&: APVal))
1559 return false;
1560 Lex(Result&: Tok);
1561 Value = APVal.getLimitedValue();
1562 return true;
1563}
1564
1565void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1566 assert(Handler && "NULL comment handler");
1567 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1568 "Comment handler already registered");
1569 CommentHandlers.push_back(x: Handler);
1570}
1571
1572void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1573 std::vector<CommentHandler *>::iterator Pos =
1574 llvm::find(Range&: CommentHandlers, Val: Handler);
1575 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1576 CommentHandlers.erase(position: Pos);
1577}
1578
1579bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1580 bool AnyPendingTokens = false;
1581 for (CommentHandler *H : CommentHandlers) {
1582 if (H->HandleComment(PP&: *this, Comment))
1583 AnyPendingTokens = true;
1584 }
1585 if (!AnyPendingTokens || getCommentRetentionState())
1586 return false;
1587 Lex(Result&: result);
1588 return true;
1589}
1590
1591void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1592 const MacroAnnotations &A =
1593 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1594 assert(A.DeprecationInfo &&
1595 "Macro deprecation warning without recorded annotation!");
1596 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1597 if (Info.Message.empty())
1598 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1599 << Identifier.getIdentifierInfo() << 0;
1600 else
1601 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1602 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1603 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 0;
1604}
1605
1606void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1607 const MacroAnnotations &A =
1608 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1609 assert(A.RestrictExpansionInfo &&
1610 "Macro restricted expansion warning without recorded annotation!");
1611 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1612 if (Info.Message.empty())
1613 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1614 << Identifier.getIdentifierInfo() << 0;
1615 else
1616 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1617 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1618 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 1;
1619}
1620
1621void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
1622 unsigned DiagSelection) const {
1623 Diag(Tok: Identifier, DiagID: diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1624}
1625
1626void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1627 bool IsUndef) const {
1628 const MacroAnnotations &A =
1629 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1630 assert(A.FinalAnnotationLoc &&
1631 "Final macro warning without recorded annotation!");
1632
1633 Diag(Tok: Identifier, DiagID: diag::warn_pragma_final_macro)
1634 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1635 Diag(Loc: *A.FinalAnnotationLoc, DiagID: diag::note_pp_macro_annotation) << 2;
1636}
1637
1638bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr,
1639 const SourceLocation &Loc) const {
1640 // The lambda that tests if a `Loc` is in an opt-out region given one opt-out
1641 // region map:
1642 auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map,
1643 const SourceLocation &Loc) -> bool {
1644 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1645 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1646 Range: Map, P: [&SourceMgr,
1647 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1648 return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc);
1649 });
1650
1651 if (FirstRegionEndingAfterLoc != Map.end()) {
1652 // To test if the start location of the found region precedes `Loc`:
1653 return SourceMgr.isBeforeInTranslationUnit(
1654 LHS: FirstRegionEndingAfterLoc->first, RHS: Loc);
1655 }
1656 // If we do not find a region whose end location passes `Loc`, we want to
1657 // check if the current region is still open:
1658 if (!Map.empty() && Map.back().first == Map.back().second)
1659 return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc);
1660 return false;
1661 };
1662
1663 // What the following does:
1664 //
1665 // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`.
1666 // Otherwise, `Loc` is from a loaded AST. We look up the
1667 // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the
1668 // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out
1669 // region w.r.t. the region map. If the region map is absent, it means there
1670 // is no opt-out pragma in that loaded AST.
1671 //
1672 // Opt-out pragmas in the local TU or a loaded AST is not visible to another
1673 // one of them. That means if you put the pragmas around a `#include
1674 // "module.h"`, where module.h is a module, it is not actually suppressing
1675 // warnings in module.h. This is fine because warnings in module.h will be
1676 // reported when module.h is compiled in isolation and nothing in module.h
1677 // will be analyzed ever again. So you will not see warnings from the file
1678 // that imports module.h anyway. And you can't even do the same thing for PCHs
1679 // because they can only be included from the command line.
1680
1681 if (SourceMgr.isLocalSourceLocation(Loc))
1682 return TestInMap(SafeBufferOptOutMap, Loc);
1683
1684 const SafeBufferOptOutRegionsTy *LoadedRegions =
1685 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr);
1686
1687 if (LoadedRegions)
1688 return TestInMap(*LoadedRegions, Loc);
1689 return false;
1690}
1691
1692bool Preprocessor::enterOrExitSafeBufferOptOutRegion(
1693 bool isEnter, const SourceLocation &Loc) {
1694 if (isEnter) {
1695 if (isPPInSafeBufferOptOutRegion())
1696 return true; // invalid enter action
1697 InSafeBufferOptOutRegion = true;
1698 CurrentSafeBufferOptOutStart = Loc;
1699
1700 // To set the start location of a new region:
1701
1702 if (!SafeBufferOptOutMap.empty()) {
1703 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1704 assert(PrevRegion->first != PrevRegion->second &&
1705 "Shall not begin a safe buffer opt-out region before closing the "
1706 "previous one.");
1707 }
1708 // If the start location equals to the end location, we call the region a
1709 // open region or a unclosed region (i.e., end location has not been set
1710 // yet).
1711 SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc);
1712 } else {
1713 if (!isPPInSafeBufferOptOutRegion())
1714 return true; // invalid enter action
1715 InSafeBufferOptOutRegion = false;
1716
1717 // To set the end location of the current open region:
1718
1719 assert(!SafeBufferOptOutMap.empty() &&
1720 "Misordered safe buffer opt-out regions");
1721 auto *CurrRegion = &SafeBufferOptOutMap.back();
1722 assert(CurrRegion->first == CurrRegion->second &&
1723 "Set end location to a closed safe buffer opt-out region");
1724 CurrRegion->second = Loc;
1725 }
1726 return false;
1727}
1728
1729bool Preprocessor::isPPInSafeBufferOptOutRegion() {
1730 return InSafeBufferOptOutRegion;
1731}
1732bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) {
1733 StartLoc = CurrentSafeBufferOptOutStart;
1734 return InSafeBufferOptOutRegion;
1735}
1736
1737SmallVector<SourceLocation, 64>
1738Preprocessor::serializeSafeBufferOptOutMap() const {
1739 assert(!InSafeBufferOptOutRegion &&
1740 "Attempt to serialize safe buffer opt-out regions before file being "
1741 "completely preprocessed");
1742
1743 SmallVector<SourceLocation, 64> SrcSeq;
1744
1745 for (const auto &[begin, end] : SafeBufferOptOutMap) {
1746 SrcSeq.push_back(Elt: begin);
1747 SrcSeq.push_back(Elt: end);
1748 }
1749 // Only `SafeBufferOptOutMap` gets serialized. No need to serialize
1750 // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every
1751 // pch/module in the pch-chain/module-DAG will be loaded one by one in order.
1752 // It means that for each loading pch/module m, it just needs to load m's own
1753 // `SafeBufferOptOutMap`.
1754 return SrcSeq;
1755}
1756
1757bool Preprocessor::setDeserializedSafeBufferOptOutMap(
1758 const SmallVectorImpl<SourceLocation> &SourceLocations) {
1759 if (SourceLocations.size() == 0)
1760 return false;
1761
1762 assert(SourceLocations.size() % 2 == 0 &&
1763 "ill-formed SourceLocation sequence");
1764
1765 auto It = SourceLocations.begin();
1766 SafeBufferOptOutRegionsTy &Regions =
1767 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr);
1768
1769 do {
1770 SourceLocation Begin = *It++;
1771 SourceLocation End = *It++;
1772
1773 Regions.emplace_back(Args&: Begin, Args&: End);
1774 } while (It != SourceLocations.end());
1775 return true;
1776}
1777
1778ModuleLoader::~ModuleLoader() = default;
1779
1780CommentHandler::~CommentHandler() = default;
1781
1782EmptylineHandler::~EmptylineHandler() = default;
1783
1784CodeCompletionHandler::~CodeCompletionHandler() = default;
1785
1786void Preprocessor::createPreprocessingRecord() {
1787 if (Record)
1788 return;
1789
1790 Record = new PreprocessingRecord(getSourceManager());
1791 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record));
1792}
1793
1794void Preprocessor::removePPCallbacks() {
1795 auto IsPreserved = [&](PPCallbacks *C) {
1796 return C == Record || C == DirTracer;
1797 };
1798 SmallVector<PPCallbacks *, 2> Released;
1799 PPCallbacks::releaseIfPreserved(CB&: Callbacks, Pred: IsPreserved, Released);
1800 Callbacks.reset();
1801 for (auto *P : Released)
1802 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(P));
1803}
1804
1805const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
1806 if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) {
1807 const SmallVector<const char *> &FileCheckPoints = It->second;
1808 auto P = llvm::upper_bound(Range: FileCheckPoints, Value&: Start);
1809 if (P == FileCheckPoints.begin())
1810 return nullptr;
1811 return *std::prev(x: P);
1812 }
1813 return nullptr;
1814}
1815
1816bool Preprocessor::hasSeenNoTrivialPPDirective() const {
1817 return DirTracer && DirTracer->hasSeenNoTrivialPPDirective();
1818}
1819
1820bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const {
1821 return SeenNoTrivialPPDirective;
1822}
1823
1824void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
1825 if (InMainFile && !SeenNoTrivialPPDirective)
1826 SeenNoTrivialPPDirective = true;
1827}
1828
1829void NoTrivialPPDirectiveTracer::LexedFileChanged(
1830 FileID FID, LexedFileChangeReason Reason,
1831 SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) {
1832 InMainFile = (FID == PP.getSourceManager().getMainFileID());
1833}
1834
1835void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok,
1836 const MacroDefinition &MD,
1837 SourceRange Range,
1838 const MacroArgs *Args) {
1839 // FIXME: Does only enable builtin macro expansion make sense?
1840 if (!MD.getMacroInfo()->isBuiltinMacro())
1841 setSeenNoTrivialPPDirective();
1842}
1843