1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Preprocessor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/Basic/FileManager.h"
30#include "clang/Basic/IdentifierTable.h"
31#include "clang/Basic/LLVM.h"
32#include "clang/Basic/LangOptions.h"
33#include "clang/Basic/Module.h"
34#include "clang/Basic/SourceLocation.h"
35#include "clang/Basic/SourceManager.h"
36#include "clang/Basic/TargetInfo.h"
37#include "clang/Lex/CodeCompletionHandler.h"
38#include "clang/Lex/DependencyDirectivesScanner.h"
39#include "clang/Lex/ExternalPreprocessorSource.h"
40#include "clang/Lex/HeaderSearch.h"
41#include "clang/Lex/LexDiagnostic.h"
42#include "clang/Lex/Lexer.h"
43#include "clang/Lex/LiteralSupport.h"
44#include "clang/Lex/MacroArgs.h"
45#include "clang/Lex/MacroInfo.h"
46#include "clang/Lex/ModuleLoader.h"
47#include "clang/Lex/NoTrivialPPDirectiveTracer.h"
48#include "clang/Lex/Pragma.h"
49#include "clang/Lex/PreprocessingRecord.h"
50#include "clang/Lex/PreprocessorLexer.h"
51#include "clang/Lex/PreprocessorOptions.h"
52#include "clang/Lex/ScratchBuffer.h"
53#include "clang/Lex/Token.h"
54#include "clang/Lex/TokenLexer.h"
55#include "clang/Support/Compiler.h"
56#include "llvm/ADT/APInt.h"
57#include "llvm/ADT/ArrayRef.h"
58#include "llvm/ADT/DenseMap.h"
59#include "llvm/ADT/STLExtras.h"
60#include "llvm/ADT/ScopeExit.h"
61#include "llvm/ADT/SmallVector.h"
62#include "llvm/ADT/StringRef.h"
63#include "llvm/Support/Capacity.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/FormatVariadic.h"
66#include "llvm/Support/MemoryBuffer.h"
67#include "llvm/Support/MemoryBufferRef.h"
68#include "llvm/Support/SaveAndRestore.h"
69#include "llvm/Support/raw_ostream.h"
70#include <algorithm>
71#include <cassert>
72#include <memory>
73#include <optional>
74#include <string>
75#include <utility>
76#include <vector>
77
78using namespace clang;
79
80/// Minimum distance between two check points, in tokens.
81static constexpr unsigned CheckPointStepSize = 1024;
82
83LLVM_INSTANTIATE_REGISTRY_EX(CLANG_ABI_EXPORT, PragmaHandlerRegistry)
84
85ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
86
87Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
88 DiagnosticsEngine &diags, const LangOptions &opts,
89 SourceManager &SM, HeaderSearch &Headers,
90 ModuleLoader &TheModuleLoader,
91 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
92 TranslationUnitKind TUKind)
93 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
94 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
95 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
96 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
97 // As the language options may have not been loaded yet (when
98 // deserializing an ASTUnit), adding keywords to the identifier table is
99 // deferred to Preprocessor::Initialize().
100 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
101 TUKind(TUKind), SkipMainFilePreamble(0, true),
102 CurSubmoduleState(&NullSubmoduleState) {
103 OwnsHeaderSearch = OwnsHeaders;
104
105 // Only record check points if we might highlight diagnostic snippets.
106 RecordCheckPoints = getDiagnostics().getShowColors();
107
108 // Default to discarding comments.
109 KeepComments = false;
110 KeepMacroComments = false;
111 SuppressIncludeNotFoundError = false;
112
113 // Macro expansion is enabled.
114 DisableMacroExpansion = false;
115 MacroExpansionInDirectivesOverride = false;
116 InMacroArgs = false;
117 ArgMacro = nullptr;
118 InMacroArgPreExpansion = false;
119 NumCachedTokenLexers = 0;
120 PragmasEnabled = true;
121 ParsingIfOrElifDirective = false;
122 PreprocessedOutput = false;
123
124 // We haven't read anything from the external source.
125 ReadMacrosFromExternalSource = false;
126
127 LastExportKeyword.startToken();
128
129 BuiltinInfo = std::make_unique<Builtin::Context>();
130
131 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
132 // a macro. They get unpoisoned where it is allowed.
133 (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned();
134 SetPoisonReason(II: Ident__VA_ARGS__,DiagID: diag::ext_pp_bad_vaargs_use);
135 (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned();
136 SetPoisonReason(II: Ident__VA_OPT__,DiagID: diag::ext_pp_bad_vaopt_use);
137
138 // Initialize the pragma handlers.
139 RegisterBuiltinPragmas();
140
141 // Initialize builtin macros like __LINE__ and friends.
142 RegisterBuiltinMacros();
143
144 if(LangOpts.Borland) {
145 Ident__exception_info = getIdentifierInfo(Name: "_exception_info");
146 Ident___exception_info = getIdentifierInfo(Name: "__exception_info");
147 Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation");
148 Ident__exception_code = getIdentifierInfo(Name: "_exception_code");
149 Ident___exception_code = getIdentifierInfo(Name: "__exception_code");
150 Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode");
151 Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination");
152 Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination");
153 Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination");
154 } else {
155 Ident__exception_info = Ident__exception_code = nullptr;
156 Ident__abnormal_termination = Ident___exception_info = nullptr;
157 Ident___exception_code = Ident___abnormal_termination = nullptr;
158 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
159 Ident_AbnormalTermination = nullptr;
160 }
161
162 // Default incremental processing to -fincremental-extensions, clients can
163 // override with `enableIncrementalProcessing` if desired.
164 IncrementalProcessing = LangOpts.IncrementalExtensions;
165
166 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
167 if (usingPCHWithPragmaHdrStop())
168 SkippingUntilPragmaHdrStop = true;
169
170 // If using a PCH with a through header, start skipping tokens.
171 if (!this->PPOpts.PCHThroughHeader.empty() &&
172 !this->PPOpts.ImplicitPCHInclude.empty())
173 SkippingUntilPCHThroughHeader = true;
174
175 if (this->PPOpts.GeneratePreamble)
176 PreambleConditionalStack.startRecording();
177
178 MaxTokens = LangOpts.MaxTokens;
179}
180
181Preprocessor::~Preprocessor() {
182 assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!");
183
184 IncludeMacroStack.clear();
185
186 // Free any cached macro expanders.
187 // This populates MacroArgCache, so all TokenLexers need to be destroyed
188 // before the code below that frees up the MacroArgCache list.
189 std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr);
190 CurTokenLexer.reset();
191
192 // Free any cached MacroArgs.
193 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
194 ArgList = ArgList->deallocate();
195
196 // Delete the header search info, if we own it.
197 if (OwnsHeaderSearch)
198 delete &HeaderInfo;
199}
200
201void Preprocessor::Initialize(const TargetInfo &Target,
202 const TargetInfo *AuxTarget) {
203 assert((!this->Target || this->Target == &Target) &&
204 "Invalid override of target information");
205 this->Target = &Target;
206
207 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
208 "Invalid override of aux target information.");
209 this->AuxTarget = AuxTarget;
210
211 // Initialize information about built-ins.
212 BuiltinInfo->InitializeTarget(Target, AuxTarget);
213 HeaderInfo.setTarget(Target);
214
215 // Populate the identifier table with info about keywords for the current language.
216 Identifiers.AddKeywords(LangOpts);
217
218 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
219 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
220
221 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
222 // Use setting from TargetInfo.
223 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod());
224 else
225 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
226 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod());
227}
228
229void Preprocessor::InitializeForModelFile() {
230 NumEnteredSourceFiles = 0;
231
232 // Reset pragmas
233 PragmaHandlersBackup = std::move(PragmaHandlers);
234 PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef());
235 RegisterBuiltinPragmas();
236
237 // Reset PredefinesFileID
238 PredefinesFileID = FileID();
239}
240
241void Preprocessor::FinalizeForModelFile() {
242 NumEnteredSourceFiles = 1;
243
244 PragmaHandlers = std::move(PragmaHandlersBackup);
245}
246
247void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
248 std::string TokenStr;
249 llvm::raw_string_ostream OS(TokenStr);
250
251 // The alignment of 16 is chosen to comfortably fit most identifiers.
252 OS << llvm::formatv(Fmt: "{0,-16} ", Vals: tok::getTokenName(Kind: Tok.getKind()));
253
254 // Annotation tokens are just markers that don't have a spelling -- they
255 // indicate where something expanded.
256 if (!Tok.isAnnotation()) {
257 OS << "'";
258 // Escape string to prevent token spelling from spanning multiple lines.
259 OS.write_escaped(Str: getSpelling(Tok));
260 OS << "'";
261 }
262
263 // The alignment of 48 (32 characters for the spelling + the 16 for
264 // the identifier name) fits most variable names, keywords and annotations.
265 llvm::errs() << llvm::formatv(Fmt: "{0,-48} ", Vals&: OS.str());
266
267 if (!DumpFlags) return;
268
269 auto Loc = Tok.getLocation();
270 llvm::errs() << "Loc=<";
271 DumpLocation(Loc);
272 llvm::errs() << ">";
273
274 // If the token points directly to a file location (i.e. not a macro
275 // expansion), then add additional padding so that trailing markers
276 // align, provided the line/column numbers are reasonably sized.
277 //
278 // Otherwise, if it's a macro expansion, don't bother with alignment,
279 // as the line will include multiple locations and be very long.
280 //
281 // NOTE: To keep this stateless, it doesn't account for filename
282 // length, so when a header starts markers will be temporarily misaligned.
283 if (Loc.isFileID()) {
284 PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
285
286 if (!PLoc.isInvalid()) {
287 int LineWidth = llvm::utostr(X: PLoc.getLine()).size();
288 int ColumnWidth = llvm::utostr(X: PLoc.getColumn()).size();
289
290 // Reserve space for lines up to 9999 and columns up to 99,
291 // which is 4 + 2 = 6 characters in total.
292 const int ReservedSpace = 6;
293
294 int LeftSpace = ReservedSpace - LineWidth - ColumnWidth;
295 int Padding = std::max<int>(a: 0, b: LeftSpace);
296
297 llvm::errs().indent(NumSpaces: Padding);
298 }
299 }
300
301 if (Tok.isAtStartOfLine())
302 llvm::errs() << " [StartOfLine]";
303 if (Tok.hasLeadingSpace())
304 llvm::errs() << " [LeadingSpace]";
305 if (Tok.isExpandDisabled())
306 llvm::errs() << " [ExpandDisabled]";
307 if (Tok.needsCleaning()) {
308 const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation());
309 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) << "']";
310 }
311}
312
313void Preprocessor::DumpLocation(SourceLocation Loc) const {
314 Loc.print(OS&: llvm::errs(), SM: SourceMgr);
315}
316
317void Preprocessor::DumpMacro(const MacroInfo &MI) const {
318 llvm::errs() << "MACRO: ";
319 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
320 DumpToken(Tok: MI.getReplacementToken(Tok: i));
321 llvm::errs() << " ";
322 }
323 llvm::errs() << "\n";
324}
325
326void Preprocessor::PrintStats() {
327 llvm::errs() << "\n*** Preprocessor Stats:\n";
328 llvm::errs() << NumDirectives << " directives found:\n";
329 llvm::errs() << " " << NumDefined << " #define.\n";
330 llvm::errs() << " " << NumUndefined << " #undef.\n";
331 llvm::errs() << " #include/#include_next/#import:\n";
332 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
333 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
334 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
335 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
336 llvm::errs() << " " << NumEndif << " #endif.\n";
337 llvm::errs() << " " << NumPragma << " #pragma.\n";
338 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
339
340 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
341 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
342 << NumFastMacroExpanded << " on the fast path.\n";
343 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
344 << " token paste (##) operations performed, "
345 << NumFastTokenPaste << " on the fast path.\n";
346
347 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
348
349 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
350 llvm::errs() << "\n Macro Expanded Tokens: "
351 << llvm::capacity_in_bytes(X: MacroExpandedTokens);
352 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
353 // FIXME: List information for all submodules.
354 llvm::errs() << "\n Macros: "
355 << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros);
356 llvm::errs() << "\n #pragma push_macro Info: "
357 << llvm::capacity_in_bytes(X: PragmaPushMacroInfo);
358 llvm::errs() << "\n Poison Reasons: "
359 << llvm::capacity_in_bytes(X: PoisonReasons);
360 llvm::errs() << "\n Comment Handlers: "
361 << llvm::capacity_in_bytes(x: CommentHandlers) << "\n";
362}
363
364llvm::iterator_range<Preprocessor::macro_iterator>
365Preprocessor::macros(bool IncludeExternalMacros) const {
366 if (IncludeExternalMacros && ExternalSource &&
367 !ReadMacrosFromExternalSource) {
368 ReadMacrosFromExternalSource = true;
369 ExternalSource->ReadDefinedMacros();
370 }
371 // Make sure we cover all macros in visible modules.
372 for (const ModuleMacro &Macro : ModuleMacros)
373 CurSubmoduleState->Macros.try_emplace(Key: Macro.II);
374
375 return CurSubmoduleState->Macros;
376}
377
378size_t Preprocessor::getTotalMemory() const {
379 return BP.getTotalMemory()
380 + llvm::capacity_in_bytes(X: MacroExpandedTokens)
381 + Predefines.capacity() /* Predefines buffer. */
382 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
383 // and ModuleMacros.
384 + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros)
385 + llvm::capacity_in_bytes(X: PragmaPushMacroInfo)
386 + llvm::capacity_in_bytes(X: PoisonReasons)
387 + llvm::capacity_in_bytes(x: CommentHandlers);
388}
389
390/// Compares macro tokens with a specified token value sequence.
391static bool MacroDefinitionEquals(const MacroInfo *MI,
392 ArrayRef<TokenValue> Tokens) {
393 return Tokens.size() == MI->getNumTokens() &&
394 std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin());
395}
396
397StringRef Preprocessor::getLastMacroWithSpelling(
398 SourceLocation Loc,
399 ArrayRef<TokenValue> Tokens) const {
400 SourceLocation BestLocation;
401 StringRef BestSpelling;
402 for (const auto &M : macros()) {
403 const MacroDirective::DefInfo Def =
404 M.second.findDirectiveAtLoc(Loc, SourceMgr);
405 if (!Def || !Def.getMacroInfo())
406 continue;
407 if (!Def.getMacroInfo()->isObjectLike())
408 continue;
409 if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens))
410 continue;
411 SourceLocation Location = Def.getLocation();
412 // Choose the macro defined latest.
413 if (BestLocation.isInvalid() ||
414 (Location.isValid() &&
415 SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) {
416 BestLocation = Location;
417 BestSpelling = M.first->getName();
418 }
419 }
420 return BestSpelling;
421}
422
423void Preprocessor::recomputeCurLexerKind() {
424 if (InCachingLexMode())
425 CurLexerCallback = CLK_CachingLexer;
426 else if (CurLexer)
427 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
428 ? CLK_DependencyDirectivesLexer
429 : CLK_Lexer;
430 else if (CurTokenLexer)
431 CurLexerCallback = CLK_TokenLexer;
432 else
433 CurLexerCallback = CLK_Lexer;
434}
435
436bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
437 unsigned CompleteLine,
438 unsigned CompleteColumn) {
439 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
440 assert(!CodeCompletionFile && "Already set");
441
442 // Load the actual file's contents.
443 std::optional<llvm::MemoryBufferRef> Buffer =
444 SourceMgr.getMemoryBufferForFileOrNone(File);
445 if (!Buffer)
446 return true;
447
448 // Find the byte position of the truncation point.
449 const char *Position = Buffer->getBufferStart();
450 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
451 for (; *Position; ++Position) {
452 if (*Position != '\r' && *Position != '\n')
453 continue;
454
455 // Eat \r\n or \n\r as a single line.
456 if ((Position[1] == '\r' || Position[1] == '\n') &&
457 Position[0] != Position[1])
458 ++Position;
459 ++Position;
460 break;
461 }
462 }
463
464 Position += CompleteColumn - 1;
465
466 // If pointing inside the preamble, adjust the position at the beginning of
467 // the file after the preamble.
468 if (SkipMainFilePreamble.first &&
469 SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) {
470 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
471 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
472 }
473
474 if (Position > Buffer->getBufferEnd())
475 Position = Buffer->getBufferEnd();
476
477 CodeCompletionFile = File;
478 CodeCompletionOffset = Position - Buffer->getBufferStart();
479
480 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
481 Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier());
482 char *NewBuf = NewBuffer->getBufferStart();
483 char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf);
484 *NewPos = '\0';
485 std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1);
486 SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer));
487
488 return false;
489}
490
491void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
492 bool IsAngled) {
493 setCodeCompletionReached();
494 if (CodeComplete)
495 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
496}
497
498void Preprocessor::CodeCompleteNaturalLanguage() {
499 setCodeCompletionReached();
500 if (CodeComplete)
501 CodeComplete->CodeCompleteNaturalLanguage();
502}
503
504/// getSpelling - This method is used to get the spelling of a token into a
505/// SmallVector. Note that the returned StringRef may not point to the
506/// supplied buffer if a copy can be avoided.
507StringRef Preprocessor::getSpelling(const Token &Tok,
508 SmallVectorImpl<char> &Buffer,
509 bool *Invalid) const {
510 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
511 if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) {
512 // Try the fast path.
513 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
514 return II->getName();
515 }
516
517 // Resize the buffer if we need to copy into it.
518 if (Tok.needsCleaning())
519 Buffer.resize(N: Tok.getLength());
520
521 const char *Ptr = Buffer.data();
522 unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid);
523 return StringRef(Ptr, Len);
524}
525
526/// CreateString - Plop the specified string into a scratch buffer and return a
527/// location for it. If specified, the source location provides a source
528/// location for the token.
529void Preprocessor::CreateString(StringRef Str, Token &Tok,
530 SourceLocation ExpansionLocStart,
531 SourceLocation ExpansionLocEnd) {
532 Tok.setLength(Str.size());
533
534 const char *DestPtr;
535 SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr);
536
537 if (ExpansionLocStart.isValid())
538 Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart,
539 ExpansionLocEnd, Length: Str.size());
540 Tok.setLocation(Loc);
541
542 // If this is a raw identifier or a literal token, set the pointer data.
543 if (Tok.is(K: tok::raw_identifier))
544 Tok.setRawIdentifierData(DestPtr);
545 else if (Tok.isLiteral())
546 Tok.setLiteralData(DestPtr);
547}
548
549SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
550 auto &SM = getSourceManager();
551 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
552 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc);
553 bool Invalid = false;
554 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
555 if (Invalid)
556 return SourceLocation();
557
558 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
559 const char *DestPtr;
560 SourceLocation Spelling =
561 ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr);
562 return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length));
563}
564
565Module *Preprocessor::getCurrentModule() {
566 if (!getLangOpts().isCompilingModule())
567 return nullptr;
568
569 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule);
570}
571
572Module *Preprocessor::getCurrentModuleImplementation() {
573 if (!getLangOpts().isCompilingModuleImplementation())
574 return nullptr;
575
576 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName);
577}
578
579//===----------------------------------------------------------------------===//
580// Preprocessor Initialization Methods
581//===----------------------------------------------------------------------===//
582
583/// EnterMainSourceFile - Enter the specified FileID as the main source file,
584/// which implicitly adds the builtin defines etc.
585void Preprocessor::EnterMainSourceFile() {
586 // We do not allow the preprocessor to reenter the main file. Doing so will
587 // cause FileID's to accumulate information from both runs (e.g. #line
588 // information) and predefined macros aren't guaranteed to be set properly.
589 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
590 FileID MainFileID = SourceMgr.getMainFileID();
591
592 // If MainFileID is loaded it means we loaded an AST file, no need to enter
593 // a main file.
594 if (!SourceMgr.isLoadedFileID(FID: MainFileID)) {
595 // Enter the main file source buffer.
596 EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
597
598 // If we've been asked to skip bytes in the main file (e.g., as part of a
599 // precompiled preamble), do so now.
600 if (SkipMainFilePreamble.first > 0)
601 CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first,
602 StartOfLine: SkipMainFilePreamble.second);
603
604 // Tell the header info that the main file was entered. If the file is later
605 // #imported, it won't be re-entered.
606 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID))
607 markIncluded(File: *FE);
608
609 // Record the first PP token in the main file. This is used to generate
610 // better diagnostics for C++ modules.
611 //
612 // // This is a comment.
613 // #define FOO int // note: add 'module;' to the start of the file
614 // ^ FirstPPToken // to introduce a global module fragment.
615 //
616 // export module M; // error: module declaration must occur
617 // // at the start of the translation unit.
618 if (getLangOpts().CPlusPlusModules) {
619 std::optional<StringRef> Input =
620 getSourceManager().getBufferDataOrNone(FID: MainFileID);
621 if (!isPreprocessedModuleFile() && Input)
622 MainFileIsPreprocessedModuleFile =
623 clang::isPreprocessedModuleFile(Source: *Input);
624 auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(args&: *this);
625 DirTracer = Tracer.get();
626 addPPCallbacks(C: std::move(Tracer));
627 std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
628 if (FirstPPTok)
629 FirstPPTokenLoc = FirstPPTok->getLocation();
630 }
631 }
632
633 // Preprocess Predefines to populate the initial preprocessor state.
634 std::unique_ptr<llvm::MemoryBuffer> SB =
635 llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>");
636 assert(SB && "Cannot create predefined source buffer");
637 FileID FID = SourceMgr.createFileID(Buffer: std::move(SB));
638 assert(FID.isValid() && "Could not create FileID for predefines?");
639 setPredefinesFileID(FID);
640
641 // Start parsing the predefines.
642 EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation());
643
644 if (!PPOpts.PCHThroughHeader.empty()) {
645 // Lookup and save the FileID for the through header. If it isn't found
646 // in the search path, it's a fatal error.
647 OptionalFileEntryRef File = LookupFile(
648 FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader,
649 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
650 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
651 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
652 /*IsFrameworkFound=*/nullptr);
653 if (!File) {
654 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_found)
655 << PPOpts.PCHThroughHeader;
656 return;
657 }
658 setPCHThroughHeaderFileID(
659 SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User));
660 }
661
662 // Skip tokens from the Predefines and if needed the main file.
663 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
664 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
665 SkipTokensWhileUsingPCH();
666}
667
668void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
669 assert(PCHThroughHeaderFileID.isInvalid() &&
670 "PCHThroughHeaderFileID already set!");
671 PCHThroughHeaderFileID = FID;
672}
673
674bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
675 assert(PCHThroughHeaderFileID.isValid() &&
676 "Invalid PCH through header FileID");
677 return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID);
678}
679
680bool Preprocessor::creatingPCHWithThroughHeader() {
681 return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
682 PCHThroughHeaderFileID.isValid();
683}
684
685bool Preprocessor::usingPCHWithThroughHeader() {
686 return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
687 PCHThroughHeaderFileID.isValid();
688}
689
690bool Preprocessor::creatingPCHWithPragmaHdrStop() {
691 return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop;
692}
693
694bool Preprocessor::usingPCHWithPragmaHdrStop() {
695 return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop;
696}
697
698/// Skip tokens until after the #include of the through header or
699/// until after a #pragma hdrstop is seen. Tokens in the predefines file
700/// and the main file may be skipped. If the end of the predefines file
701/// is reached, skipping continues into the main file. If the end of the
702/// main file is reached, it's a fatal error.
703void Preprocessor::SkipTokensWhileUsingPCH() {
704 bool ReachedMainFileEOF = false;
705 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
706 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
707 Token Tok;
708 while (true) {
709 bool InPredefines =
710 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
711 CurLexerCallback(*this, Tok);
712 if (Tok.is(K: tok::eof) && !InPredefines) {
713 ReachedMainFileEOF = true;
714 break;
715 }
716 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
717 break;
718 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
719 break;
720 }
721 if (ReachedMainFileEOF) {
722 if (UsingPCHThroughHeader)
723 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_seen)
724 << PPOpts.PCHThroughHeader << 1;
725 else if (!PPOpts.PCHWithHdrStopCreate)
726 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_pragma_hdrstop_not_seen);
727 }
728}
729
730void Preprocessor::replayPreambleConditionalStack() {
731 // Restore the conditional stack from the preamble, if there is one.
732 if (PreambleConditionalStack.isReplaying()) {
733 assert(CurPPLexer &&
734 "CurPPLexer is null when calling replayPreambleConditionalStack.");
735 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
736 PreambleConditionalStack.doneReplaying();
737 if (PreambleConditionalStack.reachedEOFWhileSkipping())
738 SkipExcludedConditionalBlock(
739 HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc,
740 IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc,
741 FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
742 FoundElse: PreambleConditionalStack.SkipInfo->FoundElse,
743 ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc);
744 }
745}
746
747void Preprocessor::EndSourceFile() {
748 // Notify the client that we reached the end of the source file.
749 if (Callbacks)
750 Callbacks->EndOfMainFile();
751}
752
753//===----------------------------------------------------------------------===//
754// Lexer Event Handling.
755//===----------------------------------------------------------------------===//
756
757/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
758/// identifier information for the token and install it into the token,
759/// updating the token kind accordingly.
760IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
761 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
762
763 // Look up this token, see if it is a macro, or if it is a language keyword.
764 IdentifierInfo *II;
765 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
766 // No cleaning needed, just use the characters from the lexed buffer.
767 II = getIdentifierInfo(Name: Identifier.getRawIdentifier());
768 } else {
769 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
770 SmallString<64> IdentifierBuffer;
771 StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer);
772
773 if (Identifier.hasUCN()) {
774 SmallString<64> UCNIdentifierBuffer;
775 expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr);
776 II = getIdentifierInfo(Name: UCNIdentifierBuffer);
777 } else {
778 II = getIdentifierInfo(Name: CleanedStr);
779 }
780 }
781
782 // Update the token info (identifier info and appropriate token kind).
783 // FIXME: the raw_identifier may contain leading whitespace which is removed
784 // from the cleaned identifier token. The SourceLocation should be updated to
785 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
786 // line continuation before 'B') is parsed as a single tok::raw_identifier and
787 // is cleaned to tok::identifier "B". After cleaning the token's length is
788 // still 3 and the SourceLocation refers to the location of the backslash.
789 Identifier.setIdentifierInfo(II);
790 Identifier.setKind(II->getTokenID());
791
792 return II;
793}
794
795void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
796 PoisonReasons[II] = DiagID;
797}
798
799void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
800 assert(Ident__exception_code && Ident__exception_info);
801 assert(Ident___exception_code && Ident___exception_info);
802 Ident__exception_code->setIsPoisoned(Poison);
803 Ident___exception_code->setIsPoisoned(Poison);
804 Ident_GetExceptionCode->setIsPoisoned(Poison);
805 Ident__exception_info->setIsPoisoned(Poison);
806 Ident___exception_info->setIsPoisoned(Poison);
807 Ident_GetExceptionInfo->setIsPoisoned(Poison);
808 Ident__abnormal_termination->setIsPoisoned(Poison);
809 Ident___abnormal_termination->setIsPoisoned(Poison);
810 Ident_AbnormalTermination->setIsPoisoned(Poison);
811}
812
813void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
814 assert(Identifier.getIdentifierInfo() &&
815 "Can't handle identifiers without identifier info!");
816 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
817 PoisonReasons.find(Val: Identifier.getIdentifierInfo());
818 if(it == PoisonReasons.end())
819 Diag(Tok: Identifier, DiagID: diag::err_pp_used_poisoned_id);
820 else
821 Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo();
822}
823
824void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const {
825 assert(II.isOutOfDate() && "not out of date");
826 assert(getExternalSource() &&
827 "getExternalSource() should not return nullptr");
828 getExternalSource()->updateOutOfDateIdentifier(II);
829}
830
831/// HandleIdentifier - This callback is invoked when the lexer reads an
832/// identifier. This callback looks up the identifier in the map and/or
833/// potentially macro expands it or turns it into a named token (like 'for').
834///
835/// Note that callers of this method are guarded by checking the
836/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
837/// IdentifierInfo methods that compute these properties will need to change to
838/// match.
839bool Preprocessor::HandleIdentifier(Token &Identifier) {
840 assert(Identifier.getIdentifierInfo() &&
841 "Can't handle identifiers without identifier info!");
842
843 IdentifierInfo &II = *Identifier.getIdentifierInfo();
844
845 // If the information about this identifier is out of date, update it from
846 // the external source.
847 // We have to treat __VA_ARGS__ in a special way, since it gets
848 // serialized with isPoisoned = true, but our preprocessor may have
849 // unpoisoned it if we're defining a C99 macro.
850 if (II.isOutOfDate()) {
851 bool CurrentIsPoisoned = false;
852 const bool IsSpecialVariadicMacro =
853 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
854 if (IsSpecialVariadicMacro)
855 CurrentIsPoisoned = II.isPoisoned();
856
857 updateOutOfDateIdentifier(II);
858 Identifier.setKind(II.getTokenID());
859
860 if (IsSpecialVariadicMacro)
861 II.setIsPoisoned(CurrentIsPoisoned);
862 }
863
864 // If this identifier was poisoned, and if it was not produced from a macro
865 // expansion, emit an error.
866 if (II.isPoisoned() && CurPPLexer) {
867 HandlePoisonedIdentifier(Identifier);
868 }
869
870 // If this is a macro to be expanded, do it.
871 if (const MacroDefinition MD = getMacroDefinition(II: &II)) {
872 const auto *MI = MD.getMacroInfo();
873 assert(MI && "macro definition with no macro info?");
874 if (!DisableMacroExpansion) {
875 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
876 // C99 6.10.3p10: If the preprocessing token immediately after the
877 // macro name isn't a '(', this macro should not be expanded.
878 if (!MI->isFunctionLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
879 return HandleMacroExpandedIdentifier(Identifier, MD);
880 } else {
881 // C99 6.10.3.4p2 says that a disabled macro may never again be
882 // expanded, even if it's in a context where it could be expanded in the
883 // future.
884 Identifier.setFlag(Token::DisableExpand);
885 if (MI->isObjectLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
886 Diag(Tok: Identifier, DiagID: diag::pp_disabled_macro_expansion);
887 }
888 }
889 }
890
891 // If this identifier is a keyword in a newer Standard or proposed Standard,
892 // produce a warning. Don't warn if we're not considering macro expansion,
893 // since this identifier might be the name of a macro.
894 // FIXME: This warning is disabled in cases where it shouldn't be, like
895 // "#define constexpr constexpr", "int constexpr;"
896 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
897 Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts()))
898 << II.getName();
899 // Don't diagnose this keyword again in this translation unit.
900 II.setIsFutureCompatKeyword(false);
901 }
902
903 // If this identifier would be a keyword in C++, diagnose as a compatibility
904 // issue.
905 if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion)
906 Diag(Tok: Identifier, DiagID: diag::warn_pp_identifier_is_cpp_keyword) << &II;
907
908 // If this is an extension token, diagnose its use.
909 // We avoid diagnosing tokens that originate from macro definitions.
910 // FIXME: This warning is disabled in cases where it shouldn't be,
911 // like "#define TY typeof", "TY(1) x".
912 if (II.isExtensionToken() && !DisableMacroExpansion)
913 Diag(Tok: Identifier, DiagID: diag::ext_token_used);
914
915 // Handle module contextual keywords.
916 if (getLangOpts().CPlusPlusModules && CurLexer &&
917 !CurLexer->isLexingRawMode() && !CurLexer->isPragmaLexer() &&
918 !CurLexer->ParsingPreprocessorDirective &&
919 Identifier.isModuleContextualKeyword() &&
920 HandleModuleContextualKeyword(Result&: Identifier)) {
921 HandleDirective(Result&: Identifier);
922 // With a fatal failure in the module loader, we abort parsing.
923 return hadModuleLoaderFatalFailure();
924 }
925
926 return true;
927}
928
929void Preprocessor::Lex(Token &Result) {
930 ++LexLevel;
931
932 // We loop here until a lex function returns a token; this avoids recursion.
933 while (!CurLexerCallback(*this, Result))
934 ;
935
936 if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure)
937 return;
938
939 if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) {
940 // Remember the identifier before code completion token.
941 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
942 setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc());
943 // Set IdenfitierInfo to null to avoid confusing code that handles both
944 // identifiers and completion tokens.
945 Result.setIdentifierInfo(nullptr);
946 }
947
948 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
949 // if this token is being produced as a result of phase 4 of translation.
950 // Update TrackGMFState to decide if we are currently in a Global Module
951 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
952 // depends on the prevailing StdCXXImportSeq state in two cases.
953 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
954 !Result.getFlag(Flag: Token::IsReinjected)) {
955 switch (Result.getKind()) {
956 case tok::l_paren: case tok::l_square: case tok::l_brace:
957 StdCXXImportSeqState.handleOpenBracket();
958 break;
959 case tok::r_paren: case tok::r_square:
960 StdCXXImportSeqState.handleCloseBracket();
961 break;
962 case tok::r_brace:
963 StdCXXImportSeqState.handleCloseBrace();
964 break;
965#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
966// For `#pragma ...` mimic ';'.
967#include "clang/Basic/TokenKinds.def"
968#undef PRAGMA_ANNOTATION
969 // This token is injected to represent the translation of '#include "a.h"'
970 // into "import a.h;". Mimic the notional ';'.
971 case tok::annot_module_include:
972 case tok::annot_repl_input_end:
973 case tok::semi:
974 TrackGMFState.handleSemi();
975 StdCXXImportSeqState.handleSemi();
976 ModuleDeclState.handleSemi();
977 break;
978 case tok::header_name:
979 case tok::annot_header_unit:
980 StdCXXImportSeqState.handleHeaderName();
981 break;
982 case tok::kw_export:
983 if (hasSeenNoTrivialPPDirective())
984 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
985 TrackGMFState.handleExport();
986 StdCXXImportSeqState.handleExport();
987 ModuleDeclState.handleExport();
988 break;
989 case tok::colon:
990 ModuleDeclState.handleColon();
991 break;
992 case tok::kw_import:
993 if (StdCXXImportSeqState.atTopLevel()) {
994 TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
995 StdCXXImportSeqState.handleImport();
996 }
997 break;
998 case tok::kw_module:
999 if (StdCXXImportSeqState.atTopLevel()) {
1000 if (hasSeenNoTrivialPPDirective())
1001 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
1002 TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
1003 ModuleDeclState.handleModule();
1004 }
1005 break;
1006 case tok::annot_module_name:
1007 ModuleDeclState.handleModuleName(
1008 NameLoc: static_cast<ModuleNameLoc *>(Result.getAnnotationValue()));
1009 if (ModuleDeclState.isModuleCandidate())
1010 break;
1011 [[fallthrough]];
1012 default:
1013 TrackGMFState.handleMisc();
1014 StdCXXImportSeqState.handleMisc();
1015 ModuleDeclState.handleMisc();
1016 break;
1017 }
1018 }
1019
1020 if (RecordCheckPoints && CurLexer &&
1021 ++CheckPointCounter == CheckPointStepSize) {
1022 CheckPoints[CurLexer->getFileID()].push_back(Elt: CurLexer->BufferPtr);
1023 CheckPointCounter = 0;
1024 }
1025
1026 if (Result.isNot(K: tok::kw_export))
1027 LastExportKeyword.startToken();
1028
1029 --LexLevel;
1030
1031 // Destroy any lexers that were deferred while we were in nested Lex calls.
1032 // This must happen after decrementing LexLevel but before any other
1033 // processing that might re-enter Lex.
1034 if (LexLevel == 0 && !PendingDestroyLexers.empty())
1035 PendingDestroyLexers.clear();
1036
1037 if ((LexLevel == 0 || PreprocessToken) &&
1038 !Result.getFlag(Flag: Token::IsReinjected)) {
1039 if (LexLevel == 0)
1040 ++TokenCount;
1041 if (OnToken)
1042 OnToken(Result);
1043 }
1044}
1045
1046void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
1047 while (1) {
1048 Token Tok;
1049 Lex(Result&: Tok);
1050 if (Tok.isOneOf(Ks: tok::unknown, Ks: tok::eof, Ks: tok::eod,
1051 Ks: tok::annot_repl_input_end))
1052 break;
1053 if (Tokens != nullptr)
1054 Tokens->push_back(x: Tok);
1055 }
1056}
1057
1058/// Lex a header-name token (including one formed from header-name-tokens if
1059/// \p AllowMacroExpansion is \c true).
1060///
1061/// \param FilenameTok Filled in with the next token. On success, this will
1062/// be either a header_name token. On failure, it will be whatever other
1063/// token was found instead.
1064/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1065/// by macro expansion (concatenating tokens as necessary if the first
1066/// token is a '<').
1067/// \return \c true if we reached EOD or EOF while looking for a > token in
1068/// a concatenated header name and diagnosed it. \c false otherwise.
1069bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1070 // Lex using header-name tokenization rules if tokens are being lexed from
1071 // a file. Just grab a token normally if we're in a macro expansion.
1072 if (CurPPLexer) {
1073 // Avoid nested header-name lexing when macro expansion recurses
1074 // __has_include(__has_include))
1075 if (CurPPLexer->ParsingFilename)
1076 LexUnexpandedToken(Result&: FilenameTok);
1077 else
1078 CurPPLexer->LexIncludeFilename(FilenameTok);
1079 } else {
1080 Lex(Result&: FilenameTok);
1081 }
1082
1083 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1084 // case, glue the tokens together into an angle_string_literal token.
1085 SmallString<128> FilenameBuffer;
1086 if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) {
1087 bool StartOfLine = FilenameTok.isAtStartOfLine();
1088 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1089 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1090
1091 SourceLocation Start = FilenameTok.getLocation();
1092 SourceLocation End;
1093 FilenameBuffer.push_back(Elt: '<');
1094
1095 // Consume tokens until we find a '>'.
1096 // FIXME: A header-name could be formed starting or ending with an
1097 // alternative token. It's not clear whether that's ill-formed in all
1098 // cases.
1099 while (FilenameTok.isNot(K: tok::greater)) {
1100 Lex(Result&: FilenameTok);
1101 if (FilenameTok.isOneOf(Ks: tok::eod, Ks: tok::eof)) {
1102 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_expected) << tok::greater;
1103 Diag(Loc: Start, DiagID: diag::note_matching) << tok::less;
1104 return true;
1105 }
1106
1107 End = FilenameTok.getLocation();
1108
1109 // FIXME: Provide code completion for #includes.
1110 if (FilenameTok.is(K: tok::code_completion)) {
1111 setCodeCompletionReached();
1112 Lex(Result&: FilenameTok);
1113 continue;
1114 }
1115
1116 // Append the spelling of this token to the buffer. If there was a space
1117 // before it, add it now.
1118 if (FilenameTok.hasLeadingSpace())
1119 FilenameBuffer.push_back(Elt: ' ');
1120
1121 // Get the spelling of the token, directly into FilenameBuffer if
1122 // possible.
1123 size_t PreAppendSize = FilenameBuffer.size();
1124 FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength());
1125
1126 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1127 unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr);
1128
1129 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1130 if (BufPtr != &FilenameBuffer[PreAppendSize])
1131 memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen);
1132
1133 // Resize FilenameBuffer to the correct size.
1134 if (FilenameTok.getLength() != ActualLen)
1135 FilenameBuffer.resize(N: PreAppendSize + ActualLen);
1136 }
1137
1138 FilenameTok.startToken();
1139 FilenameTok.setKind(tok::header_name);
1140 FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine);
1141 FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1142 FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro);
1143 CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End);
1144 } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) {
1145 // Convert a string-literal token of the form " h-char-sequence "
1146 // (produced by macro expansion) into a header-name token.
1147 //
1148 // The rules for header-names don't quite match the rules for
1149 // string-literals, but all the places where they differ result in
1150 // undefined behavior, so we can and do treat them the same.
1151 //
1152 // A string-literal with a prefix or suffix is not translated into a
1153 // header-name. This could theoretically be observable via the C++20
1154 // context-sensitive header-name formation rules.
1155 StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
1156 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1157 FilenameTok.setKind(tok::header_name);
1158 }
1159
1160 return false;
1161}
1162
1163std::optional<Token> Preprocessor::peekNextPPToken() const {
1164 // Do some quick tests for rejection cases.
1165 std::optional<Token> Val;
1166 if (CurLexer)
1167 Val = CurLexer->peekNextPPToken();
1168 else
1169 Val = CurTokenLexer->peekNextPPToken();
1170
1171 if (!Val) {
1172 // We have run off the end. If it's a source file we don't
1173 // examine enclosing ones (C99 5.1.1.2p4). Otherwise walk up the
1174 // macro stack.
1175 if (CurPPLexer)
1176 return std::nullopt;
1177 for (const IncludeStackInfo &Entry : llvm::reverse(C: IncludeMacroStack)) {
1178 if (Entry.TheLexer)
1179 Val = Entry.TheLexer->peekNextPPToken();
1180 else
1181 Val = Entry.TheTokenLexer->peekNextPPToken();
1182
1183 if (Val)
1184 break;
1185
1186 // Ran off the end of a source file?
1187 if (Entry.ThePPLexer)
1188 return std::nullopt;
1189 }
1190 }
1191
1192 // Okay, we found the token and return. Otherwise we found the end of the
1193 // translation unit.
1194 return Val;
1195}
1196
1197// We represent the primary and partition names as 'Paths' which are sections
1198// of the hierarchical access path for a clang module. However for C++20
1199// the periods in a name are just another character, and we will need to
1200// flatten them into a string.
1201std::string ModuleLoader::getFlatNameFromPath(ModuleIdPath Path) {
1202 std::string Name;
1203 if (Path.empty())
1204 return Name;
1205
1206 for (auto &Piece : Path) {
1207 assert(Piece.getIdentifierInfo() && Piece.getLoc().isValid());
1208 if (!Name.empty())
1209 Name += ".";
1210 Name += Piece.getIdentifierInfo()->getName();
1211 }
1212 return Name;
1213}
1214
1215ModuleNameLoc *ModuleNameLoc::Create(Preprocessor &PP, ModuleIdPath Path) {
1216 assert(!Path.empty() && "expect at least one identifier in a module name");
1217 void *Mem = PP.getPreprocessorAllocator().Allocate(
1218 Size: totalSizeToAlloc<IdentifierLoc>(Counts: Path.size()), Alignment: alignof(ModuleNameLoc));
1219 return new (Mem) ModuleNameLoc(Path);
1220}
1221
1222bool Preprocessor::LexModuleNameContinue(Token &Tok, SourceLocation UseLoc,
1223 SmallVectorImpl<Token> &Suffix,
1224 SmallVectorImpl<IdentifierLoc> &Path,
1225 bool AllowMacroExpansion,
1226 bool IsPartition) {
1227 auto ConsumeToken = [&]() {
1228 if (AllowMacroExpansion)
1229 Lex(Result&: Tok);
1230 else
1231 LexUnexpandedToken(Result&: Tok);
1232 Suffix.push_back(Elt: Tok);
1233 };
1234
1235 while (true) {
1236 if (Tok.isNot(K: tok::identifier)) {
1237 if (Tok.is(K: tok::code_completion)) {
1238 CurLexer->cutOffLexing();
1239 CodeComplete->CodeCompleteModuleImport(ImportLoc: UseLoc, Path);
1240 return true;
1241 }
1242
1243 Diag(Tok, DiagID: diag::err_pp_module_expected_ident) << Path.empty();
1244 return true;
1245 }
1246
1247 // [cpp.pre]/p2:
1248 // No identifier in the pp-module-name or pp-module-partition shall
1249 // currently be defined as an object-like macro.
1250 if (MacroInfo *MI = getMacroInfo(II: Tok.getIdentifierInfo());
1251 MI && MI->isObjectLike() && getLangOpts().CPlusPlus20 &&
1252 !AllowMacroExpansion) {
1253 Diag(Tok, DiagID: diag::err_pp_module_name_is_macro)
1254 << IsPartition << Tok.getIdentifierInfo();
1255 Diag(Loc: MI->getDefinitionLoc(), DiagID: diag::note_macro_here)
1256 << Tok.getIdentifierInfo();
1257 }
1258
1259 // Record this part of the module path.
1260 Path.emplace_back(Args: Tok.getLocation(), Args: Tok.getIdentifierInfo());
1261 ConsumeToken();
1262
1263 if (Tok.isNot(K: tok::period))
1264 return false;
1265
1266 ConsumeToken();
1267 }
1268}
1269
1270bool Preprocessor::HandleModuleName(StringRef DirType, SourceLocation UseLoc,
1271 Token &Tok,
1272 SmallVectorImpl<IdentifierLoc> &Path,
1273 SmallVectorImpl<Token> &DirToks,
1274 bool AllowMacroExpansion,
1275 bool IsPartition) {
1276 bool LeadingSpace = Tok.hasLeadingSpace();
1277 unsigned NumToksInDirective = DirToks.size();
1278 if (LexModuleNameContinue(Tok, UseLoc, Suffix&: DirToks, Path, AllowMacroExpansion,
1279 IsPartition)) {
1280 if (Tok.isNot(K: tok::eod))
1281 CheckEndOfDirective(DirType,
1282 /*EnableMacros=*/false, ExtraToks: &DirToks);
1283 EnterModuleSuffixTokenStream(Toks: DirToks);
1284 return true;
1285 }
1286
1287 // Clean the module-name tokens and replace these tokens with
1288 // annot_module_name.
1289 DirToks.resize(N: NumToksInDirective);
1290 ModuleNameLoc *NameLoc = ModuleNameLoc::Create(PP&: *this, Path);
1291 DirToks.emplace_back();
1292 DirToks.back().setKind(tok::annot_module_name);
1293 DirToks.back().setAnnotationRange(NameLoc->getRange());
1294 DirToks.back().setAnnotationValue(static_cast<void *>(NameLoc));
1295 DirToks.back().setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1296 DirToks.push_back(Elt: Tok);
1297 return false;
1298}
1299
1300/// [cpp.pre]/p2:
1301/// A preprocessing directive consists of a sequence of preprocessing tokens
1302/// that satisfies the following constraints: At the start of translation phase
1303/// 4, the first preprocessing token in the sequence, referred to as a
1304/// directive-introducing token, begins with the first character in the source
1305/// file (optionally after whitespace containing no new-line characters) or
1306/// follows whitespace containing at least one new-line character, and is:
1307/// - a # preprocessing token, or
1308/// - an import preprocessing token immediately followed on the same logical
1309/// source line by a header-name, <, identifier, or : preprocessing token, or
1310/// - a module preprocessing token immediately followed on the same logical
1311/// source line by an identifier, :, or ; preprocessing token, or
1312/// - an export preprocessing token immediately followed on the same logical
1313/// source line by one of the two preceding forms.
1314///
1315///
1316/// At the start of phase 4 an import or module token is treated as starting a
1317/// directive and are converted to their respective keywords iff:
1318/// - After skipping horizontal whitespace are
1319/// - at the start of a logical line, or
1320/// - preceded by an 'export' at the start of the logical line.
1321/// - Are followed by an identifier pp token (before macro expansion), or
1322/// - <, ", or : (but not ::) pp tokens for 'import', or
1323/// - ; for 'module'
1324/// Otherwise the token is treated as an identifier.
1325bool Preprocessor::HandleModuleContextualKeyword(Token &Result) {
1326 if (!getLangOpts().CPlusPlusModules || !Result.isModuleContextualKeyword())
1327 return false;
1328
1329 if (Result.is(K: tok::kw_export)) {
1330 LastExportKeyword = Result;
1331 return false;
1332 }
1333
1334 /// Trait 'module' and 'import' as a identifier when the main file is a
1335 /// preprocessed module file. We only allow '__preprocessed_module' and
1336 /// '__preprocessed_import' in this context.
1337 IdentifierInfo *II = Result.getIdentifierInfo();
1338 if (isPreprocessedModuleFile() &&
1339 (II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_import)) ||
1340 II->isStr(Str: tok::getKeywordSpelling(Kind: tok::kw_module))))
1341 return false;
1342
1343 if (LastExportKeyword.is(K: tok::kw_export)) {
1344 // The export keyword was not at the start of line, it's not a
1345 // directive-introducing token.
1346 if (!LastExportKeyword.isAtPhysicalStartOfLine())
1347 return false;
1348 // [cpp.pre]/1.4
1349 // export // not a preprocessing directive
1350 // import foo; // preprocessing directive (ill-formed at phase7)
1351 if (Result.isAtPhysicalStartOfLine())
1352 return false;
1353 } else if (!Result.isAtPhysicalStartOfLine())
1354 return false;
1355
1356 llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective(
1357 CurPPLexer->ParsingPreprocessorDirective, true);
1358
1359 // The next token may be an angled string literal after import keyword.
1360 llvm::SaveAndRestore<bool> SavedParsingFilemame(
1361 CurPPLexer->ParsingFilename,
1362 Result.getIdentifierInfo()->isImportKeyword());
1363
1364 std::optional<Token> NextTok = peekNextPPToken();
1365 if (!NextTok)
1366 return false;
1367
1368 if (NextTok->is(K: tok::raw_identifier))
1369 LookUpIdentifierInfo(Identifier&: *NextTok);
1370
1371 if (Result.getIdentifierInfo()->isImportKeyword()) {
1372 if (NextTok->isOneOf(Ks: tok::identifier, Ks: tok::less, Ks: tok::colon,
1373 Ks: tok::header_name)) {
1374 Result.setKind(tok::kw_import);
1375 ModuleImportLoc = Result.getLocation();
1376 return true;
1377 }
1378 }
1379
1380 if (Result.getIdentifierInfo()->isModuleKeyword() &&
1381 NextTok->isOneOf(Ks: tok::identifier, Ks: tok::colon, Ks: tok::semi)) {
1382 Result.setKind(tok::kw_module);
1383 ModuleDeclLoc = Result.getLocation();
1384 return true;
1385 }
1386
1387 // Ok, it's an identifier.
1388 return false;
1389}
1390
1391bool Preprocessor::CollectPPImportSuffixAndEnterStream(
1392 SmallVectorImpl<Token> &Toks, bool StopUntilEOD) {
1393 CollectPPImportSuffix(Toks);
1394 EnterModuleSuffixTokenStream(Toks);
1395 return false;
1396}
1397
1398/// Collect the tokens of a C++20 pp-import-suffix.
1399void Preprocessor::CollectPPImportSuffix(SmallVectorImpl<Token> &Toks,
1400 bool StopUntilEOD) {
1401 while (true) {
1402 Toks.emplace_back();
1403 Lex(Result&: Toks.back());
1404
1405 switch (Toks.back().getKind()) {
1406 case tok::semi:
1407 if (!StopUntilEOD)
1408 return;
1409 [[fallthrough]];
1410 case tok::eod:
1411 case tok::eof:
1412 return;
1413 default:
1414 break;
1415 }
1416 }
1417}
1418
1419// Allocate a holding buffer for a sequence of tokens and introduce it into
1420// the token stream.
1421void Preprocessor::EnterModuleSuffixTokenStream(ArrayRef<Token> Toks) {
1422 if (Toks.empty())
1423 return;
1424 auto ToksCopy = std::make_unique<Token[]>(num: Toks.size());
1425 std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get());
1426 EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(),
1427 /*DisableMacroExpansion*/ false, /*IsReinject*/ false);
1428 assert(CurTokenLexer && "Must have a TokenLexer");
1429 CurTokenLexer->setLexingCXXModuleDirective();
1430}
1431
1432void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc,
1433 bool IncludeExports) {
1434 CurSubmoduleState->VisibleModules.setVisible(
1435 M, Loc, IncludeExports, Vis: [](Module *) {},
1436 Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1437 // FIXME: Include the path in the diagnostic.
1438 // FIXME: Include the import location for the conflicting module.
1439 Diag(Loc: ModuleImportLoc, DiagID: diag::warn_module_conflict)
1440 << Path[0]->getFullModuleName()
1441 << Conflict->getFullModuleName()
1442 << Message;
1443 });
1444
1445 // Add this module to the imports list of the currently-built submodule.
1446 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1447 BuildingSubmoduleStack.back().M->Imports.push_back(Elt: M);
1448}
1449
1450bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1451 const char *DiagnosticTag,
1452 bool AllowMacroExpansion) {
1453 // We need at least one string literal.
1454 if (Result.isNot(K: tok::string_literal)) {
1455 Diag(Tok: Result, DiagID: diag::err_expected_string_literal)
1456 << /*Source='in...'*/0 << DiagnosticTag;
1457 return false;
1458 }
1459
1460 // Lex string literal tokens, optionally with macro expansion.
1461 SmallVector<Token, 4> StrToks;
1462 do {
1463 StrToks.push_back(Elt: Result);
1464
1465 if (Result.hasUDSuffix())
1466 Diag(Tok: Result, DiagID: diag::err_invalid_string_udl);
1467
1468 if (AllowMacroExpansion)
1469 Lex(Result);
1470 else
1471 LexUnexpandedToken(Result);
1472 } while (Result.is(K: tok::string_literal));
1473
1474 // Concatenate and parse the strings.
1475 StringLiteralParser Literal(StrToks, *this);
1476 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1477
1478 if (Literal.hadError)
1479 return false;
1480
1481 if (Literal.Pascal) {
1482 Diag(Loc: StrToks[0].getLocation(), DiagID: diag::err_expected_string_literal)
1483 << /*Source='in...'*/0 << DiagnosticTag;
1484 return false;
1485 }
1486
1487 String = std::string(Literal.GetString());
1488 return true;
1489}
1490
1491bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1492 assert(Tok.is(tok::numeric_constant));
1493 SmallString<8> IntegerBuffer;
1494 bool NumberInvalid = false;
1495 StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid);
1496 if (NumberInvalid)
1497 return false;
1498 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1499 getLangOpts(), getTargetInfo(),
1500 getDiagnostics());
1501 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1502 return false;
1503 llvm::APInt APVal(64, 0);
1504 if (Literal.GetIntegerValue(Val&: APVal))
1505 return false;
1506 Lex(Result&: Tok);
1507 Value = APVal.getLimitedValue();
1508 return true;
1509}
1510
1511void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1512 assert(Handler && "NULL comment handler");
1513 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1514 "Comment handler already registered");
1515 CommentHandlers.push_back(x: Handler);
1516}
1517
1518void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1519 std::vector<CommentHandler *>::iterator Pos =
1520 llvm::find(Range&: CommentHandlers, Val: Handler);
1521 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1522 CommentHandlers.erase(position: Pos);
1523}
1524
1525bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1526 bool AnyPendingTokens = false;
1527 for (CommentHandler *H : CommentHandlers) {
1528 if (H->HandleComment(PP&: *this, Comment))
1529 AnyPendingTokens = true;
1530 }
1531 if (!AnyPendingTokens || getCommentRetentionState())
1532 return false;
1533 Lex(Result&: result);
1534 return true;
1535}
1536
1537void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1538 const MacroAnnotations &A =
1539 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1540 assert(A.DeprecationInfo &&
1541 "Macro deprecation warning without recorded annotation!");
1542 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1543 if (Info.Message.empty())
1544 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1545 << Identifier.getIdentifierInfo() << 0;
1546 else
1547 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1548 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1549 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 0;
1550}
1551
1552void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1553 const MacroAnnotations &A =
1554 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1555 assert(A.RestrictExpansionInfo &&
1556 "Macro restricted expansion warning without recorded annotation!");
1557 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1558 if (Info.Message.empty())
1559 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1560 << Identifier.getIdentifierInfo() << 0;
1561 else
1562 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1563 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1564 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 1;
1565}
1566
1567void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
1568 unsigned DiagSelection) const {
1569 Diag(Tok: Identifier, DiagID: diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1570}
1571
1572void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1573 bool IsUndef) const {
1574 const MacroAnnotations &A =
1575 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1576 assert(A.FinalAnnotationLoc &&
1577 "Final macro warning without recorded annotation!");
1578
1579 Diag(Tok: Identifier, DiagID: diag::warn_pragma_final_macro)
1580 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1581 Diag(Loc: *A.FinalAnnotationLoc, DiagID: diag::note_pp_macro_annotation) << 2;
1582}
1583
1584bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr,
1585 const SourceLocation &Loc) const {
1586 // The lambda that tests if a `Loc` is in an opt-out region given one opt-out
1587 // region map:
1588 auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map,
1589 const SourceLocation &Loc) -> bool {
1590 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1591 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1592 Range: Map, P: [&SourceMgr,
1593 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1594 return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc);
1595 });
1596
1597 if (FirstRegionEndingAfterLoc != Map.end()) {
1598 // To test if the start location of the found region precedes `Loc`:
1599 return SourceMgr.isBeforeInTranslationUnit(
1600 LHS: FirstRegionEndingAfterLoc->first, RHS: Loc);
1601 }
1602 // If we do not find a region whose end location passes `Loc`, we want to
1603 // check if the current region is still open:
1604 if (!Map.empty() && Map.back().first == Map.back().second)
1605 return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc);
1606 return false;
1607 };
1608
1609 // What the following does:
1610 //
1611 // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`.
1612 // Otherwise, `Loc` is from a loaded AST. We look up the
1613 // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the
1614 // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out
1615 // region w.r.t. the region map. If the region map is absent, it means there
1616 // is no opt-out pragma in that loaded AST.
1617 //
1618 // Opt-out pragmas in the local TU or a loaded AST is not visible to another
1619 // one of them. That means if you put the pragmas around a `#include
1620 // "module.h"`, where module.h is a module, it is not actually suppressing
1621 // warnings in module.h. This is fine because warnings in module.h will be
1622 // reported when module.h is compiled in isolation and nothing in module.h
1623 // will be analyzed ever again. So you will not see warnings from the file
1624 // that imports module.h anyway. And you can't even do the same thing for PCHs
1625 // because they can only be included from the command line.
1626
1627 if (SourceMgr.isLocalSourceLocation(Loc))
1628 return TestInMap(SafeBufferOptOutMap, Loc);
1629
1630 const SafeBufferOptOutRegionsTy *LoadedRegions =
1631 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr);
1632
1633 if (LoadedRegions)
1634 return TestInMap(*LoadedRegions, Loc);
1635 return false;
1636}
1637
1638bool Preprocessor::enterOrExitSafeBufferOptOutRegion(
1639 bool isEnter, const SourceLocation &Loc) {
1640 if (isEnter) {
1641 if (isPPInSafeBufferOptOutRegion())
1642 return true; // invalid enter action
1643 InSafeBufferOptOutRegion = true;
1644 CurrentSafeBufferOptOutStart = Loc;
1645
1646 // To set the start location of a new region:
1647
1648 if (!SafeBufferOptOutMap.empty()) {
1649 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1650 assert(PrevRegion->first != PrevRegion->second &&
1651 "Shall not begin a safe buffer opt-out region before closing the "
1652 "previous one.");
1653 }
1654 // If the start location equals to the end location, we call the region a
1655 // open region or a unclosed region (i.e., end location has not been set
1656 // yet).
1657 SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc);
1658 } else {
1659 if (!isPPInSafeBufferOptOutRegion())
1660 return true; // invalid enter action
1661 InSafeBufferOptOutRegion = false;
1662
1663 // To set the end location of the current open region:
1664
1665 assert(!SafeBufferOptOutMap.empty() &&
1666 "Misordered safe buffer opt-out regions");
1667 auto *CurrRegion = &SafeBufferOptOutMap.back();
1668 assert(CurrRegion->first == CurrRegion->second &&
1669 "Set end location to a closed safe buffer opt-out region");
1670 CurrRegion->second = Loc;
1671 }
1672 return false;
1673}
1674
1675bool Preprocessor::isPPInSafeBufferOptOutRegion() {
1676 return InSafeBufferOptOutRegion;
1677}
1678bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) {
1679 StartLoc = CurrentSafeBufferOptOutStart;
1680 return InSafeBufferOptOutRegion;
1681}
1682
1683SmallVector<SourceLocation, 64>
1684Preprocessor::serializeSafeBufferOptOutMap() const {
1685 assert(!InSafeBufferOptOutRegion &&
1686 "Attempt to serialize safe buffer opt-out regions before file being "
1687 "completely preprocessed");
1688
1689 SmallVector<SourceLocation, 64> SrcSeq;
1690
1691 for (const auto &[begin, end] : SafeBufferOptOutMap) {
1692 SrcSeq.push_back(Elt: begin);
1693 SrcSeq.push_back(Elt: end);
1694 }
1695 // Only `SafeBufferOptOutMap` gets serialized. No need to serialize
1696 // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every
1697 // pch/module in the pch-chain/module-DAG will be loaded one by one in order.
1698 // It means that for each loading pch/module m, it just needs to load m's own
1699 // `SafeBufferOptOutMap`.
1700 return SrcSeq;
1701}
1702
1703bool Preprocessor::setDeserializedSafeBufferOptOutMap(
1704 const SmallVectorImpl<SourceLocation> &SourceLocations) {
1705 if (SourceLocations.size() == 0)
1706 return false;
1707
1708 assert(SourceLocations.size() % 2 == 0 &&
1709 "ill-formed SourceLocation sequence");
1710
1711 auto It = SourceLocations.begin();
1712 SafeBufferOptOutRegionsTy &Regions =
1713 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr);
1714
1715 do {
1716 SourceLocation Begin = *It++;
1717 SourceLocation End = *It++;
1718
1719 Regions.emplace_back(Args&: Begin, Args&: End);
1720 } while (It != SourceLocations.end());
1721 return true;
1722}
1723
1724ModuleLoader::~ModuleLoader() = default;
1725
1726CommentHandler::~CommentHandler() = default;
1727
1728EmptylineHandler::~EmptylineHandler() = default;
1729
1730CodeCompletionHandler::~CodeCompletionHandler() = default;
1731
1732void Preprocessor::createPreprocessingRecord() {
1733 if (Record)
1734 return;
1735
1736 Record = new PreprocessingRecord(getSourceManager());
1737 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record));
1738}
1739
1740void Preprocessor::removePPCallbacks() {
1741 auto IsPreserved = [&](PPCallbacks *C) {
1742 return C == Record || C == DirTracer;
1743 };
1744 SmallVector<PPCallbacks *, 2> Released;
1745 PPCallbacks::releaseIfPreserved(CB&: Callbacks, Pred: IsPreserved, Released);
1746 Callbacks.reset();
1747 for (auto *P : Released)
1748 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(P));
1749}
1750
1751const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
1752 if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) {
1753 const SmallVector<const char *> &FileCheckPoints = It->second;
1754 auto P = llvm::upper_bound(Range: FileCheckPoints, Value&: Start);
1755 if (P == FileCheckPoints.begin())
1756 return nullptr;
1757 return *std::prev(x: P);
1758 }
1759 return nullptr;
1760}
1761
1762bool Preprocessor::hasSeenNoTrivialPPDirective() const {
1763 return DirTracer && DirTracer->hasSeenNoTrivialPPDirective();
1764}
1765
1766bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const {
1767 return SeenNoTrivialPPDirective;
1768}
1769
1770void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
1771 if (InMainFile && !SeenNoTrivialPPDirective)
1772 SeenNoTrivialPPDirective = true;
1773}
1774
1775void NoTrivialPPDirectiveTracer::LexedFileChanged(
1776 FileID FID, LexedFileChangeReason Reason,
1777 SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) {
1778 InMainFile = (FID == PP.getSourceManager().getMainFileID());
1779}
1780
1781void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok,
1782 const MacroDefinition &MD,
1783 SourceRange Range,
1784 const MacroArgs *Args) {
1785 // FIXME: Does only enable builtin macro expansion make sense?
1786 if (!MD.getMacroInfo()->isBuiltinMacro())
1787 setSeenNoTrivialPPDirective();
1788}
1789