1//===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TokenLexer interface.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Lex/TokenLexer.h"
14#include "clang/Basic/Diagnostic.h"
15#include "clang/Basic/IdentifierTable.h"
16#include "clang/Basic/LangOptions.h"
17#include "clang/Basic/SourceLocation.h"
18#include "clang/Basic/SourceManager.h"
19#include "clang/Basic/TokenKinds.h"
20#include "clang/Lex/LexDiagnostic.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/MacroArgs.h"
23#include "clang/Lex/MacroInfo.h"
24#include "clang/Lex/Preprocessor.h"
25#include "clang/Lex/Token.h"
26#include "clang/Lex/VariadicMacroSupport.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/iterator_range.h"
31#include <cassert>
32#include <cstring>
33#include <optional>
34
35using namespace clang;
36
37/// Create a TokenLexer for the specified macro with the specified actual
38/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
39void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
40 MacroArgs *Actuals) {
41 // If the client is reusing a TokenLexer, make sure to free any memory
42 // associated with it.
43 destroy();
44
45 Macro = MI;
46 ActualArgs = Actuals;
47 CurTokenIdx = 0;
48
49 ExpandLocStart = Tok.getLocation();
50 ExpandLocEnd = ELEnd;
51 AtStartOfLine = Tok.isAtStartOfLine();
52 HasLeadingSpace = Tok.hasLeadingSpace();
53 NextTokGetsSpace = false;
54 Tokens = &*Macro->tokens_begin();
55 OwnsTokens = false;
56 DisableMacroExpansion = false;
57 IsReinject = false;
58 NumTokens = Macro->tokens_end()-Macro->tokens_begin();
59 MacroExpansionStart = SourceLocation();
60 LexingCXXModuleDirective = false;
61
62 SourceManager &SM = PP.getSourceManager();
63 MacroStartSLocOffset = SM.getNextLocalOffset();
64
65 if (NumTokens > 0) {
66 assert(Tokens[0].getLocation().isValid());
67 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
68 "Macro defined in macro?");
69 assert(ExpandLocStart.isValid());
70
71 // Reserve a source location entry chunk for the length of the macro
72 // definition. Tokens that get lexed directly from the definition will
73 // have their locations pointing inside this chunk. This is to avoid
74 // creating separate source location entries for each token.
75 MacroDefStart = SM.getExpansionLoc(Loc: Tokens[0].getLocation());
76 MacroDefLength = Macro->getDefinitionLength(SM);
77 MacroExpansionStart = SM.createExpansionLoc(SpellingLoc: MacroDefStart,
78 ExpansionLocStart: ExpandLocStart,
79 ExpansionLocEnd: ExpandLocEnd,
80 Length: MacroDefLength);
81 }
82
83 // If this is a function-like macro, expand the arguments and change
84 // Tokens to point to the expanded tokens.
85 if (Macro->isFunctionLike() && Macro->getNumParams())
86 ExpandFunctionArguments();
87
88 // Mark the macro as currently disabled, so that it is not recursively
89 // expanded. The macro must be disabled only after argument pre-expansion of
90 // function-like macro arguments occurs.
91 Macro->DisableMacro();
92}
93
94/// Create a TokenLexer for the specified token stream. This does not
95/// take ownership of the specified token vector.
96void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
97 bool disableMacroExpansion, bool ownsTokens,
98 bool isReinject) {
99 assert(!isReinject || disableMacroExpansion);
100 // If the client is reusing a TokenLexer, make sure to free any memory
101 // associated with it.
102 destroy();
103
104 Macro = nullptr;
105 ActualArgs = nullptr;
106 Tokens = TokArray;
107 OwnsTokens = ownsTokens;
108 DisableMacroExpansion = disableMacroExpansion;
109 IsReinject = isReinject;
110 NumTokens = NumToks;
111 CurTokenIdx = 0;
112 ExpandLocStart = ExpandLocEnd = SourceLocation();
113 AtStartOfLine = false;
114 HasLeadingSpace = false;
115 NextTokGetsSpace = false;
116 MacroExpansionStart = SourceLocation();
117 LexingCXXModuleDirective = false;
118
119 // Set HasLeadingSpace/AtStartOfLine so that the first token will be
120 // returned unmodified.
121 if (NumToks != 0) {
122 AtStartOfLine = TokArray[0].isAtStartOfLine();
123 HasLeadingSpace = TokArray[0].hasLeadingSpace();
124 }
125}
126
127void TokenLexer::destroy() {
128 // If this was a function-like macro that actually uses its arguments, delete
129 // the expanded tokens.
130 if (OwnsTokens) {
131 delete [] Tokens;
132 Tokens = nullptr;
133 OwnsTokens = false;
134 }
135
136 // TokenLexer owns its formal arguments.
137 if (ActualArgs) ActualArgs->destroy(PP);
138}
139
140static bool hasVaOptSupport(const LangOptions &LangOpts) {
141 return LangOpts.C23 || LangOpts.CPlusPlus20;
142}
143
144bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
145 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
146 unsigned MacroArgNo, Preprocessor &PP) {
147 // Is the macro argument __VA_ARGS__?
148 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
149 return false;
150
151 // In Microsoft-compatibility mode, a comma is removed in the expansion
152 // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
153 // not supported by gcc.
154 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
155 return false;
156
157 // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
158 // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
159 // named arguments, where it remains. In all other modes, including C99
160 // with GNU extensions, it is removed regardless of named arguments.
161 // Microsoft also appears to support this extension, unofficially.
162 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
163 && Macro->getNumParams() < 2)
164 return false;
165
166 // Is a comma available to be removed?
167 if (ResultToks.empty() || !ResultToks.back().is(K: tok::comma))
168 return false;
169
170 // Issue an extension diagnostic for the paste operator.
171 if (HasPasteOperator) {
172 PP.Diag(Loc: ResultToks.back().getLocation(), DiagID: diag::ext_paste_comma)
173 << hasVaOptSupport(LangOpts: PP.getLangOpts());
174 }
175
176 // Remove the comma.
177 ResultToks.pop_back();
178
179 if (!ResultToks.empty()) {
180 // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
181 // then removal of the comma should produce a placemarker token (in C99
182 // terms) which we model by popping off the previous ##, giving us a plain
183 // "X" when __VA_ARGS__ is empty.
184 if (ResultToks.back().is(K: tok::hashhash))
185 ResultToks.pop_back();
186
187 // Remember that this comma was elided.
188 ResultToks.back().setFlag(Token::CommaAfterElided);
189 }
190
191 // Never add a space, even if the comma, ##, or arg had a space.
192 NextTokGetsSpace = false;
193 return true;
194}
195
196void TokenLexer::stringifyVAOPTContents(
197 SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
198 const SourceLocation VAOPTClosingParenLoc) {
199 const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
200 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
201 Token *const VAOPTTokens =
202 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
203
204 SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
205 // FIXME: Should we keep track within VCtx that we did or didnot
206 // encounter pasting - and only then perform this loop.
207
208 // Perform token pasting (concatenation) prior to stringization.
209 for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
210 ++CurTokenIdx) {
211 if (VAOPTTokens[CurTokenIdx].is(K: tok::hashhash)) {
212 assert(CurTokenIdx != 0 &&
213 "Can not have __VAOPT__ contents begin with a ##");
214 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
215 pasteTokens(LHSTok&: LHS, TokenStream: llvm::ArrayRef(VAOPTTokens, NumVAOptTokens),
216 CurIdx&: CurTokenIdx);
217 // Replace the token prior to the first ## in this iteration.
218 ConcatenatedVAOPTResultToks.back() = LHS;
219 if (CurTokenIdx == NumVAOptTokens)
220 break;
221 }
222 ConcatenatedVAOPTResultToks.push_back(Elt: VAOPTTokens[CurTokenIdx]);
223 }
224
225 ConcatenatedVAOPTResultToks.push_back(Elt: VCtx.getEOFTok());
226 // Get the SourceLocation that represents the start location within
227 // the macro definition that marks where this string is substituted
228 // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
229 // macro definition, and use it to indicate that the stringified token
230 // was generated from that location.
231 const SourceLocation ExpansionLocStartWithinMacro =
232 getExpansionLocForMacroDefLoc(loc: VCtx.getVAOptLoc());
233 const SourceLocation ExpansionLocEndWithinMacro =
234 getExpansionLocForMacroDefLoc(loc: VAOPTClosingParenLoc);
235
236 Token StringifiedVAOPT = MacroArgs::StringifyArgument(
237 ArgToks: &ConcatenatedVAOPTResultToks[0], PP, Charify: VCtx.hasCharifyBefore() /*Charify*/,
238 ExpansionLocStart: ExpansionLocStartWithinMacro, ExpansionLocEnd: ExpansionLocEndWithinMacro);
239
240 if (VCtx.getLeadingSpaceForStringifiedToken())
241 StringifiedVAOPT.setFlag(Token::LeadingSpace);
242
243 StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
244 // Resize (shrink) the token stream to just capture this stringified token.
245 ResultToks.resize(N: NumToksPriorToVAOpt + 1);
246 ResultToks.back() = StringifiedVAOPT;
247}
248
249/// Expand the arguments of a function-like macro so that we can quickly
250/// return preexpanded tokens from Tokens.
251void TokenLexer::ExpandFunctionArguments() {
252 SmallVector<Token, 128> ResultToks;
253
254 // Loop through 'Tokens', expanding them into ResultToks. Keep
255 // track of whether we change anything. If not, no need to keep them. If so,
256 // we install the newly expanded sequence as the new 'Tokens' list.
257 bool MadeChange = false;
258
259 std::optional<bool> CalledWithVariadicArguments;
260
261 VAOptExpansionContext VCtx(PP);
262
263 for (unsigned I = 0, E = NumTokens; I != E; ++I) {
264 const Token &CurTok = Tokens[I];
265 // We don't want a space for the next token after a paste
266 // operator. In valid code, the token will get smooshed onto the
267 // preceding one anyway. In assembler-with-cpp mode, invalid
268 // pastes are allowed through: in this case, we do not want the
269 // extra whitespace to be added. For example, we want ". ## foo"
270 // -> ".foo" not ". foo".
271 if (I != 0 && !Tokens[I-1].is(K: tok::hashhash) && CurTok.hasLeadingSpace())
272 NextTokGetsSpace = true;
273
274 if (VCtx.isVAOptToken(T: CurTok)) {
275 MadeChange = true;
276 assert(Tokens[I + 1].is(tok::l_paren) &&
277 "__VA_OPT__ must be followed by '('");
278
279 ++I; // Skip the l_paren
280 VCtx.sawVAOptFollowedByOpeningParens(VAOptLoc: CurTok.getLocation(),
281 NumPriorTokens: ResultToks.size());
282
283 continue;
284 }
285
286 // We have entered into the __VA_OPT__ context, so handle tokens
287 // appropriately.
288 if (VCtx.isInVAOpt()) {
289 // If we are about to process a token that is either an argument to
290 // __VA_OPT__ or its closing rparen, then:
291 // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
292 // perform any necessary stringification or placemarker processing,
293 // and/or skip to the next token.
294 // 2) else if macro was invoked without variadic arguments skip this
295 // token.
296 // 3) else (macro was invoked with variadic arguments) process the token
297 // normally.
298
299 if (Tokens[I].is(K: tok::l_paren))
300 VCtx.sawOpeningParen(LParenLoc: Tokens[I].getLocation());
301 // Continue skipping tokens within __VA_OPT__ if the macro was not
302 // called with variadic arguments, else let the rest of the loop handle
303 // this token. Note sawClosingParen() returns true only if the r_paren matches
304 // the closing r_paren of the __VA_OPT__.
305 if (!Tokens[I].is(K: tok::r_paren) || !VCtx.sawClosingParen()) {
306 // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
307 if (!CalledWithVariadicArguments) {
308 CalledWithVariadicArguments =
309 ActualArgs->invokedWithVariadicArgument(MI: Macro, PP);
310 }
311 if (!*CalledWithVariadicArguments) {
312 // Skip this token.
313 continue;
314 }
315 // ... else the macro was called with variadic arguments, and we do not
316 // have a closing rparen - so process this token normally.
317 } else {
318 // Current token is the closing r_paren which marks the end of the
319 // __VA_OPT__ invocation, so handle any place-marker pasting (if
320 // empty) by removing hashhash either before (if exists) or after. And
321 // also stringify the entire contents if VAOPT was preceded by a hash,
322 // but do so only after any token concatenation that needs to occur
323 // within the contents of VAOPT.
324
325 if (VCtx.hasStringifyOrCharifyBefore()) {
326 // Replace all the tokens just added from within VAOPT into a single
327 // stringified token. This requires token-pasting to eagerly occur
328 // within these tokens. If either the contents of VAOPT were empty
329 // or the macro wasn't called with any variadic arguments, the result
330 // is a token that represents an empty string.
331 stringifyVAOPTContents(ResultToks, VCtx,
332 /*ClosingParenLoc*/ VAOPTClosingParenLoc: Tokens[I].getLocation());
333
334 } else if (/*No tokens within VAOPT*/
335 ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
336 // Treat VAOPT as a placemarker token. Eat either the '##' before the
337 // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
338 // hashhash was not a placemarker) or the '##'
339 // after VAOPT, but not both.
340
341 if (ResultToks.size() && ResultToks.back().is(K: tok::hashhash)) {
342 ResultToks.pop_back();
343 } else if ((I + 1 != E) && Tokens[I + 1].is(K: tok::hashhash)) {
344 ++I; // Skip the following hashhash.
345 }
346 } else {
347 // If there's a ## before the __VA_OPT__, we might have discovered
348 // that the __VA_OPT__ begins with a placeholder. We delay action on
349 // that to now to avoid messing up our stashed count of tokens before
350 // __VA_OPT__.
351 if (VCtx.beginsWithPlaceholder()) {
352 assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
353 ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
354 ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
355 tok::hashhash) &&
356 "no token paste before __VA_OPT__");
357 ResultToks.erase(CI: ResultToks.begin() +
358 VCtx.getNumberOfTokensPriorToVAOpt() - 1);
359 }
360 // If the expansion of __VA_OPT__ ends with a placeholder, eat any
361 // following '##' token.
362 if (VCtx.endsWithPlaceholder() && I + 1 != E &&
363 Tokens[I + 1].is(K: tok::hashhash)) {
364 ++I;
365 }
366 }
367 VCtx.reset();
368 // We processed __VA_OPT__'s closing paren (and the exit out of
369 // __VA_OPT__), so skip to the next token.
370 continue;
371 }
372 }
373
374 // If we found the stringify operator, get the argument stringified. The
375 // preprocessor already verified that the following token is a macro
376 // parameter or __VA_OPT__ when the #define was lexed.
377
378 if (CurTok.isOneOf(Ks: tok::hash, Ks: tok::hashat)) {
379 int ArgNo = Macro->getParameterNum(Arg: Tokens[I+1].getIdentifierInfo());
380 assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
381 "Token following # is not an argument or __VA_OPT__!");
382
383 if (ArgNo == -1) {
384 // Handle the __VA_OPT__ case.
385 VCtx.sawHashOrHashAtBefore(HasLeadingSpace: NextTokGetsSpace,
386 IsHashAt: CurTok.is(K: tok::hashat));
387 continue;
388 }
389 // Else handle the simple argument case.
390 SourceLocation ExpansionLocStart =
391 getExpansionLocForMacroDefLoc(loc: CurTok.getLocation());
392 SourceLocation ExpansionLocEnd =
393 getExpansionLocForMacroDefLoc(loc: Tokens[I+1].getLocation());
394
395 bool Charify = CurTok.is(K: tok::hashat);
396 const Token *UnexpArg = ActualArgs->getUnexpArgument(Arg: ArgNo);
397 Token Res = MacroArgs::StringifyArgument(
398 ArgToks: UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
399 Res.setFlag(Token::StringifiedInMacro);
400
401 // The stringified/charified string leading space flag gets set to match
402 // the #/#@ operator.
403 if (NextTokGetsSpace)
404 Res.setFlag(Token::LeadingSpace);
405
406 ResultToks.push_back(Elt: Res);
407 MadeChange = true;
408 ++I; // Skip arg name.
409 NextTokGetsSpace = false;
410 continue;
411 }
412
413 // Find out if there is a paste (##) operator before or after the token.
414 bool NonEmptyPasteBefore =
415 !ResultToks.empty() && ResultToks.back().is(K: tok::hashhash);
416 bool PasteBefore = I != 0 && Tokens[I-1].is(K: tok::hashhash);
417 bool PasteAfter = I+1 != E && Tokens[I+1].is(K: tok::hashhash);
418 bool RParenAfter = I+1 != E && Tokens[I+1].is(K: tok::r_paren);
419
420 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
421 "unexpected ## in ResultToks");
422
423 // Otherwise, if this is not an argument token, just add the token to the
424 // output buffer.
425 IdentifierInfo *II = CurTok.getIdentifierInfo();
426 int ArgNo = II ? Macro->getParameterNum(Arg: II) : -1;
427 if (ArgNo == -1) {
428 // This isn't an argument, just add it.
429 ResultToks.push_back(Elt: CurTok);
430
431 if (NextTokGetsSpace) {
432 ResultToks.back().setFlag(Token::LeadingSpace);
433 NextTokGetsSpace = false;
434 } else if (PasteBefore && !NonEmptyPasteBefore)
435 ResultToks.back().clearFlag(Flag: Token::LeadingSpace);
436
437 continue;
438 }
439
440 // An argument is expanded somehow, the result is different than the
441 // input.
442 MadeChange = true;
443
444 // Otherwise, this is a use of the argument.
445
446 // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
447 // are no trailing commas if __VA_ARGS__ is empty.
448 if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
449 MaybeRemoveCommaBeforeVaArgs(ResultToks,
450 /*HasPasteOperator=*/false,
451 Macro, MacroArgNo: ArgNo, PP))
452 continue;
453
454 // If it is not the LHS/RHS of a ## operator, we must pre-expand the
455 // argument and substitute the expanded tokens into the result. This is
456 // C99 6.10.3.1p1.
457 if (!PasteBefore && !PasteAfter) {
458 const Token *ResultArgToks;
459
460 // Only preexpand the argument if it could possibly need it. This
461 // avoids some work in common cases.
462 const Token *ArgTok = ActualArgs->getUnexpArgument(Arg: ArgNo);
463 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
464 ResultArgToks = &ActualArgs->getPreExpArgument(Arg: ArgNo, PP)[0];
465 else
466 ResultArgToks = ArgTok; // Use non-preexpanded tokens.
467
468 // If the arg token expanded into anything, append it.
469 if (ResultArgToks->isNot(K: tok::eof)) {
470 size_t FirstResult = ResultToks.size();
471 unsigned NumToks = MacroArgs::getArgLength(ArgPtr: ResultArgToks);
472 ResultToks.append(in_start: ResultArgToks, in_end: ResultArgToks+NumToks);
473
474 // In Microsoft-compatibility mode, we follow MSVC's preprocessing
475 // behavior by not considering single commas from nested macro
476 // expansions as argument separators. Set a flag on the token so we can
477 // test for this later when the macro expansion is processed.
478 if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
479 ResultToks.back().is(K: tok::comma))
480 ResultToks.back().setFlag(Token::IgnoredComma);
481
482 // If the '##' came from expanding an argument, turn it into 'unknown'
483 // to avoid pasting.
484 for (Token &Tok : llvm::drop_begin(RangeOrContainer&: ResultToks, N: FirstResult))
485 if (Tok.is(K: tok::hashhash))
486 Tok.setKind(tok::unknown);
487
488 if(ExpandLocStart.isValid()) {
489 updateLocForMacroArgTokens(ArgIdSpellLoc: CurTok.getLocation(),
490 begin_tokens: ResultToks.begin()+FirstResult,
491 end_tokens: ResultToks.end());
492 }
493
494 // If any tokens were substituted from the argument, the whitespace
495 // before the first token should match the whitespace of the arg
496 // identifier.
497 ResultToks[FirstResult].setFlagValue(Flag: Token::LeadingSpace,
498 Val: NextTokGetsSpace);
499 ResultToks[FirstResult].setFlagValue(Flag: Token::StartOfLine, Val: false);
500 NextTokGetsSpace = false;
501 } else {
502 // We're creating a placeholder token. Usually this doesn't matter,
503 // but it can affect paste behavior when at the start or end of a
504 // __VA_OPT__.
505 if (NonEmptyPasteBefore) {
506 // We're imagining a placeholder token is inserted here. If this is
507 // the first token in a __VA_OPT__ after a ##, delete the ##.
508 assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
509 VCtx.hasPlaceholderAfterHashhashAtStart();
510 } else if (RParenAfter)
511 VCtx.hasPlaceholderBeforeRParen();
512 }
513 continue;
514 }
515
516 // Okay, we have a token that is either the LHS or RHS of a paste (##)
517 // argument. It gets substituted as its non-pre-expanded tokens.
518 const Token *ArgToks = ActualArgs->getUnexpArgument(Arg: ArgNo);
519 unsigned NumToks = MacroArgs::getArgLength(ArgPtr: ArgToks);
520 if (NumToks) { // Not an empty argument?
521 bool VaArgsPseudoPaste = false;
522 // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
523 // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
524 // the expander tries to paste ',' with the first token of the __VA_ARGS__
525 // expansion.
526 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
527 ResultToks[ResultToks.size()-2].is(K: tok::comma) &&
528 (unsigned)ArgNo == Macro->getNumParams()-1 &&
529 Macro->isVariadic()) {
530 VaArgsPseudoPaste = true;
531 // Remove the paste operator, report use of the extension.
532 bool VaOptSupport = hasVaOptSupport(LangOpts: PP.getLangOpts());
533 auto Diag = PP.Diag(Loc: ResultToks.pop_back_val().getLocation(),
534 DiagID: diag::ext_paste_comma)
535 << VaOptSupport;
536 if (VaOptSupport) {
537 Diag << FixItHint::CreateReplacement(
538 RemoveRange: SourceRange(ResultToks.back().getLocation(),
539 CurTok.getLocation()),
540 Code: " __VA_OPT__(,) __VA_ARGS__");
541 }
542 }
543
544 ResultToks.append(in_start: ArgToks, in_end: ArgToks+NumToks);
545
546 // If the '##' came from expanding an argument, turn it into 'unknown'
547 // to avoid pasting.
548 for (Token &Tok : llvm::make_range(x: ResultToks.end() - NumToks,
549 y: ResultToks.end())) {
550 if (Tok.is(K: tok::hashhash))
551 Tok.setKind(tok::unknown);
552 }
553
554 if (ExpandLocStart.isValid()) {
555 updateLocForMacroArgTokens(ArgIdSpellLoc: CurTok.getLocation(),
556 begin_tokens: ResultToks.end()-NumToks, end_tokens: ResultToks.end());
557 }
558
559 // Transfer the leading whitespace information from the token
560 // (the macro argument) onto the first token of the
561 // expansion. Note that we don't do this for the GNU
562 // pseudo-paste extension ", ## __VA_ARGS__".
563 if (!VaArgsPseudoPaste) {
564 ResultToks[ResultToks.size() - NumToks].setFlagValue(Flag: Token::StartOfLine,
565 Val: false);
566 ResultToks[ResultToks.size() - NumToks].setFlagValue(
567 Flag: Token::LeadingSpace, Val: NextTokGetsSpace);
568 }
569
570 NextTokGetsSpace = false;
571 continue;
572 }
573
574 // If an empty argument is on the LHS or RHS of a paste, the standard (C99
575 // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
576 // implement this by eating ## operators when a LHS or RHS expands to
577 // empty.
578 if (PasteAfter) {
579 // Discard the argument token and skip (don't copy to the expansion
580 // buffer) the paste operator after it.
581 ++I;
582 continue;
583 }
584
585 if (RParenAfter && !NonEmptyPasteBefore)
586 VCtx.hasPlaceholderBeforeRParen();
587
588 // If this is on the RHS of a paste operator, we've already copied the
589 // paste operator to the ResultToks list, unless the LHS was empty too.
590 // Remove it.
591 assert(PasteBefore);
592 if (NonEmptyPasteBefore) {
593 assert(ResultToks.back().is(tok::hashhash));
594 // Do not remove the paste operator if it is the one before __VA_OPT__
595 // (and we are still processing tokens within VA_OPT). We handle the case
596 // of removing the paste operator if __VA_OPT__ reduces to the notional
597 // placemarker above when we encounter the closing paren of VA_OPT.
598 if (!VCtx.isInVAOpt() ||
599 ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
600 ResultToks.pop_back();
601 else
602 VCtx.hasPlaceholderAfterHashhashAtStart();
603 }
604
605 // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
606 // and if the macro had at least one real argument, and if the token before
607 // the ## was a comma, remove the comma. This is a GCC extension which is
608 // disabled when using -std=c99.
609 if (ActualArgs->isVarargsElidedUse())
610 MaybeRemoveCommaBeforeVaArgs(ResultToks,
611 /*HasPasteOperator=*/true,
612 Macro, MacroArgNo: ArgNo, PP);
613 }
614
615 // If anything changed, install this as the new Tokens list.
616 if (MadeChange) {
617 assert(!OwnsTokens && "This would leak if we already own the token list");
618 // This is deleted in the dtor.
619 NumTokens = ResultToks.size();
620 // The tokens will be added to Preprocessor's cache and will be removed
621 // when this TokenLexer finishes lexing them.
622 Tokens = PP.cacheMacroExpandedTokens(tokLexer: this, tokens: ResultToks);
623
624 // The preprocessor cache of macro expanded tokens owns these tokens,not us.
625 OwnsTokens = false;
626 }
627}
628
629/// Checks if two tokens form wide string literal.
630static bool isWideStringLiteralFromMacro(const Token &FirstTok,
631 const Token &SecondTok) {
632 return FirstTok.is(K: tok::identifier) &&
633 FirstTok.getIdentifierInfo()->isStr(Str: "L") && SecondTok.isLiteral() &&
634 SecondTok.stringifiedInMacro();
635}
636
637/// Lex - Lex and return a token from this macro stream.
638bool TokenLexer::Lex(Token &Tok) {
639 // Lexing off the end of the macro, pop this macro off the expansion stack.
640 if (isAtEnd()) {
641 // If this is a macro (not a token stream), mark the macro enabled now
642 // that it is no longer being expanded.
643 if (Macro) Macro->EnableMacro();
644
645 // CWG2947: Allow the following code:
646 //
647 // export module m; int x;
648 // extern "C++" int *y = &x;
649 //
650 // The 'extern' token should has 'StartOfLine' flag when current TokenLexer
651 // exits and propagate line start/leading space info.
652 if (!Macro && isLexingCXXModuleDirective()) {
653 AtStartOfLine = true;
654 setLexingCXXModuleDirective(false);
655 }
656
657 Tok.startToken();
658 Tok.setFlagValue(Flag: Token::StartOfLine , Val: AtStartOfLine);
659 Tok.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace || NextTokGetsSpace);
660 if (CurTokenIdx == 0)
661 Tok.setFlag(Token::LeadingEmptyMacro);
662 return PP.HandleEndOfTokenLexer(Result&: Tok);
663 }
664
665 SourceManager &SM = PP.getSourceManager();
666
667 // If this is the first token of the expanded result, we inherit spacing
668 // properties later.
669 bool isFirstToken = CurTokenIdx == 0;
670
671 // Get the next token to return.
672 Tok = Tokens[CurTokenIdx++];
673 if (IsReinject)
674 Tok.setFlag(Token::IsReinjected);
675
676 bool TokenIsFromPaste = false;
677
678 // If this token is followed by a token paste (##) operator, paste the tokens!
679 // Note that ## is a normal token when not expanding a macro.
680 if (!isAtEnd() && Macro &&
681 (Tokens[CurTokenIdx].is(K: tok::hashhash) ||
682 // Special processing of L#x macros in -fms-compatibility mode.
683 // Microsoft compiler is able to form a wide string literal from
684 // 'L#macro_arg' construct in a function-like macro.
685 (PP.getLangOpts().MSVCCompat &&
686 isWideStringLiteralFromMacro(FirstTok: Tok, SecondTok: Tokens[CurTokenIdx])))) {
687 // When handling the microsoft /##/ extension, the final token is
688 // returned by pasteTokens, not the pasted token.
689 if (pasteTokens(Tok))
690 return true;
691
692 TokenIsFromPaste = true;
693 }
694
695 // The token's current location indicate where the token was lexed from. We
696 // need this information to compute the spelling of the token, but any
697 // diagnostics for the expanded token should appear as if they came from
698 // ExpansionLoc. Pull this information together into a new SourceLocation
699 // that captures all of this.
700 if (ExpandLocStart.isValid() && // Don't do this for token streams.
701 // Check that the token's location was not already set properly.
702 SM.isBeforeInSLocAddrSpace(LHS: Tok.getLocation(), RHS: MacroStartSLocOffset)) {
703 SourceLocation instLoc;
704 if (Tok.is(K: tok::comment)) {
705 instLoc = SM.createExpansionLoc(SpellingLoc: Tok.getLocation(),
706 ExpansionLocStart: ExpandLocStart,
707 ExpansionLocEnd: ExpandLocEnd,
708 Length: Tok.getLength());
709 } else {
710 instLoc = getExpansionLocForMacroDefLoc(loc: Tok.getLocation());
711 }
712
713 Tok.setLocation(instLoc);
714 }
715
716 // If this is the first token, set the lexical properties of the token to
717 // match the lexical properties of the macro identifier.
718 if (isFirstToken) {
719 Tok.setFlagValue(Flag: Token::StartOfLine , Val: AtStartOfLine);
720 Tok.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace);
721 } else {
722 // If this is not the first token, we may still need to pass through
723 // leading whitespace if we've expanded a macro.
724 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
725 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
726 }
727 AtStartOfLine = false;
728 HasLeadingSpace = false;
729
730 // Handle recursive expansion!
731 if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr &&
732 (!PP.getLangOpts().CPlusPlusModules ||
733 !Tok.isModuleContextualKeyword())) {
734 // Change the kind of this identifier to the appropriate token kind, e.g.
735 // turning "for" into a keyword.
736 IdentifierInfo *II = Tok.getIdentifierInfo();
737 Tok.setKind(II->getTokenID());
738
739 // If this identifier was poisoned and from a paste, emit an error. This
740 // won't be handled by Preprocessor::HandleIdentifier because this is coming
741 // from a macro expansion.
742 if (II->isPoisoned() && TokenIsFromPaste) {
743 PP.HandlePoisonedIdentifier(Identifier&: Tok);
744 }
745
746 if (!DisableMacroExpansion && II->isHandleIdentifierCase())
747 return PP.HandleIdentifier(Identifier&: Tok);
748 }
749
750 // Otherwise, return a normal token.
751 return true;
752}
753
754bool TokenLexer::pasteTokens(Token &Tok) {
755 return pasteTokens(LHSTok&: Tok, TokenStream: llvm::ArrayRef(Tokens, NumTokens), CurIdx&: CurTokenIdx);
756}
757
758/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
759/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
760/// are more ## after it, chomp them iteratively. Return the result as LHSTok.
761/// If this returns true, the caller should immediately return the token.
762bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
763 unsigned int &CurIdx) {
764 assert(CurIdx > 0 && "## can not be the first token within tokens");
765 assert((TokenStream[CurIdx].is(tok::hashhash) ||
766 (PP.getLangOpts().MSVCCompat &&
767 isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
768 "Token at this Index must be ## or part of the MSVC 'L "
769 "#macro-arg' pasting pair");
770
771 // MSVC: If previous token was pasted, this must be a recovery from an invalid
772 // paste operation. Ignore spaces before this token to mimic MSVC output.
773 // Required for generating valid UUID strings in some MS headers.
774 if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
775 TokenStream[CurIdx - 2].is(K: tok::hashhash))
776 LHSTok.clearFlag(Flag: Token::LeadingSpace);
777
778 SmallString<128> Buffer;
779 const char *ResultTokStrPtr = nullptr;
780 SourceLocation StartLoc = LHSTok.getLocation();
781 SourceLocation PasteOpLoc;
782 bool HasUCNs = false;
783
784 auto IsAtEnd = [&TokenStream, &CurIdx] {
785 return TokenStream.size() == CurIdx;
786 };
787
788 do {
789 // Consume the ## operator if any.
790 PasteOpLoc = TokenStream[CurIdx].getLocation();
791 if (TokenStream[CurIdx].is(K: tok::hashhash))
792 ++CurIdx;
793 assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
794
795 // Get the RHS token.
796 const Token &RHS = TokenStream[CurIdx];
797
798 // Allocate space for the result token. This is guaranteed to be enough for
799 // the two tokens.
800 Buffer.resize(N: LHSTok.getLength() + RHS.getLength());
801
802 // Get the spelling of the LHS token in Buffer.
803 const char *BufPtr = &Buffer[0];
804 bool Invalid = false;
805 unsigned LHSLen = PP.getSpelling(Tok: LHSTok, Buffer&: BufPtr, Invalid: &Invalid);
806 if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
807 memcpy(dest: &Buffer[0], src: BufPtr, n: LHSLen);
808 if (Invalid)
809 return true;
810
811 BufPtr = Buffer.data() + LHSLen;
812 unsigned RHSLen = PP.getSpelling(Tok: RHS, Buffer&: BufPtr, Invalid: &Invalid);
813 if (Invalid)
814 return true;
815 if (RHSLen && BufPtr != &Buffer[LHSLen])
816 // Really, we want the chars in Buffer!
817 memcpy(dest: &Buffer[LHSLen], src: BufPtr, n: RHSLen);
818
819 // Trim excess space.
820 Buffer.resize(N: LHSLen+RHSLen);
821
822 // Plop the pasted result (including the trailing newline and null) into a
823 // scratch buffer where we can lex it.
824 Token ResultTokTmp;
825 ResultTokTmp.startToken();
826
827 // Claim that the tmp token is a string_literal so that we can get the
828 // character pointer back from CreateString in getLiteralData().
829 ResultTokTmp.setKind(tok::string_literal);
830 PP.CreateString(Str: Buffer, Tok&: ResultTokTmp);
831 SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
832 ResultTokStrPtr = ResultTokTmp.getLiteralData();
833
834 // Lex the resultant pasted token into Result.
835 Token Result;
836
837 if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
838 // Common paste case: identifier+identifier = identifier. Avoid creating
839 // a lexer and other overhead.
840 PP.IncrementPasteCounter(isFast: true);
841 Result.startToken();
842 Result.setKind(tok::raw_identifier);
843 Result.setRawIdentifierData(ResultTokStrPtr);
844 Result.setLocation(ResultTokLoc);
845 Result.setLength(LHSLen+RHSLen);
846 } else {
847 PP.IncrementPasteCounter(isFast: false);
848
849 assert(ResultTokLoc.isFileID() &&
850 "Should be a raw location into scratch buffer");
851 SourceManager &SourceMgr = PP.getSourceManager();
852 FileID LocFileID = SourceMgr.getFileID(SpellingLoc: ResultTokLoc);
853
854 bool Invalid = false;
855 const char *ScratchBufStart
856 = SourceMgr.getBufferData(FID: LocFileID, Invalid: &Invalid).data();
857 if (Invalid)
858 return false;
859
860 // Make a lexer to lex this string from. Lex just this one token.
861 // Make a lexer object so that we lex and expand the paste result.
862 Lexer TL(SourceMgr.getLocForStartOfFile(FID: LocFileID),
863 PP.getLangOpts(), ScratchBufStart,
864 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
865
866 // Lex a token in raw mode. This way it won't look up identifiers
867 // automatically, lexing off the end will return an eof token, and
868 // warnings are disabled. This returns true if the result token is the
869 // entire buffer.
870 bool isInvalid = !TL.LexFromRawLexer(Result);
871
872 // If we got an EOF token, we didn't form even ONE token. For example, we
873 // did "/ ## /" to get "//".
874 isInvalid |= Result.is(K: tok::eof);
875
876 // If pasting the two tokens didn't form a full new token, this is an
877 // error. This occurs with "x ## +" and other stuff. Return with LHSTok
878 // unmodified and with RHS as the next token to lex.
879 if (isInvalid) {
880 // Explicitly convert the token location to have proper expansion
881 // information so that the user knows where it came from.
882 SourceManager &SM = PP.getSourceManager();
883 SourceLocation Loc =
884 SM.createExpansionLoc(SpellingLoc: PasteOpLoc, ExpansionLocStart: ExpandLocStart, ExpansionLocEnd: ExpandLocEnd, Length: 2);
885
886 // Test for the Microsoft extension of /##/ turning into // here on the
887 // error path.
888 if (PP.getLangOpts().MicrosoftExt && LHSTok.is(K: tok::slash) &&
889 RHS.is(K: tok::slash)) {
890 HandleMicrosoftCommentPaste(Tok&: LHSTok, OpLoc: Loc);
891 return true;
892 }
893
894 // Do not emit the error when preprocessing assembler code.
895 if (!PP.getLangOpts().AsmPreprocessor) {
896 // If we're in microsoft extensions mode, downgrade this from a hard
897 // error to an extension that defaults to an error. This allows
898 // disabling it.
899 PP.Diag(Loc, DiagID: PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
900 : diag::err_pp_bad_paste)
901 << Buffer;
902 }
903
904 // An error has occurred so exit loop.
905 break;
906 }
907
908 // Turn ## into 'unknown' to avoid # ## # from looking like a paste
909 // operator.
910 if (Result.is(K: tok::hashhash))
911 Result.setKind(tok::unknown);
912 }
913
914 // Transfer properties of the LHS over the Result.
915 Result.setFlagValue(Flag: Token::StartOfLine , Val: LHSTok.isAtStartOfLine());
916 Result.setFlagValue(Flag: Token::LeadingSpace, Val: LHSTok.hasLeadingSpace());
917
918 // Finally, replace LHS with the result, consume the RHS, and iterate.
919 ++CurIdx;
920
921 // Set Token::HasUCN flag if LHS or RHS contains any UCNs.
922 HasUCNs = LHSTok.hasUCN() || RHS.hasUCN() || HasUCNs;
923 LHSTok = Result;
924 } while (!IsAtEnd() && TokenStream[CurIdx].is(K: tok::hashhash));
925
926 SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
927
928 // The token's current location indicate where the token was lexed from. We
929 // need this information to compute the spelling of the token, but any
930 // diagnostics for the expanded token should appear as if the token was
931 // expanded from the full ## expression. Pull this information together into
932 // a new SourceLocation that captures all of this.
933 SourceManager &SM = PP.getSourceManager();
934 if (StartLoc.isFileID())
935 StartLoc = getExpansionLocForMacroDefLoc(loc: StartLoc);
936 if (EndLoc.isFileID())
937 EndLoc = getExpansionLocForMacroDefLoc(loc: EndLoc);
938 FileID MacroFID = SM.getFileID(SpellingLoc: MacroExpansionStart);
939 while (SM.getFileID(SpellingLoc: StartLoc) != MacroFID)
940 StartLoc = SM.getImmediateExpansionRange(Loc: StartLoc).getBegin();
941 while (SM.getFileID(SpellingLoc: EndLoc) != MacroFID)
942 EndLoc = SM.getImmediateExpansionRange(Loc: EndLoc).getEnd();
943
944 LHSTok.setLocation(SM.createExpansionLoc(SpellingLoc: LHSTok.getLocation(), ExpansionLocStart: StartLoc, ExpansionLocEnd: EndLoc,
945 Length: LHSTok.getLength()));
946
947 // Now that we got the result token, it will be subject to expansion. Since
948 // token pasting re-lexes the result token in raw mode, identifier information
949 // isn't looked up. As such, if the result is an identifier, look up id info.
950 if (LHSTok.is(K: tok::raw_identifier)) {
951
952 // If there has any UNCs in concated token, we should mark this token
953 // with Token::HasUCN flag, then LookUpIdentifierInfo will expand UCNs in
954 // token.
955 if (HasUCNs)
956 LHSTok.setFlag(Token::HasUCN);
957
958 // Look up the identifier info for the token. We disabled identifier lookup
959 // by saying we're skipping contents, so we need to do this manually.
960 PP.LookUpIdentifierInfo(Identifier&: LHSTok);
961 }
962 return false;
963}
964
965/// isNextTokenLParen - If the next token lexed will pop this macro off the
966/// expansion stack, return std::nullopt, otherwise return the next unexpanded
967/// token.
968std::optional<Token> TokenLexer::peekNextPPToken() const {
969 // Out of tokens?
970 if (isAtEnd())
971 return std::nullopt;
972 return Tokens[CurTokenIdx];
973}
974
975/// isParsingPreprocessorDirective - Return true if we are in the middle of a
976/// preprocessor directive.
977bool TokenLexer::isParsingPreprocessorDirective() const {
978 return Tokens[NumTokens-1].is(K: tok::eod) && !isAtEnd();
979}
980
981/// setLexingCXXModuleDirective - This is set to true if this TokenLexer is
982/// created when handling C++ module directive.
983void TokenLexer::setLexingCXXModuleDirective(bool Val) {
984 LexingCXXModuleDirective = Val;
985}
986
987/// isLexingCXXModuleDirective - Return true if we are lexing a C++ module or
988/// import directive.
989bool TokenLexer::isLexingCXXModuleDirective() const {
990 return LexingCXXModuleDirective;
991}
992
993/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
994/// together to form a comment that comments out everything in the current
995/// macro, other active macros, and anything left on the current physical
996/// source line of the expanded buffer. Handle this by returning the
997/// first token on the next line.
998void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
999 PP.Diag(Loc: OpLoc, DiagID: diag::ext_comment_paste_microsoft);
1000
1001 // We 'comment out' the rest of this macro by just ignoring the rest of the
1002 // tokens that have not been lexed yet, if any.
1003
1004 // Since this must be a macro, mark the macro enabled now that it is no longer
1005 // being expanded.
1006 assert(Macro && "Token streams can't paste comments");
1007 Macro->EnableMacro();
1008
1009 PP.HandleMicrosoftCommentPaste(Tok);
1010}
1011
1012/// If \arg loc is a file ID and points inside the current macro
1013/// definition, returns the appropriate source location pointing at the
1014/// macro expansion source location entry, otherwise it returns an invalid
1015/// SourceLocation.
1016SourceLocation
1017TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
1018 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
1019 "Not appropriate for token streams");
1020 assert(loc.isValid() && loc.isFileID());
1021
1022 SourceManager &SM = PP.getSourceManager();
1023 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
1024 "Expected loc to come from the macro definition");
1025
1026 SourceLocation::UIntTy relativeOffset = 0;
1027 SM.isInSLocAddrSpace(Loc: loc, Start: MacroDefStart, Length: MacroDefLength, RelativeOffset: &relativeOffset);
1028 return MacroExpansionStart.getLocWithOffset(Offset: relativeOffset);
1029}
1030
1031/// Finds the tokens that are consecutive (from the same FileID)
1032/// creates a single SLocEntry, and assigns SourceLocations to each token that
1033/// point to that SLocEntry. e.g for
1034/// assert(foo == bar);
1035/// There will be a single SLocEntry for the "foo == bar" chunk and locations
1036/// for the 'foo', '==', 'bar' tokens will point inside that chunk.
1037///
1038/// \arg begin_tokens will be updated to a position past all the found
1039/// consecutive tokens.
1040static void updateConsecutiveMacroArgTokens(SourceManager &SM,
1041 SourceLocation ExpandLoc,
1042 Token *&begin_tokens,
1043 Token * end_tokens) {
1044 assert(begin_tokens + 1 < end_tokens);
1045 SourceLocation BeginLoc = begin_tokens->getLocation();
1046 llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens);
1047 llvm::MutableArrayRef<Token> Partition;
1048
1049 auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable {
1050 // The maximum distance between two consecutive tokens in a partition.
1051 // This is an important trick to avoid using too much SourceLocation address
1052 // space!
1053 static constexpr SourceLocation::IntTy MaxDistance = 50;
1054 auto Distance = Loc.getRawEncoding() - Last.getRawEncoding();
1055 Last = Loc;
1056 return Distance <= MaxDistance;
1057 };
1058
1059 // Partition the tokens by their FileID.
1060 // This is a hot function, and calling getFileID can be expensive, the
1061 // implementation is optimized by reducing the number of getFileID.
1062 if (BeginLoc.isFileID()) {
1063 // Consecutive tokens not written in macros must be from the same file.
1064 // (Neither #include nor eof can occur inside a macro argument.)
1065 Partition = All.take_while(Pred: [&](const Token &T) {
1066 return T.getLocation().isFileID() && NearLast(T.getLocation());
1067 });
1068 } else {
1069 // Call getFileID once to calculate the bounds, and use the cheaper
1070 // sourcelocation-against-bounds comparison.
1071 FileID BeginFID = SM.getFileID(SpellingLoc: BeginLoc);
1072 SourceLocation Limit =
1073 SM.getComposedLoc(FID: BeginFID, Offset: SM.getFileIDSize(FID: BeginFID));
1074 Partition = All.take_while(Pred: [&](const Token &T) {
1075 // NOTE: the Limit is included! The lexer recovery only ever inserts a
1076 // single token past the end of the FileID, specifically the ) when a
1077 // macro-arg containing a comma should be guarded by parentheses.
1078 //
1079 // It is safe to include the Limit here because SourceManager allocates
1080 // FileSize + 1 for each SLocEntry.
1081 //
1082 // See https://github.com/llvm/llvm-project/issues/60722.
1083 return T.getLocation() >= BeginLoc && T.getLocation() <= Limit
1084 && NearLast(T.getLocation());
1085 });
1086 }
1087 assert(!Partition.empty());
1088
1089 // For the consecutive tokens, find the length of the SLocEntry to contain
1090 // all of them.
1091 SourceLocation::UIntTy FullLength =
1092 Partition.back().getEndLoc().getRawEncoding() -
1093 Partition.front().getLocation().getRawEncoding();
1094 // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1095 SourceLocation Expansion =
1096 SM.createMacroArgExpansionLoc(SpellingLoc: BeginLoc, ExpansionLoc: ExpandLoc, Length: FullLength);
1097
1098#ifdef EXPENSIVE_CHECKS
1099 assert(llvm::all_of(Partition.drop_front(),
1100 [&SM, ID = SM.getFileID(Partition.front().getLocation())](
1101 const Token &T) {
1102 return ID == SM.getFileID(T.getLocation());
1103 }) &&
1104 "Must have the same FIleID!");
1105#endif
1106 // Change the location of the tokens from the spelling location to the new
1107 // expanded location.
1108 for (Token& T : Partition) {
1109 SourceLocation::IntTy RelativeOffset =
1110 T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();
1111 T.setLocation(Expansion.getLocWithOffset(Offset: RelativeOffset));
1112 }
1113 begin_tokens = &Partition.back() + 1;
1114}
1115
1116/// Creates SLocEntries and updates the locations of macro argument
1117/// tokens to their new expanded locations.
1118///
1119/// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1120/// definition.
1121void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1122 Token *begin_tokens,
1123 Token *end_tokens) {
1124 SourceManager &SM = PP.getSourceManager();
1125
1126 SourceLocation ExpandLoc =
1127 getExpansionLocForMacroDefLoc(loc: ArgIdSpellLoc);
1128
1129 while (begin_tokens < end_tokens) {
1130 // If there's only one token just create a SLocEntry for it.
1131 if (end_tokens - begin_tokens == 1) {
1132 Token &Tok = *begin_tokens;
1133 Tok.setLocation(SM.createMacroArgExpansionLoc(SpellingLoc: Tok.getLocation(),
1134 ExpansionLoc: ExpandLoc,
1135 Length: Tok.getLength()));
1136 return;
1137 }
1138
1139 updateConsecutiveMacroArgTokens(SM, ExpandLoc, begin_tokens, end_tokens);
1140 }
1141}
1142
1143void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1144 AtStartOfLine = Result.isAtStartOfLine();
1145 HasLeadingSpace = Result.hasLeadingSpace();
1146}
1147