1//===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class which performs incremental code compilation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "IncrementalParser.h"
14#include "IncrementalAction.h"
15
16#include "clang/AST/ASTContext.h"
17#include "clang/AST/Decl.h"
18#include "clang/AST/DeclCXX.h"
19#include "clang/AST/DeclContextInternals.h"
20#include "clang/AST/DeclTemplate.h"
21#include "clang/AST/DeclVisitor.h"
22#include "clang/Frontend/CompilerInstance.h"
23#include "clang/Interpreter/PartialTranslationUnit.h"
24#include "clang/Parse/Parser.h"
25#include "clang/Sema/Sema.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/CrashRecoveryContext.h"
29#include "llvm/Support/Error.h"
30
31#include <sstream>
32
33#define DEBUG_TYPE "clang-repl"
34
35namespace clang {
36
37// IncrementalParser::IncrementalParser() {}
38
39IncrementalParser::IncrementalParser(CompilerInstance &Instance,
40 IncrementalAction *Act, llvm::Error &Err,
41 std::list<PartialTranslationUnit> &PTUs)
42 : S(Instance.getSema()), Act(Act), PTUs(PTUs) {
43 llvm::ErrorAsOutParameter EAO(&Err);
44 Consumer = &S.getASTConsumer();
45 P.reset(p: new Parser(S.getPreprocessor(), S, /*SkipBodies=*/false));
46
47 if (ExternalASTSource *External = S.getASTContext().getExternalSource())
48 External->StartTranslationUnit(Consumer);
49
50 P->Initialize();
51}
52
53IncrementalParser::~IncrementalParser() { P.reset(); }
54
55llvm::Expected<TranslationUnitDecl *>
56IncrementalParser::ParseOrWrapTopLevelDecl() {
57 // Recover resources if we crash before exiting this method.
58 llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
59 Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true,
60 /*AtEndOfTU=*/true);
61 Sema::LocalEagerInstantiationScope LocalInstantiations(S, /*AtEndOfTU=*/true);
62
63 // Add a new PTU.
64 ASTContext &C = S.getASTContext();
65 C.addTranslationUnitDecl();
66
67 // Skip previous eof due to last incremental input.
68 if (P->getCurToken().is(K: tok::eof)) {
69 P->ConsumeAnyToken();
70 // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
71 // might want to do that around HandleEndOfTranslationUnit.
72 P->ExitScope();
73 S.CurContext = nullptr;
74 // Start a new PTU.
75 P->EnterScope(ScopeFlags: Scope::DeclScope);
76 S.ActOnTranslationUnitScope(S: P->getCurScope());
77 }
78
79 Parser::DeclGroupPtrTy ADecl;
80 Sema::ModuleImportState ImportState;
81 for (bool AtEOF = P->ParseFirstTopLevelDecl(Result&: ADecl, ImportState); !AtEOF;
82 AtEOF = P->ParseTopLevelDecl(Result&: ADecl, ImportState)) {
83 if (ADecl && !Consumer->HandleTopLevelDecl(D: ADecl.get()))
84 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
85 "The consumer rejected a decl",
86 Args: std::error_code());
87 }
88
89 DiagnosticsEngine &Diags = S.getDiagnostics();
90 if (Diags.hasErrorOccurred()) {
91 CleanUpPTU(MostRecentTU: C.getTranslationUnitDecl());
92
93 Diags.Reset(/*soft=*/true);
94 Diags.getClient()->clear();
95 return llvm::make_error<llvm::StringError>(Args: "Parsing failed.",
96 Args: std::error_code());
97 }
98
99 // Process any TopLevelDecls generated by #pragma weak.
100 for (Decl *D : S.WeakTopLevelDecls()) {
101 DeclGroupRef DGR(D);
102 Consumer->HandleTopLevelDecl(D: DGR);
103 }
104
105 LocalInstantiations.perform();
106 GlobalInstantiations.perform();
107
108 Consumer->HandleTranslationUnit(Ctx&: C);
109
110 return C.getTranslationUnitDecl();
111}
112
113llvm::Expected<TranslationUnitDecl *>
114IncrementalParser::Parse(llvm::StringRef input) {
115 Preprocessor &PP = S.getPreprocessor();
116 assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
117
118 std::ostringstream SourceName;
119 SourceName << "input_line_" << InputCount++;
120
121 // Create an uninitialized memory buffer, copy code in and append "\n"
122 size_t InputSize = input.size(); // don't include trailing 0
123 // MemBuffer size should *not* include terminating zero
124 std::unique_ptr<llvm::MemoryBuffer> MB(
125 llvm::WritableMemoryBuffer::getNewUninitMemBuffer(Size: InputSize + 1,
126 BufferName: SourceName.str()));
127 char *MBStart = const_cast<char *>(MB->getBufferStart());
128 memcpy(dest: MBStart, src: input.data(), n: InputSize);
129 MBStart[InputSize] = '\n';
130
131 SourceManager &SM = S.getSourceManager();
132
133 // FIXME: Create SourceLocation, which will allow clang to order the overload
134 // candidates for example
135 SourceLocation NewLoc = SM.getLocForStartOfFile(FID: SM.getMainFileID());
136
137 // Create FileID for the current buffer.
138 FileID FID;
139 // Create FileEntry and FileID for the current buffer.
140 FileEntryRef FE = SM.getFileManager().getVirtualFileRef(
141 Filename: SourceName.str(), Size: InputSize, ModificationTime: 0 /* mod time*/);
142 SM.overrideFileContents(SourceFile: FE, Buffer: std::move(MB));
143
144 // Ensure HeaderFileInfo exists before lookup to prevent assertion
145 HeaderSearch &HS = PP.getHeaderSearchInfo();
146 HS.getFileInfo(FE);
147
148 FID = SM.createFileID(SourceFile: FE, IncludePos: NewLoc, FileCharacter: SrcMgr::C_User);
149
150 // NewLoc only used for diags.
151 if (PP.EnterSourceFile(FID, /*DirLookup=*/Dir: nullptr, Loc: NewLoc))
152 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
153 "Cannot enter source file.",
154 Args: std::error_code());
155
156 auto PTU = ParseOrWrapTopLevelDecl();
157 if (!PTU)
158 return PTU.takeError();
159
160 if (PP.getLangOpts().DelayedTemplateParsing) {
161 // Microsoft-specific:
162 // Late parsed templates can leave unswallowed "macro"-like tokens.
163 // They will seriously confuse the Parser when entering the next
164 // source file. So lex until we are EOF.
165 Token Tok;
166 do {
167 PP.Lex(Result&: Tok);
168 } while (Tok.isNot(K: tok::eof));
169 } else {
170 Token AssertTok;
171 PP.Lex(Result&: AssertTok);
172 assert(AssertTok.is(tok::eof) &&
173 "Lexer must be EOF when starting incremental parse!");
174 }
175
176 return PTU;
177}
178
179void IncrementalParser::withdrawMostRecentTU(
180 TranslationUnitDecl *MostRecentTU) {
181 TranslationUnitDecl *Prev = MostRecentTU->getPreviousDecl();
182 if (!Prev)
183 return;
184 assert(MostRecentTU->getMostRecentDecl() == MostRecentTU &&
185 "Not the most recent translation unit!");
186
187 // Rebuild A -> ... -> Prev -> MostRecentTU as A -> ... -> Prev.
188 MostRecentTU->getFirstDecl()->RedeclLink.setLatest(Prev);
189
190 // getTranslationUnitDecl() requires the active unit to be the latest one.
191 ASTContext &C = S.getASTContext();
192 if (C.TraversalScope.size() == 1 && C.TraversalScope.back() == MostRecentTU)
193 C.TraversalScope = {Prev};
194 C.TUDecl = Prev;
195}
196
197/// Removes decls introduced in the discarding PTU and restores the
198/// redeclaration chain to previous state.
199class ASTDeclUnmerger : public DeclVisitor<ASTDeclUnmerger> {
200 Sema &S;
201 TranslationUnitDecl *DiscardedTU;
202
203 /// The newest declaration of whatever D redeclares that still lives outside
204 /// the DiscardedTU, or null if DiscardedTU introduced the name.
205 NamedDecl *findSurvivor(NamedDecl *D) const {
206 for (Decl *Prev = D->getPreviousDecl(); Prev;
207 Prev = Prev->getPreviousDecl())
208 if (Prev->getTranslationUnitDecl() != DiscardedTU)
209 return dyn_cast<NamedDecl>(Val: Prev);
210 return nullptr;
211 }
212
213 template <typename DeclT>
214 void unlinkRedeclChain(ASTContext &C, Redeclarable<DeclT> *DBase,
215 NamedDecl *PrevND) {
216 auto *Latest = static_cast<DeclT *>(DBase);
217 auto *Survivor = cast<DeclT>(PrevND);
218
219 // Rebuild First -> ... -> Survivor -> ... -> Latest as
220 // First -> ... -> Survivor.
221 Latest->getFirstDecl()->RedeclLink.setLatest(Survivor);
222
223 // The chain is circular: a withdrawn declaration still linked into it can
224 // never walk back around to itself, so redecls() on one would not
225 // terminate. Give each withdrawn declaration a chain of its own.
226 for (DeclT *Dead = Latest; Dead != Survivor;) {
227 DeclT *Next = Dead->getPreviousDecl();
228 Dead->First = Dead;
229 Dead->RedeclLink = Redeclarable<DeclT>::LatestDeclLink(C);
230 Dead = Next;
231 }
232 }
233
234 /// Remove entry from "C"'s lookup tables
235 void removeFromLookups(NamedDecl *D) {
236 if (D->getDeclName().isEmpty())
237 return;
238
239 if (D->getDeclName().isIdentifier() && D->getDeclName().getFETokenInfo() &&
240 !D->getLangOpts().ObjC && !D->getLangOpts().CPlusPlus)
241 S.IdResolver.RemoveDecl(D);
242
243 ExternCContextDecl *ECCD = S.getASTContext().getExternCContextDecl();
244 if (StoredDeclsMap *Map = ECCD->getPrimaryContext()->getLookupPtr()) {
245 auto It = Map->find(Val: D->getDeclName());
246 if (It != Map->end())
247 It->second.remove(D);
248 }
249 }
250
251 /// Remove Decls defined in this DC from the lookup table
252 /// and restore the redeclaration chain to previous state
253 void VisitDeclContext(DeclContext *DC) {
254 llvm::SmallVector<Decl *, 8> Members(DC->decls());
255 for (Decl *M : Members) {
256 Visit(D: M); // restore redecls and lookup
257 // VisitNamedDecl() removes the named decls.
258 if (!isa<NamedDecl>(Val: M))
259 DC->removeDecl(D: M);
260 }
261 }
262
263public:
264 ASTDeclUnmerger(Sema &S, TranslationUnitDecl *DiscardedTU)
265 : S(S), DiscardedTU(DiscardedTU) {}
266
267 void VisitDecl(Decl *D) {
268 if (auto *DC = dyn_cast<DeclContext>(Val: D))
269 VisitDeclContext(DC);
270 }
271
272 /// Remove ND from its context and the lookup tables, and make the
273 /// declaration it redeclares visible again. Called by VisitRedeclarable()
274 /// before it restores the redeclaration chain, which hides that declaration.
275 void VisitNamedDecl(NamedDecl *ND) {
276 // The templated decl of a template is not a member of its context: the
277 // template is.
278 DeclContext *DC = ND->getLexicalDeclContext();
279 if (!DC->containsDecl(D: ND))
280 return;
281 NamedDecl *Prev = findSurvivor(D: ND);
282 DC->removeDecl(D: ND);
283 removeFromLookups(D: ND);
284 // Like removeDecl(), use the semantic context: the out-of-line definition
285 // of a member is in DC, but it was removed from the lookup table of its
286 // class or namespace.
287 if (Prev)
288 Prev->getDeclContext()->makeDeclVisibleInContext(D: Prev);
289 }
290
291 /// Remove D like VisitNamedDecl() and restore its redeclaration chain to the
292 /// previous state. Returns the surviving previous declaration, if any.
293 /// DeclVisitor doesn't dispatch to it: Redeclarable is not a Decl class.
294 template <typename DeclT>
295 NamedDecl *VisitRedeclarable(Redeclarable<DeclT> *DBase) {
296 auto *D = static_cast<DeclT *>(DBase);
297 VisitNamedDecl(ND: D);
298 NamedDecl *Prev = findSurvivor(D);
299 if (Prev)
300 unlinkRedeclChain(S.getASTContext(), DBase, Prev);
301 return Prev;
302 }
303
304 void VisitFunctionDecl(FunctionDecl *D) { VisitRedeclarable(DBase: D); }
305 void VisitNamespaceAliasDecl(NamespaceAliasDecl *D) { VisitRedeclarable(DBase: D); }
306 void VisitTypedefNameDecl(TypedefNameDecl *D) { VisitRedeclarable(DBase: D); }
307 void VisitUsingShadowDecl(UsingShadowDecl *D) { VisitRedeclarable(DBase: D); }
308 void VisitVarDecl(VarDecl *D) { VisitRedeclarable(DBase: D); }
309
310 void VisitTagDecl(TagDecl *D) {
311 // Some members of a definition are visible outside of it: the enumerators
312 // of an unscoped enumeration, and in C the tags and enumerators declared
313 // in a structure or union.
314 if (D->isThisDeclarationADefinition())
315 VisitDeclContext(DC: D);
316
317 NamedDecl *Prev = VisitRedeclarable(DBase: D);
318
319 // A class definition is kept in DefinitionData outside the
320 // redeclaration chain
321 auto *RD = dyn_cast_if_present<CXXRecordDecl>(Val: Prev);
322 if (!RD)
323 return;
324 if (CXXRecordDecl *Def = RD->getDefinition();
325 Def && Def->getTranslationUnitDecl() == DiscardedTU)
326 for (auto *R : RD->redecls())
327 cast<CXXRecordDecl>(Val: R)->DefinitionData = nullptr;
328 }
329
330 void VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
331 VisitRedeclarable(DBase: D);
332 Visit(D: D->getTemplatedDecl());
333 }
334
335 void VisitNamespaceDecl(NamespaceDecl *D) {
336 // Handle cases of nested redeclarations like:
337 // PTU1: namespace outer { namespace ns { class Foo; } }
338 // PTU2: namespace outer { namespace ns { class Foo { ... }; error; } }
339 // Foo's redeclaration needs to be restored
340 VisitDeclContext(DC: D);
341 VisitRedeclarable(DBase: D);
342 }
343
344 void VisitTranslationUnitDecl(TranslationUnitDecl *D) { VisitDeclContext(DC: D); }
345};
346
347void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
348 ASTDeclUnmerger(S, MostRecentTU).Visit(D: MostRecentTU);
349
350 // Lookup alone is not enough: the redeclaration chain still reaches these.
351 withdrawMostRecentTU(MostRecentTU);
352}
353
354PartialTranslationUnit &
355IncrementalParser::RegisterPTU(TranslationUnitDecl *TU,
356 std::unique_ptr<llvm::Module> M /*={}*/) {
357 PTUs.emplace_back(args: PartialTranslationUnit());
358 PartialTranslationUnit &LastPTU = PTUs.back();
359 LastPTU.TUPart = TU;
360
361 if (!M)
362 M = Act->GenModule();
363
364 assert((!Act->getCodeGen() || M) && "Must have a llvm::Module at this point");
365
366 LastPTU.TheModule = std::move(M);
367 LLVM_DEBUG(llvm::dbgs() << "compile-ptu " << PTUs.size() - 1
368 << ": [TU=" << LastPTU.TUPart);
369 if (LastPTU.TheModule)
370 LLVM_DEBUG(llvm::dbgs() << ", M=" << LastPTU.TheModule.get() << " ("
371 << LastPTU.TheModule->getName() << ")");
372 LLVM_DEBUG(llvm::dbgs() << "]\n");
373 return LastPTU;
374}
375} // end namespace clang
376