1//===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class which performs incremental code compilation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "IncrementalParser.h"
14#include "IncrementalAction.h"
15
16#include "clang/AST/ASTContext.h"
17#include "clang/AST/Decl.h"
18#include "clang/AST/DeclCXX.h"
19#include "clang/AST/DeclContextInternals.h"
20#include "clang/AST/DeclTemplate.h"
21#include "clang/AST/DeclVisitor.h"
22#include "clang/Frontend/CompilerInstance.h"
23#include "clang/Interpreter/PartialTranslationUnit.h"
24#include "clang/Parse/Parser.h"
25#include "clang/Sema/Sema.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/CrashRecoveryContext.h"
29#include "llvm/Support/Error.h"
30
31#include <sstream>
32
33#define DEBUG_TYPE "clang-repl"
34
35namespace clang {
36
37// IncrementalParser::IncrementalParser() {}
38
39IncrementalParser::IncrementalParser(CompilerInstance &Instance,
40 IncrementalAction *Act, llvm::Error &Err,
41 std::list<PartialTranslationUnit> &PTUs)
42 : S(Instance.getSema()), Act(Act), PTUs(PTUs) {
43 llvm::ErrorAsOutParameter EAO(&Err);
44 Consumer = &S.getASTConsumer();
45 P.reset(p: new Parser(S.getPreprocessor(), S, /*SkipBodies=*/false));
46
47 if (ExternalASTSource *External = S.getASTContext().getExternalSource())
48 External->StartTranslationUnit(Consumer);
49
50 P->Initialize();
51}
52
53IncrementalParser::~IncrementalParser() { P.reset(); }
54
55llvm::Expected<TranslationUnitDecl *>
56IncrementalParser::ParseOrWrapTopLevelDecl() {
57 // Recover resources if we crash before exiting this method.
58 llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
59 Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true,
60 /*AtEndOfTU=*/true);
61 Sema::LocalEagerInstantiationScope LocalInstantiations(S, /*AtEndOfTU=*/true);
62
63 // Add a new PTU.
64 ASTContext &C = S.getASTContext();
65 C.addTranslationUnitDecl();
66
67 // Skip previous eof due to last incremental input.
68 if (P->getCurToken().is(K: tok::annot_repl_input_end)) {
69 P->ConsumeAnyToken();
70 // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
71 // might want to do that around HandleEndOfTranslationUnit.
72 P->ExitScope();
73 S.CurContext = nullptr;
74 // Start a new PTU.
75 P->EnterScope(ScopeFlags: Scope::DeclScope);
76 S.ActOnTranslationUnitScope(S: P->getCurScope());
77 }
78
79 Parser::DeclGroupPtrTy ADecl;
80 Sema::ModuleImportState ImportState;
81 for (bool AtEOF = P->ParseFirstTopLevelDecl(Result&: ADecl, ImportState); !AtEOF;
82 AtEOF = P->ParseTopLevelDecl(Result&: ADecl, ImportState)) {
83 if (ADecl && !Consumer->HandleTopLevelDecl(D: ADecl.get()))
84 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
85 "The consumer rejected a decl",
86 Args: std::error_code());
87 }
88
89 DiagnosticsEngine &Diags = S.getDiagnostics();
90 if (Diags.hasErrorOccurred()) {
91 CleanUpPTU(MostRecentTU: C.getTranslationUnitDecl());
92
93 Diags.Reset(/*soft=*/true);
94 Diags.getClient()->clear();
95 return llvm::make_error<llvm::StringError>(Args: "Parsing failed.",
96 Args: std::error_code());
97 }
98
99 // Process any TopLevelDecls generated by #pragma weak.
100 for (Decl *D : S.WeakTopLevelDecls()) {
101 DeclGroupRef DGR(D);
102 Consumer->HandleTopLevelDecl(D: DGR);
103 }
104
105 LocalInstantiations.perform();
106 GlobalInstantiations.perform();
107
108 Consumer->HandleTranslationUnit(Ctx&: C);
109
110 return C.getTranslationUnitDecl();
111}
112
113llvm::Expected<TranslationUnitDecl *>
114IncrementalParser::Parse(llvm::StringRef input) {
115 Preprocessor &PP = S.getPreprocessor();
116 assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
117
118 std::ostringstream SourceName;
119 SourceName << "input_line_" << InputCount++;
120
121 // Create an uninitialized memory buffer, copy code in and append "\n"
122 size_t InputSize = input.size(); // don't include trailing 0
123 // MemBuffer size should *not* include terminating zero
124 std::unique_ptr<llvm::MemoryBuffer> MB(
125 llvm::WritableMemoryBuffer::getNewUninitMemBuffer(Size: InputSize + 1,
126 BufferName: SourceName.str()));
127 char *MBStart = const_cast<char *>(MB->getBufferStart());
128 memcpy(dest: MBStart, src: input.data(), n: InputSize);
129 MBStart[InputSize] = '\n';
130
131 SourceManager &SM = S.getSourceManager();
132
133 // FIXME: Create SourceLocation, which will allow clang to order the overload
134 // candidates for example
135 SourceLocation NewLoc = SM.getLocForStartOfFile(FID: SM.getMainFileID());
136
137 // Create FileID for the current buffer.
138 FileID FID;
139 // Create FileEntry and FileID for the current buffer.
140 FileEntryRef FE = SM.getFileManager().getVirtualFileRef(
141 Filename: SourceName.str(), Size: InputSize, ModificationTime: 0 /* mod time*/);
142 SM.overrideFileContents(SourceFile: FE, Buffer: std::move(MB));
143
144 // Ensure HeaderFileInfo exists before lookup to prevent assertion
145 HeaderSearch &HS = PP.getHeaderSearchInfo();
146 HS.getFileInfo(FE);
147
148 FID = SM.createFileID(SourceFile: FE, IncludePos: NewLoc, FileCharacter: SrcMgr::C_User);
149
150 // NewLoc only used for diags.
151 if (PP.EnterSourceFile(FID, /*DirLookup=*/Dir: nullptr, Loc: NewLoc))
152 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
153 "Cannot enter source file.",
154 Args: std::error_code());
155
156 auto PTU = ParseOrWrapTopLevelDecl();
157 if (!PTU)
158 return PTU.takeError();
159
160 if (PP.getLangOpts().DelayedTemplateParsing) {
161 // Microsoft-specific:
162 // Late parsed templates can leave unswallowed "macro"-like tokens.
163 // They will seriously confuse the Parser when entering the next
164 // source file. So lex until we are EOF.
165 Token Tok;
166 do {
167 PP.Lex(Result&: Tok);
168 } while (Tok.isNot(K: tok::annot_repl_input_end));
169 } else {
170 Token AssertTok;
171 PP.Lex(Result&: AssertTok);
172 assert(AssertTok.is(tok::annot_repl_input_end) &&
173 "Lexer must be EOF when starting incremental parse!");
174 }
175
176 return PTU;
177}
178
179void IncrementalParser::withdrawMostRecentTU(
180 TranslationUnitDecl *MostRecentTU) {
181 TranslationUnitDecl *Prev = MostRecentTU->getPreviousDecl();
182 if (!Prev)
183 return;
184 assert(MostRecentTU->getMostRecentDecl() == MostRecentTU &&
185 "Not the most recent translation unit!");
186
187 // Rebuild A -> ... -> Prev -> MostRecentTU as A -> ... -> Prev.
188 MostRecentTU->getFirstDecl()->RedeclLink.setLatest(Prev);
189
190 // getTranslationUnitDecl() requires the active unit to be the latest one.
191 ASTContext &C = S.getASTContext();
192 if (C.TraversalScope.size() == 1 && C.TraversalScope.back() == MostRecentTU)
193 C.TraversalScope = {Prev};
194 C.TUDecl = Prev;
195}
196
197/// Removes decls introduced in the discarding PTU and restores the
198/// redeclaration chain to previous state.
199class ASTDeclUnmerger : public DeclVisitor<ASTDeclUnmerger> {
200 Sema &S;
201 TranslationUnitDecl *DiscardedTU;
202
203 template <typename DeclT> void withdraw(Redeclarable<DeclT> *DBase) {
204 if (NamedDecl *Prev = findSurvivor(D: static_cast<DeclT *>(DBase)))
205 unlinkRedeclChain(S.getASTContext(), DBase, Prev);
206 }
207
208 /// The newest declaration of whatever D redeclares that still lives outside
209 /// the DiscardedTU, or null if DiscardedTU introduced the name.
210 NamedDecl *findSurvivor(NamedDecl *D) const {
211 for (Decl *Prev = D->getPreviousDecl(); Prev;
212 Prev = Prev->getPreviousDecl())
213 if (Prev->getTranslationUnitDecl() != DiscardedTU)
214 return dyn_cast<NamedDecl>(Val: Prev);
215 return nullptr;
216 }
217
218 template <typename DeclT>
219 void unlinkRedeclChain(ASTContext &C, Redeclarable<DeclT> *DBase,
220 NamedDecl *PrevND) {
221 auto *Latest = static_cast<DeclT *>(DBase);
222 auto *Survivor = cast<DeclT>(PrevND);
223
224 // Rebuild First -> ... -> Survivor -> ... -> Latest as
225 // First -> ... -> Survivor.
226 Latest->getFirstDecl()->RedeclLink.setLatest(Survivor);
227
228 // The chain is circular: a withdrawn declaration still linked into it can
229 // never walk back around to itself, so redecls() on one would not
230 // terminate. Give each withdrawn declaration a chain of its own.
231 for (DeclT *Dead = Latest; Dead != Survivor;) {
232 DeclT *Next = Dead->getPreviousDecl();
233 Dead->First = Dead;
234 Dead->RedeclLink = Redeclarable<DeclT>::LatestDeclLink(C);
235 Dead = Next;
236 }
237 }
238
239 /// Remove entry from "C"'s lookup tables
240 void removeFromLookups(NamedDecl *D) {
241 if (D->getDeclName().isEmpty())
242 return;
243
244 if (D->getDeclName().isIdentifier() && D->getDeclName().getFETokenInfo() &&
245 !D->getLangOpts().ObjC && !D->getLangOpts().CPlusPlus)
246 S.IdResolver.RemoveDecl(D);
247
248 ExternCContextDecl *ECCD = S.getASTContext().getExternCContextDecl();
249 if (StoredDeclsMap *Map = ECCD->getPrimaryContext()->getLookupPtr()) {
250 auto It = Map->find(Val: D->getDeclName());
251 if (It != Map->end())
252 It->second.remove(D);
253 }
254 }
255
256 /// Remove Decls defined in this DC from the lookup table
257 /// and restore the redeclaration chain to previous state
258 void VisitDeclContext(DeclContext *DC) {
259 llvm::SmallVector<Decl *, 8> Members(DC->decls());
260 llvm::SmallVector<NamedDecl *, 8> Survivors;
261 for (Decl *M : Members) {
262 if (auto *ND = dyn_cast<NamedDecl>(Val: M))
263 if (NamedDecl *Prev = findSurvivor(D: ND))
264 Survivors.push_back(Elt: Prev);
265 Visit(D: M); // restore redecls
266 DC->removeDecl(D: M); // remove from lookup
267 if (auto *ND = dyn_cast<NamedDecl>(Val: M))
268 removeFromLookups(D: ND);
269 }
270
271 // Restore lookup for the surviving predecessor
272 // of any removed decl that had a surviving predecessor
273 DeclContext *Primary = DC->getPrimaryContext();
274 for (NamedDecl *Prev : Survivors)
275 Primary->makeDeclVisibleInContext(D: Prev);
276 }
277
278public:
279 ASTDeclUnmerger(Sema &S, TranslationUnitDecl *DiscardedTU)
280 : S(S), DiscardedTU(DiscardedTU) {}
281
282 void VisitDecl(Decl *D) {
283 if (auto *DC = dyn_cast<DeclContext>(Val: D))
284 VisitDeclContext(DC);
285 }
286
287 void VisitFunctionDecl(FunctionDecl *D) { withdraw(DBase: D); }
288 void VisitNamespaceAliasDecl(NamespaceAliasDecl *D) { withdraw(DBase: D); }
289 void VisitTypedefNameDecl(TypedefNameDecl *D) { withdraw(DBase: D); }
290 void VisitUsingShadowDecl(UsingShadowDecl *D) { withdraw(DBase: D); }
291 void VisitVarDecl(VarDecl *D) { withdraw(DBase: D); }
292
293 void VisitTagDecl(TagDecl *D) {
294 NamedDecl *Prev = findSurvivor(D);
295 if (!Prev)
296 return;
297 unlinkRedeclChain(C&: S.getASTContext(), DBase: D, PrevND: Prev);
298
299 // A class definition is kept in DefinitionData outside the
300 // redeclaration chain
301 auto *RD = dyn_cast<CXXRecordDecl>(Val: Prev);
302 if (!RD)
303 return;
304 if (CXXRecordDecl *Def = RD->getDefinition();
305 Def && Def->getTranslationUnitDecl() == DiscardedTU)
306 for (auto *R : RD->redecls())
307 cast<CXXRecordDecl>(Val: R)->DefinitionData = nullptr;
308 }
309
310 void VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
311 withdraw(DBase: D);
312 Visit(D: D->getTemplatedDecl());
313 }
314
315 void VisitNamespaceDecl(NamespaceDecl *D) {
316 // Handle cases of nested redeclarations like:
317 // PTU1: namespace outer { namespace ns { class Foo; } }
318 // PTU2: namespace outer { namespace ns { class Foo { ... }; error; } }
319 // Foo's redeclaration needs to be restored
320 VisitDeclContext(DC: D);
321 withdraw(DBase: D);
322 }
323
324 void VisitTranslationUnitDecl(TranslationUnitDecl *D) { VisitDeclContext(DC: D); }
325};
326
327void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
328 ASTDeclUnmerger(S, MostRecentTU).Visit(D: MostRecentTU);
329
330 // Lookup alone is not enough: the redeclaration chain still reaches these.
331 withdrawMostRecentTU(MostRecentTU);
332}
333
334PartialTranslationUnit &
335IncrementalParser::RegisterPTU(TranslationUnitDecl *TU,
336 std::unique_ptr<llvm::Module> M /*={}*/) {
337 PTUs.emplace_back(args: PartialTranslationUnit());
338 PartialTranslationUnit &LastPTU = PTUs.back();
339 LastPTU.TUPart = TU;
340
341 if (!M)
342 M = Act->GenModule();
343
344 assert((!Act->getCodeGen() || M) && "Must have a llvm::Module at this point");
345
346 LastPTU.TheModule = std::move(M);
347 LLVM_DEBUG(llvm::dbgs() << "compile-ptu " << PTUs.size() - 1
348 << ": [TU=" << LastPTU.TUPart);
349 if (LastPTU.TheModule)
350 LLVM_DEBUG(llvm::dbgs() << ", M=" << LastPTU.TheModule.get() << " ("
351 << LastPTU.TheModule->getName() << ")");
352 LLVM_DEBUG(llvm::dbgs() << "]\n");
353 return LastPTU;
354}
355} // end namespace clang
356