1//===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class which performs incremental code compilation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "IncrementalParser.h"
14#include "IncrementalAction.h"
15
16#include "clang/AST/ASTContext.h"
17#include "clang/AST/Decl.h"
18#include "clang/AST/DeclContextInternals.h"
19#include "clang/Frontend/CompilerInstance.h"
20#include "clang/Interpreter/PartialTranslationUnit.h"
21#include "clang/Parse/Parser.h"
22#include "clang/Sema/Sema.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/IR/Module.h"
25#include "llvm/Support/CrashRecoveryContext.h"
26#include "llvm/Support/Error.h"
27
28#include <sstream>
29
30#define DEBUG_TYPE "clang-repl"
31
32namespace clang {
33
34// IncrementalParser::IncrementalParser() {}
35
36IncrementalParser::IncrementalParser(CompilerInstance &Instance,
37 IncrementalAction *Act, llvm::Error &Err,
38 std::list<PartialTranslationUnit> &PTUs)
39 : S(Instance.getSema()), Act(Act), PTUs(PTUs) {
40 llvm::ErrorAsOutParameter EAO(&Err);
41 Consumer = &S.getASTConsumer();
42 P.reset(p: new Parser(S.getPreprocessor(), S, /*SkipBodies=*/false));
43
44 if (ExternalASTSource *External = S.getASTContext().getExternalSource())
45 External->StartTranslationUnit(Consumer);
46
47 P->Initialize();
48}
49
50IncrementalParser::~IncrementalParser() { P.reset(); }
51
52llvm::Expected<TranslationUnitDecl *>
53IncrementalParser::ParseOrWrapTopLevelDecl() {
54 // Recover resources if we crash before exiting this method.
55 llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
56 Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true,
57 /*AtEndOfTU=*/true);
58 Sema::LocalEagerInstantiationScope LocalInstantiations(S, /*AtEndOfTU=*/true);
59
60 // Add a new PTU.
61 ASTContext &C = S.getASTContext();
62 C.addTranslationUnitDecl();
63
64 // Skip previous eof due to last incremental input.
65 if (P->getCurToken().is(K: tok::annot_repl_input_end)) {
66 P->ConsumeAnyToken();
67 // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
68 // might want to do that around HandleEndOfTranslationUnit.
69 P->ExitScope();
70 S.CurContext = nullptr;
71 // Start a new PTU.
72 P->EnterScope(ScopeFlags: Scope::DeclScope);
73 S.ActOnTranslationUnitScope(S: P->getCurScope());
74 }
75
76 Parser::DeclGroupPtrTy ADecl;
77 Sema::ModuleImportState ImportState;
78 for (bool AtEOF = P->ParseFirstTopLevelDecl(Result&: ADecl, ImportState); !AtEOF;
79 AtEOF = P->ParseTopLevelDecl(Result&: ADecl, ImportState)) {
80 if (ADecl && !Consumer->HandleTopLevelDecl(D: ADecl.get()))
81 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
82 "The consumer rejected a decl",
83 Args: std::error_code());
84 }
85
86 DiagnosticsEngine &Diags = S.getDiagnostics();
87 if (Diags.hasErrorOccurred()) {
88 CleanUpPTU(MostRecentTU: C.getTranslationUnitDecl());
89
90 Diags.Reset(/*soft=*/true);
91 Diags.getClient()->clear();
92 return llvm::make_error<llvm::StringError>(Args: "Parsing failed.",
93 Args: std::error_code());
94 }
95
96 // Process any TopLevelDecls generated by #pragma weak.
97 for (Decl *D : S.WeakTopLevelDecls()) {
98 DeclGroupRef DGR(D);
99 Consumer->HandleTopLevelDecl(D: DGR);
100 }
101
102 LocalInstantiations.perform();
103 GlobalInstantiations.perform();
104
105 Consumer->HandleTranslationUnit(Ctx&: C);
106
107 return C.getTranslationUnitDecl();
108}
109
110llvm::Expected<TranslationUnitDecl *>
111IncrementalParser::Parse(llvm::StringRef input) {
112 Preprocessor &PP = S.getPreprocessor();
113 assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
114
115 std::ostringstream SourceName;
116 SourceName << "input_line_" << InputCount++;
117
118 // Create an uninitialized memory buffer, copy code in and append "\n"
119 size_t InputSize = input.size(); // don't include trailing 0
120 // MemBuffer size should *not* include terminating zero
121 std::unique_ptr<llvm::MemoryBuffer> MB(
122 llvm::WritableMemoryBuffer::getNewUninitMemBuffer(Size: InputSize + 1,
123 BufferName: SourceName.str()));
124 char *MBStart = const_cast<char *>(MB->getBufferStart());
125 memcpy(dest: MBStart, src: input.data(), n: InputSize);
126 MBStart[InputSize] = '\n';
127
128 SourceManager &SM = S.getSourceManager();
129
130 // FIXME: Create SourceLocation, which will allow clang to order the overload
131 // candidates for example
132 SourceLocation NewLoc = SM.getLocForStartOfFile(FID: SM.getMainFileID());
133
134 // Create FileID for the current buffer.
135 FileID FID;
136 // Create FileEntry and FileID for the current buffer.
137 FileEntryRef FE = SM.getFileManager().getVirtualFileRef(
138 Filename: SourceName.str(), Size: InputSize, ModificationTime: 0 /* mod time*/);
139 SM.overrideFileContents(SourceFile: FE, Buffer: std::move(MB));
140
141 // Ensure HeaderFileInfo exists before lookup to prevent assertion
142 HeaderSearch &HS = PP.getHeaderSearchInfo();
143 HS.getFileInfo(FE);
144
145 FID = SM.createFileID(SourceFile: FE, IncludePos: NewLoc, FileCharacter: SrcMgr::C_User);
146
147 // NewLoc only used for diags.
148 if (PP.EnterSourceFile(FID, /*DirLookup=*/Dir: nullptr, Loc: NewLoc))
149 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
150 "Cannot enter source file.",
151 Args: std::error_code());
152
153 auto PTU = ParseOrWrapTopLevelDecl();
154 if (!PTU)
155 return PTU.takeError();
156
157 if (PP.getLangOpts().DelayedTemplateParsing) {
158 // Microsoft-specific:
159 // Late parsed templates can leave unswallowed "macro"-like tokens.
160 // They will seriously confuse the Parser when entering the next
161 // source file. So lex until we are EOF.
162 Token Tok;
163 do {
164 PP.Lex(Result&: Tok);
165 } while (Tok.isNot(K: tok::annot_repl_input_end));
166 } else {
167 Token AssertTok;
168 PP.Lex(Result&: AssertTok);
169 assert(AssertTok.is(tok::annot_repl_input_end) &&
170 "Lexer must be EOF when starting incremental parse!");
171 }
172
173 return PTU;
174}
175
176void IncrementalParser::withdrawMostRecentTU(
177 TranslationUnitDecl *MostRecentTU) {
178 TranslationUnitDecl *Prev = MostRecentTU->getPreviousDecl();
179 if (!Prev)
180 return;
181 assert(MostRecentTU->getMostRecentDecl() == MostRecentTU &&
182 "Not the most recent translation unit!");
183
184 // Rebuild A -> ... -> Prev -> MostRecentTU as A -> ... -> Prev.
185 MostRecentTU->getFirstDecl()->RedeclLink.setLatest(Prev);
186
187 // getTranslationUnitDecl() requires the active unit to be the latest one.
188 ASTContext &C = S.getASTContext();
189 if (C.TraversalScope.size() == 1 && C.TraversalScope.back() == MostRecentTU)
190 C.TraversalScope = {Prev};
191 C.TUDecl = Prev;
192}
193
194void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
195 if (StoredDeclsMap *Map = MostRecentTU->getPrimaryContext()->getLookupPtr()) {
196 // Collect the keys to erase: erasing during iteration invalidates the map
197 // iterator under backward-shift deletion.
198 llvm::SmallVector<DeclarationName, 16> KeysToErase;
199 for (auto &&[Key, List] : *Map) {
200 DeclContextLookupResult R = List.getLookupResult();
201 std::vector<NamedDecl *> NamedDeclsToRemove;
202 bool RemoveAll = true;
203 for (NamedDecl *D : R) {
204 if (D->getTranslationUnitDecl() == MostRecentTU)
205 NamedDeclsToRemove.push_back(x: D);
206 else
207 RemoveAll = false;
208 }
209 if (LLVM_LIKELY(RemoveAll)) {
210 KeysToErase.push_back(Elt: Key);
211 } else {
212 for (NamedDecl *D : NamedDeclsToRemove)
213 List.remove(D);
214 }
215 }
216 for (DeclarationName Key : KeysToErase)
217 Map->erase(Val: Key);
218 }
219
220 // Check if we need to clean up the IdResolver chain.
221 auto RemoveFromIdResolver = [&](NamedDecl *D) {
222 if (D->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
223 !D->getLangOpts().CPlusPlus)
224 S.IdResolver.RemoveDecl(D);
225 };
226
227 ExternCContextDecl *ECCD = S.getASTContext().getExternCContextDecl();
228 if (StoredDeclsMap *Map = ECCD->getPrimaryContext()->getLookupPtr()) {
229 for (auto &&[Key, List] : *Map) {
230 DeclContextLookupResult R = List.getLookupResult();
231 llvm::SmallVector<NamedDecl *, 4> NamedDeclsToRemove;
232 for (NamedDecl *D : R) {
233 // Implicitly generated C decl is not attached to the current TU but
234 // lexically attached to the recent TU, so we need to check the lexical
235 // context.
236 DeclContext *LDC = D->getLexicalDeclContext();
237 while (LDC && !isa<TranslationUnitDecl>(Val: LDC))
238 LDC = LDC->getLexicalParent();
239 TranslationUnitDecl *TopTU = cast_or_null<TranslationUnitDecl>(Val: LDC);
240 if (TopTU == MostRecentTU)
241 NamedDeclsToRemove.push_back(Elt: D);
242 }
243 for (NamedDecl *D : NamedDeclsToRemove) {
244 List.remove(D);
245 RemoveFromIdResolver(D);
246 }
247 }
248 }
249
250 for (Decl *D : MostRecentTU->decls()) {
251 auto *ND = dyn_cast<NamedDecl>(Val: D);
252 if (!ND || ND->getDeclName().isEmpty())
253 continue;
254 RemoveFromIdResolver(ND);
255 }
256
257 // Lookup alone is not enough: the redeclaration chain still reaches these.
258 withdrawMostRecentTU(MostRecentTU);
259}
260
261PartialTranslationUnit &
262IncrementalParser::RegisterPTU(TranslationUnitDecl *TU,
263 std::unique_ptr<llvm::Module> M /*={}*/) {
264 PTUs.emplace_back(args: PartialTranslationUnit());
265 PartialTranslationUnit &LastPTU = PTUs.back();
266 LastPTU.TUPart = TU;
267
268 if (!M)
269 M = Act->GenModule();
270
271 assert((!Act->getCodeGen() || M) && "Must have a llvm::Module at this point");
272
273 LastPTU.TheModule = std::move(M);
274 LLVM_DEBUG(llvm::dbgs() << "compile-ptu " << PTUs.size() - 1
275 << ": [TU=" << LastPTU.TUPart);
276 if (LastPTU.TheModule)
277 LLVM_DEBUG(llvm::dbgs() << ", M=" << LastPTU.TheModule.get() << " ("
278 << LastPTU.TheModule->getName() << ")");
279 LLVM_DEBUG(llvm::dbgs() << "]\n");
280 return LastPTU;
281}
282} // end namespace clang
283