1//===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CrossTranslationUnit interface.
10//
11//===----------------------------------------------------------------------===//
12#include "clang/CrossTU/CrossTranslationUnit.h"
13#include "clang/AST/ASTImporter.h"
14#include "clang/AST/Decl.h"
15#include "clang/AST/ParentMapContext.h"
16#include "clang/Basic/DiagnosticDriver.h"
17#include "clang/Basic/TargetInfo.h"
18#include "clang/CrossTU/CrossTUDiagnostic.h"
19#include "clang/Driver/CreateASTUnitFromArgs.h"
20#include "clang/Frontend/ASTUnit.h"
21#include "clang/Frontend/CompilerInstance.h"
22#include "clang/Frontend/TextDiagnosticPrinter.h"
23#include "clang/UnifiedSymbolResolution/USRGeneration.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Option/ArgList.h"
26#include "llvm/Support/ErrorHandling.h"
27#include "llvm/Support/IOSandbox.h"
28#include "llvm/Support/ManagedStatic.h"
29#include "llvm/Support/Path.h"
30#include "llvm/Support/YAMLParser.h"
31#include "llvm/Support/raw_ostream.h"
32#include "llvm/TargetParser/Triple.h"
33#include <algorithm>
34#include <fstream>
35#include <optional>
36#include <sstream>
37#include <tuple>
38
39namespace clang {
40namespace cross_tu {
41
42namespace {
43
44#define DEBUG_TYPE "CrossTranslationUnit"
45STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
46STATISTIC(
47 NumNotInOtherTU,
48 "The # of getCTUDefinition called but the function is not in any other TU");
49STATISTIC(NumGetCTUSuccess,
50 "The # of getCTUDefinition successfully returned the "
51 "requested function's body");
52STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
53 "encountered an unsupported AST Node");
54STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
55 "encountered an ODR error");
56STATISTIC(NumTripleMismatch, "The # of triple mismatches");
57STATISTIC(NumLangMismatch, "The # of language mismatches");
58STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
59STATISTIC(NumASTLoadThresholdReached,
60 "The # of ASTs not loaded because of threshold");
61
62// Same as Triple's equality operator, but we check a field only if that is
63// known in both instances.
64bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
65 using llvm::Triple;
66 if (Lhs.getArch() != Triple::UnknownArch &&
67 Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
68 return false;
69 if (Lhs.getSubArch() != Triple::NoSubArch &&
70 Rhs.getSubArch() != Triple::NoSubArch &&
71 Lhs.getSubArch() != Rhs.getSubArch())
72 return false;
73 if (Lhs.getVendor() != Triple::UnknownVendor &&
74 Rhs.getVendor() != Triple::UnknownVendor &&
75 Lhs.getVendor() != Rhs.getVendor())
76 return false;
77 if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
78 Lhs.getOS() != Rhs.getOS())
79 return false;
80 if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
81 Rhs.getEnvironment() != Triple::UnknownEnvironment &&
82 Lhs.getEnvironment() != Rhs.getEnvironment())
83 return false;
84 if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
85 Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
86 Lhs.getObjectFormat() != Rhs.getObjectFormat())
87 return false;
88 return true;
89}
90
91// FIXME: This class is will be removed after the transition to llvm::Error.
92class IndexErrorCategory : public std::error_category {
93public:
94 const char *name() const noexcept override { return "clang.index"; }
95
96 std::string message(int Condition) const override {
97 switch (static_cast<index_error_code>(Condition)) {
98 case index_error_code::success:
99 // There should not be a success error. Jump to unreachable directly.
100 // Add this case to make the compiler stop complaining.
101 break;
102 case index_error_code::unspecified:
103 return "An unknown error has occurred.";
104 case index_error_code::missing_index_file:
105 return "The index file is missing.";
106 case index_error_code::invalid_index_format:
107 return "Invalid index file format.";
108 case index_error_code::multiple_definitions:
109 return "Multiple definitions in the index file.";
110 case index_error_code::missing_definition:
111 return "Missing definition from the index file.";
112 case index_error_code::failed_import:
113 return "Failed to import the definition.";
114 case index_error_code::failed_to_get_external_ast:
115 return "Failed to load external AST source.";
116 case index_error_code::failed_to_generate_usr:
117 return "Failed to generate USR.";
118 case index_error_code::triple_mismatch:
119 return "Triple mismatch";
120 case index_error_code::lang_mismatch:
121 return "Language mismatch";
122 case index_error_code::lang_dialect_mismatch:
123 return "Language dialect mismatch";
124 case index_error_code::load_threshold_reached:
125 return "Load threshold reached";
126 case index_error_code::invocation_list_ambiguous:
127 return "Invocation list file contains multiple references to the same "
128 "source file.";
129 case index_error_code::invocation_list_file_not_found:
130 return "Invocation list file is not found.";
131 case index_error_code::invocation_list_empty:
132 return "Invocation list file is empty.";
133 case index_error_code::invocation_list_wrong_format:
134 return "Invocation list file is in wrong format.";
135 case index_error_code::invocation_list_lookup_unsuccessful:
136 return "Invocation list file does not contain the requested source file.";
137 }
138 llvm_unreachable("Unrecognized index_error_code.");
139 }
140};
141
142static llvm::ManagedStatic<IndexErrorCategory> Category;
143} // end anonymous namespace
144
145/// Returns a human-readable language/dialect description for diagnostics.
146/// Checks flags from highest to lowest standard since they are cumulative
147/// (e.g. CPlusPlus20 implies CPlusPlus17).
148/// This does not cover all possible languages (e.g. Obj-C or flavors of C),
149/// because CTU currently does not differentiate between them.
150static std::string getLangDescription(const LangOptions &LO) {
151 if (!LO.CPlusPlus)
152 return "non-C++";
153 if (LO.CPlusPlus26)
154 return "C++26";
155 if (LO.CPlusPlus23)
156 return "C++23";
157 if (LO.CPlusPlus20)
158 return "C++20";
159 if (LO.CPlusPlus17)
160 return "C++17";
161 if (LO.CPlusPlus14)
162 return "C++14";
163 if (LO.CPlusPlus11)
164 return "C++11";
165 return "C++98";
166}
167
168char IndexError::ID;
169
170void IndexError::log(raw_ostream &OS) const {
171 OS << Category->message(Condition: static_cast<int>(Code)) << '\n';
172}
173
174std::error_code IndexError::convertToErrorCode() const {
175 return std::error_code(static_cast<int>(Code), *Category);
176}
177
178/// Parse one line of the input CTU index file.
179///
180/// @param[in] LineRef The input CTU index item in format
181/// "<USR-Length>:<USR> <File-Path>".
182/// @param[out] LookupName The lookup name in format "<USR-Length>:<USR>".
183/// @param[out] FilePath The file path "<File-Path>".
184static bool parseCrossTUIndexItem(StringRef LineRef, StringRef &LookupName,
185 StringRef &FilePath) {
186 // `LineRef` is "<USR-Length>:<USR> <File-Path>" now.
187
188 size_t USRLength = 0;
189 if (LineRef.consumeInteger(Radix: 10, Result&: USRLength))
190 return false;
191 assert(USRLength && "USRLength should be greater than zero.");
192
193 if (!LineRef.consume_front(Prefix: ":"))
194 return false;
195
196 // `LineRef` is now just "<USR> <File-Path>".
197
198 // Check LookupName length out of bound and incorrect delimiter.
199 if (USRLength >= LineRef.size() || ' ' != LineRef[USRLength])
200 return false;
201
202 LookupName = LineRef.substr(Start: 0, N: USRLength);
203 FilePath = LineRef.substr(Start: USRLength + 1);
204 return true;
205}
206
207llvm::Expected<llvm::StringMap<std::string>>
208parseCrossTUIndex(StringRef IndexPath) {
209 std::ifstream ExternalMapFile{std::string(IndexPath)};
210 if (!ExternalMapFile)
211 return llvm::make_error<IndexError>(Args: index_error_code::missing_index_file,
212 Args: IndexPath.str());
213
214 llvm::StringMap<std::string> Result;
215 std::string Line;
216 unsigned LineNo = 1;
217 while (std::getline(is&: ExternalMapFile, str&: Line)) {
218 // Split lookup name and file path
219 StringRef LookupName, FilePathInIndex;
220 if (!parseCrossTUIndexItem(LineRef: Line, LookupName, FilePath&: FilePathInIndex))
221 return llvm::make_error<IndexError>(
222 Args: index_error_code::invalid_index_format, Args: IndexPath.str(), Args&: LineNo);
223
224 // Store paths with posix-style directory separator.
225 SmallString<32> FilePath(FilePathInIndex);
226 llvm::sys::path::native(path&: FilePath, style: llvm::sys::path::Style::posix);
227
228 bool InsertionOccurred;
229 std::tie(args: std::ignore, args&: InsertionOccurred) =
230 Result.try_emplace(Key: LookupName, Args: FilePath.begin(), Args: FilePath.end());
231 if (!InsertionOccurred)
232 return llvm::make_error<IndexError>(
233 Args: index_error_code::multiple_definitions, Args: IndexPath.str(), Args&: LineNo);
234
235 ++LineNo;
236 }
237 return Result;
238}
239
240std::string
241createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
242 std::ostringstream Result;
243 for (const auto &E : Index)
244 Result << E.getKey().size() << ':' << E.getKey().str() << ' '
245 << E.getValue() << '\n';
246 return Result.str();
247}
248
249bool shouldImport(const VarDecl *VD, const ASTContext &ACtx) {
250 CanQualType CT = ACtx.getCanonicalType(T: VD->getType());
251 return CT.isConstQualified() && VD->getType().isTrivialType(Context: ACtx);
252}
253
254static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
255 return D->hasBody(Definition&: DefD);
256}
257static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
258 return D->getAnyInitializer(D&: DefD);
259}
260template <typename T> static bool hasBodyOrInit(const T *D) {
261 const T *Unused;
262 return hasBodyOrInit(D, Unused);
263}
264
265CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
266 : Context(CI.getASTContext()), ASTStorage(CI) {
267 if (CI.getAnalyzerOpts().ShouldEmitErrorsOnInvalidConfigValue &&
268 !CI.getAnalyzerOpts().CTUDir.empty()) {
269 auto S = CI.getVirtualFileSystem().status(Path: CI.getAnalyzerOpts().CTUDir);
270 if (!S || S->getType() != llvm::sys::fs::file_type::directory_file)
271 CI.getDiagnostics().Report(DiagID: diag::err_analyzer_config_invalid_input)
272 << "ctu-dir"
273 << "a filename";
274 }
275}
276
277CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
278
279std::optional<std::string>
280CrossTranslationUnitContext::getLookupName(const Decl *D) {
281 SmallString<128> DeclUSR;
282 bool Ret = index::generateUSRForDecl(D, Buf&: DeclUSR);
283 if (Ret)
284 return {};
285 return std::string(DeclUSR);
286}
287
288/// Recursively visits the decls of a DeclContext, and returns one with the
289/// given USR.
290template <typename T>
291const T *
292CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
293 StringRef LookupName) {
294 assert(DC && "Declaration Context must not be null");
295 for (const Decl *D : DC->decls()) {
296 const auto *SubDC = dyn_cast<DeclContext>(Val: D);
297 if (SubDC)
298 if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
299 return ND;
300
301 const auto *ND = dyn_cast<T>(D);
302 const T *ResultDecl;
303 if (!ND || !hasBodyOrInit(ND, ResultDecl))
304 continue;
305 std::optional<std::string> ResultLookupName = getLookupName(D: ResultDecl);
306 if (!ResultLookupName || *ResultLookupName != LookupName)
307 continue;
308 return ResultDecl;
309 }
310 return nullptr;
311}
312
313template <typename T>
314llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
315 const T *D, StringRef CrossTUDir, StringRef IndexName,
316 bool DisplayCTUProgress) {
317 assert(D && "D is missing, bad call to this function!");
318 assert(!hasBodyOrInit(D) &&
319 "D has a body or init in current translation unit!");
320 ++NumGetCTUCalled;
321 const std::optional<std::string> LookupName = getLookupName(D);
322 if (!LookupName)
323 return llvm::make_error<IndexError>(
324 Args: index_error_code::failed_to_generate_usr);
325 llvm::Expected<ASTUnit *> ASTUnitOrError =
326 loadExternalAST(LookupName: *LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
327 if (!ASTUnitOrError)
328 return ASTUnitOrError.takeError();
329 ASTUnit *Unit = *ASTUnitOrError;
330 assert(&Unit->getFileManager() ==
331 &Unit->getASTContext().getSourceManager().getFileManager());
332
333 const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
334 const llvm::Triple &TripleFrom =
335 Unit->getASTContext().getTargetInfo().getTriple();
336 // The imported AST had been generated for a different target.
337 // Some parts of the triple in the loaded ASTContext can be unknown while the
338 // very same parts in the target ASTContext are known. Thus we check for the
339 // known parts only.
340 if (!hasEqualKnownFields(Lhs: TripleTo, Rhs: TripleFrom)) {
341 // TODO: Pass the SourceLocation of the CallExpression for more precise
342 // diagnostics.
343 ++NumTripleMismatch;
344 return llvm::make_error<IndexError>(Args: index_error_code::triple_mismatch,
345 Args: std::string(Unit->getMainFileName()),
346 Args: TripleTo.str(), Args: TripleFrom.str());
347 }
348
349 const auto &LangTo = Context.getLangOpts();
350 const auto &LangFrom = Unit->getASTContext().getLangOpts();
351
352 // FIXME: Currenty we do not support CTU across C++ and C and across
353 // different dialects of C++.
354 if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
355 ++NumLangMismatch;
356 return llvm::make_error<IndexError>(
357 Args: index_error_code::lang_mismatch, Args: std::string(Unit->getMainFileName()),
358 Args: getLangDescription(LO: LangTo), Args: getLangDescription(LO: LangFrom));
359 }
360
361 // If CPP dialects are different then return with error.
362 //
363 // Consider this STL code:
364 // template<typename _Alloc>
365 // struct __alloc_traits
366 // #if __cplusplus >= 201103L
367 // : std::allocator_traits<_Alloc>
368 // #endif
369 // { // ...
370 // };
371 // This class template would create ODR errors during merging the two units,
372 // since in one translation unit the class template has a base class, however
373 // in the other unit it has none.
374 if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
375 LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
376 LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
377 LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
378 ++NumLangDialectMismatch;
379 return llvm::make_error<IndexError>(Args: index_error_code::lang_dialect_mismatch,
380 Args: std::string(Unit->getMainFileName()),
381 Args: getLangDescription(LO: LangTo),
382 Args: getLangDescription(LO: LangFrom));
383 }
384
385 TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
386 if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
387 return importDefinition(ResultDecl, Unit);
388 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
389}
390
391llvm::Expected<const FunctionDecl *>
392CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
393 StringRef CrossTUDir,
394 StringRef IndexName,
395 bool DisplayCTUProgress) {
396 return getCrossTUDefinitionImpl(D: FD, CrossTUDir, IndexName,
397 DisplayCTUProgress);
398}
399
400llvm::Expected<const VarDecl *>
401CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
402 StringRef CrossTUDir,
403 StringRef IndexName,
404 bool DisplayCTUProgress) {
405 return getCrossTUDefinitionImpl(D: VD, CrossTUDir, IndexName,
406 DisplayCTUProgress);
407}
408
409void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE,
410 SourceLocation Loc) {
411 switch (IE.getCode()) {
412 case index_error_code::missing_index_file:
413 case index_error_code::invocation_list_file_not_found:
414 // If the external def-map refers to source files, you must provide an
415 // invocation list file. Otherwise, CTU does not work at all, so you should
416 // check your build and analysis configuration.
417 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_error_opening)
418 << IE.getFileName();
419 return;
420
421 case index_error_code::invalid_index_format:
422 Context.getDiagnostics().Report(Loc, DiagID: diag::err_extdefmap_parsing)
423 << IE.getFileName() << IE.getLineNum();
424 return;
425
426 case index_error_code::multiple_definitions:
427 Context.getDiagnostics().Report(Loc, DiagID: diag::err_multiple_def_index)
428 << IE.getLineNum();
429 return;
430
431 case index_error_code::triple_mismatch:
432 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_triple)
433 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
434 return;
435
436 case index_error_code::missing_definition:
437 // Ignore missing definitions because it is very common to have some symbols
438 // defined outside of the analysis scope: they may be defined in 3-rd party
439 // and standard libraries, generated code, and files excluded from the
440 // analysis.
441 // Even ignoring it with Ignored diagnostic might generate too much traffic.
442 return;
443
444 case index_error_code::failed_import:
445 case index_error_code::unspecified:
446 // Not clear what happened exactly, but the outcome is a missing definition
447 // This is not a big deal, and is expected since ASTImporter is incomplete.
448 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_import_failure)
449 << Category->message(Condition: static_cast<int>(IE.getCode()));
450 return;
451
452 case index_error_code::failed_to_generate_usr:
453 // This is unlikely, so it is worth looking into, hence an error.
454 case index_error_code::failed_to_get_external_ast:
455 // This is suspicious, since the external AST is mentioned in the external
456 // defmap, so it should exist.
457 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_import_failure)
458 << Category->message(Condition: static_cast<int>(IE.getCode()));
459 return;
460
461 case index_error_code::load_threshold_reached:
462 // This is expected. It is still useful to be aware of, but it is normal
463 // operation. Emit the remark only once to avoid noise.
464 if (!HasEmittedLoadThresholdRemark) {
465 HasEmittedLoadThresholdRemark = true;
466 Context.getDiagnostics().Report(
467 Loc, DiagID: diag::remark_ctu_import_threshold_reached);
468 }
469 return;
470
471 case index_error_code::lang_mismatch:
472 case index_error_code::lang_dialect_mismatch:
473 // Similar to target triple mismatch.
474 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_lang)
475 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
476 return;
477
478 case index_error_code::invocation_list_wrong_format:
479 case index_error_code::invocation_list_empty:
480 // Without parsable invocation list, CTU cannot function.
481 Context.getDiagnostics().Report(Loc, DiagID: diag::err_invlist_parsing)
482 << IE.getFileName() << IE.getLineNum();
483 return;
484
485 case index_error_code::invocation_list_ambiguous:
486 // For automatically generated invocation lists, it is common to list
487 // multiple invocations, if a file is compiled in multiple contexts. No need
488 // to block CTU because of this.
489 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_multiple_entries_invlist)
490 << IE.getFileName();
491 return;
492
493 case index_error_code::invocation_list_lookup_unsuccessful:
494 // Some files might be missing in the invocation list. It is sad but not
495 // fatal, and CTU can take advantage of the definitions in files with known
496 // invocations.
497 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_invlist_missing_file)
498 << IE.getFileName();
499 return;
500
501 case index_error_code::success:
502 llvm_unreachable("Success is not an error.");
503 return;
504 }
505 llvm_unreachable("Unrecognized index_error_code.");
506}
507
508CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
509 CompilerInstance &CI)
510 : Loader(CI, CI.getAnalyzerOpts().CTUDir,
511 CI.getAnalyzerOpts().CTUInvocationList),
512 LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
513 ? CI.getAnalyzerOpts().CTUImportCppThreshold
514 : CI.getAnalyzerOpts().CTUImportThreshold) {}
515
516llvm::Expected<ASTUnit *>
517CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
518 StringRef FileName, bool DisplayCTUProgress) {
519 // Try the cache first.
520 auto ASTCacheEntry = FileASTUnitMap.find(Key: FileName);
521 if (ASTCacheEntry == FileASTUnitMap.end()) {
522
523 // Do not load if the limit is reached.
524 if (!LoadGuard) {
525 ++NumASTLoadThresholdReached;
526 return llvm::make_error<IndexError>(
527 Args: index_error_code::load_threshold_reached);
528 }
529
530 auto LoadAttempt = Loader.load(Identifier: FileName);
531
532 if (!LoadAttempt)
533 return LoadAttempt.takeError();
534
535 std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
536
537 // Need the raw pointer and the unique_ptr as well.
538 ASTUnit *Unit = LoadedUnit.get();
539
540 // Update the cache.
541 FileASTUnitMap[FileName] = std::move(LoadedUnit);
542
543 LoadGuard.indicateLoadSuccess();
544
545 if (DisplayCTUProgress)
546 llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
547
548 return Unit;
549
550 } else {
551 // Found in the cache.
552 return ASTCacheEntry->second.get();
553 }
554}
555
556llvm::Expected<ASTUnit *>
557CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
558 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
559 bool DisplayCTUProgress) {
560 // Try the cache first.
561 auto ASTCacheEntry = NameASTUnitMap.find(Key: FunctionName);
562 if (ASTCacheEntry == NameASTUnitMap.end()) {
563 // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
564
565 // Ensure that the Index is loaded, as we need to search in it.
566 if (llvm::Error IndexLoadError =
567 ensureCTUIndexLoaded(CrossTUDir, IndexName))
568 return std::move(IndexLoadError);
569
570 // Check if there is an entry in the index for the function.
571 auto It = NameFileMap.find(Key: FunctionName);
572 if (It == NameFileMap.end()) {
573 ++NumNotInOtherTU;
574 return llvm::make_error<IndexError>(Args: index_error_code::missing_definition);
575 }
576
577 // Search in the index for the filename where the definition of FunctionName
578 // resides.
579 if (llvm::Expected<ASTUnit *> FoundForFile =
580 getASTUnitForFile(FileName: It->second, DisplayCTUProgress)) {
581
582 // Update the cache.
583 NameASTUnitMap[FunctionName] = *FoundForFile;
584 return *FoundForFile;
585
586 } else {
587 return FoundForFile.takeError();
588 }
589 } else {
590 // Found in the cache.
591 return ASTCacheEntry->second;
592 }
593}
594
595llvm::Expected<std::string>
596CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
597 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
598 if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
599 return std::move(IndexLoadError);
600 return NameFileMap[FunctionName];
601}
602
603llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
604 StringRef CrossTUDir, StringRef IndexName) {
605 // Dont initialize if the map is filled.
606 if (!NameFileMap.empty())
607 return llvm::Error::success();
608
609 // Get the absolute path to the index file.
610 SmallString<256> IndexFile = CrossTUDir;
611 if (llvm::sys::path::is_absolute(path: IndexName))
612 IndexFile = IndexName;
613 else
614 llvm::sys::path::append(path&: IndexFile, a: IndexName);
615
616 if (auto IndexMapping = parseCrossTUIndex(IndexPath: IndexFile)) {
617 // Initialize member map.
618 NameFileMap = *IndexMapping;
619 return llvm::Error::success();
620 } else {
621 // Error while parsing CrossTU index file.
622 return IndexMapping.takeError();
623 };
624}
625
626llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
627 StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
628 bool DisplayCTUProgress) {
629 // FIXME: The current implementation only supports loading decls with
630 // a lookup name from a single translation unit. If multiple
631 // translation units contains decls with the same lookup name an
632 // error will be returned.
633
634 // Try to get the value from the heavily cached storage.
635 llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
636 FunctionName: LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
637
638 if (!Unit)
639 return Unit.takeError();
640
641 // Check whether the backing pointer of the Expected is a nullptr.
642 if (!*Unit)
643 return llvm::make_error<IndexError>(
644 Args: index_error_code::failed_to_get_external_ast);
645
646 return Unit;
647}
648
649CrossTranslationUnitContext::ASTLoader::ASTLoader(
650 CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
651 : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
652
653CrossTranslationUnitContext::LoadResultTy
654CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
655 llvm::SmallString<256> Path;
656 if (llvm::sys::path::is_absolute(path: Identifier, style: PathStyle)) {
657 Path = Identifier;
658 } else {
659 Path = CTUDir;
660 llvm::sys::path::append(path&: Path, style: PathStyle, a: Identifier);
661 }
662
663 // The path is stored in the InvocationList member in posix style. To
664 // successfully lookup an entry based on filepath, it must be converted.
665 llvm::sys::path::native(path&: Path, style: PathStyle);
666
667 // Normalize by removing relative path components.
668 llvm::sys::path::remove_dots(path&: Path, /*remove_dot_dot*/ true, style: PathStyle);
669
670 if (Path.ends_with(Suffix: ".ast"))
671 return loadFromDump(Identifier: Path);
672 else
673 return loadFromSource(Identifier: Path);
674}
675
676CrossTranslationUnitContext::LoadResultTy
677CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
678 auto DiagOpts = std::make_shared<DiagnosticOptions>();
679 TextDiagnosticPrinter *DiagClient =
680 new TextDiagnosticPrinter(llvm::errs(), *DiagOpts);
681 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(
682 A: DiagnosticIDs::create(), A&: *DiagOpts, A&: DiagClient);
683 return ASTUnit::LoadFromASTFile(
684 Filename: ASTDumpPath, PCHContainerRdr: CI.getPCHContainerOperations()->getRawReader(),
685 ToLoad: ASTUnit::LoadEverything, VFS: CI.getVirtualFileSystemPtr(), DiagOpts, Diags,
686 FileSystemOpts: CI.getFileSystemOpts(), HSOpts: CI.getHeaderSearchOpts());
687}
688
689/// Load the AST from a source-file, which is supposed to be located inside the
690/// YAML formatted invocation list file under the filesystem path specified by
691/// \p InvocationList. The invocation list should contain absolute paths.
692/// \p SourceFilePath is the absolute path of the source file that contains the
693/// function definition the analysis is looking for. The Index is built by the
694/// \p clang-extdef-mapping tool, which is also supposed to be generating
695/// absolute paths.
696///
697/// Proper diagnostic emission requires absolute paths, so even if a future
698/// change introduces the handling of relative paths, this must be taken into
699/// consideration.
700CrossTranslationUnitContext::LoadResultTy
701CrossTranslationUnitContext::ASTLoader::loadFromSource(
702 StringRef SourceFilePath) {
703
704 if (llvm::Error InitError = lazyInitInvocationList())
705 return std::move(InitError);
706 assert(InvocationList);
707
708 auto Invocation = InvocationList->find(Key: SourceFilePath);
709 if (Invocation == InvocationList->end())
710 return llvm::make_error<IndexError>(
711 Args: index_error_code::invocation_list_lookup_unsuccessful,
712 Args: SourceFilePath.str());
713
714 const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
715
716 SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
717 std::transform(first: InvocationCommand.begin(), last: InvocationCommand.end(),
718 result: CommandLineArgs.begin(),
719 unary_op: [](auto &&CmdPart) { return CmdPart.c_str(); });
720
721 auto DiagOpts = std::make_shared<DiagnosticOptions>(args&: CI.getDiagnosticOpts());
722 auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
723 IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
724 CI.getDiagnostics().getDiagnosticIDs()};
725 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(A&: DiagID, A&: *DiagOpts,
726 A&: DiagClient);
727
728 // This runs the driver which isn't expected to be free of sandbox violations.
729 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
730 return CreateASTUnitFromCommandLine(
731 ArgBegin: CommandLineArgs.begin(), ArgEnd: (CommandLineArgs.end()),
732 PCHContainerOps: CI.getPCHContainerOperations(), DiagOpts, Diags,
733 ResourceFilesPath: CI.getHeaderSearchOpts().ResourceDir);
734}
735
736llvm::Expected<InvocationListTy>
737parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle,
738 StringRef FilePath) {
739 InvocationListTy InvocationList;
740
741 /// LLVM YAML parser is used to extract information from invocation list file.
742 llvm::SourceMgr SM;
743 llvm::yaml::Stream InvocationFile(FileContent, SM);
744
745 auto GetLine = [&SM](const llvm::yaml::Node *N) -> int {
746 return N ? SM.FindLineNumber(Loc: N->getSourceRange().Start) : 0;
747 };
748 auto WrongFormatError = [&](const llvm::yaml::Node *N) {
749 return llvm::make_error<IndexError>(
750 Args: index_error_code::invocation_list_wrong_format, Args: FilePath.str(),
751 Args: GetLine(N));
752 };
753
754 /// Only the first document is processed.
755 llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
756
757 /// There has to be at least one document available.
758 if (FirstInvocationFile == InvocationFile.end())
759 return llvm::make_error<IndexError>(
760 Args: index_error_code::invocation_list_empty);
761
762 llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
763 if (!DocumentRoot)
764 return llvm::make_error<IndexError>(
765 Args: index_error_code::invocation_list_wrong_format);
766
767 /// According to the format specified the document must be a mapping, where
768 /// the keys are paths to source files, and values are sequences of invocation
769 /// parts.
770 auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(Val: DocumentRoot);
771 if (!Mappings)
772 return WrongFormatError(DocumentRoot);
773
774 for (auto &NextMapping : *Mappings) {
775 /// The keys should be strings, which represent a source-file path.
776 auto *Key = dyn_cast<llvm::yaml::ScalarNode>(Val: NextMapping.getKey());
777 if (!Key)
778 return WrongFormatError(NextMapping.getKey());
779
780 SmallString<32> ValueStorage;
781 StringRef SourcePath = Key->getValue(Storage&: ValueStorage);
782
783 // Store paths with PathStyle directory separator.
784 SmallString<32> NativeSourcePath(SourcePath);
785 llvm::sys::path::native(path&: NativeSourcePath, style: PathStyle);
786
787 StringRef InvocationKey = NativeSourcePath;
788
789 if (InvocationList.contains(Key: InvocationKey))
790 return llvm::make_error<IndexError>(
791 Args: index_error_code::invocation_list_ambiguous, Args: InvocationKey.str());
792
793 /// The values should be sequences of strings, each representing a part of
794 /// the invocation.
795 auto *Args = dyn_cast<llvm::yaml::SequenceNode>(Val: NextMapping.getValue());
796 if (!Args)
797 return WrongFormatError(NextMapping.getValue());
798
799 for (auto &Arg : *Args) {
800 auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(Val: &Arg);
801 if (!CmdString)
802 return WrongFormatError(&Arg);
803 /// Every conversion starts with an empty working storage, as it is not
804 /// clear if this is a requirement of the YAML parser.
805 ValueStorage.clear();
806 InvocationList[InvocationKey].emplace_back(
807 Args: CmdString->getValue(Storage&: ValueStorage));
808 }
809
810 if (InvocationList[InvocationKey].empty())
811 return WrongFormatError(Key);
812 }
813
814 return InvocationList;
815}
816
817llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
818 /// Lazily initialize the invocation list member used for on-demand parsing.
819 if (InvocationList)
820 return llvm::Error::success();
821 if (PreviousError)
822 return llvm::make_error<IndexError>(Args&: *PreviousError);
823
824 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
825 CI.getVirtualFileSystem().getBufferForFile(Name: InvocationListFilePath);
826 if (!FileContent) {
827 PreviousError = IndexError(index_error_code::invocation_list_file_not_found,
828 InvocationListFilePath.str());
829 return llvm::make_error<IndexError>(Args&: *PreviousError);
830 }
831 std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
832 assert(ContentBuffer && "If no error was produced after loading, the pointer "
833 "should not be nullptr.");
834
835 llvm::Expected<InvocationListTy> ExpectedInvocationList = parseInvocationList(
836 FileContent: ContentBuffer->getBuffer(), PathStyle, FilePath: InvocationListFilePath);
837
838 if (!ExpectedInvocationList) {
839 llvm::handleAllErrors(
840 E: ExpectedInvocationList.takeError(),
841 Handlers: [this](const IndexError &E) { this->PreviousError = E; });
842 return llvm::make_error<IndexError>(Args&: *PreviousError);
843 }
844
845 InvocationList = *ExpectedInvocationList;
846
847 return llvm::Error::success();
848}
849
850template <typename T>
851llvm::Expected<const T *>
852CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
853 assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
854
855 assert(&D->getASTContext() == &Unit->getASTContext() &&
856 "ASTContext of Decl and the unit should match.");
857 ASTImporter &Importer = getOrCreateASTImporter(Unit);
858
859 auto ToDeclOrError = Importer.Import(D);
860 if (!ToDeclOrError) {
861 handleAllErrors(ToDeclOrError.takeError(), [&](const ASTImportError &IE) {
862 switch (IE.Error) {
863 case ASTImportError::NameConflict:
864 ++NumNameConflicts;
865 break;
866 case ASTImportError::UnsupportedConstruct:
867 ++NumUnsupportedNodeFound;
868 break;
869 case ASTImportError::Unknown:
870 llvm_unreachable("Unknown import error happened.");
871 break;
872 }
873 });
874 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
875 }
876 auto *ToDecl = cast<T>(*ToDeclOrError);
877 assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
878 ++NumGetCTUSuccess;
879
880 // Parent map is invalidated after changing the AST.
881 ToDecl->getASTContext().getParentMapContext().clear();
882
883 return ToDecl;
884}
885
886llvm::Expected<const FunctionDecl *>
887CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
888 ASTUnit *Unit) {
889 return importDefinitionImpl(D: FD, Unit);
890}
891
892llvm::Expected<const VarDecl *>
893CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
894 ASTUnit *Unit) {
895 return importDefinitionImpl(D: VD, Unit);
896}
897
898void CrossTranslationUnitContext::lazyInitImporterSharedSt(
899 TranslationUnitDecl *ToTU) {
900 if (!ImporterSharedSt)
901 ImporterSharedSt = std::make_shared<ASTImporterSharedState>(args&: *ToTU);
902}
903
904ASTImporter &
905CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
906 ASTContext &From = Unit->getASTContext();
907
908 auto I = ASTUnitImporterMap.find(Val: From.getTranslationUnitDecl());
909 if (I != ASTUnitImporterMap.end())
910 return *I->second;
911 lazyInitImporterSharedSt(ToTU: Context.getTranslationUnitDecl());
912 ASTImporter *NewImporter = new ASTImporter(
913 Context, Context.getSourceManager().getFileManager(), From,
914 From.getSourceManager().getFileManager(), false, ImporterSharedSt);
915 ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(p: NewImporter);
916 return *NewImporter;
917}
918
919std::optional<clang::MacroExpansionContext>
920CrossTranslationUnitContext::getMacroExpansionContextForSourceLocation(
921 const clang::SourceLocation &ToLoc) const {
922 // FIXME: Implement: Record such a context for every imported ASTUnit; lookup.
923 return std::nullopt;
924}
925
926bool CrossTranslationUnitContext::isImportedAsNew(const Decl *ToDecl) const {
927 if (!ImporterSharedSt)
928 return false;
929 return ImporterSharedSt->isNewDecl(ToD: const_cast<Decl *>(ToDecl));
930}
931
932bool CrossTranslationUnitContext::hasError(const Decl *ToDecl) const {
933 if (!ImporterSharedSt)
934 return false;
935 return static_cast<bool>(
936 ImporterSharedSt->getImportDeclErrorIfAny(ToD: const_cast<Decl *>(ToDecl)));
937}
938
939} // namespace cross_tu
940} // namespace clang
941