1//===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CrossTranslationUnit interface.
10//
11//===----------------------------------------------------------------------===//
12#include "clang/CrossTU/CrossTranslationUnit.h"
13#include "clang/AST/ASTImporter.h"
14#include "clang/AST/Decl.h"
15#include "clang/AST/ParentMapContext.h"
16#include "clang/Basic/DiagnosticDriver.h"
17#include "clang/Basic/TargetInfo.h"
18#include "clang/CrossTU/CrossTUDiagnostic.h"
19#include "clang/Driver/CreateASTUnitFromArgs.h"
20#include "clang/Frontend/ASTUnit.h"
21#include "clang/Frontend/CompilerInstance.h"
22#include "clang/Frontend/TextDiagnosticPrinter.h"
23#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
24#include "clang/UnifiedSymbolResolution/USRGeneration.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/Option/ArgList.h"
27#include "llvm/Support/ErrorHandling.h"
28#include "llvm/Support/IOSandbox.h"
29#include "llvm/Support/ManagedStatic.h"
30#include "llvm/Support/Path.h"
31#include "llvm/Support/YAMLParser.h"
32#include "llvm/Support/raw_ostream.h"
33#include "llvm/TargetParser/Triple.h"
34#include <algorithm>
35#include <fstream>
36#include <optional>
37#include <sstream>
38#include <tuple>
39
40namespace clang {
41namespace cross_tu {
42
43namespace {
44
45#define DEBUG_TYPE "CrossTranslationUnit"
46STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
47STATISTIC(
48 NumNotInOtherTU,
49 "The # of getCTUDefinition called but the function is not in any other TU");
50STATISTIC(NumGetCTUSuccess,
51 "The # of getCTUDefinition successfully returned the "
52 "requested function's body");
53STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
54 "encountered an unsupported AST Node");
55STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
56 "encountered an ODR error");
57STATISTIC(NumTripleMismatch, "The # of triple mismatches");
58STATISTIC(NumLangMismatch, "The # of language mismatches");
59STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
60STATISTIC(NumASTLoadThresholdReached,
61 "The # of ASTs not loaded because of threshold");
62
63// Same as Triple's equality operator, but we check a field only if that is
64// known in both instances.
65bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
66 using llvm::Triple;
67 if (Lhs.getArch() != Triple::UnknownArch &&
68 Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
69 return false;
70 if (Lhs.getSubArch() != Triple::NoSubArch &&
71 Rhs.getSubArch() != Triple::NoSubArch &&
72 Lhs.getSubArch() != Rhs.getSubArch())
73 return false;
74 if (Lhs.getVendor() != Triple::UnknownVendor &&
75 Rhs.getVendor() != Triple::UnknownVendor &&
76 Lhs.getVendor() != Rhs.getVendor())
77 return false;
78 if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
79 Lhs.getOS() != Rhs.getOS())
80 return false;
81 if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
82 Rhs.getEnvironment() != Triple::UnknownEnvironment &&
83 Lhs.getEnvironment() != Rhs.getEnvironment())
84 return false;
85 if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
86 Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
87 Lhs.getObjectFormat() != Rhs.getObjectFormat())
88 return false;
89 return true;
90}
91
92// FIXME: This class is will be removed after the transition to llvm::Error.
93class IndexErrorCategory : public std::error_category {
94public:
95 const char *name() const noexcept override { return "clang.index"; }
96
97 std::string message(int Condition) const override {
98 switch (static_cast<index_error_code>(Condition)) {
99 case index_error_code::success:
100 // There should not be a success error. Jump to unreachable directly.
101 // Add this case to make the compiler stop complaining.
102 break;
103 case index_error_code::unspecified:
104 return "An unknown error has occurred.";
105 case index_error_code::missing_index_file:
106 return "The index file is missing.";
107 case index_error_code::invalid_index_format:
108 return "Invalid index file format.";
109 case index_error_code::multiple_definitions:
110 return "Multiple definitions in the index file.";
111 case index_error_code::missing_definition:
112 return "Missing definition from the index file.";
113 case index_error_code::failed_import:
114 return "Failed to import the definition.";
115 case index_error_code::failed_to_get_external_ast:
116 return "Failed to load external AST source.";
117 case index_error_code::failed_to_generate_usr:
118 return "Failed to generate USR.";
119 case index_error_code::triple_mismatch:
120 return "Triple mismatch";
121 case index_error_code::lang_mismatch:
122 return "Language mismatch";
123 case index_error_code::lang_dialect_mismatch:
124 return "Language dialect mismatch";
125 case index_error_code::load_threshold_reached:
126 return "Load threshold reached";
127 case index_error_code::invocation_list_ambiguous:
128 return "Invocation list file contains multiple references to the same "
129 "source file.";
130 case index_error_code::invocation_list_file_not_found:
131 return "Invocation list file is not found.";
132 case index_error_code::invocation_list_empty:
133 return "Invocation list file is empty.";
134 case index_error_code::invocation_list_wrong_format:
135 return "Invocation list file is in wrong format.";
136 case index_error_code::invocation_list_lookup_unsuccessful:
137 return "Invocation list file does not contain the requested source file.";
138 }
139 llvm_unreachable("Unrecognized index_error_code.");
140 }
141};
142
143static llvm::ManagedStatic<IndexErrorCategory> Category;
144} // end anonymous namespace
145
146/// Returns a human-readable language/dialect description for diagnostics.
147/// Checks flags from highest to lowest standard since they are cumulative
148/// (e.g. CPlusPlus20 implies CPlusPlus17).
149/// This does not cover all possible languages (e.g. Obj-C or flavors of C),
150/// because CTU currently does not differentiate between them.
151static std::string getLangDescription(const LangOptions &LO) {
152 if (!LO.CPlusPlus)
153 return "non-C++";
154 if (LO.CPlusPlus29)
155 return "C++29";
156 if (LO.CPlusPlus26)
157 return "C++26";
158 if (LO.CPlusPlus23)
159 return "C++23";
160 if (LO.CPlusPlus20)
161 return "C++20";
162 if (LO.CPlusPlus17)
163 return "C++17";
164 if (LO.CPlusPlus14)
165 return "C++14";
166 if (LO.CPlusPlus11)
167 return "C++11";
168 return "C++98";
169}
170
171char IndexError::ID;
172
173void IndexError::log(raw_ostream &OS) const {
174 OS << Category->message(Condition: static_cast<int>(Code)) << '\n';
175}
176
177std::error_code IndexError::convertToErrorCode() const {
178 return std::error_code(static_cast<int>(Code), *Category);
179}
180
181/// Parse one line of the input CTU index file.
182///
183/// @param[in] LineRef The input CTU index item in format
184/// "<USR-Length>:<USR> <File-Path>".
185/// @param[out] LookupName The lookup name in format "<USR-Length>:<USR>".
186/// @param[out] FilePath The file path "<File-Path>".
187static bool parseCrossTUIndexItem(StringRef LineRef, StringRef &LookupName,
188 StringRef &FilePath) {
189 // `LineRef` is "<USR-Length>:<USR> <File-Path>" now.
190
191 size_t USRLength = 0;
192 if (LineRef.consumeInteger(Radix: 10, Result&: USRLength))
193 return false;
194 assert(USRLength && "USRLength should be greater than zero.");
195
196 if (!LineRef.consume_front(Prefix: ":"))
197 return false;
198
199 // `LineRef` is now just "<USR> <File-Path>".
200
201 // Check LookupName length out of bound and incorrect delimiter.
202 if (USRLength >= LineRef.size() || ' ' != LineRef[USRLength])
203 return false;
204
205 LookupName = LineRef.substr(Start: 0, N: USRLength);
206 FilePath = LineRef.substr(Start: USRLength + 1);
207 return true;
208}
209
210llvm::Expected<llvm::StringMap<std::string>>
211parseCrossTUIndex(StringRef IndexPath) {
212 std::ifstream ExternalMapFile{std::string(IndexPath)};
213 if (!ExternalMapFile)
214 return llvm::make_error<IndexError>(Args: index_error_code::missing_index_file,
215 Args: IndexPath.str());
216
217 llvm::StringMap<std::string> Result;
218 std::string Line;
219 unsigned LineNo = 1;
220 while (std::getline(is&: ExternalMapFile, str&: Line)) {
221 // Split lookup name and file path
222 StringRef LookupName, FilePathInIndex;
223 if (!parseCrossTUIndexItem(LineRef: Line, LookupName, FilePath&: FilePathInIndex))
224 return llvm::make_error<IndexError>(
225 Args: index_error_code::invalid_index_format, Args: IndexPath.str(), Args&: LineNo);
226
227 // Store paths with posix-style directory separator.
228 SmallString<32> FilePath(FilePathInIndex);
229 llvm::sys::path::native(path&: FilePath, style: llvm::sys::path::Style::posix);
230
231 bool InsertionOccurred;
232 std::tie(args: std::ignore, args&: InsertionOccurred) =
233 Result.try_emplace(Key: LookupName, Args: FilePath.begin(), Args: FilePath.end());
234 if (!InsertionOccurred)
235 return llvm::make_error<IndexError>(
236 Args: index_error_code::multiple_definitions, Args: IndexPath.str(), Args&: LineNo);
237
238 ++LineNo;
239 }
240 return Result;
241}
242
243std::string
244createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
245 std::ostringstream Result;
246 for (const auto &E : Index)
247 Result << E.getKey().size() << ':' << E.getKey().str() << ' '
248 << E.getValue() << '\n';
249 return Result.str();
250}
251
252bool shouldImport(const VarDecl *VD, const ASTContext &ACtx) {
253 CanQualType CT = ACtx.getCanonicalType(T: VD->getType());
254 return CT.isConstQualified() && VD->getType().isTrivialType(Context: ACtx);
255}
256
257static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
258 return D->hasBody(Definition&: DefD);
259}
260static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
261 return D->getAnyInitializer(D&: DefD);
262}
263template <typename T> [[maybe_unused]] static bool hasBodyOrInit(const T *D) {
264 const T *Unused;
265 return hasBodyOrInit(D, Unused);
266}
267
268CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
269 : Context(CI.getASTContext()), ASTStorage(CI) {
270 if (CI.getAnalyzerOpts().ShouldEmitErrorsOnInvalidConfigValue &&
271 !CI.getAnalyzerOpts().CTUDir.empty()) {
272 auto S = CI.getVirtualFileSystem().status(Path: CI.getAnalyzerOpts().CTUDir);
273 if (!S || S->getType() != llvm::sys::fs::file_type::directory_file)
274 CI.getDiagnostics().Report(DiagID: diag::err_analyzer_config_invalid_input)
275 << "ctu-dir"
276 << "a filename";
277 }
278}
279
280CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
281
282std::optional<std::string>
283CrossTranslationUnitContext::getLookupName(const Decl *D) {
284 SmallString<128> DeclUSR;
285 bool Ret = index::generateUSRForDecl(D, Buf&: DeclUSR);
286 if (Ret)
287 return {};
288 return std::string(DeclUSR);
289}
290
291/// Recursively visits the decls of a DeclContext, and returns one with the
292/// given USR.
293template <typename T>
294const T *
295CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
296 StringRef LookupName) {
297 assert(DC && "Declaration Context must not be null");
298 for (const Decl *D : DC->decls()) {
299 const auto *SubDC = dyn_cast<DeclContext>(Val: D);
300 if (SubDC)
301 if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
302 return ND;
303
304 const auto *ND = dyn_cast<T>(D);
305 const T *ResultDecl;
306 if (!ND || !hasBodyOrInit(ND, ResultDecl))
307 continue;
308 std::optional<std::string> ResultLookupName = getLookupName(D: ResultDecl);
309 if (!ResultLookupName || *ResultLookupName != LookupName)
310 continue;
311 return ResultDecl;
312 }
313 return nullptr;
314}
315
316template <typename T>
317llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
318 const T *D, StringRef CrossTUDir, StringRef IndexName,
319 bool DisplayCTUProgress) {
320 assert(D && "D is missing, bad call to this function!");
321 assert(!hasBodyOrInit(D) &&
322 "D has a body or init in current translation unit!");
323 ++NumGetCTUCalled;
324 const std::optional<std::string> LookupName = getLookupName(D);
325 if (!LookupName)
326 return llvm::make_error<IndexError>(
327 Args: index_error_code::failed_to_generate_usr);
328 llvm::Expected<ASTUnit *> ASTUnitOrError =
329 loadExternalAST(LookupName: *LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
330 if (!ASTUnitOrError)
331 return ASTUnitOrError.takeError();
332 ASTUnit *Unit = *ASTUnitOrError;
333 assert(&Unit->getFileManager() ==
334 &Unit->getASTContext().getSourceManager().getFileManager());
335
336 const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
337 const llvm::Triple &TripleFrom =
338 Unit->getASTContext().getTargetInfo().getTriple();
339 // The imported AST had been generated for a different target.
340 // Some parts of the triple in the loaded ASTContext can be unknown while the
341 // very same parts in the target ASTContext are known. Thus we check for the
342 // known parts only.
343 if (!hasEqualKnownFields(Lhs: TripleTo, Rhs: TripleFrom)) {
344 // TODO: Pass the SourceLocation of the CallExpression for more precise
345 // diagnostics.
346 ++NumTripleMismatch;
347 return llvm::make_error<IndexError>(Args: index_error_code::triple_mismatch,
348 Args: std::string(Unit->getMainFileName()),
349 Args: TripleTo.str(), Args: TripleFrom.str());
350 }
351
352 const auto &LangTo = Context.getLangOpts();
353 const auto &LangFrom = Unit->getASTContext().getLangOpts();
354
355 // FIXME: Currenty we do not support CTU across C++ and C and across
356 // different dialects of C++.
357 if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
358 ++NumLangMismatch;
359 return llvm::make_error<IndexError>(
360 Args: index_error_code::lang_mismatch, Args: std::string(Unit->getMainFileName()),
361 Args: getLangDescription(LO: LangTo), Args: getLangDescription(LO: LangFrom));
362 }
363
364 // If CPP dialects are different then return with error.
365 //
366 // Consider this STL code:
367 // template<typename _Alloc>
368 // struct __alloc_traits
369 // #if __cplusplus >= 201103L
370 // : std::allocator_traits<_Alloc>
371 // #endif
372 // { // ...
373 // };
374 // This class template would create ODR errors during merging the two units,
375 // since in one translation unit the class template has a base class, however
376 // in the other unit it has none.
377 if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
378 LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
379 LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
380 LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
381 ++NumLangDialectMismatch;
382 return llvm::make_error<IndexError>(Args: index_error_code::lang_dialect_mismatch,
383 Args: std::string(Unit->getMainFileName()),
384 Args: getLangDescription(LO: LangTo),
385 Args: getLangDescription(LO: LangFrom));
386 }
387
388 TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
389 if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
390 return importDefinition(ResultDecl, Unit);
391 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
392}
393
394llvm::Expected<const FunctionDecl *>
395CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
396 StringRef CrossTUDir,
397 StringRef IndexName,
398 bool DisplayCTUProgress) {
399 return getCrossTUDefinitionImpl(D: FD, CrossTUDir, IndexName,
400 DisplayCTUProgress);
401}
402
403llvm::Expected<const VarDecl *>
404CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
405 StringRef CrossTUDir,
406 StringRef IndexName,
407 bool DisplayCTUProgress) {
408 return getCrossTUDefinitionImpl(D: VD, CrossTUDir, IndexName,
409 DisplayCTUProgress);
410}
411
412void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE,
413 SourceLocation Loc) {
414 switch (IE.getCode()) {
415 case index_error_code::missing_index_file:
416 case index_error_code::invocation_list_file_not_found:
417 // If the external def-map refers to source files, you must provide an
418 // invocation list file. Otherwise, CTU does not work at all, so you should
419 // check your build and analysis configuration.
420 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_error_opening)
421 << IE.getFileName();
422 return;
423
424 case index_error_code::invalid_index_format:
425 Context.getDiagnostics().Report(Loc, DiagID: diag::err_extdefmap_parsing)
426 << IE.getFileName() << IE.getLineNum();
427 return;
428
429 case index_error_code::multiple_definitions:
430 Context.getDiagnostics().Report(Loc, DiagID: diag::err_multiple_def_index)
431 << IE.getLineNum();
432 return;
433
434 case index_error_code::triple_mismatch:
435 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_triple)
436 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
437 return;
438
439 case index_error_code::missing_definition:
440 // Ignore missing definitions because it is very common to have some symbols
441 // defined outside of the analysis scope: they may be defined in 3-rd party
442 // and standard libraries, generated code, and files excluded from the
443 // analysis.
444 // Even ignoring it with Ignored diagnostic might generate too much traffic.
445 return;
446
447 case index_error_code::failed_import:
448 case index_error_code::unspecified:
449 // Not clear what happened exactly, but the outcome is a missing definition
450 // This is not a big deal, and is expected since ASTImporter is incomplete.
451 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_import_failure)
452 << Category->message(Condition: static_cast<int>(IE.getCode()));
453 return;
454
455 case index_error_code::failed_to_generate_usr:
456 // This is unlikely, so it is worth looking into, hence an error.
457 case index_error_code::failed_to_get_external_ast:
458 // This is suspicious, since the external AST is mentioned in the external
459 // defmap, so it should exist.
460 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_import_failure)
461 << Category->message(Condition: static_cast<int>(IE.getCode()));
462 return;
463
464 case index_error_code::load_threshold_reached:
465 // This is expected. It is still useful to be aware of, but it is normal
466 // operation. Emit the remark only once to avoid noise.
467 if (!HasEmittedLoadThresholdRemark) {
468 HasEmittedLoadThresholdRemark = true;
469 Context.getDiagnostics().Report(
470 Loc, DiagID: diag::remark_ctu_import_threshold_reached);
471 }
472 return;
473
474 case index_error_code::lang_mismatch:
475 case index_error_code::lang_dialect_mismatch:
476 // Similar to target triple mismatch.
477 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_lang)
478 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
479 return;
480
481 case index_error_code::invocation_list_wrong_format:
482 case index_error_code::invocation_list_empty:
483 // Without parsable invocation list, CTU cannot function.
484 Context.getDiagnostics().Report(Loc, DiagID: diag::err_invlist_parsing)
485 << IE.getFileName() << IE.getLineNum();
486 return;
487
488 case index_error_code::invocation_list_ambiguous:
489 // For automatically generated invocation lists, it is common to list
490 // multiple invocations, if a file is compiled in multiple contexts. No need
491 // to block CTU because of this.
492 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_multiple_entries_invlist)
493 << IE.getFileName();
494 return;
495
496 case index_error_code::invocation_list_lookup_unsuccessful:
497 // Some files might be missing in the invocation list. It is sad but not
498 // fatal, and CTU can take advantage of the definitions in files with known
499 // invocations.
500 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_invlist_missing_file)
501 << IE.getFileName();
502 return;
503
504 case index_error_code::success:
505 llvm_unreachable("Success is not an error.");
506 return;
507 }
508 llvm_unreachable("Unrecognized index_error_code.");
509}
510
511CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
512 CompilerInstance &CI)
513 : Loader(CI, CI.getAnalyzerOpts().CTUDir,
514 CI.getAnalyzerOpts().CTUInvocationList),
515 LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
516 ? CI.getAnalyzerOpts().CTUImportCppThreshold
517 : CI.getAnalyzerOpts().CTUImportThreshold) {}
518
519llvm::Expected<ASTUnit *>
520CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
521 StringRef FileName, bool DisplayCTUProgress) {
522 // Try the cache first.
523 auto ASTCacheEntry = FileASTUnitMap.find(Key: FileName);
524 if (ASTCacheEntry == FileASTUnitMap.end()) {
525
526 // Do not load if the limit is reached.
527 if (!LoadGuard) {
528 ++NumASTLoadThresholdReached;
529 return llvm::make_error<IndexError>(
530 Args: index_error_code::load_threshold_reached);
531 }
532
533 auto LoadAttempt = Loader.load(Identifier: FileName);
534
535 if (!LoadAttempt)
536 return LoadAttempt.takeError();
537
538 std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
539
540 // Need the raw pointer and the unique_ptr as well.
541 ASTUnit *Unit = LoadedUnit.get();
542
543 // Update the cache.
544 FileASTUnitMap[FileName] = std::move(LoadedUnit);
545
546 LoadGuard.indicateLoadSuccess();
547
548 if (DisplayCTUProgress)
549 llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
550
551 return Unit;
552
553 } else {
554 // Found in the cache.
555 return ASTCacheEntry->second.get();
556 }
557}
558
559llvm::Expected<ASTUnit *>
560CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
561 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
562 bool DisplayCTUProgress) {
563 // Try the cache first.
564 auto ASTCacheEntry = NameASTUnitMap.find(Key: FunctionName);
565 if (ASTCacheEntry == NameASTUnitMap.end()) {
566 // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
567
568 // Ensure that the Index is loaded, as we need to search in it.
569 if (llvm::Error IndexLoadError =
570 ensureCTUIndexLoaded(CrossTUDir, IndexName))
571 return std::move(IndexLoadError);
572
573 // Check if there is an entry in the index for the function.
574 auto It = NameFileMap.find(Key: FunctionName);
575 if (It == NameFileMap.end()) {
576 ++NumNotInOtherTU;
577 return llvm::make_error<IndexError>(Args: index_error_code::missing_definition);
578 }
579
580 // Search in the index for the filename where the definition of FunctionName
581 // resides.
582 if (llvm::Expected<ASTUnit *> FoundForFile =
583 getASTUnitForFile(FileName: It->second, DisplayCTUProgress)) {
584
585 // Update the cache.
586 NameASTUnitMap[FunctionName] = *FoundForFile;
587 return *FoundForFile;
588
589 } else {
590 return FoundForFile.takeError();
591 }
592 } else {
593 // Found in the cache.
594 return ASTCacheEntry->second;
595 }
596}
597
598llvm::Expected<std::string>
599CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
600 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
601 if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
602 return std::move(IndexLoadError);
603 return NameFileMap[FunctionName];
604}
605
606llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
607 StringRef CrossTUDir, StringRef IndexName) {
608 // Dont initialize if the map is filled.
609 if (!NameFileMap.empty())
610 return llvm::Error::success();
611
612 // Get the absolute path to the index file.
613 SmallString<256> IndexFile = CrossTUDir;
614 if (llvm::sys::path::is_absolute(path: IndexName))
615 IndexFile = IndexName;
616 else
617 llvm::sys::path::append(path&: IndexFile, a: IndexName);
618
619 if (auto IndexMapping = parseCrossTUIndex(IndexPath: IndexFile)) {
620 // Initialize member map.
621 NameFileMap = *IndexMapping;
622 return llvm::Error::success();
623 } else {
624 // Error while parsing CrossTU index file.
625 return IndexMapping.takeError();
626 };
627}
628
629llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
630 StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
631 bool DisplayCTUProgress) {
632 // FIXME: The current implementation only supports loading decls with
633 // a lookup name from a single translation unit. If multiple
634 // translation units contains decls with the same lookup name an
635 // error will be returned.
636
637 // Try to get the value from the heavily cached storage.
638 llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
639 FunctionName: LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
640
641 if (!Unit)
642 return Unit.takeError();
643
644 // Check whether the backing pointer of the Expected is a nullptr.
645 if (!*Unit)
646 return llvm::make_error<IndexError>(
647 Args: index_error_code::failed_to_get_external_ast);
648
649 return Unit;
650}
651
652CrossTranslationUnitContext::ASTLoader::ASTLoader(
653 CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
654 : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
655
656CrossTranslationUnitContext::LoadResultTy
657CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
658 llvm::SmallString<256> Path;
659 if (llvm::sys::path::is_absolute(path: Identifier, style: PathStyle)) {
660 Path = Identifier;
661 } else {
662 Path = CTUDir;
663 llvm::sys::path::append(path&: Path, style: PathStyle, a: Identifier);
664 }
665
666 // The path is stored in the InvocationList member in posix style. To
667 // successfully lookup an entry based on filepath, it must be converted.
668 llvm::sys::path::native(path&: Path, style: PathStyle);
669
670 // Normalize by removing relative path components.
671 llvm::sys::path::remove_dots(path&: Path, /*remove_dot_dot*/ true, style: PathStyle);
672
673 if (Path.ends_with(Suffix: ".ast"))
674 return loadFromDump(Identifier: Path);
675 else
676 return loadFromSource(Identifier: Path);
677}
678
679CrossTranslationUnitContext::LoadResultTy
680CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
681 auto DiagOpts = std::make_shared<DiagnosticOptions>();
682 TextDiagnosticPrinter *DiagClient =
683 new TextDiagnosticPrinter(llvm::errs(), *DiagOpts);
684 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(
685 A: DiagnosticIDs::create(), A&: *DiagOpts, A&: DiagClient);
686 return ASTUnit::LoadFromASTFile(
687 Filename: ASTDumpPath, PCHContainerRdr: CI.getPCHContainerOperations()->getRawReader(),
688 ToLoad: ASTUnit::LoadEverything, VFS: CI.getVirtualFileSystemPtr(), DiagOpts, Diags,
689 FileSystemOpts: CI.getFileSystemOpts(), HSOpts: CI.getHeaderSearchOpts());
690}
691
692/// Load the AST from a source-file, which is supposed to be located inside the
693/// YAML formatted invocation list file under the filesystem path specified by
694/// \p InvocationList. The invocation list should contain absolute paths.
695/// \p SourceFilePath is the absolute path of the source file that contains the
696/// function definition the analysis is looking for. The Index is built by the
697/// \p clang-extdef-mapping tool, which is also supposed to be generating
698/// absolute paths.
699///
700/// Proper diagnostic emission requires absolute paths, so even if a future
701/// change introduces the handling of relative paths, this must be taken into
702/// consideration.
703CrossTranslationUnitContext::LoadResultTy
704CrossTranslationUnitContext::ASTLoader::loadFromSource(
705 StringRef SourceFilePath) {
706
707 if (llvm::Error InitError = lazyInitInvocationList())
708 return std::move(InitError);
709 assert(InvocationList);
710
711 auto Invocation = InvocationList->find(Key: SourceFilePath);
712 if (Invocation == InvocationList->end())
713 return llvm::make_error<IndexError>(
714 Args: index_error_code::invocation_list_lookup_unsuccessful,
715 Args: SourceFilePath.str());
716
717 const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
718
719 SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
720 std::transform(first: InvocationCommand.begin(), last: InvocationCommand.end(),
721 result: CommandLineArgs.begin(),
722 unary_op: [](auto &&CmdPart) { return CmdPart.c_str(); });
723
724 auto DiagOpts = std::make_shared<DiagnosticOptions>(args&: CI.getDiagnosticOpts());
725 auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
726 IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
727 CI.getDiagnostics().getDiagnosticIDs()};
728 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(A&: DiagID, A&: *DiagOpts,
729 A&: DiagClient);
730
731 // This runs the driver which isn't expected to be free of sandbox violations.
732 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
733 return CreateASTUnitFromCommandLine(
734 ArgBegin: CommandLineArgs.begin(), ArgEnd: (CommandLineArgs.end()),
735 PCHContainerOps: CI.getPCHContainerOperations(), DiagOpts, Diags,
736 ResourceFilesPath: CI.getHeaderSearchOpts().ResourceDir);
737}
738
739llvm::Expected<InvocationListTy>
740parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle,
741 StringRef FilePath) {
742 InvocationListTy InvocationList;
743
744 /// LLVM YAML parser is used to extract information from invocation list file.
745 llvm::SourceMgr SM;
746 llvm::yaml::Stream InvocationFile(FileContent, SM);
747
748 auto GetLine = [&SM](const llvm::yaml::Node *N) -> int {
749 return N ? SM.FindLineNumber(Loc: N->getSourceRange().Start) : 0;
750 };
751 auto WrongFormatError = [&](const llvm::yaml::Node *N) {
752 return llvm::make_error<IndexError>(
753 Args: index_error_code::invocation_list_wrong_format, Args: FilePath.str(),
754 Args: GetLine(N));
755 };
756
757 /// Only the first document is processed.
758 llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
759
760 /// There has to be at least one document available.
761 if (FirstInvocationFile == InvocationFile.end())
762 return llvm::make_error<IndexError>(
763 Args: index_error_code::invocation_list_empty);
764
765 llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
766 if (!DocumentRoot)
767 return llvm::make_error<IndexError>(
768 Args: index_error_code::invocation_list_wrong_format);
769
770 /// According to the format specified the document must be a mapping, where
771 /// the keys are paths to source files, and values are sequences of invocation
772 /// parts.
773 auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(Val: DocumentRoot);
774 if (!Mappings)
775 return WrongFormatError(DocumentRoot);
776
777 for (auto &NextMapping : *Mappings) {
778 /// The keys should be strings, which represent a source-file path.
779 auto *Key =
780 dyn_cast_if_present<llvm::yaml::ScalarNode>(Val: NextMapping.getKey());
781 if (!Key)
782 return WrongFormatError(NextMapping.getKey());
783
784 SmallString<32> ValueStorage;
785 StringRef SourcePath = Key->getValue(Storage&: ValueStorage);
786
787 // Store paths with PathStyle directory separator.
788 SmallString<32> NativeSourcePath(SourcePath);
789 llvm::sys::path::native(path&: NativeSourcePath, style: PathStyle);
790
791 StringRef InvocationKey = NativeSourcePath;
792
793 if (InvocationList.contains(Key: InvocationKey))
794 return llvm::make_error<IndexError>(
795 Args: index_error_code::invocation_list_ambiguous, Args: InvocationKey.str());
796
797 /// The values should be sequences of strings, each representing a part of
798 /// the invocation.
799 auto *Args =
800 dyn_cast_if_present<llvm::yaml::SequenceNode>(Val: NextMapping.getValue());
801 if (!Args)
802 return WrongFormatError(NextMapping.getValue());
803
804 for (auto &Arg : *Args) {
805 auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(Val: &Arg);
806 if (!CmdString)
807 return WrongFormatError(&Arg);
808 /// Every conversion starts with an empty working storage, as it is not
809 /// clear if this is a requirement of the YAML parser.
810 ValueStorage.clear();
811 InvocationList[InvocationKey].emplace_back(
812 Args: CmdString->getValue(Storage&: ValueStorage));
813 }
814
815 if (InvocationList[InvocationKey].empty())
816 return WrongFormatError(Key);
817 }
818
819 return InvocationList;
820}
821
822llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
823 /// Lazily initialize the invocation list member used for on-demand parsing.
824 if (InvocationList)
825 return llvm::Error::success();
826 if (PreviousError)
827 return llvm::make_error<IndexError>(Args&: *PreviousError);
828
829 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
830 CI.getVirtualFileSystem().getBufferForFile(Name: InvocationListFilePath);
831 if (!FileContent) {
832 PreviousError = IndexError(index_error_code::invocation_list_file_not_found,
833 InvocationListFilePath.str());
834 return llvm::make_error<IndexError>(Args&: *PreviousError);
835 }
836 std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
837 assert(ContentBuffer && "If no error was produced after loading, the pointer "
838 "should not be nullptr.");
839
840 llvm::Expected<InvocationListTy> ExpectedInvocationList = parseInvocationList(
841 FileContent: ContentBuffer->getBuffer(), PathStyle, FilePath: InvocationListFilePath);
842
843 if (!ExpectedInvocationList) {
844 llvm::handleAllErrors(
845 E: ExpectedInvocationList.takeError(),
846 Handlers: [this](const IndexError &E) { this->PreviousError = E; });
847 return llvm::make_error<IndexError>(Args&: *PreviousError);
848 }
849
850 InvocationList = *ExpectedInvocationList;
851
852 return llvm::Error::success();
853}
854
855template <typename T>
856llvm::Expected<const T *>
857CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
858 assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
859
860 assert(&D->getASTContext() == &Unit->getASTContext() &&
861 "ASTContext of Decl and the unit should match.");
862 ASTImporter &Importer = getOrCreateASTImporter(Unit);
863
864 auto ToDeclOrError = Importer.Import(D);
865 if (!ToDeclOrError) {
866 handleAllErrors(ToDeclOrError.takeError(), [&](const ASTImportError &IE) {
867 switch (IE.Error) {
868 case ASTImportError::NameConflict:
869 ++NumNameConflicts;
870 break;
871 case ASTImportError::UnsupportedConstruct:
872 ++NumUnsupportedNodeFound;
873 break;
874 case ASTImportError::Unknown:
875 llvm_unreachable("Unknown import error happened.");
876 break;
877 }
878 });
879 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
880 }
881 auto *ToDecl = cast<T>(*ToDeclOrError);
882 assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
883 ++NumGetCTUSuccess;
884
885 // Parent map is invalidated after changing the AST.
886 ToDecl->getASTContext().getParentMapContext().clear();
887
888 return ToDecl;
889}
890
891llvm::Expected<const FunctionDecl *>
892CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
893 ASTUnit *Unit) {
894 return importDefinitionImpl(D: FD, Unit);
895}
896
897llvm::Expected<const VarDecl *>
898CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
899 ASTUnit *Unit) {
900 return importDefinitionImpl(D: VD, Unit);
901}
902
903void CrossTranslationUnitContext::lazyInitImporterSharedSt(
904 TranslationUnitDecl *ToTU) {
905 if (!ImporterSharedSt)
906 ImporterSharedSt = std::make_shared<ASTImporterSharedState>(args&: *ToTU);
907}
908
909ASTImporter &
910CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
911 ASTContext &From = Unit->getASTContext();
912
913 auto I = ASTUnitImporterMap.find(Val: From.getTranslationUnitDecl());
914 if (I != ASTUnitImporterMap.end())
915 return *I->second;
916 lazyInitImporterSharedSt(ToTU: Context.getTranslationUnitDecl());
917 ASTImporter *NewImporter = new ASTImporter(
918 Context, Context.getSourceManager().getFileManager(), From,
919 From.getSourceManager().getFileManager(), false, ImporterSharedSt);
920 ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(p: NewImporter);
921 return *NewImporter;
922}
923
924std::optional<clang::MacroExpansionContext>
925CrossTranslationUnitContext::getMacroExpansionContextForSourceLocation(
926 const clang::SourceLocation &ToLoc) const {
927 // FIXME: Implement: Record such a context for every imported ASTUnit; lookup.
928 return std::nullopt;
929}
930
931bool CrossTranslationUnitContext::isImportedAsNew(const Decl *ToDecl) const {
932 if (!ImporterSharedSt)
933 return false;
934 return ImporterSharedSt->isNewDecl(ToD: const_cast<Decl *>(ToDecl));
935}
936
937bool CrossTranslationUnitContext::hasError(const Decl *ToDecl) const {
938 if (!ImporterSharedSt)
939 return false;
940 return static_cast<bool>(
941 ImporterSharedSt->getImportDeclErrorIfAny(ToD: const_cast<Decl *>(ToDecl)));
942}
943
944} // namespace cross_tu
945} // namespace clang
946