1//===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CrossTranslationUnit interface.
10//
11//===----------------------------------------------------------------------===//
12#include "clang/CrossTU/CrossTranslationUnit.h"
13#include "clang/AST/ASTImporter.h"
14#include "clang/AST/Decl.h"
15#include "clang/AST/ParentMapContext.h"
16#include "clang/Basic/DiagnosticDriver.h"
17#include "clang/Basic/TargetInfo.h"
18#include "clang/CrossTU/CrossTUDiagnostic.h"
19#include "clang/Driver/CreateASTUnitFromArgs.h"
20#include "clang/Frontend/ASTUnit.h"
21#include "clang/Frontend/CompilerInstance.h"
22#include "clang/Frontend/TextDiagnosticPrinter.h"
23#include "clang/UnifiedSymbolResolution/USRGeneration.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Option/ArgList.h"
26#include "llvm/Support/ErrorHandling.h"
27#include "llvm/Support/IOSandbox.h"
28#include "llvm/Support/ManagedStatic.h"
29#include "llvm/Support/Path.h"
30#include "llvm/Support/YAMLParser.h"
31#include "llvm/Support/raw_ostream.h"
32#include "llvm/TargetParser/Triple.h"
33#include <algorithm>
34#include <fstream>
35#include <optional>
36#include <sstream>
37#include <tuple>
38
39namespace clang {
40namespace cross_tu {
41
42namespace {
43
44#define DEBUG_TYPE "CrossTranslationUnit"
45STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
46STATISTIC(
47 NumNotInOtherTU,
48 "The # of getCTUDefinition called but the function is not in any other TU");
49STATISTIC(NumGetCTUSuccess,
50 "The # of getCTUDefinition successfully returned the "
51 "requested function's body");
52STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
53 "encountered an unsupported AST Node");
54STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
55 "encountered an ODR error");
56STATISTIC(NumTripleMismatch, "The # of triple mismatches");
57STATISTIC(NumLangMismatch, "The # of language mismatches");
58STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
59STATISTIC(NumASTLoadThresholdReached,
60 "The # of ASTs not loaded because of threshold");
61
62// Same as Triple's equality operator, but we check a field only if that is
63// known in both instances.
64bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
65 using llvm::Triple;
66 if (Lhs.getArch() != Triple::UnknownArch &&
67 Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
68 return false;
69 if (Lhs.getSubArch() != Triple::NoSubArch &&
70 Rhs.getSubArch() != Triple::NoSubArch &&
71 Lhs.getSubArch() != Rhs.getSubArch())
72 return false;
73 if (Lhs.getVendor() != Triple::UnknownVendor &&
74 Rhs.getVendor() != Triple::UnknownVendor &&
75 Lhs.getVendor() != Rhs.getVendor())
76 return false;
77 if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
78 Lhs.getOS() != Rhs.getOS())
79 return false;
80 if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
81 Rhs.getEnvironment() != Triple::UnknownEnvironment &&
82 Lhs.getEnvironment() != Rhs.getEnvironment())
83 return false;
84 if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
85 Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
86 Lhs.getObjectFormat() != Rhs.getObjectFormat())
87 return false;
88 return true;
89}
90
91// FIXME: This class is will be removed after the transition to llvm::Error.
92class IndexErrorCategory : public std::error_category {
93public:
94 const char *name() const noexcept override { return "clang.index"; }
95
96 std::string message(int Condition) const override {
97 switch (static_cast<index_error_code>(Condition)) {
98 case index_error_code::success:
99 // There should not be a success error. Jump to unreachable directly.
100 // Add this case to make the compiler stop complaining.
101 break;
102 case index_error_code::unspecified:
103 return "An unknown error has occurred.";
104 case index_error_code::missing_index_file:
105 return "The index file is missing.";
106 case index_error_code::invalid_index_format:
107 return "Invalid index file format.";
108 case index_error_code::multiple_definitions:
109 return "Multiple definitions in the index file.";
110 case index_error_code::missing_definition:
111 return "Missing definition from the index file.";
112 case index_error_code::failed_import:
113 return "Failed to import the definition.";
114 case index_error_code::failed_to_get_external_ast:
115 return "Failed to load external AST source.";
116 case index_error_code::failed_to_generate_usr:
117 return "Failed to generate USR.";
118 case index_error_code::triple_mismatch:
119 return "Triple mismatch";
120 case index_error_code::lang_mismatch:
121 return "Language mismatch";
122 case index_error_code::lang_dialect_mismatch:
123 return "Language dialect mismatch";
124 case index_error_code::load_threshold_reached:
125 return "Load threshold reached";
126 case index_error_code::invocation_list_ambiguous:
127 return "Invocation list file contains multiple references to the same "
128 "source file.";
129 case index_error_code::invocation_list_file_not_found:
130 return "Invocation list file is not found.";
131 case index_error_code::invocation_list_empty:
132 return "Invocation list file is empty.";
133 case index_error_code::invocation_list_wrong_format:
134 return "Invocation list file is in wrong format.";
135 case index_error_code::invocation_list_lookup_unsuccessful:
136 return "Invocation list file does not contain the requested source file.";
137 }
138 llvm_unreachable("Unrecognized index_error_code.");
139 }
140};
141
142static llvm::ManagedStatic<IndexErrorCategory> Category;
143} // end anonymous namespace
144
145/// Returns a human-readable language/dialect description for diagnostics.
146/// Checks flags from highest to lowest standard since they are cumulative
147/// (e.g. CPlusPlus20 implies CPlusPlus17).
148/// This does not cover all possible languages (e.g. Obj-C or flavors of C),
149/// because CTU currently does not differentiate between them.
150static std::string getLangDescription(const LangOptions &LO) {
151 if (!LO.CPlusPlus)
152 return "non-C++";
153 if (LO.CPlusPlus29)
154 return "C++29";
155 if (LO.CPlusPlus26)
156 return "C++26";
157 if (LO.CPlusPlus23)
158 return "C++23";
159 if (LO.CPlusPlus20)
160 return "C++20";
161 if (LO.CPlusPlus17)
162 return "C++17";
163 if (LO.CPlusPlus14)
164 return "C++14";
165 if (LO.CPlusPlus11)
166 return "C++11";
167 return "C++98";
168}
169
170char IndexError::ID;
171
172void IndexError::log(raw_ostream &OS) const {
173 OS << Category->message(Condition: static_cast<int>(Code)) << '\n';
174}
175
176std::error_code IndexError::convertToErrorCode() const {
177 return std::error_code(static_cast<int>(Code), *Category);
178}
179
180/// Parse one line of the input CTU index file.
181///
182/// @param[in] LineRef The input CTU index item in format
183/// "<USR-Length>:<USR> <File-Path>".
184/// @param[out] LookupName The lookup name in format "<USR-Length>:<USR>".
185/// @param[out] FilePath The file path "<File-Path>".
186static bool parseCrossTUIndexItem(StringRef LineRef, StringRef &LookupName,
187 StringRef &FilePath) {
188 // `LineRef` is "<USR-Length>:<USR> <File-Path>" now.
189
190 size_t USRLength = 0;
191 if (LineRef.consumeInteger(Radix: 10, Result&: USRLength))
192 return false;
193 assert(USRLength && "USRLength should be greater than zero.");
194
195 if (!LineRef.consume_front(Prefix: ":"))
196 return false;
197
198 // `LineRef` is now just "<USR> <File-Path>".
199
200 // Check LookupName length out of bound and incorrect delimiter.
201 if (USRLength >= LineRef.size() || ' ' != LineRef[USRLength])
202 return false;
203
204 LookupName = LineRef.substr(Start: 0, N: USRLength);
205 FilePath = LineRef.substr(Start: USRLength + 1);
206 return true;
207}
208
209llvm::Expected<llvm::StringMap<std::string>>
210parseCrossTUIndex(StringRef IndexPath) {
211 std::ifstream ExternalMapFile{std::string(IndexPath)};
212 if (!ExternalMapFile)
213 return llvm::make_error<IndexError>(Args: index_error_code::missing_index_file,
214 Args: IndexPath.str());
215
216 llvm::StringMap<std::string> Result;
217 std::string Line;
218 unsigned LineNo = 1;
219 while (std::getline(is&: ExternalMapFile, str&: Line)) {
220 // Split lookup name and file path
221 StringRef LookupName, FilePathInIndex;
222 if (!parseCrossTUIndexItem(LineRef: Line, LookupName, FilePath&: FilePathInIndex))
223 return llvm::make_error<IndexError>(
224 Args: index_error_code::invalid_index_format, Args: IndexPath.str(), Args&: LineNo);
225
226 // Store paths with posix-style directory separator.
227 SmallString<32> FilePath(FilePathInIndex);
228 llvm::sys::path::native(path&: FilePath, style: llvm::sys::path::Style::posix);
229
230 bool InsertionOccurred;
231 std::tie(args: std::ignore, args&: InsertionOccurred) =
232 Result.try_emplace(Key: LookupName, Args: FilePath.begin(), Args: FilePath.end());
233 if (!InsertionOccurred)
234 return llvm::make_error<IndexError>(
235 Args: index_error_code::multiple_definitions, Args: IndexPath.str(), Args&: LineNo);
236
237 ++LineNo;
238 }
239 return Result;
240}
241
242std::string
243createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
244 std::ostringstream Result;
245 for (const auto &E : Index)
246 Result << E.getKey().size() << ':' << E.getKey().str() << ' '
247 << E.getValue() << '\n';
248 return Result.str();
249}
250
251bool shouldImport(const VarDecl *VD, const ASTContext &ACtx) {
252 CanQualType CT = ACtx.getCanonicalType(T: VD->getType());
253 return CT.isConstQualified() && VD->getType().isTrivialType(Context: ACtx);
254}
255
256static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
257 return D->hasBody(Definition&: DefD);
258}
259static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
260 return D->getAnyInitializer(D&: DefD);
261}
262template <typename T> [[maybe_unused]] static bool hasBodyOrInit(const T *D) {
263 const T *Unused;
264 return hasBodyOrInit(D, Unused);
265}
266
267CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
268 : Context(CI.getASTContext()), ASTStorage(CI) {
269 if (CI.getAnalyzerOpts().ShouldEmitErrorsOnInvalidConfigValue &&
270 !CI.getAnalyzerOpts().CTUDir.empty()) {
271 auto S = CI.getVirtualFileSystem().status(Path: CI.getAnalyzerOpts().CTUDir);
272 if (!S || S->getType() != llvm::sys::fs::file_type::directory_file)
273 CI.getDiagnostics().Report(DiagID: diag::err_analyzer_config_invalid_input)
274 << "ctu-dir"
275 << "a filename";
276 }
277}
278
279CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
280
281std::optional<std::string>
282CrossTranslationUnitContext::getLookupName(const Decl *D) {
283 SmallString<128> DeclUSR;
284 bool Ret = index::generateUSRForDecl(D, Buf&: DeclUSR);
285 if (Ret)
286 return {};
287 return std::string(DeclUSR);
288}
289
290/// Recursively visits the decls of a DeclContext, and returns one with the
291/// given USR.
292template <typename T>
293const T *
294CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
295 StringRef LookupName) {
296 assert(DC && "Declaration Context must not be null");
297 for (const Decl *D : DC->decls()) {
298 const auto *SubDC = dyn_cast<DeclContext>(Val: D);
299 if (SubDC)
300 if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
301 return ND;
302
303 const auto *ND = dyn_cast<T>(D);
304 const T *ResultDecl;
305 if (!ND || !hasBodyOrInit(ND, ResultDecl))
306 continue;
307 std::optional<std::string> ResultLookupName = getLookupName(D: ResultDecl);
308 if (!ResultLookupName || *ResultLookupName != LookupName)
309 continue;
310 return ResultDecl;
311 }
312 return nullptr;
313}
314
315template <typename T>
316llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
317 const T *D, StringRef CrossTUDir, StringRef IndexName,
318 bool DisplayCTUProgress) {
319 assert(D && "D is missing, bad call to this function!");
320 assert(!hasBodyOrInit(D) &&
321 "D has a body or init in current translation unit!");
322 ++NumGetCTUCalled;
323 const std::optional<std::string> LookupName = getLookupName(D);
324 if (!LookupName)
325 return llvm::make_error<IndexError>(
326 Args: index_error_code::failed_to_generate_usr);
327 llvm::Expected<ASTUnit *> ASTUnitOrError =
328 loadExternalAST(LookupName: *LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
329 if (!ASTUnitOrError)
330 return ASTUnitOrError.takeError();
331 ASTUnit *Unit = *ASTUnitOrError;
332 assert(&Unit->getFileManager() ==
333 &Unit->getASTContext().getSourceManager().getFileManager());
334
335 const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
336 const llvm::Triple &TripleFrom =
337 Unit->getASTContext().getTargetInfo().getTriple();
338 // The imported AST had been generated for a different target.
339 // Some parts of the triple in the loaded ASTContext can be unknown while the
340 // very same parts in the target ASTContext are known. Thus we check for the
341 // known parts only.
342 if (!hasEqualKnownFields(Lhs: TripleTo, Rhs: TripleFrom)) {
343 // TODO: Pass the SourceLocation of the CallExpression for more precise
344 // diagnostics.
345 ++NumTripleMismatch;
346 return llvm::make_error<IndexError>(Args: index_error_code::triple_mismatch,
347 Args: std::string(Unit->getMainFileName()),
348 Args: TripleTo.str(), Args: TripleFrom.str());
349 }
350
351 const auto &LangTo = Context.getLangOpts();
352 const auto &LangFrom = Unit->getASTContext().getLangOpts();
353
354 // FIXME: Currenty we do not support CTU across C++ and C and across
355 // different dialects of C++.
356 if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
357 ++NumLangMismatch;
358 return llvm::make_error<IndexError>(
359 Args: index_error_code::lang_mismatch, Args: std::string(Unit->getMainFileName()),
360 Args: getLangDescription(LO: LangTo), Args: getLangDescription(LO: LangFrom));
361 }
362
363 // If CPP dialects are different then return with error.
364 //
365 // Consider this STL code:
366 // template<typename _Alloc>
367 // struct __alloc_traits
368 // #if __cplusplus >= 201103L
369 // : std::allocator_traits<_Alloc>
370 // #endif
371 // { // ...
372 // };
373 // This class template would create ODR errors during merging the two units,
374 // since in one translation unit the class template has a base class, however
375 // in the other unit it has none.
376 if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
377 LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
378 LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
379 LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
380 ++NumLangDialectMismatch;
381 return llvm::make_error<IndexError>(Args: index_error_code::lang_dialect_mismatch,
382 Args: std::string(Unit->getMainFileName()),
383 Args: getLangDescription(LO: LangTo),
384 Args: getLangDescription(LO: LangFrom));
385 }
386
387 TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
388 if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
389 return importDefinition(ResultDecl, Unit);
390 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
391}
392
393llvm::Expected<const FunctionDecl *>
394CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
395 StringRef CrossTUDir,
396 StringRef IndexName,
397 bool DisplayCTUProgress) {
398 return getCrossTUDefinitionImpl(D: FD, CrossTUDir, IndexName,
399 DisplayCTUProgress);
400}
401
402llvm::Expected<const VarDecl *>
403CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
404 StringRef CrossTUDir,
405 StringRef IndexName,
406 bool DisplayCTUProgress) {
407 return getCrossTUDefinitionImpl(D: VD, CrossTUDir, IndexName,
408 DisplayCTUProgress);
409}
410
411void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE,
412 SourceLocation Loc) {
413 switch (IE.getCode()) {
414 case index_error_code::missing_index_file:
415 case index_error_code::invocation_list_file_not_found:
416 // If the external def-map refers to source files, you must provide an
417 // invocation list file. Otherwise, CTU does not work at all, so you should
418 // check your build and analysis configuration.
419 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_error_opening)
420 << IE.getFileName();
421 return;
422
423 case index_error_code::invalid_index_format:
424 Context.getDiagnostics().Report(Loc, DiagID: diag::err_extdefmap_parsing)
425 << IE.getFileName() << IE.getLineNum();
426 return;
427
428 case index_error_code::multiple_definitions:
429 Context.getDiagnostics().Report(Loc, DiagID: diag::err_multiple_def_index)
430 << IE.getLineNum();
431 return;
432
433 case index_error_code::triple_mismatch:
434 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_triple)
435 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
436 return;
437
438 case index_error_code::missing_definition:
439 // Ignore missing definitions because it is very common to have some symbols
440 // defined outside of the analysis scope: they may be defined in 3-rd party
441 // and standard libraries, generated code, and files excluded from the
442 // analysis.
443 // Even ignoring it with Ignored diagnostic might generate too much traffic.
444 return;
445
446 case index_error_code::failed_import:
447 case index_error_code::unspecified:
448 // Not clear what happened exactly, but the outcome is a missing definition
449 // This is not a big deal, and is expected since ASTImporter is incomplete.
450 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_import_failure)
451 << Category->message(Condition: static_cast<int>(IE.getCode()));
452 return;
453
454 case index_error_code::failed_to_generate_usr:
455 // This is unlikely, so it is worth looking into, hence an error.
456 case index_error_code::failed_to_get_external_ast:
457 // This is suspicious, since the external AST is mentioned in the external
458 // defmap, so it should exist.
459 Context.getDiagnostics().Report(Loc, DiagID: diag::err_ctu_import_failure)
460 << Category->message(Condition: static_cast<int>(IE.getCode()));
461 return;
462
463 case index_error_code::load_threshold_reached:
464 // This is expected. It is still useful to be aware of, but it is normal
465 // operation. Emit the remark only once to avoid noise.
466 if (!HasEmittedLoadThresholdRemark) {
467 HasEmittedLoadThresholdRemark = true;
468 Context.getDiagnostics().Report(
469 Loc, DiagID: diag::remark_ctu_import_threshold_reached);
470 }
471 return;
472
473 case index_error_code::lang_mismatch:
474 case index_error_code::lang_dialect_mismatch:
475 // Similar to target triple mismatch.
476 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_ctu_incompat_lang)
477 << IE.getFileName() << IE.getConfigToName() << IE.getConfigFromName();
478 return;
479
480 case index_error_code::invocation_list_wrong_format:
481 case index_error_code::invocation_list_empty:
482 // Without parsable invocation list, CTU cannot function.
483 Context.getDiagnostics().Report(Loc, DiagID: diag::err_invlist_parsing)
484 << IE.getFileName() << IE.getLineNum();
485 return;
486
487 case index_error_code::invocation_list_ambiguous:
488 // For automatically generated invocation lists, it is common to list
489 // multiple invocations, if a file is compiled in multiple contexts. No need
490 // to block CTU because of this.
491 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_multiple_entries_invlist)
492 << IE.getFileName();
493 return;
494
495 case index_error_code::invocation_list_lookup_unsuccessful:
496 // Some files might be missing in the invocation list. It is sad but not
497 // fatal, and CTU can take advantage of the definitions in files with known
498 // invocations.
499 Context.getDiagnostics().Report(Loc, DiagID: diag::warn_invlist_missing_file)
500 << IE.getFileName();
501 return;
502
503 case index_error_code::success:
504 llvm_unreachable("Success is not an error.");
505 return;
506 }
507 llvm_unreachable("Unrecognized index_error_code.");
508}
509
510CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
511 CompilerInstance &CI)
512 : Loader(CI, CI.getAnalyzerOpts().CTUDir,
513 CI.getAnalyzerOpts().CTUInvocationList),
514 LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
515 ? CI.getAnalyzerOpts().CTUImportCppThreshold
516 : CI.getAnalyzerOpts().CTUImportThreshold) {}
517
518llvm::Expected<ASTUnit *>
519CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
520 StringRef FileName, bool DisplayCTUProgress) {
521 // Try the cache first.
522 auto ASTCacheEntry = FileASTUnitMap.find(Key: FileName);
523 if (ASTCacheEntry == FileASTUnitMap.end()) {
524
525 // Do not load if the limit is reached.
526 if (!LoadGuard) {
527 ++NumASTLoadThresholdReached;
528 return llvm::make_error<IndexError>(
529 Args: index_error_code::load_threshold_reached);
530 }
531
532 auto LoadAttempt = Loader.load(Identifier: FileName);
533
534 if (!LoadAttempt)
535 return LoadAttempt.takeError();
536
537 std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
538
539 // Need the raw pointer and the unique_ptr as well.
540 ASTUnit *Unit = LoadedUnit.get();
541
542 // Update the cache.
543 FileASTUnitMap[FileName] = std::move(LoadedUnit);
544
545 LoadGuard.indicateLoadSuccess();
546
547 if (DisplayCTUProgress)
548 llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
549
550 return Unit;
551
552 } else {
553 // Found in the cache.
554 return ASTCacheEntry->second.get();
555 }
556}
557
558llvm::Expected<ASTUnit *>
559CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
560 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
561 bool DisplayCTUProgress) {
562 // Try the cache first.
563 auto ASTCacheEntry = NameASTUnitMap.find(Key: FunctionName);
564 if (ASTCacheEntry == NameASTUnitMap.end()) {
565 // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
566
567 // Ensure that the Index is loaded, as we need to search in it.
568 if (llvm::Error IndexLoadError =
569 ensureCTUIndexLoaded(CrossTUDir, IndexName))
570 return std::move(IndexLoadError);
571
572 // Check if there is an entry in the index for the function.
573 auto It = NameFileMap.find(Key: FunctionName);
574 if (It == NameFileMap.end()) {
575 ++NumNotInOtherTU;
576 return llvm::make_error<IndexError>(Args: index_error_code::missing_definition);
577 }
578
579 // Search in the index for the filename where the definition of FunctionName
580 // resides.
581 if (llvm::Expected<ASTUnit *> FoundForFile =
582 getASTUnitForFile(FileName: It->second, DisplayCTUProgress)) {
583
584 // Update the cache.
585 NameASTUnitMap[FunctionName] = *FoundForFile;
586 return *FoundForFile;
587
588 } else {
589 return FoundForFile.takeError();
590 }
591 } else {
592 // Found in the cache.
593 return ASTCacheEntry->second;
594 }
595}
596
597llvm::Expected<std::string>
598CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
599 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
600 if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
601 return std::move(IndexLoadError);
602 return NameFileMap[FunctionName];
603}
604
605llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
606 StringRef CrossTUDir, StringRef IndexName) {
607 // Dont initialize if the map is filled.
608 if (!NameFileMap.empty())
609 return llvm::Error::success();
610
611 // Get the absolute path to the index file.
612 SmallString<256> IndexFile = CrossTUDir;
613 if (llvm::sys::path::is_absolute(path: IndexName))
614 IndexFile = IndexName;
615 else
616 llvm::sys::path::append(path&: IndexFile, a: IndexName);
617
618 if (auto IndexMapping = parseCrossTUIndex(IndexPath: IndexFile)) {
619 // Initialize member map.
620 NameFileMap = *IndexMapping;
621 return llvm::Error::success();
622 } else {
623 // Error while parsing CrossTU index file.
624 return IndexMapping.takeError();
625 };
626}
627
628llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
629 StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
630 bool DisplayCTUProgress) {
631 // FIXME: The current implementation only supports loading decls with
632 // a lookup name from a single translation unit. If multiple
633 // translation units contains decls with the same lookup name an
634 // error will be returned.
635
636 // Try to get the value from the heavily cached storage.
637 llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
638 FunctionName: LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
639
640 if (!Unit)
641 return Unit.takeError();
642
643 // Check whether the backing pointer of the Expected is a nullptr.
644 if (!*Unit)
645 return llvm::make_error<IndexError>(
646 Args: index_error_code::failed_to_get_external_ast);
647
648 return Unit;
649}
650
651CrossTranslationUnitContext::ASTLoader::ASTLoader(
652 CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
653 : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
654
655CrossTranslationUnitContext::LoadResultTy
656CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
657 llvm::SmallString<256> Path;
658 if (llvm::sys::path::is_absolute(path: Identifier, style: PathStyle)) {
659 Path = Identifier;
660 } else {
661 Path = CTUDir;
662 llvm::sys::path::append(path&: Path, style: PathStyle, a: Identifier);
663 }
664
665 // The path is stored in the InvocationList member in posix style. To
666 // successfully lookup an entry based on filepath, it must be converted.
667 llvm::sys::path::native(path&: Path, style: PathStyle);
668
669 // Normalize by removing relative path components.
670 llvm::sys::path::remove_dots(path&: Path, /*remove_dot_dot*/ true, style: PathStyle);
671
672 if (Path.ends_with(Suffix: ".ast"))
673 return loadFromDump(Identifier: Path);
674 else
675 return loadFromSource(Identifier: Path);
676}
677
678CrossTranslationUnitContext::LoadResultTy
679CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
680 auto DiagOpts = std::make_shared<DiagnosticOptions>();
681 TextDiagnosticPrinter *DiagClient =
682 new TextDiagnosticPrinter(llvm::errs(), *DiagOpts);
683 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(
684 A: DiagnosticIDs::create(), A&: *DiagOpts, A&: DiagClient);
685 return ASTUnit::LoadFromASTFile(
686 Filename: ASTDumpPath, PCHContainerRdr: CI.getPCHContainerOperations()->getRawReader(),
687 ToLoad: ASTUnit::LoadEverything, VFS: CI.getVirtualFileSystemPtr(), DiagOpts, Diags,
688 FileSystemOpts: CI.getFileSystemOpts(), HSOpts: CI.getHeaderSearchOpts());
689}
690
691/// Load the AST from a source-file, which is supposed to be located inside the
692/// YAML formatted invocation list file under the filesystem path specified by
693/// \p InvocationList. The invocation list should contain absolute paths.
694/// \p SourceFilePath is the absolute path of the source file that contains the
695/// function definition the analysis is looking for. The Index is built by the
696/// \p clang-extdef-mapping tool, which is also supposed to be generating
697/// absolute paths.
698///
699/// Proper diagnostic emission requires absolute paths, so even if a future
700/// change introduces the handling of relative paths, this must be taken into
701/// consideration.
702CrossTranslationUnitContext::LoadResultTy
703CrossTranslationUnitContext::ASTLoader::loadFromSource(
704 StringRef SourceFilePath) {
705
706 if (llvm::Error InitError = lazyInitInvocationList())
707 return std::move(InitError);
708 assert(InvocationList);
709
710 auto Invocation = InvocationList->find(Key: SourceFilePath);
711 if (Invocation == InvocationList->end())
712 return llvm::make_error<IndexError>(
713 Args: index_error_code::invocation_list_lookup_unsuccessful,
714 Args: SourceFilePath.str());
715
716 const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
717
718 SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
719 std::transform(first: InvocationCommand.begin(), last: InvocationCommand.end(),
720 result: CommandLineArgs.begin(),
721 unary_op: [](auto &&CmdPart) { return CmdPart.c_str(); });
722
723 auto DiagOpts = std::make_shared<DiagnosticOptions>(args&: CI.getDiagnosticOpts());
724 auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
725 IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
726 CI.getDiagnostics().getDiagnosticIDs()};
727 auto Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(A&: DiagID, A&: *DiagOpts,
728 A&: DiagClient);
729
730 // This runs the driver which isn't expected to be free of sandbox violations.
731 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
732 return CreateASTUnitFromCommandLine(
733 ArgBegin: CommandLineArgs.begin(), ArgEnd: (CommandLineArgs.end()),
734 PCHContainerOps: CI.getPCHContainerOperations(), DiagOpts, Diags,
735 ResourceFilesPath: CI.getHeaderSearchOpts().ResourceDir);
736}
737
738llvm::Expected<InvocationListTy>
739parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle,
740 StringRef FilePath) {
741 InvocationListTy InvocationList;
742
743 /// LLVM YAML parser is used to extract information from invocation list file.
744 llvm::SourceMgr SM;
745 llvm::yaml::Stream InvocationFile(FileContent, SM);
746
747 auto GetLine = [&SM](const llvm::yaml::Node *N) -> int {
748 return N ? SM.FindLineNumber(Loc: N->getSourceRange().Start) : 0;
749 };
750 auto WrongFormatError = [&](const llvm::yaml::Node *N) {
751 return llvm::make_error<IndexError>(
752 Args: index_error_code::invocation_list_wrong_format, Args: FilePath.str(),
753 Args: GetLine(N));
754 };
755
756 /// Only the first document is processed.
757 llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
758
759 /// There has to be at least one document available.
760 if (FirstInvocationFile == InvocationFile.end())
761 return llvm::make_error<IndexError>(
762 Args: index_error_code::invocation_list_empty);
763
764 llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
765 if (!DocumentRoot)
766 return llvm::make_error<IndexError>(
767 Args: index_error_code::invocation_list_wrong_format);
768
769 /// According to the format specified the document must be a mapping, where
770 /// the keys are paths to source files, and values are sequences of invocation
771 /// parts.
772 auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(Val: DocumentRoot);
773 if (!Mappings)
774 return WrongFormatError(DocumentRoot);
775
776 for (auto &NextMapping : *Mappings) {
777 /// The keys should be strings, which represent a source-file path.
778 auto *Key =
779 dyn_cast_if_present<llvm::yaml::ScalarNode>(Val: NextMapping.getKey());
780 if (!Key)
781 return WrongFormatError(NextMapping.getKey());
782
783 SmallString<32> ValueStorage;
784 StringRef SourcePath = Key->getValue(Storage&: ValueStorage);
785
786 // Store paths with PathStyle directory separator.
787 SmallString<32> NativeSourcePath(SourcePath);
788 llvm::sys::path::native(path&: NativeSourcePath, style: PathStyle);
789
790 StringRef InvocationKey = NativeSourcePath;
791
792 if (InvocationList.contains(Key: InvocationKey))
793 return llvm::make_error<IndexError>(
794 Args: index_error_code::invocation_list_ambiguous, Args: InvocationKey.str());
795
796 /// The values should be sequences of strings, each representing a part of
797 /// the invocation.
798 auto *Args =
799 dyn_cast_if_present<llvm::yaml::SequenceNode>(Val: NextMapping.getValue());
800 if (!Args)
801 return WrongFormatError(NextMapping.getValue());
802
803 for (auto &Arg : *Args) {
804 auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(Val: &Arg);
805 if (!CmdString)
806 return WrongFormatError(&Arg);
807 /// Every conversion starts with an empty working storage, as it is not
808 /// clear if this is a requirement of the YAML parser.
809 ValueStorage.clear();
810 InvocationList[InvocationKey].emplace_back(
811 Args: CmdString->getValue(Storage&: ValueStorage));
812 }
813
814 if (InvocationList[InvocationKey].empty())
815 return WrongFormatError(Key);
816 }
817
818 return InvocationList;
819}
820
821llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
822 /// Lazily initialize the invocation list member used for on-demand parsing.
823 if (InvocationList)
824 return llvm::Error::success();
825 if (PreviousError)
826 return llvm::make_error<IndexError>(Args&: *PreviousError);
827
828 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
829 CI.getVirtualFileSystem().getBufferForFile(Name: InvocationListFilePath);
830 if (!FileContent) {
831 PreviousError = IndexError(index_error_code::invocation_list_file_not_found,
832 InvocationListFilePath.str());
833 return llvm::make_error<IndexError>(Args&: *PreviousError);
834 }
835 std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
836 assert(ContentBuffer && "If no error was produced after loading, the pointer "
837 "should not be nullptr.");
838
839 llvm::Expected<InvocationListTy> ExpectedInvocationList = parseInvocationList(
840 FileContent: ContentBuffer->getBuffer(), PathStyle, FilePath: InvocationListFilePath);
841
842 if (!ExpectedInvocationList) {
843 llvm::handleAllErrors(
844 E: ExpectedInvocationList.takeError(),
845 Handlers: [this](const IndexError &E) { this->PreviousError = E; });
846 return llvm::make_error<IndexError>(Args&: *PreviousError);
847 }
848
849 InvocationList = *ExpectedInvocationList;
850
851 return llvm::Error::success();
852}
853
854template <typename T>
855llvm::Expected<const T *>
856CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
857 assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
858
859 assert(&D->getASTContext() == &Unit->getASTContext() &&
860 "ASTContext of Decl and the unit should match.");
861 ASTImporter &Importer = getOrCreateASTImporter(Unit);
862
863 auto ToDeclOrError = Importer.Import(D);
864 if (!ToDeclOrError) {
865 handleAllErrors(ToDeclOrError.takeError(), [&](const ASTImportError &IE) {
866 switch (IE.Error) {
867 case ASTImportError::NameConflict:
868 ++NumNameConflicts;
869 break;
870 case ASTImportError::UnsupportedConstruct:
871 ++NumUnsupportedNodeFound;
872 break;
873 case ASTImportError::Unknown:
874 llvm_unreachable("Unknown import error happened.");
875 break;
876 }
877 });
878 return llvm::make_error<IndexError>(Args: index_error_code::failed_import);
879 }
880 auto *ToDecl = cast<T>(*ToDeclOrError);
881 assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
882 ++NumGetCTUSuccess;
883
884 // Parent map is invalidated after changing the AST.
885 ToDecl->getASTContext().getParentMapContext().clear();
886
887 return ToDecl;
888}
889
890llvm::Expected<const FunctionDecl *>
891CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
892 ASTUnit *Unit) {
893 return importDefinitionImpl(D: FD, Unit);
894}
895
896llvm::Expected<const VarDecl *>
897CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
898 ASTUnit *Unit) {
899 return importDefinitionImpl(D: VD, Unit);
900}
901
902void CrossTranslationUnitContext::lazyInitImporterSharedSt(
903 TranslationUnitDecl *ToTU) {
904 if (!ImporterSharedSt)
905 ImporterSharedSt = std::make_shared<ASTImporterSharedState>(args&: *ToTU);
906}
907
908ASTImporter &
909CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
910 ASTContext &From = Unit->getASTContext();
911
912 auto I = ASTUnitImporterMap.find(Val: From.getTranslationUnitDecl());
913 if (I != ASTUnitImporterMap.end())
914 return *I->second;
915 lazyInitImporterSharedSt(ToTU: Context.getTranslationUnitDecl());
916 ASTImporter *NewImporter = new ASTImporter(
917 Context, Context.getSourceManager().getFileManager(), From,
918 From.getSourceManager().getFileManager(), false, ImporterSharedSt);
919 ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(p: NewImporter);
920 return *NewImporter;
921}
922
923std::optional<clang::MacroExpansionContext>
924CrossTranslationUnitContext::getMacroExpansionContextForSourceLocation(
925 const clang::SourceLocation &ToLoc) const {
926 // FIXME: Implement: Record such a context for every imported ASTUnit; lookup.
927 return std::nullopt;
928}
929
930bool CrossTranslationUnitContext::isImportedAsNew(const Decl *ToDecl) const {
931 if (!ImporterSharedSt)
932 return false;
933 return ImporterSharedSt->isNewDecl(ToD: const_cast<Decl *>(ToDecl));
934}
935
936bool CrossTranslationUnitContext::hasError(const Decl *ToDecl) const {
937 if (!ImporterSharedSt)
938 return false;
939 return static_cast<bool>(
940 ImporterSharedSt->getImportDeclErrorIfAny(ToD: const_cast<Decl *>(ToDecl)));
941}
942
943} // namespace cross_tu
944} // namespace clang
945