1//===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the ExtractAPIAction, and ASTConsumer to collect API
11/// information.
12///
13//===----------------------------------------------------------------------===//
14
15#include "clang/AST/ASTConcept.h"
16#include "clang/AST/ASTConsumer.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/DeclObjC.h"
19#include "clang/Basic/DiagnosticFrontend.h"
20#include "clang/Basic/FileEntry.h"
21#include "clang/Basic/SourceLocation.h"
22#include "clang/Basic/SourceManager.h"
23#include "clang/Basic/TargetInfo.h"
24#include "clang/ExtractAPI/API.h"
25#include "clang/ExtractAPI/APIIgnoresList.h"
26#include "clang/ExtractAPI/ExtractAPIVisitor.h"
27#include "clang/ExtractAPI/FrontendActions.h"
28#include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
29#include "clang/Frontend/ASTConsumers.h"
30#include "clang/Frontend/CompilerInstance.h"
31#include "clang/Frontend/FrontendOptions.h"
32#include "clang/Frontend/MultiplexConsumer.h"
33#include "clang/Index/USRGeneration.h"
34#include "clang/InstallAPI/HeaderFile.h"
35#include "clang/Lex/MacroInfo.h"
36#include "clang/Lex/PPCallbacks.h"
37#include "clang/Lex/Preprocessor.h"
38#include "clang/Lex/PreprocessorOptions.h"
39#include "llvm/ADT/DenseSet.h"
40#include "llvm/ADT/STLExtras.h"
41#include "llvm/ADT/SmallString.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/ADT/StringRef.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Error.h"
46#include "llvm/Support/MemoryBuffer.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/Regex.h"
49#include "llvm/Support/raw_ostream.h"
50#include <memory>
51#include <optional>
52#include <utility>
53
54using namespace clang;
55using namespace extractapi;
56
57namespace {
58
59std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
60 StringRef File,
61 bool *IsQuoted = nullptr) {
62 assert(CI.hasFileManager() &&
63 "CompilerInstance does not have a FileNamager!");
64
65 using namespace llvm::sys;
66 const auto &FS = CI.getVirtualFileSystem();
67
68 SmallString<128> FilePath(File.begin(), File.end());
69 FS.makeAbsolute(Path&: FilePath);
70 path::remove_dots(path&: FilePath, remove_dot_dot: true);
71 FilePath = path::convert_to_slash(path: FilePath);
72 File = FilePath;
73
74 // Checks whether `Dir` is a strict path prefix of `File`. If so returns
75 // the prefix length. Otherwise return 0.
76 auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {
77 llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());
78 FS.makeAbsolute(Path&: DirPath);
79 path::remove_dots(path&: DirPath, remove_dot_dot: true);
80 Dir = DirPath;
81 for (auto NI = path::begin(path: File), NE = path::end(path: File),
82 DI = path::begin(path: Dir), DE = path::end(path: Dir);
83 /*termination condition in loop*/; ++NI, ++DI) {
84 // '.' components in File are ignored.
85 while (NI != NE && *NI == ".")
86 ++NI;
87 if (NI == NE)
88 break;
89
90 // '.' components in Dir are ignored.
91 while (DI != DE && *DI == ".")
92 ++DI;
93
94 // Dir is a prefix of File, up to '.' components and choice of path
95 // separators.
96 if (DI == DE)
97 return NI - path::begin(path: File);
98
99 // Consider all path separators equal.
100 if (NI->size() == 1 && DI->size() == 1 &&
101 path::is_separator(value: NI->front()) && path::is_separator(value: DI->front()))
102 continue;
103
104 // Special case Apple .sdk folders since the search path is typically a
105 // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
106 // located in `iPhoneSimulator.sdk` (the real folder).
107 if (NI->ends_with(Suffix: ".sdk") && DI->ends_with(Suffix: ".sdk")) {
108 StringRef NBasename = path::stem(path: *NI);
109 StringRef DBasename = path::stem(path: *DI);
110 if (DBasename.starts_with(Prefix: NBasename))
111 continue;
112 }
113
114 if (*NI != *DI)
115 break;
116 }
117 return 0;
118 };
119
120 unsigned PrefixLength = 0;
121
122 // Go through the search paths and find the first one that is a prefix of
123 // the header.
124 for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {
125 // Note whether the match is found in a quoted entry.
126 if (IsQuoted)
127 *IsQuoted = Entry.Group == frontend::Quoted;
128
129 if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Filename: Entry.Path)) {
130 if (auto HMap = HeaderMap::Create(FE: *EntryFile, FM&: CI.getFileManager())) {
131 // If this is a headermap entry, try to reverse lookup the full path
132 // for a spelled name before mapping.
133 StringRef SpelledFilename = HMap->reverseLookupFilename(DestPath: File);
134 if (!SpelledFilename.empty())
135 return SpelledFilename.str();
136
137 // No matching mapping in this headermap, try next search entry.
138 continue;
139 }
140 }
141
142 // Entry is a directory search entry, try to check if it's a prefix of File.
143 PrefixLength = CheckDir(Entry.Path);
144 if (PrefixLength > 0) {
145 // The header is found in a framework path, construct the framework-style
146 // include name `<Framework/Header.h>`
147 if (Entry.IsFramework) {
148 SmallVector<StringRef, 4> Matches;
149 clang::installapi::HeaderFile::getFrameworkIncludeRule().match(
150 String: File, Matches: &Matches);
151 // Returned matches are always in stable order.
152 if (Matches.size() != 4)
153 return std::nullopt;
154
155 return path::convert_to_slash(
156 path: (Matches[1].drop_front(N: Matches[1].rfind(C: '/') + 1) + "/" +
157 Matches[3])
158 .str());
159 }
160
161 // The header is found in a normal search path, strip the search path
162 // prefix to get an include name.
163 return path::convert_to_slash(path: File.drop_front(N: PrefixLength));
164 }
165 }
166
167 // Couldn't determine a include name, use full path instead.
168 return std::nullopt;
169}
170
171std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
172 FileEntryRef FE,
173 bool *IsQuoted = nullptr) {
174 return getRelativeIncludeName(CI, File: FE.getNameAsRequested(), IsQuoted);
175}
176
177struct LocationFileChecker {
178 bool operator()(SourceLocation Loc) {
179 // If the loc refers to a macro expansion we need to first get the file
180 // location of the expansion.
181 auto &SM = CI.getSourceManager();
182 auto FileLoc = SM.getFileLoc(Loc);
183 FileID FID = SM.getFileID(SpellingLoc: FileLoc);
184 if (FID.isInvalid())
185 return false;
186
187 OptionalFileEntryRef File = SM.getFileEntryRefForID(FID);
188 if (!File)
189 return false;
190
191 if (KnownFileEntries.count(V: *File))
192 return true;
193
194 if (ExternalFileEntries.count(V: *File))
195 return false;
196
197 // Try to reduce the include name the same way we tried to include it.
198 bool IsQuoted = false;
199 if (auto IncludeName = getRelativeIncludeName(CI, FE: *File, IsQuoted: &IsQuoted))
200 if (llvm::any_of(Range&: KnownFiles,
201 P: [&IsQuoted, &IncludeName](const auto &KnownFile) {
202 return KnownFile.first.equals(*IncludeName) &&
203 KnownFile.second == IsQuoted;
204 })) {
205 KnownFileEntries.insert(V: *File);
206 return true;
207 }
208
209 // Record that the file was not found to avoid future reverse lookup for
210 // the same file.
211 ExternalFileEntries.insert(V: *File);
212 return false;
213 }
214
215 LocationFileChecker(const CompilerInstance &CI,
216 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)
217 : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {
218 for (const auto &KnownFile : KnownFiles)
219 if (auto FE = CI.getFileManager().getOptionalFileRef(Filename: KnownFile.first))
220 KnownFileEntries.insert(V: *FE);
221 }
222
223private:
224 const CompilerInstance &CI;
225 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;
226 llvm::DenseSet<const FileEntry *> KnownFileEntries;
227 llvm::DenseSet<const FileEntry *> ExternalFileEntries;
228};
229
230struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> {
231 bool shouldDeclBeIncluded(const Decl *D) const {
232 bool ShouldBeIncluded = true;
233 // Check that we have the definition for redeclarable types.
234 if (auto *TD = llvm::dyn_cast<TagDecl>(Val: D))
235 ShouldBeIncluded = TD->isThisDeclarationADefinition();
236 else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(Val: D))
237 ShouldBeIncluded = Interface->isThisDeclarationADefinition();
238 else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(Val: D))
239 ShouldBeIncluded = Protocol->isThisDeclarationADefinition();
240
241 ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation());
242 return ShouldBeIncluded;
243 }
244
245 BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context,
246 APISet &API)
247 : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {}
248
249private:
250 LocationFileChecker &LCF;
251};
252
253class WrappingExtractAPIConsumer : public ASTConsumer {
254public:
255 WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)
256 : Visitor(Context, API) {}
257
258 void HandleTranslationUnit(ASTContext &Context) override {
259 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
260 Visitor.TraverseDecl(D: Context.getTranslationUnitDecl());
261 }
262
263private:
264 ExtractAPIVisitor<> Visitor;
265};
266
267class ExtractAPIConsumer : public ASTConsumer {
268public:
269 ExtractAPIConsumer(ASTContext &Context,
270 std::unique_ptr<LocationFileChecker> LCF, APISet &API)
271 : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {}
272
273 void HandleTranslationUnit(ASTContext &Context) override {
274 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
275 Visitor.TraverseDecl(D: Context.getTranslationUnitDecl());
276 }
277
278private:
279 BatchExtractAPIVisitor Visitor;
280 std::unique_ptr<LocationFileChecker> LCF;
281};
282
283class MacroCallback : public PPCallbacks {
284public:
285 MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)
286 : SM(SM), API(API), PP(PP) {}
287
288 void EndOfMainFile() override {
289 for (const auto &M : PP.macros()) {
290 auto *II = M.getFirst();
291 auto MD = PP.getMacroDefinition(II);
292 auto *MI = MD.getMacroInfo();
293
294 if (!MI)
295 continue;
296
297 // Ignore header guard macros
298 if (MI->isUsedForHeaderGuard())
299 continue;
300
301 // Ignore builtin macros and ones defined via the command line.
302 if (MI->isBuiltinMacro())
303 continue;
304
305 auto DefLoc = MI->getDefinitionLoc();
306
307 if (SM.isInPredefinedFile(Loc: DefLoc))
308 continue;
309
310 auto AssociatedModuleMacros = MD.getModuleMacros();
311 StringRef OwningModuleName;
312 if (!AssociatedModuleMacros.empty())
313 OwningModuleName = AssociatedModuleMacros.back()
314 ->getOwningModule()
315 ->getTopLevelModuleName();
316
317 if (!shouldMacroBeIncluded(MacroLoc: DefLoc, ModuleName: OwningModuleName))
318 continue;
319
320 StringRef Name = II->getName();
321 PresumedLoc Loc = SM.getPresumedLoc(Loc: DefLoc);
322 SmallString<128> USR;
323 index::generateUSRForMacro(MacroName: Name, Loc: DefLoc, SM, Buf&: USR);
324 API.createRecord<extractapi::MacroDefinitionRecord>(
325 USR, Name, CtorArgs: SymbolReference(), CtorArgs&: Loc,
326 CtorArgs: DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI),
327 CtorArgs: DeclarationFragmentsBuilder::getSubHeadingForMacro(Name),
328 CtorArgs: SM.isInSystemHeader(Loc: DefLoc));
329 }
330 }
331
332 virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
333 StringRef ModuleName) {
334 return true;
335 }
336
337 const SourceManager &SM;
338 APISet &API;
339 Preprocessor &PP;
340};
341
342class APIMacroCallback : public MacroCallback {
343public:
344 APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,
345 LocationFileChecker &LCF)
346 : MacroCallback(SM, API, PP), LCF(LCF) {}
347
348 bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
349 StringRef ModuleName) override {
350 // Do not include macros from external files
351 return LCF(MacroLoc);
352 }
353
354private:
355 LocationFileChecker &LCF;
356};
357
358std::unique_ptr<llvm::raw_pwrite_stream>
359createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) {
360 auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir;
361
362 SmallString<256> FileName;
363 llvm::sys::path::append(path&: FileName, a: OutputDirectory,
364 b: BaseName + ".symbols.json");
365 return CI.createOutputFile(
366 OutputPath: FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false,
367 /*UseTemporary*/ true, /*CreateMissingDirectories*/ true);
368}
369
370} // namespace
371
372void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) {
373 SymbolGraphSerializerOption SerializationOptions;
374 SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs;
375 SerializationOptions.EmitSymbolLabelsForTesting =
376 CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting;
377
378 if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) {
379 auto ConstructOutputFile = [&CI](Twine BaseName) {
380 return createAdditionalSymbolGraphFile(CI, BaseName);
381 };
382
383 SymbolGraphSerializer::serializeWithExtensionGraphs(
384 MainOutput&: *OS, API: *API, IgnoresList, CreateOutputStream: ConstructOutputFile, Options: SerializationOptions);
385 } else {
386 SymbolGraphSerializer::serializeMainSymbolGraph(OS&: *OS, API: *API, IgnoresList,
387 Options: SerializationOptions);
388 }
389
390 // Flush the stream and close the main output stream.
391 OS.reset();
392}
393
394std::unique_ptr<ASTConsumer>
395ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
396 auto ProductName = CI.getFrontendOpts().ProductName;
397
398 if (CI.getFrontendOpts().SymbolGraphOutputDir.empty())
399 OS = CI.createDefaultOutputFile(/*Binary*/ false, BaseInput: InFile,
400 /*Extension*/ "symbols.json",
401 /*RemoveFileOnSignal*/ false,
402 /*CreateMissingDirectories*/ true);
403 else
404 OS = createAdditionalSymbolGraphFile(CI, BaseName: ProductName);
405
406 if (!OS)
407 return nullptr;
408
409 // Now that we have enough information about the language options and the
410 // target triple, let's create the APISet before anyone uses it.
411 API = std::make_unique<APISet>(
412 args: CI.getTarget().getTriple(),
413 args: CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), args&: ProductName);
414
415 auto LCF = std::make_unique<LocationFileChecker>(args&: CI, args&: KnownInputFiles);
416
417 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<APIMacroCallback>(
418 args&: CI.getSourceManager(), args&: *API, args&: CI.getPreprocessor(), args&: *LCF));
419
420 // Do not include location in anonymous decls.
421 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
422 Policy.AnonymousTagLocations = false;
423 CI.getASTContext().setPrintingPolicy(Policy);
424
425 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
426 llvm::handleAllErrors(
427 E: APIIgnoresList::create(IgnoresFilePathList: CI.getFrontendOpts().ExtractAPIIgnoresFileList,
428 FM&: CI.getFileManager())
429 .moveInto(Value&: IgnoresList),
430 Handlers: [&CI](const IgnoresFileNotFound &Err) {
431 CI.getDiagnostics().Report(
432 DiagID: diag::err_extract_api_ignores_file_not_found)
433 << Err.Path;
434 });
435 }
436
437 return std::make_unique<ExtractAPIConsumer>(args&: CI.getASTContext(),
438 args: std::move(LCF), args&: *API);
439}
440
441bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
442 auto &Inputs = CI.getFrontendOpts().Inputs;
443 if (Inputs.empty())
444 return true;
445
446 if (!CI.hasFileManager())
447 if (!CI.createFileManager())
448 return false;
449
450 auto Kind = Inputs[0].getKind();
451
452 // Convert the header file inputs into a single input buffer.
453 SmallString<256> HeaderContents;
454 bool IsQuoted = false;
455 for (const FrontendInputFile &FIF : Inputs) {
456 if (Kind.isObjectiveC())
457 HeaderContents += "#import";
458 else
459 HeaderContents += "#include";
460
461 StringRef FilePath = FIF.getFile();
462 if (auto RelativeName = getRelativeIncludeName(CI, File: FilePath, IsQuoted: &IsQuoted)) {
463 if (IsQuoted)
464 HeaderContents += " \"";
465 else
466 HeaderContents += " <";
467
468 HeaderContents += *RelativeName;
469
470 if (IsQuoted)
471 HeaderContents += "\"\n";
472 else
473 HeaderContents += ">\n";
474 KnownInputFiles.emplace_back(Args: static_cast<SmallString<32>>(*RelativeName),
475 Args&: IsQuoted);
476 } else {
477 HeaderContents += " \"";
478 HeaderContents += FilePath;
479 HeaderContents += "\"\n";
480 KnownInputFiles.emplace_back(Args&: FilePath, Args: true);
481 }
482 }
483
484 if (CI.getHeaderSearchOpts().Verbose)
485 CI.getVerboseOutputStream() << getInputBufferName() << ":\n"
486 << HeaderContents << "\n";
487
488 Buffer = llvm::MemoryBuffer::getMemBufferCopy(InputData: HeaderContents,
489 BufferName: getInputBufferName());
490
491 // Set that buffer up as our "real" input in the CompilerInstance.
492 Inputs.clear();
493 Inputs.emplace_back(Args: Buffer->getMemBufferRef(), Args&: Kind, /*IsSystem*/ Args: false);
494
495 return true;
496}
497
498void ExtractAPIAction::EndSourceFileAction() {
499 ImplEndSourceFileAction(CI&: getCompilerInstance());
500}
501
502std::unique_ptr<ASTConsumer>
503WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,
504 StringRef InFile) {
505 auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
506 if (!OtherConsumer)
507 return nullptr;
508
509 CreatedASTConsumer = true;
510
511 ProductName = CI.getFrontendOpts().ProductName;
512 auto InputFilename = llvm::sys::path::filename(path: InFile);
513 OS = createAdditionalSymbolGraphFile(CI, BaseName: InputFilename);
514
515 // Now that we have enough information about the language options and the
516 // target triple, let's create the APISet before anyone uses it.
517 API = std::make_unique<APISet>(
518 args: CI.getTarget().getTriple(),
519 args: CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), args&: ProductName);
520
521 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<MacroCallback>(
522 args&: CI.getSourceManager(), args&: *API, args&: CI.getPreprocessor()));
523
524 // Do not include location in anonymous decls.
525 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
526 Policy.AnonymousTagLocations = false;
527 CI.getASTContext().setPrintingPolicy(Policy);
528
529 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
530 llvm::handleAllErrors(
531 E: APIIgnoresList::create(IgnoresFilePathList: CI.getFrontendOpts().ExtractAPIIgnoresFileList,
532 FM&: CI.getFileManager())
533 .moveInto(Value&: IgnoresList),
534 Handlers: [&CI](const IgnoresFileNotFound &Err) {
535 CI.getDiagnostics().Report(
536 DiagID: diag::err_extract_api_ignores_file_not_found)
537 << Err.Path;
538 });
539 }
540
541 auto WrappingConsumer =
542 std::make_unique<WrappingExtractAPIConsumer>(args&: CI.getASTContext(), args&: *API);
543 std::vector<std::unique_ptr<ASTConsumer>> Consumers;
544 Consumers.push_back(x: std::move(OtherConsumer));
545 Consumers.push_back(x: std::move(WrappingConsumer));
546
547 return std::make_unique<MultiplexConsumer>(args: std::move(Consumers));
548}
549
550void WrappingExtractAPIAction::EndSourceFileAction() {
551 // Invoke wrapped action's method.
552 WrapperFrontendAction::EndSourceFileAction();
553
554 if (CreatedASTConsumer) {
555 ImplEndSourceFileAction(CI&: getCompilerInstance());
556 }
557}
558