1//===-- HeaderIncludeGen.cpp - Generate Header Includes -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Basic/DiagnosticFrontend.h"
10#include "clang/Basic/SourceManager.h"
11#include "clang/Frontend/DependencyOutputOptions.h"
12#include "clang/Frontend/Utils.h"
13#include "clang/Lex/Preprocessor.h"
14#include "llvm/ADT/SmallString.h"
15#include "llvm/Support/JSON.h"
16#include "llvm/Support/raw_ostream.h"
17using namespace clang;
18
19namespace {
20class HeaderIncludesCallback : public PPCallbacks {
21 SourceManager &SM;
22 raw_ostream *OutputFile;
23 const DependencyOutputOptions &DepOpts;
24 unsigned CurrentIncludeDepth;
25 bool HasProcessedPredefines;
26 bool OwnsOutputFile;
27 bool ShowAllHeaders;
28 bool ShowDepth;
29 bool MSStyle;
30
31public:
32 HeaderIncludesCallback(const Preprocessor *PP, bool ShowAllHeaders_,
33 raw_ostream *OutputFile_,
34 const DependencyOutputOptions &DepOpts,
35 bool OwnsOutputFile_, bool ShowDepth_, bool MSStyle_)
36 : SM(PP->getSourceManager()), OutputFile(OutputFile_), DepOpts(DepOpts),
37 CurrentIncludeDepth(0), HasProcessedPredefines(false),
38 OwnsOutputFile(OwnsOutputFile_), ShowAllHeaders(ShowAllHeaders_),
39 ShowDepth(ShowDepth_), MSStyle(MSStyle_) {}
40
41 ~HeaderIncludesCallback() override {
42 if (OwnsOutputFile)
43 delete OutputFile;
44 }
45
46 HeaderIncludesCallback(const HeaderIncludesCallback &) = delete;
47 HeaderIncludesCallback &operator=(const HeaderIncludesCallback &) = delete;
48
49 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
50 SrcMgr::CharacteristicKind FileType,
51 FileID PrevFID) override;
52
53 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
54 SrcMgr::CharacteristicKind FileType) override;
55
56private:
57 bool ShouldShowHeader(SrcMgr::CharacteristicKind HeaderType) {
58 if (!DepOpts.IncludeSystemHeaders && isSystem(CK: HeaderType))
59 return false;
60
61 // Show the current header if we are (a) past the predefines, or (b) showing
62 // all headers and in the predefines at a depth past the initial file and
63 // command line buffers.
64 return (HasProcessedPredefines ||
65 (ShowAllHeaders && CurrentIncludeDepth > 2));
66 }
67};
68
69/// A callback for emitting header usage information to a file in JSON. Each
70/// line in the file is a JSON object that includes the source file name and
71/// the list of headers directly or indirectly included from it. For example:
72///
73/// {"source":"/tmp/foo.c",
74/// "includes":["/usr/include/stdio.h", "/usr/include/stdlib.h"]}
75///
76/// To reduce the amount of data written to the file, we only record system
77/// headers that are directly included from a file that isn't in the system
78/// directory.
79class HeaderIncludesJSONCallback : public PPCallbacks {
80 SourceManager &SM;
81 raw_ostream *OutputFile;
82 bool OwnsOutputFile;
83 SmallVector<std::string, 16> IncludedHeaders;
84
85public:
86 HeaderIncludesJSONCallback(const Preprocessor *PP, raw_ostream *OutputFile_,
87 bool OwnsOutputFile_)
88 : SM(PP->getSourceManager()), OutputFile(OutputFile_),
89 OwnsOutputFile(OwnsOutputFile_) {}
90
91 ~HeaderIncludesJSONCallback() override {
92 if (OwnsOutputFile)
93 delete OutputFile;
94 }
95
96 HeaderIncludesJSONCallback(const HeaderIncludesJSONCallback &) = delete;
97 HeaderIncludesJSONCallback &
98 operator=(const HeaderIncludesJSONCallback &) = delete;
99
100 void EndOfMainFile() override;
101
102 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
103 SrcMgr::CharacteristicKind FileType,
104 FileID PrevFID) override;
105
106 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
107 SrcMgr::CharacteristicKind FileType) override;
108};
109
110/// A callback for emitting direct header and module usage information to a
111/// file in JSON. The output format is like HeaderIncludesJSONCallback but has
112/// an array of separate entries, one for each non-system source file used in
113/// the compilation showing only the direct includes and imports from that file.
114class HeaderIncludesDirectPerFileCallback : public PPCallbacks {
115 struct HeaderIncludeInfo {
116 SourceLocation Location;
117 FileEntryRef File;
118 const Module *ImportedModule;
119
120 HeaderIncludeInfo(SourceLocation Location, FileEntryRef File,
121 const Module *ImportedModule)
122 : Location(Location), File(File), ImportedModule(ImportedModule) {}
123 };
124
125 SourceManager &SM;
126 HeaderSearch &HSI;
127 raw_ostream *OutputFile;
128 bool OwnsOutputFile;
129 using DependencyMap =
130 llvm::DenseMap<FileEntryRef, SmallVector<HeaderIncludeInfo>>;
131 DependencyMap Dependencies;
132
133public:
134 HeaderIncludesDirectPerFileCallback(const Preprocessor *PP,
135 raw_ostream *OutputFile_,
136 bool OwnsOutputFile_)
137 : SM(PP->getSourceManager()), HSI(PP->getHeaderSearchInfo()),
138 OutputFile(OutputFile_), OwnsOutputFile(OwnsOutputFile_) {}
139
140 ~HeaderIncludesDirectPerFileCallback() override {
141 if (OwnsOutputFile)
142 delete OutputFile;
143 }
144
145 HeaderIncludesDirectPerFileCallback(
146 const HeaderIncludesDirectPerFileCallback &) = delete;
147 HeaderIncludesDirectPerFileCallback &
148 operator=(const HeaderIncludesDirectPerFileCallback &) = delete;
149
150 void EndOfMainFile() override;
151
152 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
153 StringRef FileName, bool IsAngled,
154 CharSourceRange FilenameRange,
155 OptionalFileEntryRef File, StringRef SearchPath,
156 StringRef RelativePath, const Module *SuggestedModule,
157 bool ModuleImported,
158 SrcMgr::CharacteristicKind FileType) override;
159
160 void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
161 const Module *Imported) override;
162};
163}
164
165static void PrintHeaderInfo(raw_ostream *OutputFile, StringRef Filename,
166 bool ShowDepth, unsigned CurrentIncludeDepth,
167 bool MSStyle) {
168 // Write to a temporary string to avoid unnecessary flushing on errs().
169 SmallString<512> Pathname(Filename);
170 if (!MSStyle)
171 Lexer::Stringify(Str&: Pathname);
172
173 SmallString<256> Msg;
174 if (MSStyle)
175 Msg += "Note: including file:";
176
177 if (ShowDepth) {
178 // The main source file is at depth 1, so skip one dot.
179 for (unsigned i = 1; i != CurrentIncludeDepth; ++i)
180 Msg += MSStyle ? ' ' : '.';
181
182 if (!MSStyle)
183 Msg += ' ';
184 }
185 Msg += Pathname;
186 Msg += '\n';
187
188 *OutputFile << Msg;
189 OutputFile->flush();
190}
191
192void clang::AttachHeaderIncludeGen(Preprocessor &PP,
193 const DependencyOutputOptions &DepOpts,
194 bool ShowAllHeaders, StringRef OutputPath,
195 bool ShowDepth, bool MSStyle) {
196 raw_ostream *OutputFile = &llvm::errs();
197 bool OwnsOutputFile = false;
198
199 // Choose output stream, when printing in cl.exe /showIncludes style.
200 if (MSStyle) {
201 switch (DepOpts.ShowIncludesDest) {
202 default:
203 llvm_unreachable("Invalid destination for /showIncludes output!");
204 case ShowIncludesDestination::Stderr:
205 OutputFile = &llvm::errs();
206 break;
207 case ShowIncludesDestination::Stdout:
208 OutputFile = &llvm::outs();
209 break;
210 }
211 }
212
213 // Open the output file, if used.
214 if (!OutputPath.empty()) {
215 std::error_code EC;
216 llvm::raw_fd_ostream *OS = new llvm::raw_fd_ostream(
217 OutputPath.str(), EC,
218 llvm::sys::fs::OF_Append | llvm::sys::fs::OF_TextWithCRLF);
219 if (EC) {
220 PP.getDiagnostics().Report(DiagID: clang::diag::warn_fe_cc_print_header_failure)
221 << EC.message();
222 delete OS;
223 } else {
224 OS->SetUnbuffered();
225 OutputFile = OS;
226 OwnsOutputFile = true;
227 }
228 }
229
230 switch (DepOpts.HeaderIncludeFormat) {
231 case HIFMT_None:
232 llvm_unreachable("unexpected header format kind");
233 case HIFMT_Textual: {
234 assert(DepOpts.HeaderIncludeFiltering == HIFIL_None &&
235 "header filtering is currently always disabled when output format is"
236 "textual");
237 // Print header info for extra headers, pretending they were discovered by
238 // the regular preprocessor. The primary use case is to support proper
239 // generation of Make / Ninja file dependencies for implicit includes, such
240 // as sanitizer ignorelists. It's only important for cl.exe compatibility,
241 // the GNU way to generate rules is -M / -MM / -MD / -MMD.
242 for (const auto &Header : DepOpts.ExtraDeps)
243 PrintHeaderInfo(OutputFile, Filename: Header.first, ShowDepth, CurrentIncludeDepth: 2, MSStyle);
244 PP.addPPCallbacks(C: std::make_unique<HeaderIncludesCallback>(
245 args: &PP, args&: ShowAllHeaders, args&: OutputFile, args: DepOpts, args&: OwnsOutputFile, args&: ShowDepth,
246 args&: MSStyle));
247 break;
248 }
249 case HIFMT_JSON:
250 switch (DepOpts.HeaderIncludeFiltering) {
251 default:
252 llvm_unreachable("Unknown HeaderIncludeFilteringKind enum");
253 case HIFIL_Only_Direct_System:
254 PP.addPPCallbacks(C: std::make_unique<HeaderIncludesJSONCallback>(
255 args: &PP, args&: OutputFile, args&: OwnsOutputFile));
256 break;
257 case HIFIL_Direct_Per_File:
258 PP.addPPCallbacks(C: std::make_unique<HeaderIncludesDirectPerFileCallback>(
259 args: &PP, args&: OutputFile, args&: OwnsOutputFile));
260 break;
261 }
262 break;
263 }
264}
265
266void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
267 FileChangeReason Reason,
268 SrcMgr::CharacteristicKind NewFileType,
269 FileID PrevFID) {
270 // Unless we are exiting a #include, make sure to skip ahead to the line the
271 // #include directive was at.
272 PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
273 if (UserLoc.isInvalid())
274 return;
275
276 // Adjust the current include depth.
277 if (Reason == PPCallbacks::EnterFile) {
278 ++CurrentIncludeDepth;
279 } else if (Reason == PPCallbacks::ExitFile) {
280 if (CurrentIncludeDepth)
281 --CurrentIncludeDepth;
282
283 // We track when we are done with the predefines by watching for the first
284 // place where we drop back to a nesting depth of 1.
285 if (CurrentIncludeDepth == 1 && !HasProcessedPredefines)
286 HasProcessedPredefines = true;
287
288 return;
289 } else {
290 return;
291 }
292
293 if (!ShouldShowHeader(HeaderType: NewFileType))
294 return;
295
296 unsigned IncludeDepth = CurrentIncludeDepth;
297 if (!HasProcessedPredefines)
298 --IncludeDepth; // Ignore indent from <built-in>.
299
300 // FIXME: Identify headers in a more robust way than comparing their name to
301 // "<command line>" and "<built-in>" in a bunch of places.
302 if (Reason == PPCallbacks::EnterFile &&
303 UserLoc.getFilename() != StringRef("<command line>")) {
304 PrintHeaderInfo(OutputFile, Filename: UserLoc.getFilename(), ShowDepth, CurrentIncludeDepth: IncludeDepth,
305 MSStyle);
306 }
307}
308
309void HeaderIncludesCallback::FileSkipped(const FileEntryRef &SkippedFile,
310 const Token &FilenameTok,
311 SrcMgr::CharacteristicKind FileType) {
312 if (!DepOpts.ShowSkippedHeaderIncludes)
313 return;
314
315 if (!ShouldShowHeader(HeaderType: FileType))
316 return;
317
318 PrintHeaderInfo(OutputFile, Filename: SkippedFile.getName(), ShowDepth,
319 CurrentIncludeDepth: CurrentIncludeDepth + 1, MSStyle);
320}
321
322void HeaderIncludesJSONCallback::EndOfMainFile() {
323 OptionalFileEntryRef FE = SM.getFileEntryRefForID(FID: SM.getMainFileID());
324 SmallString<256> MainFile;
325 if (FE) {
326 MainFile += FE->getName();
327 SM.getFileManager().makeAbsolutePath(Path&: MainFile);
328 }
329
330 std::string Str;
331 llvm::raw_string_ostream OS(Str);
332 llvm::json::OStream JOS(OS);
333 JOS.object(Contents: [&] {
334 JOS.attribute(Key: "source", Contents: MainFile.c_str());
335 JOS.attributeArray(Key: "includes", Contents: [&] {
336 llvm::StringSet<> SeenHeaders;
337 for (const std::string &H : IncludedHeaders)
338 if (SeenHeaders.insert(key: H).second)
339 JOS.value(V: H);
340 });
341 });
342 OS << "\n";
343
344 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) {
345 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile);
346 if (auto L = FDS->lock())
347 *OutputFile << Str;
348 } else
349 *OutputFile << Str;
350}
351
352/// Determine whether the header file should be recorded. The header file should
353/// be recorded only if the header file is a system header and the current file
354/// isn't a system header.
355static bool shouldRecordNewFile(SrcMgr::CharacteristicKind NewFileType,
356 SourceLocation PrevLoc, SourceManager &SM) {
357 return SrcMgr::isSystem(CK: NewFileType) && !SM.isInSystemHeader(Loc: PrevLoc);
358}
359
360void HeaderIncludesJSONCallback::FileChanged(
361 SourceLocation Loc, FileChangeReason Reason,
362 SrcMgr::CharacteristicKind NewFileType, FileID PrevFID) {
363 if (PrevFID.isInvalid() ||
364 !shouldRecordNewFile(NewFileType, PrevLoc: SM.getLocForStartOfFile(FID: PrevFID), SM))
365 return;
366
367 // Unless we are exiting a #include, make sure to skip ahead to the line the
368 // #include directive was at.
369 PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
370 if (UserLoc.isInvalid())
371 return;
372
373 if (Reason == PPCallbacks::EnterFile &&
374 UserLoc.getFilename() != StringRef("<command line>"))
375 IncludedHeaders.push_back(Elt: UserLoc.getFilename());
376}
377
378void HeaderIncludesJSONCallback::FileSkipped(
379 const FileEntryRef &SkippedFile, const Token &FilenameTok,
380 SrcMgr::CharacteristicKind FileType) {
381 if (!shouldRecordNewFile(NewFileType: FileType, PrevLoc: FilenameTok.getLocation(), SM))
382 return;
383
384 IncludedHeaders.push_back(Elt: SkippedFile.getName().str());
385}
386
387void HeaderIncludesDirectPerFileCallback::EndOfMainFile() {
388 if (Dependencies.empty())
389 return;
390
391 // Sort the files so that the output does not depend on the DenseMap order.
392 SmallVector<FileEntryRef> SourceFiles;
393 for (auto F = Dependencies.begin(), FEnd = Dependencies.end(); F != FEnd;
394 ++F) {
395 SourceFiles.push_back(Elt: F->first);
396 }
397 llvm::sort(C&: SourceFiles, Comp: [](const FileEntryRef &LHS, const FileEntryRef &RHS) {
398 return LHS.getUID() < RHS.getUID();
399 });
400
401 std::string Str;
402 llvm::raw_string_ostream OS(Str);
403 llvm::json::OStream JOS(OS);
404 JOS.object(Contents: [&] {
405 JOS.attribute(Key: "version", Contents: "2.0.0");
406 JOS.attributeArray(Key: "dependencies", Contents: [&] {
407 for (const auto &S : SourceFiles) {
408 JOS.object(Contents: [&] {
409 SmallVector<HeaderIncludeInfo> &Deps = Dependencies[S];
410 JOS.attribute(Key: "source", Contents: S.getName().str());
411 JOS.attributeArray(Key: "includes", Contents: [&] {
412 for (unsigned I = 0, N = Deps.size(); I != N; ++I) {
413 if (!Deps[I].ImportedModule) {
414 JOS.object(Contents: [&] {
415 JOS.attribute(Key: "location", Contents: Deps[I].Location.printToString(SM));
416 JOS.attribute(Key: "file", Contents: Deps[I].File.getName());
417 });
418 }
419 }
420 });
421 JOS.attributeArray(Key: "imports", Contents: [&] {
422 for (unsigned I = 0, N = Deps.size(); I != N; ++I) {
423 if (Deps[I].ImportedModule) {
424 JOS.object(Contents: [&] {
425 JOS.attribute(Key: "location", Contents: Deps[I].Location.printToString(SM));
426 JOS.attribute(
427 Key: "module",
428 Contents: Deps[I].ImportedModule->getTopLevelModuleName());
429 JOS.attribute(Key: "file", Contents: Deps[I].File.getName());
430 });
431 }
432 }
433 });
434 });
435 }
436 });
437 });
438
439 OS << "\n";
440
441 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) {
442 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile);
443 if (auto L = FDS->lock())
444 *OutputFile << Str;
445 } else
446 *OutputFile << Str;
447}
448
449void HeaderIncludesDirectPerFileCallback::InclusionDirective(
450 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
451 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
452 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
453 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
454 if (!File)
455 return;
456
457 SourceLocation Loc = SM.getExpansionLoc(Loc: HashLoc);
458 if (SM.isInSystemHeader(Loc))
459 return;
460 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(FID: SM.getFileID(SpellingLoc: Loc));
461 if (!FromFile)
462 return;
463
464 FileEntryRef HeaderOrModuleMapFile = *File;
465 if (ModuleImported && SuggestedModule) {
466 OptionalFileEntryRef ModuleMapFile =
467 HSI.getModuleMap().getModuleMapFileForUniquing(M: SuggestedModule);
468 if (ModuleMapFile) {
469 HeaderOrModuleMapFile = *ModuleMapFile;
470 }
471 }
472
473 HeaderIncludeInfo DependenciesEntry(
474 Loc, HeaderOrModuleMapFile, (ModuleImported ? SuggestedModule : nullptr));
475 Dependencies[*FromFile].push_back(Elt: DependenciesEntry);
476}
477
478void HeaderIncludesDirectPerFileCallback::moduleImport(SourceLocation ImportLoc,
479 ModuleIdPath Path,
480 const Module *Imported) {
481 if (!Imported)
482 return;
483
484 SourceLocation Loc = SM.getExpansionLoc(Loc: ImportLoc);
485 if (SM.isInSystemHeader(Loc))
486 return;
487 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(FID: SM.getFileID(SpellingLoc: Loc));
488 if (!FromFile)
489 return;
490
491 OptionalFileEntryRef ModuleMapFile =
492 HSI.getModuleMap().getModuleMapFileForUniquing(M: Imported);
493 if (!ModuleMapFile)
494 return;
495
496 HeaderIncludeInfo DependenciesEntry(Loc, *ModuleMapFile, Imported);
497 Dependencies[*FromFile].push_back(Elt: DependenciesEntry);
498}
499