| 1 | //===- DependencyScanningWorker.cpp - Thread-Safe Scanning Worker ---------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "clang/DependencyScanning/DependencyScanningWorker.h" |
| 10 | #include "clang/Basic/Diagnostic.h" |
| 11 | #include "clang/Basic/DiagnosticFrontend.h" |
| 12 | #include "clang/Basic/DiagnosticSerialization.h" |
| 13 | #include "clang/DependencyScanning/DependencyActionController.h" |
| 14 | #include "clang/DependencyScanning/DependencyConsumer.h" |
| 15 | #include "clang/DependencyScanning/DependencyScanningFilesystem.h" |
| 16 | #include "clang/Frontend/FrontendActions.h" |
| 17 | #include "clang/Lex/HeaderSearch.h" |
| 18 | #include "clang/Lex/Preprocessor.h" |
| 19 | #include "clang/Sema/SemaOpenACC.h" |
| 20 | #include "clang/Serialization/ObjectFilePCHContainerReader.h" |
| 21 | #include "llvm/ADT/IntrusiveRefCntPtr.h" |
| 22 | #include "llvm/ADT/ScopeExit.h" |
| 23 | #include "llvm/Option/Option.h" |
| 24 | #include "llvm/Support/AdvisoryLock.h" |
| 25 | #include "llvm/Support/CrashRecoveryContext.h" |
| 26 | #include "llvm/Support/VirtualFileSystem.h" |
| 27 | #include "llvm/TargetParser/Host.h" |
| 28 | #include <mutex> |
| 29 | #include <thread> |
| 30 | |
| 31 | using namespace clang; |
| 32 | using namespace dependencies; |
| 33 | |
| 34 | static bool (const HeaderSearchOptions &HSOpts, |
| 35 | const HeaderSearchOptions &ExistingHSOpts, |
| 36 | DiagnosticsEngine *Diags, |
| 37 | const LangOptions &LangOpts) { |
| 38 | if (LangOpts.Modules) { |
| 39 | if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { |
| 40 | if (Diags) { |
| 41 | Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch); |
| 42 | auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) { |
| 43 | if (VFSOverlays.empty()) { |
| 44 | Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type; |
| 45 | } else { |
| 46 | std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n" ); |
| 47 | Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files; |
| 48 | } |
| 49 | }; |
| 50 | VFSNote(0, HSOpts.VFSOverlayFiles); |
| 51 | VFSNote(1, ExistingHSOpts.VFSOverlayFiles); |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | return false; |
| 56 | } |
| 57 | namespace { |
| 58 | using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); |
| 59 | |
| 60 | /// A listener that collects the imported modules and the input |
| 61 | /// files. While visiting, collect vfsoverlays and file inputs that determine |
| 62 | /// whether prebuilt modules fully resolve in stable directories. |
| 63 | class PrebuiltModuleListener : public ASTReaderListener { |
| 64 | public: |
| 65 | PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, |
| 66 | llvm::SmallVector<std::string> &NewModuleFiles, |
| 67 | PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, |
| 68 | const HeaderSearchOptions &HSOpts, |
| 69 | const LangOptions &LangOpts, DiagnosticsEngine &Diags, |
| 70 | const ArrayRef<StringRef> StableDirs) |
| 71 | : PrebuiltModuleFiles(PrebuiltModuleFiles), |
| 72 | NewModuleFiles(NewModuleFiles), |
| 73 | PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts), |
| 74 | ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {} |
| 75 | |
| 76 | bool needsImportVisitation() const override { return true; } |
| 77 | bool needsInputFileVisitation() override { return true; } |
| 78 | bool needsSystemInputFileVisitation() override { return true; } |
| 79 | |
| 80 | /// Accumulate the modules are transitively depended on by the initial |
| 81 | /// prebuilt module. |
| 82 | void visitImport(StringRef ModuleName, StringRef Filename) override { |
| 83 | if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second) |
| 84 | NewModuleFiles.push_back(Elt: Filename.str()); |
| 85 | |
| 86 | auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename); |
| 87 | PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| 88 | if (PrebuiltMapEntry.second) |
| 89 | PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| 90 | |
| 91 | if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile); |
| 92 | It != PrebuiltModulesASTMap.end() && CurrentFile != Filename) |
| 93 | PrebuiltModule.addDependent(ModuleFile: It->getKey()); |
| 94 | } |
| 95 | |
| 96 | /// For each input file discovered, check whether it's external path is in a |
| 97 | /// stable directory. Traversal is stopped if the current module is not |
| 98 | /// considered stable. |
| 99 | bool visitInputFileAsRequested(StringRef FilenameAsRequested, |
| 100 | StringRef Filename, bool isSystem, |
| 101 | bool isOverridden, time_t StoredTime, |
| 102 | bool isExplicitModule) override { |
| 103 | if (StableDirs.empty()) |
| 104 | return false; |
| 105 | auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile); |
| 106 | if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) || |
| 107 | (!PrebuiltEntryIt->second.isInStableDir())) |
| 108 | return false; |
| 109 | |
| 110 | PrebuiltEntryIt->second.setInStableDir( |
| 111 | isPathInStableDir(Directories: StableDirs, Input: Filename)); |
| 112 | return PrebuiltEntryIt->second.isInStableDir(); |
| 113 | } |
| 114 | |
| 115 | /// Update which module that is being actively traversed. |
| 116 | void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind, |
| 117 | bool DirectlyImported) override { |
| 118 | // If the CurrentFile is not |
| 119 | // considered stable, update any of it's transitive dependents. |
| 120 | auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile); |
| 121 | if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) && |
| 122 | !PrebuiltEntryIt->second.isInStableDir()) |
| 123 | PrebuiltEntryIt->second.updateDependentsNotInStableDirs( |
| 124 | PrebuiltModulesMap&: PrebuiltModulesASTMap); |
| 125 | CurrentFile = Filename.str(); |
| 126 | } |
| 127 | |
| 128 | /// Check the header search options for a given module when considering |
| 129 | /// if the module comes from stable directories. |
| 130 | bool (const HeaderSearchOptions &HSOpts, |
| 131 | StringRef ModuleFilename, StringRef ContextHash, |
| 132 | bool Complain) override { |
| 133 | |
| 134 | auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile); |
| 135 | PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| 136 | if (PrebuiltMapEntry.second) |
| 137 | PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| 138 | |
| 139 | if (PrebuiltModule.isInStableDir()) |
| 140 | PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts)); |
| 141 | |
| 142 | return false; |
| 143 | } |
| 144 | |
| 145 | /// Accumulate vfsoverlays used to build these prebuilt modules. |
| 146 | bool (const HeaderSearchOptions &HSOpts, |
| 147 | bool Complain) override { |
| 148 | |
| 149 | auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile); |
| 150 | PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| 151 | if (PrebuiltMapEntry.second) |
| 152 | PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| 153 | |
| 154 | PrebuiltModule.setVFS( |
| 155 | llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles)); |
| 156 | |
| 157 | return checkHeaderSearchPaths( |
| 158 | HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts); |
| 159 | } |
| 160 | |
| 161 | private: |
| 162 | PrebuiltModuleFilesT &PrebuiltModuleFiles; |
| 163 | llvm::SmallVector<std::string> &NewModuleFiles; |
| 164 | PrebuiltModulesAttrsMap &PrebuiltModulesASTMap; |
| 165 | const HeaderSearchOptions &ExistingHSOpts; |
| 166 | const LangOptions &ExistingLangOpts; |
| 167 | DiagnosticsEngine &Diags; |
| 168 | std::string CurrentFile; |
| 169 | const ArrayRef<StringRef> StableDirs; |
| 170 | }; |
| 171 | } // namespace |
| 172 | |
| 173 | /// Visit the given prebuilt module and collect all of the modules it |
| 174 | /// transitively imports and contributing input files. |
| 175 | static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, |
| 176 | CompilerInstance &CI, |
| 177 | PrebuiltModuleFilesT &ModuleFiles, |
| 178 | PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, |
| 179 | DiagnosticsEngine &Diags, |
| 180 | const ArrayRef<StringRef> StableDirs) { |
| 181 | // List of module files to be processed. |
| 182 | llvm::SmallVector<std::string> Worklist; |
| 183 | |
| 184 | PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap, |
| 185 | CI.getHeaderSearchOpts(), CI.getLangOpts(), |
| 186 | Diags, StableDirs); |
| 187 | |
| 188 | Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: PrebuiltModuleFilename), |
| 189 | Kind: serialization::MK_ExplicitModule, |
| 190 | /*DirectlyImported=*/true); |
| 191 | if (ASTReader::readASTFileControlBlock( |
| 192 | Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(), |
| 193 | PCHContainerRdr: CI.getPCHContainerReader(), |
| 194 | /*FindModuleFileExtensions=*/false, Listener, |
| 195 | /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate)) |
| 196 | return true; |
| 197 | |
| 198 | while (!Worklist.empty()) { |
| 199 | // FIXME: This is assuming the PCH only refers to explicitly-built modules, |
| 200 | // which technically is not guaranteed. To remove the assumption, we'd need |
| 201 | // to also rework how the module files are handled to the scan, specifically |
| 202 | // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain |
| 203 | // paths to ModuleFileName. |
| 204 | Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: Worklist.back()), |
| 205 | Kind: serialization::MK_ExplicitModule, |
| 206 | /*DirectlyImported=*/false); |
| 207 | if (ASTReader::readASTFileControlBlock( |
| 208 | Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(), |
| 209 | PCHContainerRdr: CI.getPCHContainerReader(), |
| 210 | /*FindModuleFileExtensions=*/false, Listener, |
| 211 | /*ValidateDiagnosticOptions=*/false)) |
| 212 | return true; |
| 213 | } |
| 214 | return false; |
| 215 | } |
| 216 | |
| 217 | /// Transform arbitrary file name into an object-like file name. |
| 218 | static std::string makeObjFileName(StringRef FileName) { |
| 219 | SmallString<128> ObjFileName(FileName); |
| 220 | llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o" ); |
| 221 | return std::string(ObjFileName); |
| 222 | } |
| 223 | |
| 224 | /// Deduce the dependency target based on the output file and input files. |
| 225 | static std::string |
| 226 | deduceDepTarget(const std::string &OutputFile, |
| 227 | const SmallVectorImpl<FrontendInputFile> &InputFiles) { |
| 228 | if (OutputFile != "-" ) |
| 229 | return OutputFile; |
| 230 | |
| 231 | if (InputFiles.empty() || !InputFiles.front().isFile()) |
| 232 | return "clang-scan-deps\\ dependency" ; |
| 233 | |
| 234 | return makeObjFileName(FileName: InputFiles.front().getFile()); |
| 235 | } |
| 236 | |
| 237 | // Clang implements -D and -U by splatting text into a predefines buffer. This |
| 238 | // allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and |
| 239 | // define the same macro, or adding C++ style comments before the macro name. |
| 240 | // |
| 241 | // This function checks that the first non-space characters in the macro |
| 242 | // obviously form an identifier that can be uniqued on without lexing. Failing |
| 243 | // to do this could lead to changing the final definition of a macro. |
| 244 | // |
| 245 | // We could set up a preprocessor and actually lex the name, but that's very |
| 246 | // heavyweight for a situation that will almost never happen in practice. |
| 247 | static std::optional<StringRef> getSimpleMacroName(StringRef Macro) { |
| 248 | StringRef Name = Macro.split(Separator: "=" ).first.ltrim(Chars: " \t" ); |
| 249 | std::size_t I = 0; |
| 250 | |
| 251 | auto FinishName = [&]() -> std::optional<StringRef> { |
| 252 | StringRef SimpleName = Name.slice(Start: 0, End: I); |
| 253 | if (SimpleName.empty()) |
| 254 | return std::nullopt; |
| 255 | return SimpleName; |
| 256 | }; |
| 257 | |
| 258 | for (; I != Name.size(); ++I) { |
| 259 | switch (Name[I]) { |
| 260 | case '(': // Start of macro parameter list |
| 261 | case ' ': // End of macro name |
| 262 | case '\t': |
| 263 | return FinishName(); |
| 264 | case '_': |
| 265 | continue; |
| 266 | default: |
| 267 | if (llvm::isAlnum(C: Name[I])) |
| 268 | continue; |
| 269 | return std::nullopt; |
| 270 | } |
| 271 | } |
| 272 | return FinishName(); |
| 273 | } |
| 274 | |
| 275 | static void canonicalizeDefines(PreprocessorOptions &PPOpts) { |
| 276 | using MacroOpt = std::pair<StringRef, std::size_t>; |
| 277 | std::vector<MacroOpt> SimpleNames; |
| 278 | SimpleNames.reserve(n: PPOpts.Macros.size()); |
| 279 | std::size_t Index = 0; |
| 280 | for (const auto &M : PPOpts.Macros) { |
| 281 | auto SName = getSimpleMacroName(Macro: M.first); |
| 282 | // Skip optimizing if we can't guarantee we can preserve relative order. |
| 283 | if (!SName) |
| 284 | return; |
| 285 | SimpleNames.emplace_back(args&: *SName, args&: Index); |
| 286 | ++Index; |
| 287 | } |
| 288 | |
| 289 | llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first()); |
| 290 | // Keep the last instance of each macro name by going in reverse |
| 291 | auto NewEnd = std::unique( |
| 292 | first: SimpleNames.rbegin(), last: SimpleNames.rend(), |
| 293 | binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; }); |
| 294 | SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base()); |
| 295 | |
| 296 | // Apply permutation. |
| 297 | decltype(PPOpts.Macros) NewMacros; |
| 298 | NewMacros.reserve(n: SimpleNames.size()); |
| 299 | for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) { |
| 300 | std::size_t OriginalIndex = SimpleNames[I].second; |
| 301 | // We still emit undefines here as they may be undefining a predefined macro |
| 302 | NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex])); |
| 303 | } |
| 304 | std::swap(x&: PPOpts.Macros, y&: NewMacros); |
| 305 | } |
| 306 | |
| 307 | namespace { |
| 308 | class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { |
| 309 | DependencyScanningWorkerFilesystem *DepFS; |
| 310 | |
| 311 | public: |
| 312 | ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) { |
| 313 | FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) { |
| 314 | auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS); |
| 315 | if (DFS) { |
| 316 | assert(!DepFS && "Found multiple scanning VFSs" ); |
| 317 | DepFS = DFS; |
| 318 | } |
| 319 | }); |
| 320 | assert(DepFS && "Did not find scanning VFS" ); |
| 321 | } |
| 322 | |
| 323 | std::unique_ptr<DependencyDirectivesGetter> |
| 324 | cloneFor(FileManager &FileMgr) override { |
| 325 | return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr); |
| 326 | } |
| 327 | |
| 328 | std::optional<ArrayRef<dependency_directives_scan::Directive>> |
| 329 | operator()(FileEntryRef File) override { |
| 330 | return DepFS->getDirectiveTokens(Path: File.getName()); |
| 331 | } |
| 332 | }; |
| 333 | } // namespace |
| 334 | |
| 335 | /// Sanitize diagnostic options for dependency scan. |
| 336 | static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { |
| 337 | // Don't print 'X warnings and Y errors generated'. |
| 338 | DiagOpts.ShowCarets = false; |
| 339 | // Don't write out diagnostic file. |
| 340 | DiagOpts.DiagnosticSerializationFile.clear(); |
| 341 | // Don't emit warnings except for scanning specific warnings. |
| 342 | // TODO: It would be useful to add a more principled way to ignore all |
| 343 | // warnings that come from source code. The issue is that we need to |
| 344 | // ignore warnings that could be surpressed by |
| 345 | // `#pragma clang diagnostic`, while still allowing some scanning |
| 346 | // warnings for things we're not ready to turn into errors yet. |
| 347 | // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example. |
| 348 | llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) { |
| 349 | return llvm::StringSwitch<bool>(Warning) |
| 350 | .Cases(CaseStrings: {"pch-vfs-diff" , "error=pch-vfs-diff" }, Value: false) |
| 351 | .StartsWith(S: "no-error=" , Value: false) |
| 352 | .Default(Value: true); |
| 353 | }); |
| 354 | } |
| 355 | |
| 356 | static std::unique_ptr<CompilerInvocation> |
| 357 | createCompilerInvocation(ArrayRef<std::string> CommandLine, |
| 358 | DiagnosticsEngine &Diags) { |
| 359 | llvm::opt::ArgStringList Argv; |
| 360 | for (const std::string &Str : ArrayRef(CommandLine).drop_front()) |
| 361 | Argv.push_back(Elt: Str.c_str()); |
| 362 | |
| 363 | auto Invocation = std::make_unique<CompilerInvocation>(); |
| 364 | if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) { |
| 365 | // FIXME: Should we just go on like cc1_main does? |
| 366 | return nullptr; |
| 367 | } |
| 368 | return Invocation; |
| 369 | } |
| 370 | |
| 371 | static void initializeScanCompilerInstance( |
| 372 | CompilerInstance &ScanInstance, |
| 373 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, |
| 374 | DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, |
| 375 | IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) { |
| 376 | ScanInstance.setBuildingModule(false); |
| 377 | ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer); |
| 378 | ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false); |
| 379 | if (!Service.getOpts().EmitWarnings) |
| 380 | ScanInstance.getDiagnostics().setIgnoreAllWarnings(true); |
| 381 | ScanInstance.createFileManager(); |
| 382 | ScanInstance.createSourceManager(); |
| 383 | |
| 384 | // Use DepFS for getting the dependency directives if requested to do so. |
| 385 | if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) |
| 386 | ScanInstance.setDependencyDirectivesGetter( |
| 387 | std::make_unique<ScanningDependencyDirectivesGetter>( |
| 388 | args&: ScanInstance.getFileManager())); |
| 389 | } |
| 390 | |
| 391 | static std::shared_ptr<CompilerInvocation> |
| 392 | createScanCompilerInvocation(const CompilerInvocation &Invocation, |
| 393 | const DependencyScanningService &Service, |
| 394 | DependencyActionController &Controller) { |
| 395 | auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation); |
| 396 | |
| 397 | sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts()); |
| 398 | |
| 399 | ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = |
| 400 | true; |
| 401 | |
| 402 | if (ScanInvocation->getHeaderSearchOpts() |
| 403 | .ModulesValidateOncePerBuildSession || |
| 404 | Service.getOpts().ValidateAgainstInvalidatedPaths) |
| 405 | ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp = |
| 406 | Service.getOpts().BuildSessionTimestamp; |
| 407 | if (Service.getOpts().ValidateAgainstInvalidatedPaths) |
| 408 | ScanInvocation->getHeaderSearchOpts().ModulesValidateDirectoryDependencies = |
| 409 | true; |
| 410 | |
| 411 | ScanInvocation->getFrontendOpts().DisableFree = false; |
| 412 | ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false; |
| 413 | ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false; |
| 414 | ScanInvocation->getFrontendOpts().GenReducedBMI = false; |
| 415 | ScanInvocation->getFrontendOpts().ModuleOutputPath.clear(); |
| 416 | // This will prevent us compiling individual modules asynchronously since |
| 417 | // FileManager is not thread-safe, but it does improve performance for now. |
| 418 | ScanInvocation->getFrontendOpts().ModulesShareFileManager = true; |
| 419 | ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw" ; |
| 420 | ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage = |
| 421 | any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS); |
| 422 | |
| 423 | // Consider different header search and diagnostic options to create |
| 424 | // different modules. This avoids the unsound aliasing of module PCMs. |
| 425 | // |
| 426 | // TODO: Implement diagnostic bucketing to reduce the impact of strict |
| 427 | // context hashing. |
| 428 | ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true; |
| 429 | ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; |
| 430 | ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; |
| 431 | ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; |
| 432 | ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = |
| 433 | true; |
| 434 | ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; |
| 435 | |
| 436 | // FIXME: Do this even with PCHs by marking the option as something like |
| 437 | // "preprocessor benign" in LangOptions.def so that it passes the |
| 438 | // compatibility checks in ASTReader. |
| 439 | if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) { |
| 440 | // Application extension only affects the handling of availability |
| 441 | // attributes, which cannot change the dependencies. |
| 442 | ScanInvocation->getLangOpts().AppExt = false; |
| 443 | } |
| 444 | |
| 445 | // Ensure that the scanner does not create new dependency collectors, |
| 446 | // and thus won't write out the extra '.d' files to disk. |
| 447 | ScanInvocation->getDependencyOutputOpts() = {}; |
| 448 | |
| 449 | Controller.initializeScanInvocation(ScanInvocation&: *ScanInvocation); |
| 450 | |
| 451 | return ScanInvocation; |
| 452 | } |
| 453 | |
| 454 | static llvm::SmallVector<StringRef> |
| 455 | getInitialStableDirs(const CompilerInstance &ScanInstance) { |
| 456 | // Create a collection of stable directories derived from the ScanInstance |
| 457 | // for determining whether module dependencies would fully resolve from |
| 458 | // those directories. |
| 459 | llvm::SmallVector<StringRef> StableDirs; |
| 460 | const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; |
| 461 | if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot)) |
| 462 | StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; |
| 463 | return StableDirs; |
| 464 | } |
| 465 | |
| 466 | static std::optional<PrebuiltModulesAttrsMap> |
| 467 | computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, |
| 468 | llvm::SmallVector<StringRef> &StableDirs) { |
| 469 | // Store a mapping of prebuilt module files and their properties like header |
| 470 | // search options. This will prevent the implicit build to create duplicate |
| 471 | // modules and will force reuse of the existing prebuilt module files |
| 472 | // instead. |
| 473 | PrebuiltModulesAttrsMap PrebuiltModulesASTMap; |
| 474 | |
| 475 | if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) |
| 476 | if (visitPrebuiltModule( |
| 477 | PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance, |
| 478 | ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, |
| 479 | PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs)) |
| 480 | return {}; |
| 481 | |
| 482 | return PrebuiltModulesASTMap; |
| 483 | } |
| 484 | |
| 485 | static std::unique_ptr<DependencyOutputOptions> |
| 486 | createDependencyOutputOptions(const CompilerInvocation &Invocation) { |
| 487 | auto Opts = std::make_unique<DependencyOutputOptions>( |
| 488 | args: Invocation.getDependencyOutputOpts()); |
| 489 | // We need at least one -MT equivalent for the generator of make dependency |
| 490 | // files to work. |
| 491 | if (Opts->Targets.empty()) |
| 492 | Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile, |
| 493 | InputFiles: Invocation.getFrontendOpts().Inputs)}; |
| 494 | Opts->IncludeSystemHeaders = true; |
| 495 | |
| 496 | return Opts; |
| 497 | } |
| 498 | |
| 499 | static std::shared_ptr<ModuleDepCollector> |
| 500 | initializeScanInstanceDependencyCollector( |
| 501 | CompilerInstance &ScanInstance, |
| 502 | std::unique_ptr<DependencyOutputOptions> DepOutputOpts, |
| 503 | DependencyScanningService &Service, CompilerInvocation &Inv, |
| 504 | DependencyActionController &Controller, |
| 505 | PrebuiltModulesAttrsMap PrebuiltModulesASTMap, |
| 506 | SmallVector<StringRef> &StableDirs) { |
| 507 | auto MDC = std::make_shared<ModuleDepCollector>( |
| 508 | args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Controller, args&: Inv, |
| 509 | args: std::move(PrebuiltModulesASTMap), args&: StableDirs); |
| 510 | ScanInstance.addDependencyCollector(Listener: MDC); |
| 511 | return MDC; |
| 512 | } |
| 513 | |
| 514 | namespace { |
| 515 | /// Manages (and terminates) the asynchronous compilation of modules. |
| 516 | class AsyncModuleCompiles { |
| 517 | std::mutex Mutex; |
| 518 | bool Stop = false; |
| 519 | // FIXME: Have the service own a thread pool and use that instead. |
| 520 | std::vector<std::thread> Compiles; |
| 521 | |
| 522 | public: |
| 523 | /// Registers the module compilation, unless this instance is about to be |
| 524 | /// destroyed. |
| 525 | void add(llvm::unique_function<void()> Compile) { |
| 526 | std::lock_guard<std::mutex> Lock(Mutex); |
| 527 | if (!Stop) |
| 528 | Compiles.emplace_back(args: std::move(Compile)); |
| 529 | } |
| 530 | |
| 531 | ~AsyncModuleCompiles() { |
| 532 | { |
| 533 | std::lock_guard<std::mutex> Lock(Mutex); |
| 534 | Stop = true; |
| 535 | } |
| 536 | for (std::thread &Compile : Compiles) |
| 537 | Compile.join(); |
| 538 | } |
| 539 | }; |
| 540 | |
| 541 | struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction { |
| 542 | DependencyScanningService &Service; |
| 543 | DependencyActionController &Controller; |
| 544 | AsyncModuleCompiles &Compiles; |
| 545 | |
| 546 | SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service, |
| 547 | DependencyActionController &Controller, |
| 548 | AsyncModuleCompiles &Compiles) |
| 549 | : Service(Service), Controller(Controller), Compiles(Compiles) {} |
| 550 | |
| 551 | bool BeginSourceFileAction(CompilerInstance &CI) override; |
| 552 | }; |
| 553 | |
| 554 | /// Runs the preprocessor on a TU with single-module-parse-mode and compiles |
| 555 | /// modules asynchronously without blocking or importing them. |
| 556 | struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction { |
| 557 | DependencyScanningService &Service; |
| 558 | DependencyActionController &Controller; |
| 559 | AsyncModuleCompiles &Compiles; |
| 560 | |
| 561 | SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service, |
| 562 | DependencyActionController &Controller, |
| 563 | AsyncModuleCompiles &Compiles) |
| 564 | : Service(Service), Controller(Controller), Compiles(Compiles) {} |
| 565 | |
| 566 | bool BeginSourceFileAction(CompilerInstance &CI) override; |
| 567 | }; |
| 568 | |
| 569 | /// The preprocessor callback that takes care of initiating an asynchronous |
| 570 | /// module compilation if needed. |
| 571 | struct AsyncModuleCompile : PPCallbacks { |
| 572 | CompilerInstance &CI; |
| 573 | DependencyScanningService &Service; |
| 574 | DependencyActionController &Controller; |
| 575 | AsyncModuleCompiles &Compiles; |
| 576 | |
| 577 | AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service, |
| 578 | DependencyActionController &Controller, |
| 579 | AsyncModuleCompiles &Compiles) |
| 580 | : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {} |
| 581 | |
| 582 | void moduleLoadSkipped(Module *M) override { |
| 583 | M = M->getTopLevelModule(); |
| 584 | |
| 585 | HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); |
| 586 | ModuleCache &ModCache = CI.getModuleCache(); |
| 587 | ModuleFileName ModuleFileName = HS.getCachedModuleFileName(Module: M); |
| 588 | |
| 589 | uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName); |
| 590 | // Someone else already built/validated the PCM. |
| 591 | if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp) |
| 592 | return; |
| 593 | |
| 594 | if (!CI.getASTReader()) |
| 595 | CI.createASTReader(); |
| 596 | SmallVector<ASTReader::ImportedModule, 0> Imported; |
| 597 | // Only calling ReadASTCore() to avoid the expensive eager deserialization |
| 598 | // of the clang::Module objects in ReadAST(). |
| 599 | // FIXME: Consider doing this in the new thread depending on how expensive |
| 600 | // the read turns out to be. |
| 601 | switch (CI.getASTReader()->ReadASTCore( |
| 602 | FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(), |
| 603 | ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {}, |
| 604 | ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing | |
| 605 | ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) { |
| 606 | case ASTReader::Success: |
| 607 | // We successfully read a valid, up-to-date PCM. |
| 608 | // FIXME: This could update the timestamp. Regular calls to |
| 609 | // ASTReader::ReadAST() would do so unless they encountered corrupted |
| 610 | // AST block, corrupted extension block, or did not read the expected |
| 611 | // top-level module. |
| 612 | return; |
| 613 | case ASTReader::OutOfDate: |
| 614 | case ASTReader::Missing: |
| 615 | // The most interesting case. |
| 616 | break; |
| 617 | default: |
| 618 | // Let the regular scan diagnose this. |
| 619 | return; |
| 620 | } |
| 621 | |
| 622 | auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName); |
| 623 | bool Owned; |
| 624 | llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned); |
| 625 | // Someone else is building the PCM right now. |
| 626 | if (!LockErr && !Owned) |
| 627 | return; |
| 628 | // We should build the PCM. |
| 629 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = |
| 630 | llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>( |
| 631 | A&: Service, A: Service.getOpts().MakeVFS()); |
| 632 | VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(), |
| 633 | Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS)); |
| 634 | auto DC = std::make_unique<DiagnosticConsumer>(); |
| 635 | auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(), |
| 636 | Logger&: Service.getLogger()); |
| 637 | CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC, |
| 638 | std::move(MC)); |
| 639 | auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName, |
| 640 | ThreadSafeConfig: CloneConfig); |
| 641 | auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName, |
| 642 | ThreadSafeConfig: CloneConfig); |
| 643 | |
| 644 | auto ModController = Controller.clone(); |
| 645 | |
| 646 | // Note: This lock belongs to a module cache that might not outlive the |
| 647 | // thread. This works, because the in-process lock only refers to an object |
| 648 | // managed by the service, which does outlive the thread. |
| 649 | Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1), |
| 650 | ModCI2 = std::move(ModCI2), DC = std::move(DC), |
| 651 | ModController = std::move(ModController), Service = &Service, |
| 652 | Compiles = &Compiles] { |
| 653 | llvm::CrashRecoveryContext CRC; |
| 654 | (void)CRC.RunSafely(Fn: [&] { |
| 655 | // Quickly discovers and compiles modules for the real scan below. |
| 656 | SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController, |
| 657 | *Compiles); |
| 658 | (void)ModCI1->ExecuteAction(Act&: Action1); |
| 659 | // The real scan below. |
| 660 | ModCI2->getPreprocessorOpts().SingleModuleParseMode = false; |
| 661 | GenerateModuleFromModuleMapAction Action2; |
| 662 | (void)ModCI2->ExecuteAction(Act&: Action2); |
| 663 | }); |
| 664 | }); |
| 665 | } |
| 666 | }; |
| 667 | |
| 668 | bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction( |
| 669 | CompilerInstance &CI) { |
| 670 | CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; |
| 671 | CI.getPreprocessor().addPPCallbacks( |
| 672 | C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles)); |
| 673 | return true; |
| 674 | } |
| 675 | |
| 676 | bool SingleTUWithAsyncModuleCompiles::BeginSourceFileAction( |
| 677 | CompilerInstance &CI) { |
| 678 | CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; |
| 679 | CI.getPreprocessor().addPPCallbacks( |
| 680 | C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles)); |
| 681 | return true; |
| 682 | } |
| 683 | } // namespace |
| 684 | |
| 685 | static void runTUModulePrescan(CompilerInstance &PrescanCI, |
| 686 | DependencyScanningService &Service, |
| 687 | DependencyActionController &Controller, |
| 688 | AsyncModuleCompiles &Compiles) { |
| 689 | SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles); |
| 690 | (void)PrescanCI.ExecuteAction(Act&: Action); |
| 691 | } |
| 692 | |
| 693 | namespace clang { |
| 694 | namespace dependencies { |
| 695 | |
| 696 | std::unique_ptr<DiagnosticOptions> |
| 697 | createScanningDiagOptions(ArrayRef<std::string> CommandLine) { |
| 698 | std::vector<const char *> CCommandLine(CommandLine.size(), nullptr); |
| 699 | llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(), |
| 700 | F: [](const std::string &Str) { return Str.c_str(); }); |
| 701 | auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine); |
| 702 | sanitizeDiagOpts(DiagOpts&: *DiagOpts); |
| 703 | return DiagOpts; |
| 704 | } |
| 705 | |
| 706 | namespace { |
| 707 | /// Find the module maps that define modules without loading them. |
| 708 | class ModuleMapLookup { |
| 709 | public: |
| 710 | explicit ModuleMapLookup(CompilerInstance &CI) |
| 711 | : CI(CI), HSOpts(CI.getHeaderSearchOpts()), |
| 712 | Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()), |
| 713 | SourceMgr(Diags, CI.getFileManager()), |
| 714 | HS(HSOpts, SourceMgr, Diags, CI.getLangOpts(), &CI.getTarget()) { |
| 715 | HSOpts.LazyLoadModuleMaps = true; |
| 716 | // Ignore verbose mode as it's already respected by the real scan. |
| 717 | HSOpts.Verbose = false; |
| 718 | ApplyHeaderSearchOptions(HS, HSOpts, Lang: CI.getLangOpts(), |
| 719 | triple: CI.getTarget().getTriple()); |
| 720 | } |
| 721 | |
| 722 | llvm::StringMap<std::string> |
| 723 | computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) { |
| 724 | // Resolve any modulemaps passed directly first. |
| 725 | for (const std::string &Filename : CI.getFrontendOpts().ModuleMapFiles) |
| 726 | if (OptionalFileEntryRef File = |
| 727 | CI.getFileManager().getOptionalFileRef(Filename)) |
| 728 | HS.parseModuleMapFile(File: *File, /*IsSystem=*/false, |
| 729 | /*ImplicitlyDiscovered=*/false); |
| 730 | |
| 731 | const bool HasPrebuiltModules = !HSOpts.PrebuiltModuleFiles.empty() || |
| 732 | !HSOpts.PrebuiltModulePaths.empty(); |
| 733 | |
| 734 | llvm::StringMap<std::string> Result; |
| 735 | // Then resolve modulemaps through header search. |
| 736 | for (StringRef Name : ModuleNames) { |
| 737 | // Skip over prebuilt modules. |
| 738 | if (HasPrebuiltModules && !HS.getPrebuiltModuleFileName(ModuleName: Name).empty()) |
| 739 | continue; |
| 740 | OptionalFileEntryRef File = HS.lookupModuleMapFileForModule( |
| 741 | ModuleName: Name, /*AllowExtraModuleMapSearch=*/true); |
| 742 | // Skip over modules that didn't find a modulemap. |
| 743 | if (!File) |
| 744 | continue; |
| 745 | // Canonicalize the path the same way as for the real scan. |
| 746 | SmallString<128> Path = File->getNameAsRequested(); |
| 747 | HS.getModuleMap().canonicalizeModuleMapPath(Path); |
| 748 | Result[Name] = std::string(Path); |
| 749 | } |
| 750 | return Result; |
| 751 | } |
| 752 | |
| 753 | private: |
| 754 | CompilerInstance &CI; |
| 755 | HeaderSearchOptions HSOpts; |
| 756 | DiagnosticOptions DiagOpts; |
| 757 | DiagnosticsEngine Diags; |
| 758 | SourceManager SourceMgr; |
| 759 | HeaderSearch HS; |
| 760 | }; |
| 761 | } // namespace |
| 762 | |
| 763 | class CompilerInstanceWithContext { |
| 764 | // Context |
| 765 | DependencyScanningWorker &Worker; |
| 766 | llvm::StringRef CWD; |
| 767 | std::vector<std::string> CommandLine; |
| 768 | |
| 769 | // Context - compiler invocation |
| 770 | std::unique_ptr<CompilerInvocation> OriginalInvocation; |
| 771 | |
| 772 | // Context - output options |
| 773 | std::unique_ptr<DependencyOutputOptions> OutputOpts; |
| 774 | |
| 775 | // Context - stable directory handling |
| 776 | llvm::SmallVector<StringRef> StableDirs; |
| 777 | PrebuiltModulesAttrsMap PrebuiltModuleASTMap; |
| 778 | |
| 779 | // Context - used by AsyncScan's prescan pass |
| 780 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS; |
| 781 | |
| 782 | // Compiler Instance |
| 783 | std::unique_ptr<CompilerInstance> CIPtr; |
| 784 | |
| 785 | // Source location offset. |
| 786 | int32_t SrcLocOffset = 0; |
| 787 | |
| 788 | CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD, |
| 789 | ArrayRef<std::string> CMD) |
| 790 | : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {} |
| 791 | |
| 792 | bool initialize(DependencyActionController &Controller, |
| 793 | DiagnosticsEngine &DiagEngine, |
| 794 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { |
| 795 | { |
| 796 | auto LogLine = Worker.Service.getLogger().log(); |
| 797 | LogLine.logArray(Prefix: "init_compiler_instance_with_context:" , Sep: " " , |
| 798 | Arr: CommandLine); |
| 799 | } |
| 800 | |
| 801 | ScanFS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS: std::move(OverlayFS)); |
| 802 | OriginalInvocation = createCompilerInvocation(CommandLine, Diags&: DiagEngine); |
| 803 | if (!OriginalInvocation) { |
| 804 | DiagEngine.Report(DiagID: diag::err_fe_expected_compiler_job) |
| 805 | << llvm::join(R&: CommandLine, Separator: " " ); |
| 806 | return false; |
| 807 | } |
| 808 | |
| 809 | return initializeScanInstance(Controller, DiagConsumer: DiagEngine.getClient()); |
| 810 | } |
| 811 | |
| 812 | bool initializeScanInstance(DependencyActionController &Controller, |
| 813 | DiagnosticConsumer *DiagConsumer) { |
| 814 | assert(OriginalInvocation && ScanFS && |
| 815 | "OriginalInvocation and ScanFS must be set before this call" ); |
| 816 | |
| 817 | if (any(Val: Worker.Service.getOpts().OptimizeArgs & |
| 818 | ScanningOptimizations::Macros)) |
| 819 | canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts()); |
| 820 | |
| 821 | // Create the CompilerInstance. |
| 822 | std::shared_ptr<ModuleCache> ModCache = makeInProcessModuleCache( |
| 823 | Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger()); |
| 824 | CIPtr = std::make_unique<CompilerInstance>( |
| 825 | args: createScanCompilerInvocation(Invocation: *OriginalInvocation, Service: Worker.Service, |
| 826 | Controller), |
| 827 | args&: Worker.PCHContainerOps, args: std::move(ModCache)); |
| 828 | auto &CI = *CIPtr; |
| 829 | |
| 830 | initializeScanCompilerInstance(ScanInstance&: CI, FS: ScanFS, DiagConsumer, Service&: Worker.Service, |
| 831 | DepFS: Worker.DepFS); |
| 832 | |
| 833 | StableDirs = getInitialStableDirs(ScanInstance: CI); |
| 834 | auto MaybePrebuiltModulesASTMap = |
| 835 | computePrebuiltModulesASTMap(ScanInstance&: CI, StableDirs); |
| 836 | if (!MaybePrebuiltModulesASTMap) |
| 837 | return false; |
| 838 | |
| 839 | PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap); |
| 840 | OutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation); |
| 841 | |
| 842 | // We do not create the target in initializeScanCompilerInstance because |
| 843 | // setting it here is unique for by-name lookups. We create the target only |
| 844 | // once here, and the information is reused for all computeDependencies |
| 845 | // calls. We do not need to call createTarget explicitly if we go through |
| 846 | // CompilerInstance::ExecuteAction to perform scanning. |
| 847 | return CI.createTarget(); |
| 848 | } |
| 849 | |
| 850 | bool prescanModulesAsync(AsyncModuleCompiles &Compiles, |
| 851 | DependencyActionController &Controller) { |
| 852 | auto ModCache = makeInProcessModuleCache( |
| 853 | Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger()); |
| 854 | CompilerInstance PrescanCI( |
| 855 | std::make_shared<CompilerInvocation>(args&: CIPtr->getInvocation()), |
| 856 | Worker.PCHContainerOps, std::move(ModCache)); |
| 857 | |
| 858 | DiagnosticConsumer DiagConsumer; |
| 859 | initializeScanCompilerInstance(ScanInstance&: PrescanCI, FS: ScanFS, DiagConsumer: &DiagConsumer, |
| 860 | Service&: Worker.Service, DepFS: Worker.DepFS); |
| 861 | |
| 862 | // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in |
| 863 | // initialize(). |
| 864 | SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(ScanInstance: PrescanCI); |
| 865 | if (!computePrebuiltModulesASTMap(ScanInstance&: PrescanCI, StableDirs&: PrescanStableDirs)) |
| 866 | return false; |
| 867 | |
| 868 | if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH) |
| 869 | PrescanCI.getLangOpts().CompilingPCH = true; |
| 870 | |
| 871 | runTUModulePrescan(PrescanCI, Service&: Worker.Service, Controller, Compiles); |
| 872 | return true; |
| 873 | } |
| 874 | |
| 875 | public: |
| 876 | static std::optional<CompilerInstanceWithContext> |
| 877 | initializeFromCC1Commandline( |
| 878 | DependencyScanningWorker &Worker, StringRef CWD, |
| 879 | ArrayRef<std::string> CC1CommandLine, DiagnosticsEngine &DiagEngine, |
| 880 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, |
| 881 | DependencyActionController &Controller) { |
| 882 | CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine); |
| 883 | if (!CIWC.initialize(Controller, DiagEngine, OverlayFS: std::move(OverlayFS))) |
| 884 | return std::nullopt; |
| 885 | return std::move(CIWC); |
| 886 | } |
| 887 | |
| 888 | /// Initialize from a cc1 command line, reporting diagnostics to |
| 889 | /// \p DiagConsumer. |
| 890 | static std::optional<CompilerInstanceWithContext> |
| 891 | initializeFromCC1Commandline( |
| 892 | DependencyScanningWorker &Worker, StringRef CWD, |
| 893 | ArrayRef<std::string> CC1CommandLine, DiagnosticConsumer &DiagConsumer, |
| 894 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, |
| 895 | DependencyActionController &Controller) { |
| 896 | auto FS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS); |
| 897 | auto DiagOpts = createScanningDiagOptions(CommandLine: CC1CommandLine); |
| 898 | auto DiagEngine = |
| 899 | CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer, |
| 900 | /*ShouldOwnClient=*/false); |
| 901 | return initializeFromCC1Commandline(Worker, CWD, CC1CommandLine, |
| 902 | DiagEngine&: *DiagEngine, OverlayFS: std::move(OverlayFS), |
| 903 | Controller); |
| 904 | } |
| 905 | |
| 906 | bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer, |
| 907 | DependencyActionController &Controller) { |
| 908 | Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName; |
| 909 | llvm::scope_exit ExitLogging([&] { |
| 910 | Worker.Service.getLogger().log() << "finish scan_by_name: " << ModuleName; |
| 911 | }); |
| 912 | if (SrcLocOffset >= DependencyScanningWorker::MaxNumOfByNameQueries) |
| 913 | llvm::report_fatal_error(reason: "exceeded maximum by-name scans for worker" ); |
| 914 | |
| 915 | assert(CIPtr && "CIPtr must be initialized before calling this method" ); |
| 916 | auto &CI = *CIPtr; |
| 917 | |
| 918 | // We need to reset the diagnostics, so that the diagnostics issued |
| 919 | // during a previous computeDependencies call do not affect the current |
| 920 | // call. If we do not reset, we may inherit fatal errors from a previous |
| 921 | // call. |
| 922 | CI.getDiagnostics().Reset(); |
| 923 | |
| 924 | // We create this cleanup object because computeDependencies may exit |
| 925 | // early with errors. |
| 926 | llvm::scope_exit CleanUp([&]() { |
| 927 | CI.clearDependencyCollectors(); |
| 928 | // The preprocessor may not be created at the entry of this method, |
| 929 | // but it must have been created when this method returns, whether |
| 930 | // there are errors during scanning or not. |
| 931 | CI.getPreprocessor().removePPCallbacks(); |
| 932 | }); |
| 933 | |
| 934 | auto MDC = initializeScanInstanceDependencyCollector( |
| 935 | ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts), |
| 936 | Service&: Worker.Service, |
| 937 | /* The MDC's constructor makes a copy of the OriginalInvocation, so |
| 938 | we can pass it in without worrying that it might be changed across |
| 939 | invocations of computeDependencies. */ |
| 940 | Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, StableDirs); |
| 941 | |
| 942 | CompilerInvocation ModuleInvocation(*OriginalInvocation); |
| 943 | if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: ModuleInvocation)) |
| 944 | return false; |
| 945 | |
| 946 | if (!SrcLocOffset) { |
| 947 | // When SrcLocOffset is zero, we are at the beginning of the fake source |
| 948 | // file. In this case, we call BeginSourceFile to initialize. |
| 949 | std::unique_ptr<FrontendAction> Action = |
| 950 | std::make_unique<PreprocessOnlyAction>(); |
| 951 | auto *InputFile = CI.getFrontendOpts().Inputs.begin(); |
| 952 | bool ActionBeginSucceeded = Action->BeginSourceFile(CI, Input: *InputFile); |
| 953 | assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed" ); |
| 954 | (void)ActionBeginSucceeded; |
| 955 | } |
| 956 | |
| 957 | Preprocessor &PP = CI.getPreprocessor(); |
| 958 | SourceManager &SM = PP.getSourceManager(); |
| 959 | FileID MainFileID = SM.getMainFileID(); |
| 960 | SourceLocation FileStart = SM.getLocForStartOfFile(FID: MainFileID); |
| 961 | SourceLocation IDLocation = FileStart.getLocWithOffset(Offset: SrcLocOffset); |
| 962 | PPCallbacks *CB = nullptr; |
| 963 | if (!SrcLocOffset) { |
| 964 | // We need to call EnterSourceFile when SrcLocOffset is zero to initialize |
| 965 | // the preprocessor. |
| 966 | bool PPFailed = PP.EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation()); |
| 967 | assert(!PPFailed && "Preprocess must be able to enter the main file." ); |
| 968 | (void)PPFailed; |
| 969 | CB = MDC->getPPCallbacks(); |
| 970 | } else { |
| 971 | // When SrcLocOffset is non-zero, the preprocessor has already been |
| 972 | // initialized through a previous call of computeDependencies. We want to |
| 973 | // preserve the PP's state, hence we do not call EnterSourceFile again. |
| 974 | MDC->attachToPreprocessor(PP); |
| 975 | CB = MDC->getPPCallbacks(); |
| 976 | |
| 977 | FileID PrevFID; |
| 978 | SrcMgr::CharacteristicKind FileType = |
| 979 | SM.getFileCharacteristic(Loc: IDLocation); |
| 980 | CB->LexedFileChanged(FID: MainFileID, |
| 981 | Reason: PPChainedCallbacks::LexedFileChangeReason::EnterFile, |
| 982 | FileType, PrevFID, Loc: IDLocation); |
| 983 | } |
| 984 | |
| 985 | // FIXME: Scan modules asynchronously here as well. |
| 986 | |
| 987 | SrcLocOffset++; |
| 988 | SmallVector<IdentifierLoc, 2> Path; |
| 989 | IdentifierInfo *ModuleID = PP.getIdentifierInfo(Name: ModuleName); |
| 990 | Path.emplace_back(Args&: IDLocation, Args&: ModuleID); |
| 991 | auto ModResult = CI.loadModule(ImportLoc: IDLocation, Path, Visibility: Module::Hidden, |
| 992 | /*IsInclusionDirective=*/false); |
| 993 | |
| 994 | assert(CB && "Must have PPCallbacks after module loading" ); |
| 995 | CB->moduleImport(ImportLoc: SourceLocation(), Path, Imported: ModResult); |
| 996 | |
| 997 | if (!ModResult) |
| 998 | return false; |
| 999 | |
| 1000 | if (CI.getDiagnostics().hasErrorOccurred()) |
| 1001 | return false; |
| 1002 | |
| 1003 | MDC->run(Consumer); |
| 1004 | MDC->applyDiscoveredDependencies(CI&: ModuleInvocation); |
| 1005 | |
| 1006 | bool Success = ModuleInvocation.withCowRef<bool>( |
| 1007 | Fn: [&](CowCompilerInvocation &CowModuleInvocation) { |
| 1008 | return Controller.finalize(ScanInstance&: CI, NewInvocation&: CowModuleInvocation); |
| 1009 | }); |
| 1010 | if (!Success) |
| 1011 | return false; |
| 1012 | |
| 1013 | Consumer.handleBuildCommand( |
| 1014 | Cmd: {.Executable: CommandLine[0], .Arguments: ModuleInvocation.getCC1CommandLine()}); |
| 1015 | |
| 1016 | return true; |
| 1017 | } |
| 1018 | |
| 1019 | llvm::StringMap<std::string> |
| 1020 | computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) { |
| 1021 | assert(CIPtr && "CIPtr must be initialized before calling this method" ); |
| 1022 | return ModuleMapLookup(*CIPtr).computeModuleMapsForModules(ModuleNames); |
| 1023 | } |
| 1024 | |
| 1025 | std::shared_ptr<ModuleDepCollector> |
| 1026 | scanTranslationUnit(DependencyConsumer &Consumer, |
| 1027 | DependencyActionController &Controller) { |
| 1028 | assert(CIPtr && "CIPtr must be initialized before calling this method" ); |
| 1029 | auto &CI = *CIPtr; |
| 1030 | |
| 1031 | std::optional<AsyncModuleCompiles> AsyncCompiles; |
| 1032 | if (Worker.Service.getOpts().AsyncScanModules) { |
| 1033 | AsyncCompiles.emplace(); |
| 1034 | if (!prescanModulesAsync(Compiles&: *AsyncCompiles, Controller)) |
| 1035 | return nullptr; |
| 1036 | } |
| 1037 | |
| 1038 | auto MDC = initializeScanInstanceDependencyCollector( |
| 1039 | ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts), |
| 1040 | Service&: Worker.Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, |
| 1041 | StableDirs); |
| 1042 | |
| 1043 | if (CI.getDiagnostics().hasErrorOccurred()) |
| 1044 | return nullptr; |
| 1045 | |
| 1046 | if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: *OriginalInvocation)) |
| 1047 | return nullptr; |
| 1048 | |
| 1049 | ReadPCHAndPreprocessAction Action; |
| 1050 | if (!CI.ExecuteAction(Act&: Action)) |
| 1051 | return nullptr; |
| 1052 | |
| 1053 | MDC->run(Consumer); |
| 1054 | if (!applyAndReport(MDC&: *MDC, ModuleInvocation&: *OriginalInvocation, Consumer, Controller, |
| 1055 | Executable: CommandLine[0])) |
| 1056 | return nullptr; |
| 1057 | return MDC; |
| 1058 | } |
| 1059 | |
| 1060 | bool applyAndReport(ModuleDepCollector &MDC, |
| 1061 | CompilerInvocation &ModuleInvocation, |
| 1062 | DependencyConsumer &Consumer, |
| 1063 | DependencyActionController &Controller, |
| 1064 | StringRef Executable) { |
| 1065 | MDC.applyDiscoveredDependencies(CI&: ModuleInvocation); |
| 1066 | bool Success = ModuleInvocation.withCowRef<bool>( |
| 1067 | Fn: [&](CowCompilerInvocation &CowModuleInvocation) { |
| 1068 | return Controller.finalize(ScanInstance&: *CIPtr, NewInvocation&: CowModuleInvocation); |
| 1069 | }); |
| 1070 | if (!Success) |
| 1071 | return false; |
| 1072 | Consumer.handleBuildCommand( |
| 1073 | Cmd: {.Executable: Executable.str(), .Arguments: ModuleInvocation.getCC1CommandLine()}); |
| 1074 | return true; |
| 1075 | } |
| 1076 | }; |
| 1077 | } // namespace dependencies |
| 1078 | } // namespace clang |
| 1079 | |
| 1080 | DependencyScanningWorker::DependencyScanningWorker( |
| 1081 | DependencyScanningService &Service, |
| 1082 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> LocalFS) |
| 1083 | : Service(Service) { |
| 1084 | PCHContainerOps = std::make_shared<PCHContainerOperations>(); |
| 1085 | // We need to read object files from PCH built outside the scanner. |
| 1086 | PCHContainerOps->registerReader( |
| 1087 | Reader: std::make_unique<ObjectFilePCHContainerReader>()); |
| 1088 | // The scanner itself writes only raw ast files. |
| 1089 | PCHContainerOps->registerWriter(Writer: std::make_unique<RawPCHContainerWriter>()); |
| 1090 | |
| 1091 | auto BaseFS = Service.getOpts().MakeVFS(); |
| 1092 | |
| 1093 | if (Service.getOpts().TraceVFS) { |
| 1094 | TracingFS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>( |
| 1095 | A: std::move(BaseFS)); |
| 1096 | BaseFS = TracingFS; |
| 1097 | } |
| 1098 | |
| 1099 | DepFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>( |
| 1100 | A&: Service, A: std::move(BaseFS), A: std::move(LocalFS)); |
| 1101 | } |
| 1102 | |
| 1103 | DependencyScanningWorker::~DependencyScanningWorker() = default; |
| 1104 | |
| 1105 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> |
| 1106 | DependencyScanningWorker::makeEffectiveVFS( |
| 1107 | StringRef WorkingDirectory, |
| 1108 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) const { |
| 1109 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = DepFS; |
| 1110 | if (OverlayFS) { |
| 1111 | auto NewFS = |
| 1112 | llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(A: std::move(FS)); |
| 1113 | NewFS->pushOverlay(FS: std::move(OverlayFS)); |
| 1114 | FS = std::move(NewFS); |
| 1115 | } |
| 1116 | FS->setCurrentWorkingDirectory(WorkingDirectory); |
| 1117 | return FS; |
| 1118 | } |
| 1119 | |
| 1120 | bool DependencyScanningWorker::computeDependencies( |
| 1121 | StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, |
| 1122 | DependencyConsumer &DepConsumer, DependencyActionController &Controller, |
| 1123 | DiagnosticConsumer &DiagConsumer, |
| 1124 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { |
| 1125 | auto FS = makeEffectiveVFS(WorkingDirectory, OverlayFS); |
| 1126 | |
| 1127 | bool Scanned = false; |
| 1128 | std::shared_ptr<ModuleDepCollector> MDC; |
| 1129 | std::optional<CompilerInstanceWithContext> CIWC; |
| 1130 | |
| 1131 | const bool Success = llvm::all_of(Range&: CommandLines, P: [&](const auto &Cmd) { |
| 1132 | if (StringRef(Cmd[1]) != "-cc1" ) { |
| 1133 | // Non-clang command. Just pass through to the dependency consumer. |
| 1134 | DepConsumer.handleBuildCommand( |
| 1135 | Cmd: {Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); |
| 1136 | return true; |
| 1137 | } |
| 1138 | |
| 1139 | Service.getLogger().log().logArray("starting scanning command:" , " " , Cmd); |
| 1140 | llvm::scope_exit ExitLogging([&] { |
| 1141 | Service.getLogger().log().logArray("finished scanning command:" , " " , |
| 1142 | Cmd); |
| 1143 | }); |
| 1144 | |
| 1145 | auto DiagOpts = createScanningDiagOptions(Cmd); |
| 1146 | auto DiagEngine = |
| 1147 | CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, |
| 1148 | /*ShouldOwnClient=*/false); |
| 1149 | if (!Scanned) { |
| 1150 | // Scanning runs once for the first -cc1 invocation in a chain of driver |
| 1151 | // jobs. |
| 1152 | // For any dependent jobs, reuse the scanning result and just update the |
| 1153 | // new invocation. |
| 1154 | // FIXME: to support multi-arch builds, each arch requires a separate |
| 1155 | // scan. |
| 1156 | Scanned = true; |
| 1157 | auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline( |
| 1158 | *this, WorkingDirectory, Cmd, *DiagEngine, OverlayFS, Controller); |
| 1159 | if (!Result) |
| 1160 | return false; |
| 1161 | CIWC.emplace(std::move(*Result)); |
| 1162 | MDC = CIWC->scanTranslationUnit(Consumer&: DepConsumer, Controller); |
| 1163 | return MDC != nullptr; |
| 1164 | } |
| 1165 | |
| 1166 | auto Invocation = createCompilerInvocation(Cmd, *DiagEngine); |
| 1167 | if (!Invocation) |
| 1168 | return false; |
| 1169 | |
| 1170 | // The first cc1 is canonicalized in initializeScanInstance; each sibling |
| 1171 | // invocation must likewise be canonicalized before its cc1 command line is |
| 1172 | // emitted. This is mostly relevant for multi-arch jobs where we currently |
| 1173 | // do not do re-scans. |
| 1174 | if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros)) |
| 1175 | canonicalizeDefines(Invocation->getPreprocessorOpts()); |
| 1176 | |
| 1177 | assert(CIWC && "Must have an initialized CIWC" ); |
| 1178 | return CIWC->applyAndReport(MDC&: *MDC, ModuleInvocation&: *Invocation, Consumer&: DepConsumer, Controller, |
| 1179 | Executable: Cmd.front()); |
| 1180 | }); |
| 1181 | |
| 1182 | return Success && Scanned; |
| 1183 | } |
| 1184 | |
| 1185 | bool DependencyScanningWorker::computeDependenciesByName( |
| 1186 | StringRef CWD, ArrayRef<std::string> CC1CommandLine, |
| 1187 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, |
| 1188 | DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, |
| 1189 | llvm::function_ref<std::optional<std::string>()> getNextName, |
| 1190 | DependencyConsumer &DepConsumer) { |
| 1191 | std::optional<CompilerInstanceWithContext> CIWC = |
| 1192 | CompilerInstanceWithContext::initializeFromCC1Commandline( |
| 1193 | Worker&: *this, CWD, CC1CommandLine, DiagConsumer, OverlayFS: std::move(OverlayFS), |
| 1194 | Controller); |
| 1195 | if (!CIWC) |
| 1196 | return false; |
| 1197 | |
| 1198 | bool AllScansSucceeded = true; |
| 1199 | while (std::optional<std::string> NextName = getNextName()) { |
| 1200 | bool Success = |
| 1201 | CIWC->computeDependencies(ModuleName: *NextName, Consumer&: DepConsumer, Controller); |
| 1202 | DepConsumer.finishQuery(ModuleName: *NextName, Success); |
| 1203 | AllScansSucceeded = AllScansSucceeded && Success; |
| 1204 | } |
| 1205 | return AllScansSucceeded; |
| 1206 | } |
| 1207 | |
| 1208 | std::optional<llvm::StringMap<std::string>> |
| 1209 | DependencyScanningWorker::computeModuleMapsForModules( |
| 1210 | StringRef CWD, ArrayRef<std::string> CC1CommandLine, |
| 1211 | IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, |
| 1212 | DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, |
| 1213 | ArrayRef<StringRef> ModuleNames) { |
| 1214 | std::optional<CompilerInstanceWithContext> CIWC = |
| 1215 | CompilerInstanceWithContext::initializeFromCC1Commandline( |
| 1216 | Worker&: *this, CWD, CC1CommandLine, DiagConsumer, OverlayFS: std::move(OverlayFS), |
| 1217 | Controller); |
| 1218 | if (!CIWC) |
| 1219 | return std::nullopt; |
| 1220 | return CIWC->computeModuleMapsForModules(ModuleNames); |
| 1221 | } |
| 1222 | |