1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/AliasAnalysis.h"
21#include "llvm/Analysis/ModuleSummaryAnalysis.h"
22#include "llvm/Analysis/ProfileSummaryInfo.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Bitcode/BitcodeReader.h"
25#include "llvm/Bitcode/BitcodeWriter.h"
26#include "llvm/Bitcode/BitcodeWriterPass.h"
27#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/LegacyPassManager.h"
31#include "llvm/IR/LLVMContext.h"
32#include "llvm/IR/LLVMRemarkStreamer.h"
33#include "llvm/IR/Mangler.h"
34#include "llvm/IR/PassTimingInfo.h"
35#include "llvm/IR/Verifier.h"
36#include "llvm/IRReader/IRReader.h"
37#include "llvm/LTO/LTO.h"
38#include "llvm/LTO/SummaryBasedOptimizations.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Object/IRObjectFile.h"
41#include "llvm/Passes/PassBuilder.h"
42#include "llvm/Passes/StandardInstrumentations.h"
43#include "llvm/Remarks/HotnessThresholdParser.h"
44#include "llvm/Support/CachePruning.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/FileSystem.h"
48#include "llvm/Support/FormatVariadic.h"
49#include "llvm/Support/Path.h"
50#include "llvm/Support/SHA1.h"
51#include "llvm/Support/SmallVectorMemoryBuffer.h"
52#include "llvm/Support/ThreadPool.h"
53#include "llvm/Support/Threading.h"
54#include "llvm/Support/ToolOutputFile.h"
55#include "llvm/Support/raw_ostream.h"
56#include "llvm/Target/TargetMachine.h"
57#include "llvm/TargetParser/SubtargetFeature.h"
58#include "llvm/Transforms/IPO/FunctionAttrs.h"
59#include "llvm/Transforms/IPO/FunctionImport.h"
60#include "llvm/Transforms/IPO/Internalize.h"
61#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
62#include "llvm/Transforms/ObjCARC.h"
63#include "llvm/Transforms/Utils/FunctionImportUtils.h"
64
65#include <numeric>
66
67#if !defined(_MSC_VER) && !defined(__MINGW32__)
68#include <unistd.h>
69#else
70#include <io.h>
71#endif
72
73using namespace llvm;
74
75#define DEBUG_TYPE "thinlto"
76
77namespace llvm {
78// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
79extern cl::opt<bool> LTODiscardValueNames;
80extern cl::opt<std::string> RemarksFilename;
81extern cl::opt<std::string> RemarksPasses;
82extern cl::opt<bool> RemarksWithHotness;
83extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
84 RemarksHotnessThreshold;
85extern cl::opt<std::string> RemarksFormat;
86}
87
88namespace {
89
90// Default to using all available threads in the system, but using only one
91// thred per core, as indicated by the usage of
92// heavyweight_hardware_concurrency() below.
93static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
94
95// Simple helper to save temporary files for debug.
96static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
97 unsigned count, StringRef Suffix) {
98 if (TempDir.empty())
99 return;
100 // User asked to save temps, let dump the bitcode file after import.
101 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
102 std::error_code EC;
103 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
104 if (EC)
105 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
106 " to save optimized bitcode\n");
107 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
108}
109
110static const GlobalValueSummary *
111getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
112 // If there is any strong definition anywhere, get it.
113 auto StrongDefForLinker = llvm::find_if(
114 Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
115 auto Linkage = Summary->linkage();
116 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
117 !GlobalValue::isWeakForLinker(Linkage);
118 });
119 if (StrongDefForLinker != GVSummaryList.end())
120 return StrongDefForLinker->get();
121 // Get the first *linker visible* definition for this global in the summary
122 // list.
123 auto FirstDefForLinker = llvm::find_if(
124 Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
125 auto Linkage = Summary->linkage();
126 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
127 });
128 // Extern templates can be emitted as available_externally.
129 if (FirstDefForLinker == GVSummaryList.end())
130 return nullptr;
131 return FirstDefForLinker->get();
132}
133
134// Populate map of GUID to the prevailing copy for any multiply defined
135// symbols. Currently assume first copy is prevailing, or any strong
136// definition. Can be refined with Linker information in the future.
137static void computePrevailingCopies(
138 const ModuleSummaryIndex &Index,
139 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
140 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
141 return GVSummaryList.size() > 1;
142 };
143
144 for (auto &I : Index) {
145 if (HasMultipleCopies(I.second.SummaryList))
146 PrevailingCopy[I.first] =
147 getFirstDefinitionForLinker(GVSummaryList: I.second.SummaryList);
148 }
149}
150
151static StringMap<lto::InputFile *>
152generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
153 StringMap<lto::InputFile *> ModuleMap;
154 for (auto &M : Modules) {
155 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
156 assert(!ModuleMap.contains(M->getName()) &&
157 "Expect unique Buffer Identifier");
158 ModuleMap[M->getName()] = M.get();
159 }
160 return ModuleMap;
161}
162
163static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
164 bool ClearDSOLocalOnDeclarations) {
165 if (renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations))
166 report_fatal_error(reason: "renameModuleForThinLTO failed");
167}
168
169namespace {
170class ThinLTODiagnosticInfo : public DiagnosticInfo {
171 const Twine &Msg;
172public:
173 ThinLTODiagnosticInfo(const Twine &DiagMsg,
174 DiagnosticSeverity Severity = DS_Error)
175 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
176 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
177};
178}
179
180/// Verify the module and strip broken debug info.
181static void verifyLoadedModule(Module &TheModule) {
182 bool BrokenDebugInfo = false;
183 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
184 report_fatal_error(reason: "Broken module found, compilation aborted!");
185 if (BrokenDebugInfo) {
186 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
187 "Invalid debug info found, debug info will be stripped", DS_Warning));
188 StripDebugInfo(M&: TheModule);
189 }
190}
191
192static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
193 LLVMContext &Context,
194 bool Lazy,
195 bool IsImporting) {
196 auto &Mod = Input->getSingleBitcodeModule();
197 SMDiagnostic Err;
198 Expected<std::unique_ptr<Module>> ModuleOrErr =
199 Lazy ? Mod.getLazyModule(Context,
200 /* ShouldLazyLoadMetadata */ true, IsImporting)
201 : Mod.parseModule(Context);
202 if (!ModuleOrErr) {
203 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
204 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
205 SourceMgr::DK_Error, EIB.message());
206 Err.print(ProgName: "ThinLTO", S&: errs());
207 });
208 report_fatal_error(reason: "Can't load module, abort.");
209 }
210 if (!Lazy)
211 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
212 return std::move(*ModuleOrErr);
213}
214
215static void
216crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
217 StringMap<lto::InputFile *> &ModuleMap,
218 const FunctionImporter::ImportMapTy &ImportList,
219 bool ClearDSOLocalOnDeclarations) {
220 auto Loader = [&](StringRef Identifier) {
221 auto &Input = ModuleMap[Identifier];
222 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
223 /*Lazy=*/true, /*IsImporting*/ true);
224 };
225
226 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
227 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
228 if (!Result) {
229 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
230 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
231 SourceMgr::DK_Error, EIB.message());
232 Err.print(ProgName: "ThinLTO", S&: errs());
233 });
234 report_fatal_error(reason: "importFunctions failed");
235 }
236 // Verify again after cross-importing.
237 verifyLoadedModule(TheModule);
238}
239
240static void optimizeModule(Module &TheModule, TargetMachine &TM,
241 unsigned OptLevel, bool Freestanding,
242 bool DebugPassManager, ModuleSummaryIndex *Index) {
243 std::optional<PGOOptions> PGOOpt;
244 LoopAnalysisManager LAM;
245 FunctionAnalysisManager FAM;
246 CGSCCAnalysisManager CGAM;
247 ModuleAnalysisManager MAM;
248
249 PassInstrumentationCallbacks PIC;
250 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
251 SI.registerCallbacks(PIC, MAM: &MAM);
252 PipelineTuningOptions PTO;
253 PTO.LoopVectorization = true;
254 PTO.SLPVectorization = true;
255 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
256
257 std::unique_ptr<TargetLibraryInfoImpl> TLII(
258 new TargetLibraryInfoImpl(Triple(TM.getTargetTriple())));
259 if (Freestanding)
260 TLII->disableAllFunctions();
261 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
262
263 // Register all the basic analyses with the managers.
264 PB.registerModuleAnalyses(MAM);
265 PB.registerCGSCCAnalyses(CGAM);
266 PB.registerFunctionAnalyses(FAM);
267 PB.registerLoopAnalyses(LAM);
268 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
269
270 ModulePassManager MPM;
271
272 OptimizationLevel OL;
273
274 switch (OptLevel) {
275 default:
276 llvm_unreachable("Invalid optimization level");
277 case 0:
278 OL = OptimizationLevel::O0;
279 break;
280 case 1:
281 OL = OptimizationLevel::O1;
282 break;
283 case 2:
284 OL = OptimizationLevel::O2;
285 break;
286 case 3:
287 OL = OptimizationLevel::O3;
288 break;
289 }
290
291 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
292
293 MPM.run(IR&: TheModule, AM&: MAM);
294}
295
296static void
297addUsedSymbolToPreservedGUID(const lto::InputFile &File,
298 DenseSet<GlobalValue::GUID> &PreservedGUID) {
299 for (const auto &Sym : File.symbols()) {
300 if (Sym.isUsed())
301 PreservedGUID.insert(V: GlobalValue::getGUID(GlobalName: Sym.getIRName()));
302 }
303}
304
305// Convert the PreservedSymbols map from "Name" based to "GUID" based.
306static void computeGUIDPreservedSymbols(const lto::InputFile &File,
307 const StringSet<> &PreservedSymbols,
308 const Triple &TheTriple,
309 DenseSet<GlobalValue::GUID> &GUIDs) {
310 // Iterate the symbols in the input file and if the input has preserved symbol
311 // compute the GUID for the symbol.
312 for (const auto &Sym : File.symbols()) {
313 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
314 GUIDs.insert(V: GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier(
315 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: "")));
316 }
317}
318
319static DenseSet<GlobalValue::GUID>
320computeGUIDPreservedSymbols(const lto::InputFile &File,
321 const StringSet<> &PreservedSymbols,
322 const Triple &TheTriple) {
323 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
324 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
325 GUIDs&: GUIDPreservedSymbols);
326 return GUIDPreservedSymbols;
327}
328
329std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
330 TargetMachine &TM) {
331 SmallVector<char, 128> OutputBuffer;
332
333 // CodeGen
334 {
335 raw_svector_ostream OS(OutputBuffer);
336 legacy::PassManager PM;
337
338 // If the bitcode files contain ARC code and were compiled with optimization,
339 // the ObjCARCContractPass must be run, so do it unconditionally here.
340 PM.add(P: createObjCARCContractPass());
341
342 // Setup the codegen now.
343 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
344 /* DisableVerify */ true))
345 report_fatal_error(reason: "Failed to setup codegen");
346
347 // Run codegen now. resulting binary is in OutputBuffer.
348 PM.run(M&: TheModule);
349 }
350 return std::make_unique<SmallVectorMemoryBuffer>(
351 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
352}
353
354/// Manage caching for a single Module.
355class ModuleCacheEntry {
356 SmallString<128> EntryPath;
357
358public:
359 // Create a cache entry. This compute a unique hash for the Module considering
360 // the current list of export/import, and offer an interface to query to
361 // access the content in the cache.
362 ModuleCacheEntry(
363 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
364 const FunctionImporter::ImportMapTy &ImportList,
365 const FunctionImporter::ExportSetTy &ExportList,
366 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
367 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
368 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
369 if (CachePath.empty())
370 return;
371
372 if (!Index.modulePaths().count(Key: ModuleID))
373 // The module does not have an entry, it can't have a hash at all
374 return;
375
376 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
377 P: [](uint32_t V) { return V == 0; }))
378 // No hash entry, no caching!
379 return;
380
381 llvm::lto::Config Conf;
382 Conf.OptLevel = OptLevel;
383 Conf.Options = TMBuilder.Options;
384 Conf.CPU = TMBuilder.MCpu;
385 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
386 Conf.RelocModel = TMBuilder.RelocModel;
387 Conf.CGOptLevel = TMBuilder.CGOptLevel;
388 Conf.Freestanding = Freestanding;
389 SmallString<40> Key;
390 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
391 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
392
393 // This choice of file name allows the cache to be pruned (see pruneCache()
394 // in include/llvm/Support/CachePruning.h).
395 sys::path::append(path&: EntryPath, a: CachePath, b: "llvmcache-" + Key);
396 }
397
398 // Access the path to this entry in the cache.
399 StringRef getEntryPath() { return EntryPath; }
400
401 // Try loading the buffer for this cache entry.
402 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
403 if (EntryPath.empty())
404 return std::error_code();
405 SmallString<64> ResultPath;
406 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
407 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
408 if (!FDOrErr)
409 return errorToErrorCode(Err: FDOrErr.takeError());
410 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
411 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
412 sys::fs::closeFile(F&: *FDOrErr);
413 return MBOrErr;
414 }
415
416 // Cache the Produced object file
417 void write(const MemoryBuffer &OutputBuffer) {
418 if (EntryPath.empty())
419 return;
420
421 if (auto Err = llvm::writeToOutput(
422 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
423 OS << OutputBuffer.getBuffer();
424 return llvm::Error::success();
425 }))
426 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
427 Vals&: EntryPath,
428 Vals: toString(E: std::move(Err)).c_str()));
429 }
430};
431
432static std::unique_ptr<MemoryBuffer>
433ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
434 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
435 const FunctionImporter::ImportMapTy &ImportList,
436 const FunctionImporter::ExportSetTy &ExportList,
437 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
438 const GVSummaryMapTy &DefinedGlobals,
439 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
440 bool DisableCodeGen, StringRef SaveTempsDir,
441 bool Freestanding, unsigned OptLevel, unsigned count,
442 bool DebugPassManager) {
443 // "Benchmark"-like optimization: single-source case
444 bool SingleModule = (ModuleMap.size() == 1);
445
446 // When linking an ELF shared object, dso_local should be dropped. We
447 // conservatively do this for -fpic.
448 bool ClearDSOLocalOnDeclarations =
449 TM.getTargetTriple().isOSBinFormatELF() &&
450 TM.getRelocationModel() != Reloc::Static &&
451 TheModule.getPIELevel() == PIELevel::Default;
452
453 if (!SingleModule) {
454 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
455
456 // Apply summary-based prevailing-symbol resolution decisions.
457 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
458
459 // Save temps: after promotion.
460 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
461 }
462
463 // Be friendly and don't nuke totally the module when the client didn't
464 // supply anything to preserve.
465 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
466 // Apply summary-based internalization decisions.
467 thinLTOInternalizeModule(TheModule, DefinedGlobals);
468 }
469
470 // Save internalized bitcode
471 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
472
473 if (!SingleModule)
474 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
475 ClearDSOLocalOnDeclarations);
476
477 // Do this after any importing so that imported code is updated.
478 // See comment at call to updateVCallVisibilityInIndex() for why
479 // WholeProgramVisibilityEnabledInLTO is false.
480 updatePublicTypeTestCalls(M&: TheModule,
481 /* WholeProgramVisibilityEnabledInLTO */ false);
482
483 // Save temps: after cross-module import.
484 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
485
486 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
487 Index: &Index);
488
489 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
490
491 if (DisableCodeGen) {
492 // Configured to stop before CodeGen, serialize the bitcode and return.
493 SmallVector<char, 128> OutputBuffer;
494 {
495 raw_svector_ostream OS(OutputBuffer);
496 ProfileSummaryInfo PSI(TheModule);
497 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
498 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
499 }
500 return std::make_unique<SmallVectorMemoryBuffer>(
501 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
502 }
503
504 return codegenModule(TheModule, TM);
505}
506
507/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
508/// for caching, and in the \p Index for application during the ThinLTO
509/// backends. This is needed for correctness for exported symbols (ensure
510/// at least one copy kept) and a compile-time optimization (to drop duplicate
511/// copies when possible).
512static void resolvePrevailingInIndex(
513 ModuleSummaryIndex &Index,
514 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
515 &ResolvedODR,
516 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
517 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
518 &PrevailingCopy) {
519
520 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
521 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
522 // Not in map means that there was only one copy, which must be prevailing.
523 if (Prevailing == PrevailingCopy.end())
524 return true;
525 return Prevailing->second == S;
526 };
527
528 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
529 GlobalValue::GUID GUID,
530 GlobalValue::LinkageTypes NewLinkage) {
531 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
532 };
533
534 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
535 lto::Config Conf;
536 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
537 GUIDPreservedSymbols);
538}
539
540// Initialize the TargetMachine builder for a given Triple
541static void initTMBuilder(TargetMachineBuilder &TMBuilder,
542 const Triple &TheTriple) {
543 if (TMBuilder.MCpu.empty())
544 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
545 TMBuilder.TheTriple = std::move(TheTriple);
546}
547
548} // end anonymous namespace
549
550void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
551 MemoryBufferRef Buffer(Data, Identifier);
552
553 auto InputOrError = lto::InputFile::create(Object: Buffer);
554 if (!InputOrError)
555 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
556 toString(E: InputOrError.takeError()));
557
558 auto TripleStr = (*InputOrError)->getTargetTriple();
559 Triple TheTriple(TripleStr);
560
561 if (Modules.empty())
562 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
563 else if (TMBuilder.TheTriple != TheTriple) {
564 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
565 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
566 "supported");
567 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
568 }
569
570 Modules.emplace_back(args: std::move(*InputOrError));
571}
572
573void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
574 PreservedSymbols.insert(key: Name);
575}
576
577void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
578 // FIXME: At the moment, we don't take advantage of this extra information,
579 // we're conservatively considering cross-references as preserved.
580 // CrossReferencedSymbols.insert(Name);
581 PreservedSymbols.insert(key: Name);
582}
583
584// TargetMachine factory
585std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
586 std::string ErrMsg;
587 const Target *TheTarget =
588 TargetRegistry::lookupTarget(Triple: TheTriple.str(), Error&: ErrMsg);
589 if (!TheTarget) {
590 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
591 }
592
593 // Use MAttr as the default set of features.
594 SubtargetFeatures Features(MAttr);
595 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
596 std::string FeatureStr = Features.getString();
597
598 std::unique_ptr<TargetMachine> TM(
599 TheTarget->createTargetMachine(TT: TheTriple.str(), CPU: MCpu, Features: FeatureStr, Options,
600 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
601 assert(TM && "Cannot create target machine");
602
603 return TM;
604}
605
606/**
607 * Produce the combined summary index from all the bitcode files:
608 * "thin-link".
609 */
610std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
611 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
612 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
613 for (auto &Mod : Modules) {
614 auto &M = Mod->getSingleBitcodeModule();
615 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
616 // FIXME diagnose
617 logAllUnhandledErrors(
618 E: std::move(Err), OS&: errs(),
619 ErrorBanner: "error: can't create module summary index for buffer: ");
620 return nullptr;
621 }
622 }
623 return CombinedIndex;
624}
625
626namespace {
627struct IsExported {
628 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
629 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
630
631 IsExported(
632 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
633 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
634 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
635
636 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
637 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
638 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
639 GUIDPreservedSymbols.count(V: VI.getGUID());
640 }
641};
642
643struct IsPrevailing {
644 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
645 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
646 &PrevailingCopy)
647 : PrevailingCopy(PrevailingCopy) {}
648
649 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
650 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
651 // Not in map means that there was only one copy, which must be prevailing.
652 if (Prevailing == PrevailingCopy.end())
653 return true;
654 return Prevailing->second == S;
655 };
656};
657} // namespace
658
659static void computeDeadSymbolsInIndex(
660 ModuleSummaryIndex &Index,
661 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
662 // We have no symbols resolution available. And can't do any better now in the
663 // case where the prevailing symbol is in a native object. It can be refined
664 // with linker information in the future.
665 auto isPrevailing = [&](GlobalValue::GUID G) {
666 return PrevailingType::Unknown;
667 };
668 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
669 /* ImportEnabled = */ true);
670}
671
672/**
673 * Perform promotion and renaming of exported internal functions.
674 * Index is updated to reflect linkage changes from weak resolution.
675 */
676void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
677 const lto::InputFile &File) {
678 auto ModuleCount = Index.modulePaths().size();
679 auto ModuleIdentifier = TheModule.getModuleIdentifier();
680
681 // Collect for each module the list of function it defines (GUID -> Summary).
682 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
683 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
684
685 // Convert the preserved symbols set from string to GUID
686 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
687 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
688
689 // Add used symbol to the preserved symbols.
690 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
691
692 // Compute "dead" symbols, we don't want to import/export these!
693 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
694
695 // Compute prevailing symbols
696 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
697 computePrevailingCopies(Index, PrevailingCopy);
698
699 // Generate import/export list
700 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
701 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
702 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
703 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
704 ExportLists);
705
706 // Resolve prevailing symbols
707 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
708 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
709 PrevailingCopy);
710
711 thinLTOFinalizeInModule(TheModule,
712 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
713 /*PropagateAttrs=*/false);
714
715 // Promote the exported values in the index, so that they are promoted
716 // in the module.
717 thinLTOInternalizeAndPromoteInIndex(
718 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
719 isPrevailing: IsPrevailing(PrevailingCopy));
720
721 // FIXME Set ClearDSOLocalOnDeclarations.
722 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
723}
724
725/**
726 * Perform cross-module importing for the module identified by ModuleIdentifier.
727 */
728void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
729 ModuleSummaryIndex &Index,
730 const lto::InputFile &File) {
731 auto ModuleMap = generateModuleMap(Modules);
732 auto ModuleCount = Index.modulePaths().size();
733
734 // Collect for each module the list of function it defines (GUID -> Summary).
735 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
736 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
737
738 // Convert the preserved symbols set from string to GUID
739 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
740 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
741
742 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
743
744 // Compute "dead" symbols, we don't want to import/export these!
745 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
746
747 // Compute prevailing symbols
748 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
749 computePrevailingCopies(Index, PrevailingCopy);
750
751 // Generate import/export list
752 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
753 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
754 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
755 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
756 ExportLists);
757 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
758
759 // FIXME Set ClearDSOLocalOnDeclarations.
760 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
761 /*ClearDSOLocalOnDeclarations=*/false);
762}
763
764/**
765 * Compute the list of summaries needed for importing into module.
766 */
767void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
768 Module &TheModule, ModuleSummaryIndex &Index,
769 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
770 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
771 auto ModuleCount = Index.modulePaths().size();
772 auto ModuleIdentifier = TheModule.getModuleIdentifier();
773
774 // Collect for each module the list of function it defines (GUID -> Summary).
775 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
776 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
777
778 // Convert the preserved symbols set from string to GUID
779 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
780 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
781
782 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
783
784 // Compute "dead" symbols, we don't want to import/export these!
785 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
786
787 // Compute prevailing symbols
788 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
789 computePrevailingCopies(Index, PrevailingCopy);
790
791 // Generate import/export list
792 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
793 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
794 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
795 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
796 ExportLists);
797
798 llvm::gatherImportedSummariesForModule(
799 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
800 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
801}
802
803/**
804 * Emit the list of files needed for importing into module.
805 */
806void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
807 ModuleSummaryIndex &Index,
808 const lto::InputFile &File) {
809 auto ModuleCount = Index.modulePaths().size();
810 auto ModuleIdentifier = TheModule.getModuleIdentifier();
811
812 // Collect for each module the list of function it defines (GUID -> Summary).
813 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
814 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
815
816 // Convert the preserved symbols set from string to GUID
817 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
818 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
819
820 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
821
822 // Compute "dead" symbols, we don't want to import/export these!
823 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
824
825 // Compute prevailing symbols
826 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
827 computePrevailingCopies(Index, PrevailingCopy);
828
829 // Generate import/export list
830 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
831 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
832 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
833 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
834 ExportLists);
835
836 // 'EmitImportsFiles' emits the list of modules from which to import from, and
837 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
838 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportFiles`.
839 GVSummaryPtrSet DecSummaries;
840 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
841 llvm::gatherImportedSummariesForModule(
842 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
843 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
844
845 std::error_code EC;
846 if ((EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
847 ModuleToSummariesForIndex)))
848 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
849 " to save imports lists\n");
850}
851
852/**
853 * Perform internalization. Runs promote and internalization together.
854 * Index is updated to reflect linkage changes.
855 */
856void ThinLTOCodeGenerator::internalize(Module &TheModule,
857 ModuleSummaryIndex &Index,
858 const lto::InputFile &File) {
859 initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple()));
860 auto ModuleCount = Index.modulePaths().size();
861 auto ModuleIdentifier = TheModule.getModuleIdentifier();
862
863 // Convert the preserved symbols set from string to GUID
864 auto GUIDPreservedSymbols =
865 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
866
867 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
868
869 // Collect for each module the list of function it defines (GUID -> Summary).
870 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
871 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
872
873 // Compute "dead" symbols, we don't want to import/export these!
874 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
875
876 // Compute prevailing symbols
877 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
878 computePrevailingCopies(Index, PrevailingCopy);
879
880 // Generate import/export list
881 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
882 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
883 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
884 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
885 ExportLists);
886 auto &ExportList = ExportLists[ModuleIdentifier];
887
888 // Be friendly and don't nuke totally the module when the client didn't
889 // supply anything to preserve.
890 if (ExportList.empty() && GUIDPreservedSymbols.empty())
891 return;
892
893 // Resolve prevailing symbols
894 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
895 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
896 PrevailingCopy);
897
898 // Promote the exported values in the index, so that they are promoted
899 // in the module.
900 thinLTOInternalizeAndPromoteInIndex(
901 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
902 isPrevailing: IsPrevailing(PrevailingCopy));
903
904 // FIXME Set ClearDSOLocalOnDeclarations.
905 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
906
907 // Internalization
908 thinLTOFinalizeInModule(TheModule,
909 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
910 /*PropagateAttrs=*/false);
911
912 thinLTOInternalizeModule(TheModule,
913 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
914}
915
916/**
917 * Perform post-importing ThinLTO optimizations.
918 */
919void ThinLTOCodeGenerator::optimize(Module &TheModule) {
920 initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple()));
921
922 // Optimize now
923 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
924 DebugPassManager, Index: nullptr);
925}
926
927/// Write out the generated object file, either from CacheEntryPath or from
928/// OutputBuffer, preferring hard-link when possible.
929/// Returns the path to the generated file in SavedObjectsDirectoryPath.
930std::string
931ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
932 const MemoryBuffer &OutputBuffer) {
933 auto ArchName = TMBuilder.TheTriple.getArchName();
934 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
935 llvm::sys::path::append(path&: OutputPath,
936 a: Twine(count) + "." + ArchName + ".thinlto.o");
937 OutputPath.c_str(); // Ensure the string is null terminated.
938 if (sys::fs::exists(Path: OutputPath))
939 sys::fs::remove(path: OutputPath);
940
941 // We don't return a memory buffer to the linker, just a list of files.
942 if (!CacheEntryPath.empty()) {
943 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
944 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
945 if (!Err)
946 return std::string(OutputPath);
947 // Hard linking failed, try to copy.
948 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
949 if (!Err)
950 return std::string(OutputPath);
951 // Copy failed (could be because the CacheEntry was removed from the cache
952 // in the meantime by another process), fall back and try to write down the
953 // buffer to the output.
954 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
955 << "' to '" << OutputPath << "'\n";
956 }
957 // No cache entry, just write out the buffer.
958 std::error_code Err;
959 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
960 if (Err)
961 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
962 OS << OutputBuffer.getBuffer();
963 return std::string(OutputPath);
964}
965
966// Main entry point for the ThinLTO processing
967void ThinLTOCodeGenerator::run() {
968 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
969 auto TimeTraceScopeExit = llvm::make_scope_exit(F: []() {
970 if (llvm::timeTraceProfilerEnabled())
971 llvm::timeTraceProfilerEnd();
972 });
973 // Prepare the resulting object vector
974 assert(ProducedBinaries.empty() && "The generator should not be reused");
975 if (SavedObjectsDirectoryPath.empty())
976 ProducedBinaries.resize(new_size: Modules.size());
977 else {
978 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
979 bool IsDir;
980 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
981 if (!IsDir)
982 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
983 ProducedBinaryFiles.resize(new_size: Modules.size());
984 }
985
986 if (CodeGenOnly) {
987 // Perform only parallel codegen and return.
988 DefaultThreadPool Pool;
989 int count = 0;
990 for (auto &Mod : Modules) {
991 Pool.async(F: [&](int count) {
992 LLVMContext Context;
993 Context.setDiscardValueNames(LTODiscardValueNames);
994
995 // Parse module now
996 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
997 /*IsImporting*/ false);
998
999 // CodeGen
1000 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1001 if (SavedObjectsDirectoryPath.empty())
1002 ProducedBinaries[count] = std::move(OutputBuffer);
1003 else
1004 ProducedBinaryFiles[count] =
1005 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1006 }, ArgList: count++);
1007 }
1008
1009 return;
1010 }
1011
1012 // Sequential linking phase
1013 auto Index = linkCombinedIndex();
1014
1015 // Save temps: index.
1016 if (!SaveTempsDir.empty()) {
1017 auto SaveTempPath = SaveTempsDir + "index.bc";
1018 std::error_code EC;
1019 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1020 if (EC)
1021 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1022 " to save optimized bitcode\n");
1023 writeIndexToFile(Index: *Index, Out&: OS);
1024 }
1025
1026
1027 // Prepare the module map.
1028 auto ModuleMap = generateModuleMap(Modules);
1029 auto ModuleCount = Modules.size();
1030
1031 // Collect for each module the list of function it defines (GUID -> Summary).
1032 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1033 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1034
1035 // Convert the preserved symbols set from string to GUID, this is needed for
1036 // computing the caching hash and the internalization.
1037 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1038 for (const auto &M : Modules)
1039 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1040 GUIDs&: GUIDPreservedSymbols);
1041
1042 // Add used symbol from inputs to the preserved symbols.
1043 for (const auto &M : Modules)
1044 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1045
1046 // Compute "dead" symbols, we don't want to import/export these!
1047 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1048
1049 // Synthesize entry counts for functions in the combined index.
1050 computeSyntheticCounts(Index&: *Index);
1051
1052 // Currently there is no support for enabling whole program visibility via a
1053 // linker option in the old LTO API, but this call allows it to be specified
1054 // via the internal option. Must be done before WPD below.
1055 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1056 Index->setWithWholeProgramVisibility();
1057
1058 // FIXME: This needs linker information via a TBD new interface
1059 updateVCallVisibilityInIndex(Index&: *Index,
1060 /*WholeProgramVisibilityEnabledInLTO=*/false,
1061 // FIXME: These need linker information via a
1062 // TBD new interface.
1063 /*DynamicExportSymbols=*/{},
1064 /*VisibleToRegularObjSymbols=*/{});
1065
1066 // Perform index-based WPD. This will return immediately if there are
1067 // no index entries in the typeIdMetadata map (e.g. if we are instead
1068 // performing IR-based WPD in hybrid regular/thin LTO mode).
1069 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1070 std::set<GlobalValue::GUID> ExportedGUIDs;
1071 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1072 for (auto GUID : ExportedGUIDs)
1073 GUIDPreservedSymbols.insert(V: GUID);
1074
1075 // Compute prevailing symbols
1076 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1077 computePrevailingCopies(Index: *Index, PrevailingCopy);
1078
1079 // Collect the import/export lists for all modules from the call-graph in the
1080 // combined index.
1081 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1082 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1083 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1084 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1085 ExportLists);
1086
1087 // We use a std::map here to be able to have a defined ordering when
1088 // producing a hash for the cache entry.
1089 // FIXME: we should be able to compute the caching hash for the entry based
1090 // on the index, and nuke this map.
1091 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1092
1093 // Resolve prevailing symbols, this has to be computed early because it
1094 // impacts the caching.
1095 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1096 PrevailingCopy);
1097
1098 // Use global summary-based analysis to identify symbols that can be
1099 // internalized (because they aren't exported or preserved as per callback).
1100 // Changes are made in the index, consumed in the ThinLTO backends.
1101 updateIndexWPDForExports(Summary&: *Index,
1102 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1103 LocalWPDTargetsMap);
1104 thinLTOInternalizeAndPromoteInIndex(
1105 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1106 isPrevailing: IsPrevailing(PrevailingCopy));
1107
1108 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1109
1110 // Make sure that every module has an entry in the ExportLists, ImportList,
1111 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1112 // below.
1113 for (auto &Module : Modules) {
1114 auto ModuleIdentifier = Module->getName();
1115 ExportLists[ModuleIdentifier];
1116 ImportLists[ModuleIdentifier];
1117 ResolvedODR[ModuleIdentifier];
1118 ModuleToDefinedGVSummaries[ModuleIdentifier];
1119 }
1120
1121 std::vector<BitcodeModule *> ModulesVec;
1122 ModulesVec.reserve(n: Modules.size());
1123 for (auto &Mod : Modules)
1124 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1125 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1126
1127 if (llvm::timeTraceProfilerEnabled())
1128 llvm::timeTraceProfilerEnd();
1129
1130 TimeTraceScopeExit.release();
1131
1132 // Parallel optimizer + codegen
1133 {
1134 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1135 for (auto IndexCount : ModulesOrdering) {
1136 auto &Mod = Modules[IndexCount];
1137 Pool.async(F: [&](int count) {
1138 auto ModuleIdentifier = Mod->getName();
1139 auto &ExportList = ExportLists[ModuleIdentifier];
1140
1141 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1142
1143 // The module may be cached, this helps handling it.
1144 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1145 ImportLists[ModuleIdentifier], ExportList,
1146 ResolvedODR[ModuleIdentifier],
1147 DefinedGVSummaries, OptLevel, Freestanding,
1148 TMBuilder);
1149 auto CacheEntryPath = CacheEntry.getEntryPath();
1150
1151 {
1152 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1153 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1154 << " '" << CacheEntryPath << "' for buffer "
1155 << count << " " << ModuleIdentifier << "\n");
1156
1157 if (ErrOrBuffer) {
1158 // Cache Hit!
1159 if (SavedObjectsDirectoryPath.empty())
1160 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1161 else
1162 ProducedBinaryFiles[count] = writeGeneratedObject(
1163 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1164 return;
1165 }
1166 }
1167
1168 LLVMContext Context;
1169 Context.setDiscardValueNames(LTODiscardValueNames);
1170 Context.enableDebugTypeODRUniquing();
1171 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1172 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1173 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1174 if (!DiagFileOrErr) {
1175 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1176 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1177 "remarks");
1178 }
1179
1180 // Parse module now
1181 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1182 /*IsImporting*/ false);
1183
1184 // Save temps: original file.
1185 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1186
1187 auto &ImportList = ImportLists[ModuleIdentifier];
1188 // Run the main process now, and generates a binary
1189 auto OutputBuffer = ProcessThinLTOModule(
1190 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1191 ExportList, GUIDPreservedSymbols,
1192 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1193 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1194 DebugPassManager);
1195
1196 // Commit to the cache (if enabled)
1197 CacheEntry.write(OutputBuffer: *OutputBuffer);
1198
1199 if (SavedObjectsDirectoryPath.empty()) {
1200 // We need to generated a memory buffer for the linker.
1201 if (!CacheEntryPath.empty()) {
1202 // When cache is enabled, reload from the cache if possible.
1203 // Releasing the buffer from the heap and reloading it from the
1204 // cache file with mmap helps us to lower memory pressure.
1205 // The freed memory can be used for the next input file.
1206 // The final binary link will read from the VFS cache (hopefully!)
1207 // or from disk (if the memory pressure was too high).
1208 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1209 if (auto EC = ReloadedBufferOrErr.getError()) {
1210 // On error, keep the preexisting buffer and print a diagnostic.
1211 errs() << "remark: can't reload cached file '" << CacheEntryPath
1212 << "': " << EC.message() << "\n";
1213 } else {
1214 OutputBuffer = std::move(*ReloadedBufferOrErr);
1215 }
1216 }
1217 ProducedBinaries[count] = std::move(OutputBuffer);
1218 return;
1219 }
1220 ProducedBinaryFiles[count] = writeGeneratedObject(
1221 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1222 }, ArgList&: IndexCount);
1223 }
1224 }
1225
1226 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1227
1228 // If statistics were requested, print them out now.
1229 if (llvm::AreStatisticsEnabled())
1230 llvm::PrintStatistics();
1231 reportAndResetTimings();
1232}
1233