1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/ModuleSummaryAnalysis.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/Bitcode/BitcodeWriterPass.h"
26#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/LLVMRemarkStreamer.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/Mangler.h"
33#include "llvm/IR/PassTimingInfo.h"
34#include "llvm/IR/Verifier.h"
35#include "llvm/IRReader/IRReader.h"
36#include "llvm/LTO/LTO.h"
37#include "llvm/MC/TargetRegistry.h"
38#include "llvm/Object/IRObjectFile.h"
39#include "llvm/Passes/PassBuilder.h"
40#include "llvm/Passes/StandardInstrumentations.h"
41#include "llvm/Remarks/HotnessThresholdParser.h"
42#include "llvm/Support/CachePruning.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/FormatVariadic.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/SHA1.h"
49#include "llvm/Support/SmallVectorMemoryBuffer.h"
50#include "llvm/Support/ThreadPool.h"
51#include "llvm/Support/Threading.h"
52#include "llvm/Support/ToolOutputFile.h"
53#include "llvm/Support/raw_ostream.h"
54#include "llvm/Target/TargetMachine.h"
55#include "llvm/TargetParser/SubtargetFeature.h"
56#include "llvm/Transforms/IPO/FunctionAttrs.h"
57#include "llvm/Transforms/IPO/FunctionImport.h"
58#include "llvm/Transforms/IPO/Internalize.h"
59#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
60#include "llvm/Transforms/Utils/FunctionImportUtils.h"
61
62#if !defined(_MSC_VER) && !defined(__MINGW32__)
63#include <unistd.h>
64#else
65#include <io.h>
66#endif
67
68using namespace llvm;
69using namespace ThinLTOCodeGeneratorImpl;
70
71#define DEBUG_TYPE "thinlto"
72
73namespace llvm {
74// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
75extern cl::opt<bool> LTODiscardValueNames;
76extern cl::opt<std::string> RemarksFilename;
77extern cl::opt<std::string> RemarksPasses;
78extern cl::opt<bool> RemarksWithHotness;
79extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
80 RemarksHotnessThreshold;
81extern cl::opt<std::string> RemarksFormat;
82extern cl::opt<bool> LTORunCSIRInstr;
83extern cl::opt<std::string> LTOCSIRProfile;
84extern cl::opt<std::string> SampleProfileFile;
85}
86
87// Default to using all available threads in the system, but using only one
88// thred per core, as indicated by the usage of
89// heavyweight_hardware_concurrency() below.
90static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
91
92// Simple helper to save temporary files for debug.
93static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
94 unsigned count, StringRef Suffix) {
95 if (TempDir.empty())
96 return;
97 // User asked to save temps, let dump the bitcode file after import.
98 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
99 std::error_code EC;
100 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
101 if (EC)
102 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
103 " to save optimized bitcode\n");
104 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
105}
106
107static const GlobalValueSummary *getFirstDefinitionForLinker(
108 ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
109 // If there is any strong definition anywhere, get it.
110 auto StrongDefForLinker = llvm::find_if(
111 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
112 auto Linkage = Summary->linkage();
113 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
114 !GlobalValue::isWeakForLinker(Linkage);
115 });
116 if (StrongDefForLinker != GVSummaryList.end())
117 return StrongDefForLinker->get();
118 // Get the first *linker visible* definition for this global in the summary
119 // list.
120 auto FirstDefForLinker = llvm::find_if(
121 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
122 auto Linkage = Summary->linkage();
123 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
124 });
125 // Extern templates can be emitted as available_externally.
126 if (FirstDefForLinker == GVSummaryList.end())
127 return nullptr;
128 return FirstDefForLinker->get();
129}
130
131// Populate map of GUID to the prevailing copy for any multiply defined
132// symbols. Currently assume first copy is prevailing, or any strong
133// definition. Can be refined with Linker information in the future.
134static void computePrevailingCopies(
135 const ModuleSummaryIndex &Index,
136 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
137 auto HasMultipleCopies =
138 [&](ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
139 return GVSummaryList.size() > 1;
140 };
141
142 for (auto &I : Index) {
143 if (HasMultipleCopies(I.second.getSummaryList()))
144 PrevailingCopy[I.first] =
145 getFirstDefinitionForLinker(GVSummaryList: I.second.getSummaryList());
146 }
147}
148
149static StringMap<lto::InputFile *>
150generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
151 StringMap<lto::InputFile *> ModuleMap;
152 for (auto &M : Modules) {
153 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
154 assert(!ModuleMap.contains(M->getName()) &&
155 "Expect unique Buffer Identifier");
156 ModuleMap[M->getName()] = M.get();
157 }
158 return ModuleMap;
159}
160
161static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
162 bool ClearDSOLocalOnDeclarations) {
163 renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations);
164}
165
166namespace {
167class ThinLTODiagnosticInfo : public DiagnosticInfo {
168 const Twine &Msg;
169public:
170 ThinLTODiagnosticInfo(const Twine &DiagMsg LLVM_LIFETIME_BOUND,
171 DiagnosticSeverity Severity = DS_Error)
172 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
173 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
174};
175}
176
177/// Verify the module and strip broken debug info.
178static void verifyLoadedModule(Module &TheModule) {
179 bool BrokenDebugInfo = false;
180 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
181 report_fatal_error(reason: "Broken module found, compilation aborted!");
182 if (BrokenDebugInfo) {
183 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
184 "Invalid debug info found, debug info will be stripped", DS_Warning));
185 StripDebugInfo(M&: TheModule);
186 }
187}
188
189static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
190 LLVMContext &Context,
191 bool Lazy,
192 bool IsImporting) {
193 auto &Mod = Input->getSingleBitcodeModule();
194 SMDiagnostic Err;
195 Expected<std::unique_ptr<Module>> ModuleOrErr =
196 Lazy ? Mod.getLazyModule(Context,
197 /* ShouldLazyLoadMetadata */ true, IsImporting)
198 : Mod.parseModule(Context);
199 if (!ModuleOrErr) {
200 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
201 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
202 SourceMgr::DK_Error, EIB.message());
203 Err.print(ProgName: "ThinLTO", S&: errs());
204 });
205 report_fatal_error(reason: "Can't load module, abort.");
206 }
207 if (!Lazy)
208 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
209 return std::move(*ModuleOrErr);
210}
211
212static void
213crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
214 StringMap<lto::InputFile *> &ModuleMap,
215 const FunctionImporter::ImportMapTy &ImportList,
216 bool ClearDSOLocalOnDeclarations) {
217 auto Loader = [&](StringRef Identifier) {
218 auto &Input = ModuleMap[Identifier];
219 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
220 /*Lazy=*/true, /*IsImporting*/ true);
221 };
222
223 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
224 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
225 if (!Result) {
226 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
227 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
228 SourceMgr::DK_Error, EIB.message());
229 Err.print(ProgName: "ThinLTO", S&: errs());
230 });
231 report_fatal_error(reason: "importFunctions failed");
232 }
233 // Verify again after cross-importing.
234 verifyLoadedModule(TheModule);
235}
236
237static void optimizeModule(Module &TheModule, TargetMachine &TM,
238 unsigned OptLevel, bool Freestanding,
239 bool DebugPassManager, ModuleSummaryIndex *Index) {
240 std::optional<PGOOptions> PGOOpt;
241 if (LTORunCSIRInstr) {
242 PGOOpt =
243 PGOOptions("", LTOCSIRProfile, "",
244 /*MemoryProfile=*/"", PGOOptions::IRUse,
245 PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default);
246 } else if (!LTOCSIRProfile.empty()) {
247 PGOOpt = PGOOptions(LTOCSIRProfile, "", "",
248 /*MemoryProfile=*/"", PGOOptions::IRUse,
249 PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default);
250 } else if (!SampleProfileFile.empty()) {
251 PGOOpt =
252 PGOOptions(SampleProfileFile, "", "",
253 /*MemoryProfile=*/"", PGOOptions::SampleUse,
254 PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default);
255 }
256 LoopAnalysisManager LAM;
257 FunctionAnalysisManager FAM;
258 CGSCCAnalysisManager CGAM;
259 ModuleAnalysisManager MAM;
260
261 PassInstrumentationCallbacks PIC;
262 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
263 SI.registerCallbacks(PIC, MAM: &MAM);
264 PipelineTuningOptions PTO;
265 PTO.LoopVectorization = true;
266 PTO.SLPVectorization = true;
267 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
268
269 std::unique_ptr<TargetLibraryInfoImpl> TLII(
270 new TargetLibraryInfoImpl(TM.getTargetTriple(), TM.Options.VecLib));
271 if (Freestanding)
272 TLII->disableAllFunctions();
273 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
274
275 // Register all the basic analyses with the managers.
276 PB.registerModuleAnalyses(MAM);
277 PB.registerCGSCCAnalyses(CGAM);
278 PB.registerFunctionAnalyses(FAM);
279 PB.registerLoopAnalyses(LAM);
280 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
281
282 ModulePassManager MPM;
283
284 OptimizationLevel OL;
285
286 switch (OptLevel) {
287 default:
288 llvm_unreachable("Invalid optimization level");
289 case 0:
290 OL = OptimizationLevel::O0;
291 break;
292 case 1:
293 OL = OptimizationLevel::O1;
294 break;
295 case 2:
296 OL = OptimizationLevel::O2;
297 break;
298 case 3:
299 OL = OptimizationLevel::O3;
300 break;
301 }
302
303 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
304
305 MPM.run(IR&: TheModule, AM&: MAM);
306}
307
308static void
309addUsedSymbolToPreservedGUID(const lto::InputFile &File,
310 DenseSet<GlobalValue::GUID> &PreservedGUID) {
311 Triple TT(File.getTargetTriple());
312 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
313 TargetLibraryInfoImpl TLII(TT);
314 TargetLibraryInfo TLI(TLII);
315 for (const auto &Sym : File.symbols())
316 if (Sym.isUsed() || Sym.isLibcall(TLI, Libcalls))
317 PreservedGUID.insert(
318 V: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Sym.getIRName()));
319}
320
321// Convert the PreservedSymbols map from "Name" based to "GUID" based.
322static void computeGUIDPreservedSymbols(const lto::InputFile &File,
323 const StringSet<> &PreservedSymbols,
324 const Triple &TheTriple,
325 DenseSet<GlobalValue::GUID> &GUIDs) {
326 // Iterate the symbols in the input file and if the input has preserved symbol
327 // compute the GUID for the symbol.
328 for (const auto &Sym : File.symbols()) {
329 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
330 GUIDs.insert(V: GlobalValue::getGUIDAssumingExternalLinkage(
331 GlobalName: GlobalValue::getGlobalIdentifier(Name: Sym.getIRName(),
332 Linkage: GlobalValue::ExternalLinkage, FileName: "")));
333 }
334}
335
336static DenseSet<GlobalValue::GUID>
337computeGUIDPreservedSymbols(const lto::InputFile &File,
338 const StringSet<> &PreservedSymbols,
339 const Triple &TheTriple) {
340 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
341 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
342 GUIDs&: GUIDPreservedSymbols);
343 return GUIDPreservedSymbols;
344}
345
346static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
347 TargetMachine &TM) {
348 SmallVector<char, 128> OutputBuffer;
349
350 // CodeGen
351 {
352 raw_svector_ostream OS(OutputBuffer);
353 legacy::PassManager PM;
354
355 // Setup the codegen now.
356 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
357 /* DisableVerify */ true))
358 report_fatal_error(reason: "Failed to setup codegen");
359
360 // Run codegen now. resulting binary is in OutputBuffer.
361 PM.run(M&: TheModule);
362 }
363 return std::make_unique<SmallVectorMemoryBuffer>(
364 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
365}
366
367namespace {
368/// Manage caching for a single Module.
369class ModuleCacheEntry {
370 SmallString<128> EntryPath;
371
372public:
373 // Create a cache entry. This compute a unique hash for the Module considering
374 // the current list of export/import, and offer an interface to query to
375 // access the content in the cache.
376 ModuleCacheEntry(
377 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
378 const FunctionImporter::ImportMapTy &ImportList,
379 const FunctionImporter::ExportSetTy &ExportList,
380 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
381 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
382 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
383 if (CachePath.empty())
384 return;
385
386 if (!Index.modulePaths().count(Key: ModuleID))
387 // The module does not have an entry, it can't have a hash at all
388 return;
389
390 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
391 P: [](uint32_t V) { return V == 0; }))
392 // No hash entry, no caching!
393 return;
394
395 llvm::lto::Config Conf;
396 Conf.OptLevel = OptLevel;
397 Conf.Options = TMBuilder.Options;
398 Conf.CPU = TMBuilder.MCpu;
399 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
400 Conf.RelocModel = TMBuilder.RelocModel;
401 Conf.CGOptLevel = TMBuilder.CGOptLevel;
402 Conf.Freestanding = Freestanding;
403 std::string Key =
404 computeLTOCacheKey(Conf, Index, ModuleID, ImportList, ExportList,
405 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
406
407 // This choice of file name allows the cache to be pruned (see pruneCache()
408 // in include/llvm/Support/CachePruning.h).
409 sys::path::append(path&: EntryPath, a: CachePath, b: Twine("llvmcache-", Key));
410 }
411
412 // Access the path to this entry in the cache.
413 StringRef getEntryPath() { return EntryPath; }
414
415 // Try loading the buffer for this cache entry.
416 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
417 if (EntryPath.empty())
418 return std::error_code();
419 SmallString<64> ResultPath;
420 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
421 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
422 if (!FDOrErr)
423 return errorToErrorCode(Err: FDOrErr.takeError());
424 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
425 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
426 sys::fs::closeFile(F&: *FDOrErr);
427 return MBOrErr;
428 }
429
430 // Cache the Produced object file
431 void write(const MemoryBuffer &OutputBuffer) {
432 if (EntryPath.empty())
433 return;
434
435 if (auto Err = llvm::writeToOutput(
436 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
437 OS << OutputBuffer.getBuffer();
438 return llvm::Error::success();
439 }))
440 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
441 Vals&: EntryPath,
442 Vals: toString(E: std::move(Err)).c_str()));
443 }
444};
445} // end anonymous namespace
446
447static std::unique_ptr<MemoryBuffer>
448ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
449 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
450 const FunctionImporter::ImportMapTy &ImportList,
451 const FunctionImporter::ExportSetTy &ExportList,
452 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
453 const GVSummaryMapTy &DefinedGlobals,
454 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
455 bool DisableCodeGen, StringRef SaveTempsDir,
456 bool Freestanding, unsigned OptLevel, unsigned count,
457 bool DebugPassManager) {
458 // "Benchmark"-like optimization: single-source case
459 bool SingleModule = (ModuleMap.size() == 1);
460
461 // When linking an ELF shared object, dso_local should be dropped. We
462 // conservatively do this for -fpic.
463 bool ClearDSOLocalOnDeclarations =
464 TM.getTargetTriple().isOSBinFormatELF() &&
465 TM.getRelocationModel() != Reloc::Static &&
466 TheModule.getPIELevel() == PIELevel::Default;
467
468 if (!SingleModule) {
469 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
470
471 // Apply summary-based prevailing-symbol resolution decisions.
472 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
473
474 // Save temps: after promotion.
475 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
476 }
477
478 // Be friendly and don't nuke totally the module when the client didn't
479 // supply anything to preserve.
480 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
481 // Apply summary-based internalization decisions.
482 thinLTOInternalizeModule(TheModule, DefinedGlobals);
483 }
484
485 // Save internalized bitcode
486 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
487
488 if (!SingleModule)
489 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
490 ClearDSOLocalOnDeclarations);
491
492 // Do this after any importing so that imported code is updated.
493 // See comment at call to updateVCallVisibilityInIndex() for why
494 // WholeProgramVisibilityEnabledInLTO is false.
495 updatePublicTypeTestCalls(M&: TheModule,
496 /* WholeProgramVisibilityEnabledInLTO */ false);
497
498 // Save temps: after cross-module import.
499 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
500
501 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
502 Index: &Index);
503
504 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
505
506 if (DisableCodeGen) {
507 // Configured to stop before CodeGen, serialize the bitcode and return.
508 SmallVector<char, 128> OutputBuffer;
509 {
510 raw_svector_ostream OS(OutputBuffer);
511 ProfileSummaryInfo PSI(TheModule);
512 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
513 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
514 }
515 return std::make_unique<SmallVectorMemoryBuffer>(
516 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
517 }
518
519 return codegenModule(TheModule, TM);
520}
521
522/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
523/// for caching, and in the \p Index for application during the ThinLTO
524/// backends. This is needed for correctness for exported symbols (ensure
525/// at least one copy kept) and a compile-time optimization (to drop duplicate
526/// copies when possible).
527static void resolvePrevailingInIndex(
528 ModuleSummaryIndex &Index,
529 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
530 &ResolvedODR,
531 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
532 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
533 &PrevailingCopy) {
534
535 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
536 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
537 // Not in map means that there was only one copy, which must be prevailing.
538 if (Prevailing == PrevailingCopy.end())
539 return true;
540 return Prevailing->second == S;
541 };
542
543 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
544 GlobalValue::GUID GUID,
545 GlobalValue::LinkageTypes NewLinkage) {
546 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
547 };
548
549 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
550 lto::Config Conf;
551 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
552 GUIDPreservedSymbols);
553}
554
555// Initialize the TargetMachine builder for a given Triple
556static void initTMBuilder(TargetMachineBuilder &TMBuilder,
557 const Triple &TheTriple) {
558 if (TMBuilder.MCpu.empty())
559 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
560 TMBuilder.TheTriple = std::move(TheTriple);
561}
562
563void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
564 MemoryBufferRef Buffer(Data, Identifier);
565
566 auto InputOrError = lto::InputFile::create(Object: Buffer);
567 if (!InputOrError)
568 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
569 toString(E: InputOrError.takeError()));
570
571 auto TripleStr = (*InputOrError)->getTargetTriple();
572 Triple TheTriple(TripleStr);
573
574 if (Modules.empty())
575 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
576 else if (TMBuilder.TheTriple != TheTriple) {
577 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
578 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
579 "supported");
580 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
581 }
582
583 Modules.emplace_back(args: std::move(*InputOrError));
584}
585
586void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
587 PreservedSymbols.insert(key: Name);
588}
589
590void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
591 // FIXME: At the moment, we don't take advantage of this extra information,
592 // we're conservatively considering cross-references as preserved.
593 // CrossReferencedSymbols.insert(Name);
594 PreservedSymbols.insert(key: Name);
595}
596
597// TargetMachine factory
598std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
599 std::string ErrMsg;
600 const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error&: ErrMsg);
601 if (!TheTarget) {
602 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
603 }
604
605 // Use MAttr as the default set of features.
606 SubtargetFeatures Features(MAttr);
607 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
608 std::string FeatureStr = Features.getString();
609
610 std::unique_ptr<TargetMachine> TM(
611 TheTarget->createTargetMachine(TT: TheTriple, CPU: MCpu, Features: FeatureStr, Options,
612 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
613 assert(TM && "Cannot create target machine");
614
615 return TM;
616}
617
618/**
619 * Produce the combined summary index from all the bitcode files:
620 * "thin-link".
621 */
622std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
623 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
624 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
625 for (auto &Mod : Modules) {
626 auto &M = Mod->getSingleBitcodeModule();
627 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
628 // FIXME diagnose
629 logAllUnhandledErrors(
630 E: std::move(Err), OS&: errs(),
631 ErrorBanner: "error: can't create module summary index for buffer: ");
632 return nullptr;
633 }
634 }
635 return CombinedIndex;
636}
637
638namespace {
639struct IsExported {
640 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
641 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
642
643 IsExported(
644 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
645 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
646 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
647
648 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
649 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
650 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
651 GUIDPreservedSymbols.count(V: VI.getGUID());
652 }
653};
654
655struct IsPrevailing {
656 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
657 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
658 &PrevailingCopy)
659 : PrevailingCopy(PrevailingCopy) {}
660
661 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
662 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
663 // Not in map means that there was only one copy, which must be prevailing.
664 if (Prevailing == PrevailingCopy.end())
665 return true;
666 return Prevailing->second == S;
667 };
668};
669} // namespace
670
671static void computeDeadSymbolsInIndex(
672 ModuleSummaryIndex &Index,
673 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
674 // We have no symbols resolution available. And can't do any better now in the
675 // case where the prevailing symbol is in a native object. It can be refined
676 // with linker information in the future.
677 auto isPrevailing = [&](GlobalValue::GUID G) {
678 return PrevailingType::Unknown;
679 };
680 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
681 /* ImportEnabled = */ true);
682}
683
684/**
685 * Perform promotion and renaming of exported internal functions.
686 * Index is updated to reflect linkage changes from weak resolution.
687 */
688void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
689 const lto::InputFile &File) {
690 auto ModuleCount = Index.modulePaths().size();
691 auto ModuleIdentifier = TheModule.getModuleIdentifier();
692
693 // Collect for each module the list of function it defines (GUID -> Summary).
694 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
695 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
696
697 // Convert the preserved symbols set from string to GUID
698 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
699 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
700
701 // Add used symbol to the preserved symbols.
702 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
703
704 // Compute "dead" symbols, we don't want to import/export these!
705 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
706
707 // Compute prevailing symbols
708 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
709 computePrevailingCopies(Index, PrevailingCopy);
710
711 // Generate import/export list
712 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
713 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
714 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
715 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
716 ExportLists);
717
718 // Resolve prevailing symbols
719 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
720 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
721 PrevailingCopy);
722
723 thinLTOFinalizeInModule(TheModule,
724 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
725 /*PropagateAttrs=*/false);
726
727 // Promote the exported values in the index, so that they are promoted
728 // in the module.
729 thinLTOInternalizeAndPromoteInIndex(
730 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
731 isPrevailing: IsPrevailing(PrevailingCopy));
732
733 // FIXME Set ClearDSOLocalOnDeclarations.
734 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
735}
736
737/**
738 * Perform cross-module importing for the module identified by ModuleIdentifier.
739 */
740void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
741 ModuleSummaryIndex &Index,
742 const lto::InputFile &File) {
743 auto ModuleMap = generateModuleMap(Modules);
744 auto ModuleCount = Index.modulePaths().size();
745
746 // Collect for each module the list of function it defines (GUID -> Summary).
747 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
748 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
749
750 // Convert the preserved symbols set from string to GUID
751 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
752 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
753
754 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
755
756 // Compute "dead" symbols, we don't want to import/export these!
757 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
758
759 // Compute prevailing symbols
760 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
761 computePrevailingCopies(Index, PrevailingCopy);
762
763 // Generate import/export list
764 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
765 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
766 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
767 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
768 ExportLists);
769 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
770
771 // FIXME Set ClearDSOLocalOnDeclarations.
772 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
773 /*ClearDSOLocalOnDeclarations=*/false);
774}
775
776/**
777 * Compute the list of summaries needed for importing into module.
778 */
779void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
780 Module &TheModule, ModuleSummaryIndex &Index,
781 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
782 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
783 auto ModuleCount = Index.modulePaths().size();
784 auto ModuleIdentifier = TheModule.getModuleIdentifier();
785
786 // Collect for each module the list of function it defines (GUID -> Summary).
787 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
788 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
789
790 // Convert the preserved symbols set from string to GUID
791 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
792 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
793
794 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
795
796 // Compute "dead" symbols, we don't want to import/export these!
797 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
798
799 // Compute prevailing symbols
800 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
801 computePrevailingCopies(Index, PrevailingCopy);
802
803 // Generate import/export list
804 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
805 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
806 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
807 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
808 ExportLists);
809
810 llvm::gatherImportedSummariesForModule(
811 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
812 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
813}
814
815/**
816 * Emit the list of files needed for importing into module.
817 */
818void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
819 ModuleSummaryIndex &Index,
820 const lto::InputFile &File) {
821 auto ModuleCount = Index.modulePaths().size();
822 auto ModuleIdentifier = TheModule.getModuleIdentifier();
823
824 // Collect for each module the list of function it defines (GUID -> Summary).
825 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
826 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
827
828 // Convert the preserved symbols set from string to GUID
829 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
830 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
831
832 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
833
834 // Compute "dead" symbols, we don't want to import/export these!
835 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
836
837 // Compute prevailing symbols
838 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
839 computePrevailingCopies(Index, PrevailingCopy);
840
841 // Generate import/export list
842 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
843 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
844 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
845 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
846 ExportLists);
847
848 // 'EmitImportsFiles' emits the list of modules from which to import from, and
849 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
850 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportsFiles`.
851 GVSummaryPtrSet DecSummaries;
852 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
853 llvm::gatherImportedSummariesForModule(
854 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
855 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
856
857 if (Error EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
858 ModuleToSummariesForIndex))
859 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
860 " to save imports lists\n");
861}
862
863/**
864 * Perform internalization. Runs promote and internalization together.
865 * Index is updated to reflect linkage changes.
866 */
867void ThinLTOCodeGenerator::internalize(Module &TheModule,
868 ModuleSummaryIndex &Index,
869 const lto::InputFile &File) {
870 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
871 auto ModuleCount = Index.modulePaths().size();
872 auto ModuleIdentifier = TheModule.getModuleIdentifier();
873
874 // Convert the preserved symbols set from string to GUID
875 auto GUIDPreservedSymbols =
876 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
877
878 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
879
880 // Collect for each module the list of function it defines (GUID -> Summary).
881 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
882 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
883
884 // Compute "dead" symbols, we don't want to import/export these!
885 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
886
887 // Compute prevailing symbols
888 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
889 computePrevailingCopies(Index, PrevailingCopy);
890
891 // Generate import/export list
892 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
893 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
894 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
895 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
896 ExportLists);
897 auto &ExportList = ExportLists[ModuleIdentifier];
898
899 // Be friendly and don't nuke totally the module when the client didn't
900 // supply anything to preserve.
901 if (ExportList.empty() && GUIDPreservedSymbols.empty())
902 return;
903
904 // Resolve prevailing symbols
905 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
906 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
907 PrevailingCopy);
908
909 // Promote the exported values in the index, so that they are promoted
910 // in the module.
911 thinLTOInternalizeAndPromoteInIndex(
912 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
913 isPrevailing: IsPrevailing(PrevailingCopy));
914
915 // FIXME Set ClearDSOLocalOnDeclarations.
916 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
917
918 // Internalization
919 thinLTOFinalizeInModule(TheModule,
920 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
921 /*PropagateAttrs=*/false);
922
923 thinLTOInternalizeModule(TheModule,
924 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
925}
926
927/**
928 * Perform post-importing ThinLTO optimizations.
929 */
930void ThinLTOCodeGenerator::optimize(Module &TheModule) {
931 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
932
933 // Optimize now
934 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
935 DebugPassManager, Index: nullptr);
936}
937
938/// Write out the generated object file, either from CacheEntryPath or from
939/// OutputBuffer, preferring hard-link when possible.
940/// Returns the path to the generated file in SavedObjectsDirectoryPath.
941std::string
942ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
943 const MemoryBuffer &OutputBuffer) {
944 auto ArchName = TMBuilder.TheTriple.getArchName();
945 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
946 llvm::sys::path::append(path&: OutputPath,
947 a: Twine(count) + "." + ArchName + ".thinlto.o");
948 OutputPath.c_str(); // Ensure the string is null terminated.
949 if (sys::fs::exists(Path: OutputPath))
950 sys::fs::remove(path: OutputPath);
951
952 // We don't return a memory buffer to the linker, just a list of files.
953 if (!CacheEntryPath.empty()) {
954 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
955 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
956 if (!Err)
957 return std::string(OutputPath);
958 // Hard linking failed, try to copy.
959 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
960 if (!Err)
961 return std::string(OutputPath);
962 // Copy failed (could be because the CacheEntry was removed from the cache
963 // in the meantime by another process), fall back and try to write down the
964 // buffer to the output.
965 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
966 << "' to '" << OutputPath << "'\n";
967 }
968 // No cache entry, just write out the buffer.
969 std::error_code Err;
970 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
971 if (Err)
972 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
973 OS << OutputBuffer.getBuffer();
974 return std::string(OutputPath);
975}
976
977// Main entry point for the ThinLTO processing
978void ThinLTOCodeGenerator::run() {
979 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
980 llvm::scope_exit TimeTraceScopeExit([]() {
981 if (llvm::timeTraceProfilerEnabled())
982 llvm::timeTraceProfilerEnd();
983 });
984 // Prepare the resulting object vector
985 assert(ProducedBinaries.empty() && "The generator should not be reused");
986 if (SavedObjectsDirectoryPath.empty())
987 ProducedBinaries.resize(new_size: Modules.size());
988 else {
989 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
990 bool IsDir;
991 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
992 if (!IsDir)
993 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
994 ProducedBinaryFiles.resize(new_size: Modules.size());
995 }
996
997 if (CodeGenOnly) {
998 // Perform only parallel codegen and return.
999 DefaultThreadPool Pool;
1000 int count = 0;
1001 for (auto &Mod : Modules) {
1002 Pool.async(F: [&](int count) {
1003 LLVMContext Context;
1004 Context.setDiscardValueNames(LTODiscardValueNames);
1005
1006 // Parse module now
1007 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1008 /*IsImporting*/ false);
1009
1010 // CodeGen
1011 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1012 if (SavedObjectsDirectoryPath.empty())
1013 ProducedBinaries[count] = std::move(OutputBuffer);
1014 else
1015 ProducedBinaryFiles[count] =
1016 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1017 }, ArgList: count++);
1018 }
1019
1020 return;
1021 }
1022
1023 // Sequential linking phase
1024 auto Index = linkCombinedIndex();
1025
1026 // Save temps: index.
1027 if (!SaveTempsDir.empty()) {
1028 auto SaveTempPath = SaveTempsDir + "index.bc";
1029 std::error_code EC;
1030 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1031 if (EC)
1032 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1033 " to save optimized bitcode\n");
1034 writeIndexToFile(Index: *Index, Out&: OS);
1035 }
1036
1037
1038 // Prepare the module map.
1039 auto ModuleMap = generateModuleMap(Modules);
1040 auto ModuleCount = Modules.size();
1041
1042 // Collect for each module the list of function it defines (GUID -> Summary).
1043 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1044 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1045
1046 // Convert the preserved symbols set from string to GUID, this is needed for
1047 // computing the caching hash and the internalization.
1048 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1049 for (const auto &M : Modules)
1050 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1051 GUIDs&: GUIDPreservedSymbols);
1052
1053 // Add used symbol from inputs to the preserved symbols.
1054 for (const auto &M : Modules)
1055 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1056
1057 // Compute "dead" symbols, we don't want to import/export these!
1058 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1059
1060 // Currently there is no support for enabling whole program visibility via a
1061 // linker option in the old LTO API, but this call allows it to be specified
1062 // via the internal option. Must be done before WPD below.
1063 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1064 Index->setWithWholeProgramVisibility();
1065
1066 // FIXME: This needs linker information via a TBD new interface
1067 updateVCallVisibilityInIndex(Index&: *Index,
1068 /*WholeProgramVisibilityEnabledInLTO=*/false,
1069 // FIXME: These need linker information via a
1070 // TBD new interface.
1071 /*DynamicExportSymbols=*/{},
1072 /*VisibleToRegularObjSymbols=*/{});
1073
1074 // Perform index-based WPD. This will return immediately if there are
1075 // no index entries in the typeIdMetadata map (e.g. if we are instead
1076 // performing IR-based WPD in hybrid regular/thin LTO mode).
1077 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1078 std::set<GlobalValue::GUID> ExportedGUIDs;
1079 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1080 GUIDPreservedSymbols.insert_range(R&: ExportedGUIDs);
1081
1082 // Compute prevailing symbols
1083 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1084 computePrevailingCopies(Index: *Index, PrevailingCopy);
1085
1086 // Collect the import/export lists for all modules from the call-graph in the
1087 // combined index.
1088 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
1089 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1090 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1091 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1092 ExportLists);
1093
1094 // We use a std::map here to be able to have a defined ordering when
1095 // producing a hash for the cache entry.
1096 // FIXME: we should be able to compute the caching hash for the entry based
1097 // on the index, and nuke this map.
1098 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1099
1100 // Resolve prevailing symbols, this has to be computed early because it
1101 // impacts the caching.
1102 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1103 PrevailingCopy);
1104
1105 // Use global summary-based analysis to identify symbols that can be
1106 // internalized (because they aren't exported or preserved as per callback).
1107 // Changes are made in the index, consumed in the ThinLTO backends.
1108 updateIndexWPDForExports(Summary&: *Index,
1109 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1110 LocalWPDTargetsMap);
1111 thinLTOInternalizeAndPromoteInIndex(
1112 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1113 isPrevailing: IsPrevailing(PrevailingCopy));
1114
1115 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1116
1117 // Make sure that every module has an entry in the ExportLists, ImportList,
1118 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1119 // below.
1120 for (auto &Module : Modules) {
1121 auto ModuleIdentifier = Module->getName();
1122 ExportLists[ModuleIdentifier];
1123 ImportLists[ModuleIdentifier];
1124 ResolvedODR[ModuleIdentifier];
1125 ModuleToDefinedGVSummaries[ModuleIdentifier];
1126 }
1127
1128 std::vector<BitcodeModule *> ModulesVec;
1129 ModulesVec.reserve(n: Modules.size());
1130 for (auto &Mod : Modules)
1131 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1132 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1133
1134 if (llvm::timeTraceProfilerEnabled())
1135 llvm::timeTraceProfilerEnd();
1136
1137 TimeTraceScopeExit.release();
1138
1139 // Parallel optimizer + codegen
1140 {
1141 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1142 for (auto IndexCount : ModulesOrdering) {
1143 auto &Mod = Modules[IndexCount];
1144 Pool.async(F: [&](int count) {
1145 auto ModuleIdentifier = Mod->getName();
1146 auto &ExportList = ExportLists[ModuleIdentifier];
1147
1148 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1149
1150 // The module may be cached, this helps handling it.
1151 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1152 ImportLists[ModuleIdentifier], ExportList,
1153 ResolvedODR[ModuleIdentifier],
1154 DefinedGVSummaries, OptLevel, Freestanding,
1155 TMBuilder);
1156 auto CacheEntryPath = CacheEntry.getEntryPath();
1157
1158 {
1159 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1160 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1161 << " '" << CacheEntryPath << "' for buffer "
1162 << count << " " << ModuleIdentifier << "\n");
1163
1164 if (ErrOrBuffer) {
1165 // Cache Hit!
1166 if (SavedObjectsDirectoryPath.empty())
1167 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1168 else
1169 ProducedBinaryFiles[count] = writeGeneratedObject(
1170 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1171 return;
1172 }
1173 }
1174
1175 LLVMContext Context;
1176 Context.setDiscardValueNames(LTODiscardValueNames);
1177 Context.enableDebugTypeODRUniquing();
1178 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1179 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1180 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1181 if (!DiagFileOrErr) {
1182 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1183 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1184 "remarks");
1185 }
1186
1187 // Parse module now
1188 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1189 /*IsImporting*/ false);
1190
1191 // Save temps: original file.
1192 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1193
1194 auto &ImportList = ImportLists[ModuleIdentifier];
1195 // Run the main process now, and generates a binary
1196 auto OutputBuffer = ProcessThinLTOModule(
1197 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1198 ExportList, GUIDPreservedSymbols,
1199 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1200 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1201 DebugPassManager);
1202
1203 // Commit to the cache (if enabled)
1204 CacheEntry.write(OutputBuffer: *OutputBuffer);
1205
1206 if (SavedObjectsDirectoryPath.empty()) {
1207 // We need to generated a memory buffer for the linker.
1208 if (!CacheEntryPath.empty()) {
1209 // When cache is enabled, reload from the cache if possible.
1210 // Releasing the buffer from the heap and reloading it from the
1211 // cache file with mmap helps us to lower memory pressure.
1212 // The freed memory can be used for the next input file.
1213 // The final binary link will read from the VFS cache (hopefully!)
1214 // or from disk (if the memory pressure was too high).
1215 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1216 if (auto EC = ReloadedBufferOrErr.getError()) {
1217 // On error, keep the preexisting buffer and print a diagnostic.
1218 errs() << "remark: can't reload cached file '" << CacheEntryPath
1219 << "': " << EC.message() << "\n";
1220 } else {
1221 OutputBuffer = std::move(*ReloadedBufferOrErr);
1222 }
1223 }
1224 ProducedBinaries[count] = std::move(OutputBuffer);
1225 return;
1226 }
1227 ProducedBinaryFiles[count] = writeGeneratedObject(
1228 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1229 }, ArgList&: IndexCount);
1230 }
1231 }
1232
1233 Expected<bool> PrunedOrErr =
1234 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1235 if (!PrunedOrErr) {
1236 errs() << "Error: " << toString(E: PrunedOrErr.takeError()) << "\n";
1237 report_fatal_error(reason: "ThinLTO: failure to prune cache");
1238 }
1239
1240 // If statistics were requested, print them out now.
1241 if (llvm::AreStatisticsEnabled())
1242 llvm::PrintStatistics();
1243 reportAndResetTimings();
1244}
1245