1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/ModuleSummaryAnalysis.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/Bitcode/BitcodeWriterPass.h"
26#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/LLVMRemarkStreamer.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/Mangler.h"
33#include "llvm/IR/PassTimingInfo.h"
34#include "llvm/IR/Verifier.h"
35#include "llvm/IRReader/IRReader.h"
36#include "llvm/LTO/LTO.h"
37#include "llvm/MC/TargetRegistry.h"
38#include "llvm/Object/IRObjectFile.h"
39#include "llvm/Passes/PassBuilder.h"
40#include "llvm/Passes/StandardInstrumentations.h"
41#include "llvm/Remarks/HotnessThresholdParser.h"
42#include "llvm/Support/CachePruning.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/FormatVariadic.h"
47#include "llvm/Support/LibraryOptionsContext.h"
48#include "llvm/Support/Path.h"
49#include "llvm/Support/SHA1.h"
50#include "llvm/Support/SmallVectorMemoryBuffer.h"
51#include "llvm/Support/ThreadPool.h"
52#include "llvm/Support/Threading.h"
53#include "llvm/Support/ToolOutputFile.h"
54#include "llvm/Support/raw_ostream.h"
55#include "llvm/Target/TargetMachine.h"
56#include "llvm/TargetParser/SubtargetFeature.h"
57#include "llvm/Transforms/IPO/FunctionAttrs.h"
58#include "llvm/Transforms/IPO/FunctionImport.h"
59#include "llvm/Transforms/IPO/Internalize.h"
60#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
61#include "llvm/Transforms/Utils/FunctionImportUtils.h"
62
63#if !defined(_MSC_VER) && !defined(__MINGW32__)
64#include <unistd.h>
65#else
66#include <io.h>
67#endif
68
69using namespace llvm;
70using namespace ThinLTOCodeGeneratorImpl;
71
72#define DEBUG_TYPE "thinlto"
73
74namespace llvm {
75// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
76extern cl::opt<bool> LTODiscardValueNames;
77extern cl::opt<std::string> RemarksFilename;
78extern cl::opt<std::string> RemarksPasses;
79extern cl::opt<bool> RemarksWithHotness;
80extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
81 RemarksHotnessThreshold;
82extern cl::opt<std::string> RemarksFormat;
83extern cl::opt<bool> LTORunCSIRInstr;
84extern cl::opt<std::string> LTOCSIRProfile;
85extern cl::opt<std::string> SampleProfileFile;
86}
87
88// Default to using all available threads in the system, but using only one
89// thred per core, as indicated by the usage of
90// heavyweight_hardware_concurrency() below.
91static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
92
93// Simple helper to save temporary files for debug.
94static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
95 unsigned count, StringRef Suffix) {
96 if (TempDir.empty())
97 return;
98 // User asked to save temps, let dump the bitcode file after import.
99 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
100 std::error_code EC;
101 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
102 if (EC)
103 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
104 " to save optimized bitcode\n");
105 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
106}
107
108static const GlobalValueSummary *getFirstDefinitionForLinker(
109 ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
110 // If there is any strong definition anywhere, get it.
111 auto StrongDefForLinker = llvm::find_if(
112 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
113 auto Linkage = Summary->linkage();
114 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
115 !GlobalValue::isWeakForLinker(Linkage);
116 });
117 if (StrongDefForLinker != GVSummaryList.end())
118 return StrongDefForLinker->get();
119 // Get the first *linker visible* definition for this global in the summary
120 // list.
121 auto FirstDefForLinker = llvm::find_if(
122 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
123 auto Linkage = Summary->linkage();
124 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
125 });
126 // Extern templates can be emitted as available_externally.
127 if (FirstDefForLinker == GVSummaryList.end())
128 return nullptr;
129 return FirstDefForLinker->get();
130}
131
132// Populate map of GUID to the prevailing copy for any multiply defined
133// symbols. Currently assume first copy is prevailing, or any strong
134// definition. Can be refined with Linker information in the future.
135static void computePrevailingCopies(
136 const ModuleSummaryIndex &Index,
137 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
138 auto HasMultipleCopies =
139 [&](ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
140 return GVSummaryList.size() > 1;
141 };
142
143 for (auto &I : Index) {
144 if (HasMultipleCopies(I.second.getSummaryList()))
145 PrevailingCopy[I.first] =
146 getFirstDefinitionForLinker(GVSummaryList: I.second.getSummaryList());
147 }
148}
149
150static StringMap<lto::InputFile *>
151generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
152 StringMap<lto::InputFile *> ModuleMap;
153 for (auto &M : Modules) {
154 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
155 assert(!ModuleMap.contains(M->getName()) &&
156 "Expect unique Buffer Identifier");
157 ModuleMap[M->getName()] = M.get();
158 }
159 return ModuleMap;
160}
161
162static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
163 bool ClearDSOLocalOnDeclarations) {
164 renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations);
165}
166
167namespace {
168class ThinLTODiagnosticInfo : public DiagnosticInfo {
169 const Twine &Msg;
170public:
171 ThinLTODiagnosticInfo(const Twine &DiagMsg LLVM_LIFETIME_BOUND,
172 DiagnosticSeverity Severity = DS_Error)
173 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
174 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
175};
176}
177
178/// Verify the module and strip broken debug info.
179static void verifyLoadedModule(Module &TheModule) {
180 bool BrokenDebugInfo = false;
181 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
182 report_fatal_error(reason: "Broken module found, compilation aborted!");
183 if (BrokenDebugInfo) {
184 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
185 "Invalid debug info found, debug info will be stripped", DS_Warning));
186 StripDebugInfo(M&: TheModule);
187 }
188}
189
190static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
191 LLVMContext &Context,
192 bool Lazy,
193 bool IsImporting) {
194 auto &Mod = Input->getSingleBitcodeModule();
195 SMDiagnostic Err;
196 Expected<std::unique_ptr<Module>> ModuleOrErr =
197 Lazy ? Mod.getLazyModule(Context,
198 /* ShouldLazyLoadMetadata */ true, IsImporting)
199 : Mod.parseModule(Context);
200 if (!ModuleOrErr) {
201 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
202 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
203 SourceMgr::DK_Error, EIB.message());
204 Err.print(ProgName: "ThinLTO", S&: errs());
205 });
206 report_fatal_error(reason: "Can't load module, abort.");
207 }
208 if (!Lazy)
209 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
210 return std::move(*ModuleOrErr);
211}
212
213static void
214crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
215 StringMap<lto::InputFile *> &ModuleMap,
216 const FunctionImporter::ImportMapTy &ImportList,
217 bool ClearDSOLocalOnDeclarations) {
218 auto Loader = [&](StringRef Identifier) {
219 auto &Input = ModuleMap[Identifier];
220 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
221 /*Lazy=*/true, /*IsImporting*/ true);
222 };
223
224 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
225 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
226 if (!Result) {
227 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
228 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
229 SourceMgr::DK_Error, EIB.message());
230 Err.print(ProgName: "ThinLTO", S&: errs());
231 });
232 report_fatal_error(reason: "importFunctions failed");
233 }
234 // Verify again after cross-importing.
235 verifyLoadedModule(TheModule);
236}
237
238static void optimizeModule(Module &TheModule, TargetMachine &TM,
239 unsigned OptLevel, bool Freestanding,
240 bool DebugPassManager, ModuleSummaryIndex *Index) {
241 std::optional<PGOOptions> PGOOpt;
242 if (LTORunCSIRInstr) {
243 PGOOpt =
244 PGOOptions("", LTOCSIRProfile, "",
245 /*MemoryProfile=*/"", PGOOptions::IRUse,
246 PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default);
247 } else if (!LTOCSIRProfile.empty()) {
248 PGOOpt = PGOOptions(LTOCSIRProfile, "", "",
249 /*MemoryProfile=*/"", PGOOptions::IRUse,
250 PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default);
251 } else if (!SampleProfileFile.empty()) {
252 PGOOpt =
253 PGOOptions(SampleProfileFile, "", "",
254 /*MemoryProfile=*/"", PGOOptions::SampleUse,
255 PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default);
256 }
257 LoopAnalysisManager LAM;
258 FunctionAnalysisManager FAM;
259 CGSCCAnalysisManager CGAM;
260 ModuleAnalysisManager MAM;
261
262 PassInstrumentationCallbacks PIC;
263 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
264 SI.registerCallbacks(PIC, MAM: &MAM);
265 PipelineTuningOptions PTO(&TheModule.getContext().getLibraryOptions());
266 PTO.LoopVectorization = true;
267 PTO.SLPVectorization = true;
268 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
269
270 std::unique_ptr<TargetLibraryInfoImpl> TLII(
271 new TargetLibraryInfoImpl(TM.getTargetTriple(), TM.Options.VecLib));
272 if (Freestanding)
273 TLII->disableAllFunctions();
274 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
275
276 // Register all the basic analyses with the managers.
277 PB.registerModuleAnalyses(MAM);
278 PB.registerCGSCCAnalyses(CGAM);
279 PB.registerFunctionAnalyses(FAM);
280 PB.registerLoopAnalyses(LAM);
281 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
282
283 ModulePassManager MPM;
284
285 OptimizationLevel OL;
286
287 switch (OptLevel) {
288 default:
289 llvm_unreachable("Invalid optimization level");
290 case 0:
291 OL = OptimizationLevel::O0;
292 break;
293 case 1:
294 OL = OptimizationLevel::O1;
295 break;
296 case 2:
297 OL = OptimizationLevel::O2;
298 break;
299 case 3:
300 OL = OptimizationLevel::O3;
301 break;
302 }
303
304 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
305
306 MPM.run(IR&: TheModule, AM&: MAM);
307}
308
309static void
310addUsedSymbolToPreservedGUID(const lto::InputFile &File,
311 DenseSet<GlobalValue::GUID> &PreservedGUID) {
312 Triple TT(File.getTargetTriple());
313 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
314 TargetLibraryInfoImpl TLII(TT);
315 TargetLibraryInfo TLI(TLII);
316 for (const auto &Sym : File.symbols())
317 if (Sym.isUsed() || Sym.isLibcall(TLI, Libcalls))
318 PreservedGUID.insert(
319 V: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Sym.getIRName()));
320}
321
322// Convert the PreservedSymbols map from "Name" based to "GUID" based.
323static void computeGUIDPreservedSymbols(const lto::InputFile &File,
324 const StringSet<> &PreservedSymbols,
325 const Triple &TheTriple,
326 DenseSet<GlobalValue::GUID> &GUIDs) {
327 // Iterate the symbols in the input file and if the input has preserved symbol
328 // compute the GUID for the symbol.
329 for (const auto &Sym : File.symbols()) {
330 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
331 GUIDs.insert(V: GlobalValue::getGUIDAssumingExternalLinkage(
332 GlobalName: GlobalValue::getGlobalIdentifier(Name: Sym.getIRName(),
333 Linkage: GlobalValue::ExternalLinkage, FileName: "")));
334 }
335}
336
337static DenseSet<GlobalValue::GUID>
338computeGUIDPreservedSymbols(const lto::InputFile &File,
339 const StringSet<> &PreservedSymbols,
340 const Triple &TheTriple) {
341 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
342 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
343 GUIDs&: GUIDPreservedSymbols);
344 return GUIDPreservedSymbols;
345}
346
347static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
348 TargetMachine &TM) {
349 SmallVector<char, 128> OutputBuffer;
350
351 // CodeGen
352 {
353 raw_svector_ostream OS(OutputBuffer);
354 legacy::PassManager PM;
355
356 // Setup the codegen now.
357 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
358 /* DisableVerify */ true))
359 report_fatal_error(reason: "Failed to setup codegen");
360
361 // Run codegen now. resulting binary is in OutputBuffer.
362 PM.run(M&: TheModule);
363 }
364 return std::make_unique<SmallVectorMemoryBuffer>(
365 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
366}
367
368namespace {
369/// Manage caching for a single Module.
370class ModuleCacheEntry {
371 SmallString<128> EntryPath;
372
373public:
374 // Create a cache entry. This compute a unique hash for the Module considering
375 // the current list of export/import, and offer an interface to query to
376 // access the content in the cache.
377 ModuleCacheEntry(
378 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
379 const FunctionImporter::ImportMapTy &ImportList,
380 const FunctionImporter::ExportSetTy &ExportList,
381 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
382 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
383 bool Freestanding, const TargetMachineBuilder &TMBuilder,
384 ArrayRef<std::string> MllvmArgs) {
385 if (CachePath.empty())
386 return;
387
388 if (!Index.modulePaths().count(Key: ModuleID))
389 // The module does not have an entry, it can't have a hash at all
390 return;
391
392 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
393 P: [](uint32_t V) { return V == 0; }))
394 // No hash entry, no caching!
395 return;
396
397 llvm::lto::Config Conf;
398 Conf.OptLevel = OptLevel;
399 Conf.Options = TMBuilder.Options;
400 Conf.CPU = TMBuilder.MCpu;
401 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
402 Conf.RelocModel = TMBuilder.RelocModel;
403 Conf.CGOptLevel = TMBuilder.CGOptLevel;
404 Conf.Freestanding = Freestanding;
405 append_range(C&: Conf.MllvmArgs, R&: MllvmArgs);
406 std::string Key =
407 computeLTOCacheKey(Conf, Index, ModuleID, ImportList, ExportList,
408 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
409
410 // This choice of file name allows the cache to be pruned (see pruneCache()
411 // in include/llvm/Support/CachePruning.h).
412 sys::path::append(path&: EntryPath, a: CachePath, b: Twine("llvmcache-", Key));
413 }
414
415 // Access the path to this entry in the cache.
416 StringRef getEntryPath() { return EntryPath; }
417
418 // Try loading the buffer for this cache entry.
419 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
420 if (EntryPath.empty())
421 return std::error_code();
422 SmallString<64> ResultPath;
423 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
424 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
425 if (!FDOrErr)
426 return errorToErrorCode(Err: FDOrErr.takeError());
427 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
428 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
429 sys::fs::closeFile(F&: *FDOrErr);
430 return MBOrErr;
431 }
432
433 // Cache the Produced object file
434 void write(const MemoryBuffer &OutputBuffer) {
435 if (EntryPath.empty())
436 return;
437
438 if (auto Err = llvm::writeToOutput(
439 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
440 OS << OutputBuffer.getBuffer();
441 return llvm::Error::success();
442 }))
443 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
444 Vals&: EntryPath,
445 Vals: toString(E: std::move(Err)).c_str()));
446 }
447};
448} // end anonymous namespace
449
450static std::unique_ptr<MemoryBuffer>
451ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
452 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
453 const FunctionImporter::ImportMapTy &ImportList,
454 const FunctionImporter::ExportSetTy &ExportList,
455 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
456 const GVSummaryMapTy &DefinedGlobals,
457 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
458 bool DisableCodeGen, StringRef SaveTempsDir,
459 bool Freestanding, unsigned OptLevel, unsigned count,
460 bool DebugPassManager) {
461 // "Benchmark"-like optimization: single-source case
462 bool SingleModule = (ModuleMap.size() == 1);
463
464 // When linking an ELF shared object, dso_local should be dropped. We
465 // conservatively do this for -fpic.
466 bool ClearDSOLocalOnDeclarations =
467 TheModule.getTargetTriple().isOSBinFormatELF() &&
468 TM.getRelocationModel() != Reloc::Static &&
469 TheModule.getPIELevel() == PIELevel::Default;
470
471 if (!SingleModule) {
472 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
473
474 // Apply summary-based prevailing-symbol resolution decisions.
475 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
476
477 // Save temps: after promotion.
478 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
479 }
480
481 // Be friendly and don't nuke totally the module when the client didn't
482 // supply anything to preserve.
483 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
484 // Apply summary-based internalization decisions.
485 thinLTOInternalizeModule(TheModule, DefinedGlobals);
486 }
487
488 // Save internalized bitcode
489 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
490
491 if (!SingleModule)
492 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
493 ClearDSOLocalOnDeclarations);
494
495 // Do this after any importing so that imported code is updated.
496 // See comment at call to updateVCallVisibilityInIndex() for why
497 // WholeProgramVisibilityEnabledInLTO is false.
498 updatePublicTypeTestCalls(M&: TheModule,
499 /* WholeProgramVisibilityEnabledInLTO */ false);
500
501 // Save temps: after cross-module import.
502 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
503
504 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
505 Index: &Index);
506
507 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
508
509 if (DisableCodeGen) {
510 // Configured to stop before CodeGen, serialize the bitcode and return.
511 SmallVector<char, 128> OutputBuffer;
512 {
513 raw_svector_ostream OS(OutputBuffer);
514 ProfileSummaryInfo PSI(TheModule);
515 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
516 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
517 }
518 return std::make_unique<SmallVectorMemoryBuffer>(
519 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
520 }
521
522 return codegenModule(TheModule, TM);
523}
524
525/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
526/// for caching, and in the \p Index for application during the ThinLTO
527/// backends. This is needed for correctness for exported symbols (ensure
528/// at least one copy kept) and a compile-time optimization (to drop duplicate
529/// copies when possible).
530static void resolvePrevailingInIndex(
531 ModuleSummaryIndex &Index,
532 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
533 &ResolvedODR,
534 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
535 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
536 &PrevailingCopy) {
537
538 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
539 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
540 // Not in map means that there was only one copy, which must be prevailing.
541 if (Prevailing == PrevailingCopy.end())
542 return true;
543 return Prevailing->second == S;
544 };
545
546 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
547 GlobalValue::GUID GUID,
548 GlobalValue::LinkageTypes NewLinkage) {
549 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
550 };
551
552 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
553 lto::Config Conf;
554 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
555 GUIDPreservedSymbols);
556}
557
558// Initialize the TargetMachine builder for a given Triple
559static void initTMBuilder(TargetMachineBuilder &TMBuilder,
560 const Triple &TheTriple) {
561 if (TMBuilder.MCpu.empty())
562 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
563 TMBuilder.TheTriple = std::move(TheTriple);
564}
565
566void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
567 MemoryBufferRef Buffer(Data, Identifier);
568
569 auto InputOrError = lto::InputFile::create(Object: Buffer);
570 if (!InputOrError)
571 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
572 toString(E: InputOrError.takeError()));
573
574 auto TripleStr = (*InputOrError)->getTargetTriple();
575 Triple TheTriple(TripleStr);
576
577 if (Modules.empty())
578 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
579 else if (TMBuilder.TheTriple != TheTriple) {
580 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
581 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
582 "supported");
583 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
584 }
585
586 Modules.emplace_back(args: std::move(*InputOrError));
587}
588
589void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
590 PreservedSymbols.insert(key: Name);
591}
592
593void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
594 // FIXME: At the moment, we don't take advantage of this extra information,
595 // we're conservatively considering cross-references as preserved.
596 // CrossReferencedSymbols.insert(Name);
597 PreservedSymbols.insert(key: Name);
598}
599
600// TargetMachine factory
601std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
602 std::string ErrMsg;
603 const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error&: ErrMsg);
604 if (!TheTarget) {
605 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
606 }
607
608 // Use MAttr as the default set of features.
609 SubtargetFeatures Features(MAttr);
610 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
611 std::string FeatureStr = Features.getString();
612
613 std::unique_ptr<TargetMachine> TM(
614 TheTarget->createTargetMachine(TT: TheTriple, CPU: MCpu, Features: FeatureStr, Options,
615 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
616 assert(TM && "Cannot create target machine");
617
618 return TM;
619}
620
621/**
622 * Produce the combined summary index from all the bitcode files:
623 * "thin-link".
624 */
625std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
626 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
627 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
628 for (auto &Mod : Modules) {
629 auto &M = Mod->getSingleBitcodeModule();
630 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
631 // FIXME diagnose
632 logAllUnhandledErrors(
633 E: std::move(Err), OS&: errs(),
634 ErrorBanner: "error: can't create module summary index for buffer: ");
635 return nullptr;
636 }
637 }
638 return CombinedIndex;
639}
640
641namespace {
642struct IsExported {
643 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
644 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
645
646 IsExported(
647 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
648 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
649 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
650
651 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
652 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
653 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
654 GUIDPreservedSymbols.count(V: VI.getGUID());
655 }
656};
657
658struct IsPrevailing {
659 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
660 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
661 &PrevailingCopy)
662 : PrevailingCopy(PrevailingCopy) {}
663
664 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
665 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
666 // Not in map means that there was only one copy, which must be prevailing.
667 if (Prevailing == PrevailingCopy.end())
668 return true;
669 return Prevailing->second == S;
670 };
671};
672} // namespace
673
674static void computeDeadSymbolsInIndex(
675 ModuleSummaryIndex &Index,
676 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
677 // We have no symbols resolution available. And can't do any better now in the
678 // case where the prevailing symbol is in a native object. It can be refined
679 // with linker information in the future.
680 auto isPrevailing = [&](GlobalValue::GUID G) {
681 return PrevailingType::Unknown;
682 };
683 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
684 /* ImportEnabled = */ true);
685}
686
687/**
688 * Perform promotion and renaming of exported internal functions.
689 * Index is updated to reflect linkage changes from weak resolution.
690 */
691void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
692 const lto::InputFile &File) {
693 auto ModuleCount = Index.modulePaths().size();
694 auto ModuleIdentifier = TheModule.getModuleIdentifier();
695
696 // Collect for each module the list of function it defines (GUID -> Summary).
697 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
698 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
699
700 // Convert the preserved symbols set from string to GUID
701 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
702 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
703
704 // Add used symbol to the preserved symbols.
705 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
706
707 // Compute "dead" symbols, we don't want to import/export these!
708 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
709
710 // Compute prevailing symbols
711 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
712 computePrevailingCopies(Index, PrevailingCopy);
713
714 // Generate import/export list
715 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
716 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
717 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
718 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
719 ExportLists);
720
721 // Resolve prevailing symbols
722 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
723 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
724 PrevailingCopy);
725
726 thinLTOFinalizeInModule(TheModule,
727 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
728 /*PropagateAttrs=*/false);
729
730 // Promote the exported values in the index, so that they are promoted
731 // in the module.
732 thinLTOInternalizeAndPromoteInIndex(
733 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
734 isPrevailing: IsPrevailing(PrevailingCopy));
735
736 // FIXME Set ClearDSOLocalOnDeclarations.
737 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
738}
739
740/**
741 * Perform cross-module importing for the module identified by ModuleIdentifier.
742 */
743void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
744 ModuleSummaryIndex &Index,
745 const lto::InputFile &File) {
746 auto ModuleMap = generateModuleMap(Modules);
747 auto ModuleCount = Index.modulePaths().size();
748
749 // Collect for each module the list of function it defines (GUID -> Summary).
750 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
751 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
752
753 // Convert the preserved symbols set from string to GUID
754 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
755 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
756
757 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
758
759 // Compute "dead" symbols, we don't want to import/export these!
760 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
761
762 // Compute prevailing symbols
763 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
764 computePrevailingCopies(Index, PrevailingCopy);
765
766 // Generate import/export list
767 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
768 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
769 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
770 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
771 ExportLists);
772 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
773
774 // FIXME Set ClearDSOLocalOnDeclarations.
775 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
776 /*ClearDSOLocalOnDeclarations=*/false);
777}
778
779/**
780 * Compute the list of summaries needed for importing into module.
781 */
782void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
783 Module &TheModule, ModuleSummaryIndex &Index,
784 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
785 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
786 auto ModuleCount = Index.modulePaths().size();
787 auto ModuleIdentifier = TheModule.getModuleIdentifier();
788
789 // Collect for each module the list of function it defines (GUID -> Summary).
790 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
791 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
792
793 // Convert the preserved symbols set from string to GUID
794 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
795 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
796
797 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
798
799 // Compute "dead" symbols, we don't want to import/export these!
800 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
801
802 // Compute prevailing symbols
803 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
804 computePrevailingCopies(Index, PrevailingCopy);
805
806 // Generate import/export list
807 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
808 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
809 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
810 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
811 ExportLists);
812
813 llvm::gatherImportedSummariesForModule(
814 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
815 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
816}
817
818/**
819 * Emit the list of files needed for importing into module.
820 */
821void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
822 ModuleSummaryIndex &Index,
823 const lto::InputFile &File) {
824 auto ModuleCount = Index.modulePaths().size();
825 auto ModuleIdentifier = TheModule.getModuleIdentifier();
826
827 // Collect for each module the list of function it defines (GUID -> Summary).
828 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
829 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
830
831 // Convert the preserved symbols set from string to GUID
832 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
833 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
834
835 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
836
837 // Compute "dead" symbols, we don't want to import/export these!
838 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
839
840 // Compute prevailing symbols
841 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
842 computePrevailingCopies(Index, PrevailingCopy);
843
844 // Generate import/export list
845 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
846 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
847 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
848 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
849 ExportLists);
850
851 // 'EmitImportsFiles' emits the list of modules from which to import from, and
852 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
853 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportsFiles`.
854 GVSummaryPtrSet DecSummaries;
855 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
856 llvm::gatherImportedSummariesForModule(
857 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
858 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
859
860 if (Error EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
861 ModuleToSummariesForIndex))
862 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
863 " to save imports lists\n");
864}
865
866/**
867 * Perform internalization. Runs promote and internalization together.
868 * Index is updated to reflect linkage changes.
869 */
870void ThinLTOCodeGenerator::internalize(Module &TheModule,
871 ModuleSummaryIndex &Index,
872 const lto::InputFile &File) {
873 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
874 auto ModuleCount = Index.modulePaths().size();
875 auto ModuleIdentifier = TheModule.getModuleIdentifier();
876
877 // Convert the preserved symbols set from string to GUID
878 auto GUIDPreservedSymbols =
879 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
880
881 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
882
883 // Collect for each module the list of function it defines (GUID -> Summary).
884 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
885 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
886
887 // Compute "dead" symbols, we don't want to import/export these!
888 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
889
890 // Compute prevailing symbols
891 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
892 computePrevailingCopies(Index, PrevailingCopy);
893
894 // Generate import/export list
895 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
896 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
897 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
898 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
899 ExportLists);
900 auto &ExportList = ExportLists[ModuleIdentifier];
901
902 // Be friendly and don't nuke totally the module when the client didn't
903 // supply anything to preserve.
904 if (ExportList.empty() && GUIDPreservedSymbols.empty())
905 return;
906
907 // Resolve prevailing symbols
908 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
909 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
910 PrevailingCopy);
911
912 // Promote the exported values in the index, so that they are promoted
913 // in the module.
914 thinLTOInternalizeAndPromoteInIndex(
915 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
916 isPrevailing: IsPrevailing(PrevailingCopy));
917
918 // FIXME Set ClearDSOLocalOnDeclarations.
919 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
920
921 // Internalization
922 thinLTOFinalizeInModule(TheModule,
923 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
924 /*PropagateAttrs=*/false);
925
926 thinLTOInternalizeModule(TheModule,
927 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
928}
929
930/**
931 * Perform post-importing ThinLTO optimizations.
932 */
933void ThinLTOCodeGenerator::optimize(Module &TheModule) {
934 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
935
936 // Optimize now
937 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
938 DebugPassManager, Index: nullptr);
939}
940
941/// Write out the generated object file, either from CacheEntryPath or from
942/// OutputBuffer, preferring hard-link when possible.
943/// Returns the path to the generated file in SavedObjectsDirectoryPath.
944std::string
945ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
946 const MemoryBuffer &OutputBuffer) {
947 auto ArchName = TMBuilder.TheTriple.getArchName();
948 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
949 llvm::sys::path::append(path&: OutputPath,
950 a: Twine(count) + "." + ArchName + ".thinlto.o");
951 OutputPath.c_str(); // Ensure the string is null terminated.
952 if (sys::fs::exists(Path: OutputPath))
953 sys::fs::remove(path: OutputPath);
954
955 // We don't return a memory buffer to the linker, just a list of files.
956 if (!CacheEntryPath.empty()) {
957 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
958 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
959 if (!Err)
960 return std::string(OutputPath);
961 // Hard linking failed, try to copy.
962 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
963 if (!Err)
964 return std::string(OutputPath);
965 // Copy failed (could be because the CacheEntry was removed from the cache
966 // in the meantime by another process), fall back and try to write down the
967 // buffer to the output.
968 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
969 << "' to '" << OutputPath << "'\n";
970 }
971 // No cache entry, just write out the buffer.
972 std::error_code Err;
973 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
974 if (Err)
975 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
976 OS << OutputBuffer.getBuffer();
977 return std::string(OutputPath);
978}
979
980// Main entry point for the ThinLTO processing
981void ThinLTOCodeGenerator::run() {
982 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
983 llvm::scope_exit TimeTraceScopeExit([]() {
984 if (llvm::timeTraceProfilerEnabled())
985 llvm::timeTraceProfilerEnd();
986 });
987 // Prepare the resulting object vector
988 assert(ProducedBinaries.empty() && "The generator should not be reused");
989 if (SavedObjectsDirectoryPath.empty())
990 ProducedBinaries.resize(new_size: Modules.size());
991 else {
992 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
993 bool IsDir;
994 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
995 if (!IsDir)
996 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
997 ProducedBinaryFiles.resize(new_size: Modules.size());
998 }
999
1000 if (CodeGenOnly) {
1001 // Perform only parallel codegen and return.
1002 DefaultThreadPool Pool;
1003 int count = 0;
1004 for (auto &Mod : Modules) {
1005 Pool.async(F: [&](int count) {
1006 LLVMContext Context;
1007 Context.getLibraryOptions().assign(Other: LibraryOptions);
1008 Context.setDiscardValueNames(LTODiscardValueNames);
1009
1010 // Parse module now
1011 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1012 /*IsImporting*/ false);
1013
1014 // CodeGen
1015 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1016 if (SavedObjectsDirectoryPath.empty())
1017 ProducedBinaries[count] = std::move(OutputBuffer);
1018 else
1019 ProducedBinaryFiles[count] =
1020 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1021 }, ArgList: count++);
1022 }
1023
1024 return;
1025 }
1026
1027 // Sequential linking phase
1028 auto Index = linkCombinedIndex();
1029
1030 // Save temps: index.
1031 if (!SaveTempsDir.empty()) {
1032 auto SaveTempPath = SaveTempsDir + "index.bc";
1033 std::error_code EC;
1034 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1035 if (EC)
1036 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1037 " to save optimized bitcode\n");
1038 writeIndexToFile(Index: *Index, Out&: OS);
1039 }
1040
1041
1042 // Prepare the module map.
1043 auto ModuleMap = generateModuleMap(Modules);
1044 auto ModuleCount = Modules.size();
1045
1046 // Collect for each module the list of function it defines (GUID -> Summary).
1047 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1048 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1049
1050 // Convert the preserved symbols set from string to GUID, this is needed for
1051 // computing the caching hash and the internalization.
1052 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1053 for (const auto &M : Modules)
1054 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1055 GUIDs&: GUIDPreservedSymbols);
1056
1057 // Add used symbol from inputs to the preserved symbols.
1058 for (const auto &M : Modules)
1059 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1060
1061 // Compute "dead" symbols, we don't want to import/export these!
1062 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1063
1064 // Currently there is no support for enabling whole program visibility via a
1065 // linker option in the old LTO API, but this call allows it to be specified
1066 // via the internal option. Must be done before WPD below.
1067 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1068 Index->setWithWholeProgramVisibility();
1069
1070 // FIXME: This needs linker information via a TBD new interface
1071 updateVCallVisibilityInIndex(Index&: *Index,
1072 /*WholeProgramVisibilityEnabledInLTO=*/false,
1073 // FIXME: These need linker information via a
1074 // TBD new interface.
1075 /*DynamicExportSymbols=*/{},
1076 /*VisibleToRegularObjSymbols=*/{});
1077
1078 // Perform index-based WPD. This will return immediately if there are
1079 // no index entries in the typeIdMetadata map (e.g. if we are instead
1080 // performing IR-based WPD in hybrid regular/thin LTO mode).
1081 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1082 std::set<GlobalValue::GUID> ExportedGUIDs;
1083 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1084 GUIDPreservedSymbols.insert_range(R&: ExportedGUIDs);
1085
1086 // Compute prevailing symbols
1087 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1088 computePrevailingCopies(Index: *Index, PrevailingCopy);
1089
1090 // Collect the import/export lists for all modules from the call-graph in the
1091 // combined index.
1092 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
1093 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1094 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1095 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1096 ExportLists);
1097
1098 // We use a std::map here to be able to have a defined ordering when
1099 // producing a hash for the cache entry.
1100 // FIXME: we should be able to compute the caching hash for the entry based
1101 // on the index, and nuke this map.
1102 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1103
1104 // Resolve prevailing symbols, this has to be computed early because it
1105 // impacts the caching.
1106 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1107 PrevailingCopy);
1108
1109 // Use global summary-based analysis to identify symbols that can be
1110 // internalized (because they aren't exported or preserved as per callback).
1111 // Changes are made in the index, consumed in the ThinLTO backends.
1112 updateIndexWPDForExports(Summary&: *Index,
1113 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1114 LocalWPDTargetsMap);
1115 thinLTOInternalizeAndPromoteInIndex(
1116 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1117 isPrevailing: IsPrevailing(PrevailingCopy));
1118
1119 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1120
1121 // Make sure that every module has an entry in the ExportLists, ImportList,
1122 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1123 // below.
1124 for (auto &Module : Modules) {
1125 auto ModuleIdentifier = Module->getName();
1126 ExportLists[ModuleIdentifier];
1127 ImportLists[ModuleIdentifier];
1128 ResolvedODR[ModuleIdentifier];
1129 ModuleToDefinedGVSummaries[ModuleIdentifier];
1130 }
1131
1132 std::vector<BitcodeModule *> ModulesVec;
1133 ModulesVec.reserve(n: Modules.size());
1134 for (auto &Mod : Modules)
1135 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1136 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1137
1138 if (llvm::timeTraceProfilerEnabled())
1139 llvm::timeTraceProfilerEnd();
1140
1141 TimeTraceScopeExit.release();
1142
1143 // Parallel optimizer + codegen
1144 {
1145 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1146 for (auto IndexCount : ModulesOrdering) {
1147 auto &Mod = Modules[IndexCount];
1148 Pool.async(F: [&](int count) {
1149 auto ModuleIdentifier = Mod->getName();
1150 auto &ExportList = ExportLists[ModuleIdentifier];
1151
1152 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1153
1154 // The module may be cached, this helps handling it.
1155 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1156 ImportLists[ModuleIdentifier], ExportList,
1157 ResolvedODR[ModuleIdentifier],
1158 DefinedGVSummaries, OptLevel, Freestanding,
1159 TMBuilder, MllvmArgs);
1160 auto CacheEntryPath = CacheEntry.getEntryPath();
1161
1162 {
1163 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1164 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1165 << " '" << CacheEntryPath << "' for buffer "
1166 << count << " " << ModuleIdentifier << "\n");
1167
1168 if (ErrOrBuffer) {
1169 // Cache Hit!
1170 if (SavedObjectsDirectoryPath.empty())
1171 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1172 else
1173 ProducedBinaryFiles[count] = writeGeneratedObject(
1174 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1175 return;
1176 }
1177 }
1178
1179 LLVMContext Context;
1180 Context.getLibraryOptions().assign(Other: LibraryOptions);
1181 Context.setDiscardValueNames(LTODiscardValueNames);
1182 Context.enableDebugTypeODRUniquing();
1183 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1184 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1185 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1186 if (!DiagFileOrErr) {
1187 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1188 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1189 "remarks");
1190 }
1191
1192 // Parse module now
1193 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1194 /*IsImporting*/ false);
1195
1196 // Save temps: original file.
1197 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1198
1199 auto &ImportList = ImportLists[ModuleIdentifier];
1200 // Run the main process now, and generates a binary
1201 auto OutputBuffer = ProcessThinLTOModule(
1202 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1203 ExportList, GUIDPreservedSymbols,
1204 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1205 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1206 DebugPassManager);
1207
1208 // Commit to the cache (if enabled)
1209 CacheEntry.write(OutputBuffer: *OutputBuffer);
1210
1211 if (SavedObjectsDirectoryPath.empty()) {
1212 // We need to generated a memory buffer for the linker.
1213 if (!CacheEntryPath.empty()) {
1214 // When cache is enabled, reload from the cache if possible.
1215 // Releasing the buffer from the heap and reloading it from the
1216 // cache file with mmap helps us to lower memory pressure.
1217 // The freed memory can be used for the next input file.
1218 // The final binary link will read from the VFS cache (hopefully!)
1219 // or from disk (if the memory pressure was too high).
1220 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1221 if (auto EC = ReloadedBufferOrErr.getError()) {
1222 // On error, keep the preexisting buffer and print a diagnostic.
1223 errs() << "remark: can't reload cached file '" << CacheEntryPath
1224 << "': " << EC.message() << "\n";
1225 } else {
1226 OutputBuffer = std::move(*ReloadedBufferOrErr);
1227 }
1228 }
1229 ProducedBinaries[count] = std::move(OutputBuffer);
1230 return;
1231 }
1232 ProducedBinaryFiles[count] = writeGeneratedObject(
1233 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1234 }, ArgList&: IndexCount);
1235 }
1236 }
1237
1238 Expected<bool> PrunedOrErr =
1239 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1240 if (!PrunedOrErr) {
1241 errs() << "Error: " << toString(E: PrunedOrErr.takeError()) << "\n";
1242 report_fatal_error(reason: "ThinLTO: failure to prune cache");
1243 }
1244
1245 // If statistics were requested, print them out now.
1246 if (llvm::AreStatisticsEnabled())
1247 llvm::PrintStatistics();
1248 reportAndResetTimings();
1249}
1250