1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/ModuleSummaryAnalysis.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/Bitcode/BitcodeWriterPass.h"
26#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/LLVMRemarkStreamer.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/Mangler.h"
33#include "llvm/IR/PassTimingInfo.h"
34#include "llvm/IR/Verifier.h"
35#include "llvm/IRReader/IRReader.h"
36#include "llvm/LTO/LTO.h"
37#include "llvm/MC/TargetRegistry.h"
38#include "llvm/Object/IRObjectFile.h"
39#include "llvm/Passes/PassBuilder.h"
40#include "llvm/Passes/StandardInstrumentations.h"
41#include "llvm/Remarks/HotnessThresholdParser.h"
42#include "llvm/Support/CachePruning.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/FormatVariadic.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/SHA1.h"
49#include "llvm/Support/SmallVectorMemoryBuffer.h"
50#include "llvm/Support/ThreadPool.h"
51#include "llvm/Support/Threading.h"
52#include "llvm/Support/ToolOutputFile.h"
53#include "llvm/Support/raw_ostream.h"
54#include "llvm/Target/TargetMachine.h"
55#include "llvm/TargetParser/SubtargetFeature.h"
56#include "llvm/Transforms/IPO/FunctionAttrs.h"
57#include "llvm/Transforms/IPO/FunctionImport.h"
58#include "llvm/Transforms/IPO/Internalize.h"
59#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
60#include "llvm/Transforms/Utils/FunctionImportUtils.h"
61
62#if !defined(_MSC_VER) && !defined(__MINGW32__)
63#include <unistd.h>
64#else
65#include <io.h>
66#endif
67
68using namespace llvm;
69using namespace ThinLTOCodeGeneratorImpl;
70
71#define DEBUG_TYPE "thinlto"
72
73namespace llvm {
74// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
75extern cl::opt<bool> LTODiscardValueNames;
76extern cl::opt<std::string> RemarksFilename;
77extern cl::opt<std::string> RemarksPasses;
78extern cl::opt<bool> RemarksWithHotness;
79extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
80 RemarksHotnessThreshold;
81extern cl::opt<std::string> RemarksFormat;
82}
83
84// Default to using all available threads in the system, but using only one
85// thred per core, as indicated by the usage of
86// heavyweight_hardware_concurrency() below.
87static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
88
89// Simple helper to save temporary files for debug.
90static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
91 unsigned count, StringRef Suffix) {
92 if (TempDir.empty())
93 return;
94 // User asked to save temps, let dump the bitcode file after import.
95 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
96 std::error_code EC;
97 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
98 if (EC)
99 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
100 " to save optimized bitcode\n");
101 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
102}
103
104static const GlobalValueSummary *getFirstDefinitionForLinker(
105 ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
106 // If there is any strong definition anywhere, get it.
107 auto StrongDefForLinker = llvm::find_if(
108 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
109 auto Linkage = Summary->linkage();
110 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
111 !GlobalValue::isWeakForLinker(Linkage);
112 });
113 if (StrongDefForLinker != GVSummaryList.end())
114 return StrongDefForLinker->get();
115 // Get the first *linker visible* definition for this global in the summary
116 // list.
117 auto FirstDefForLinker = llvm::find_if(
118 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
119 auto Linkage = Summary->linkage();
120 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
121 });
122 // Extern templates can be emitted as available_externally.
123 if (FirstDefForLinker == GVSummaryList.end())
124 return nullptr;
125 return FirstDefForLinker->get();
126}
127
128// Populate map of GUID to the prevailing copy for any multiply defined
129// symbols. Currently assume first copy is prevailing, or any strong
130// definition. Can be refined with Linker information in the future.
131static void computePrevailingCopies(
132 const ModuleSummaryIndex &Index,
133 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
134 auto HasMultipleCopies =
135 [&](ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
136 return GVSummaryList.size() > 1;
137 };
138
139 for (auto &I : Index) {
140 if (HasMultipleCopies(I.second.getSummaryList()))
141 PrevailingCopy[I.first] =
142 getFirstDefinitionForLinker(GVSummaryList: I.second.getSummaryList());
143 }
144}
145
146static StringMap<lto::InputFile *>
147generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
148 StringMap<lto::InputFile *> ModuleMap;
149 for (auto &M : Modules) {
150 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
151 assert(!ModuleMap.contains(M->getName()) &&
152 "Expect unique Buffer Identifier");
153 ModuleMap[M->getName()] = M.get();
154 }
155 return ModuleMap;
156}
157
158static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
159 bool ClearDSOLocalOnDeclarations) {
160 renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations);
161}
162
163namespace {
164class ThinLTODiagnosticInfo : public DiagnosticInfo {
165 const Twine &Msg;
166public:
167 ThinLTODiagnosticInfo(const Twine &DiagMsg LLVM_LIFETIME_BOUND,
168 DiagnosticSeverity Severity = DS_Error)
169 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
170 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
171};
172}
173
174/// Verify the module and strip broken debug info.
175static void verifyLoadedModule(Module &TheModule) {
176 bool BrokenDebugInfo = false;
177 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
178 report_fatal_error(reason: "Broken module found, compilation aborted!");
179 if (BrokenDebugInfo) {
180 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
181 "Invalid debug info found, debug info will be stripped", DS_Warning));
182 StripDebugInfo(M&: TheModule);
183 }
184}
185
186static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
187 LLVMContext &Context,
188 bool Lazy,
189 bool IsImporting) {
190 auto &Mod = Input->getSingleBitcodeModule();
191 SMDiagnostic Err;
192 Expected<std::unique_ptr<Module>> ModuleOrErr =
193 Lazy ? Mod.getLazyModule(Context,
194 /* ShouldLazyLoadMetadata */ true, IsImporting)
195 : Mod.parseModule(Context);
196 if (!ModuleOrErr) {
197 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
198 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
199 SourceMgr::DK_Error, EIB.message());
200 Err.print(ProgName: "ThinLTO", S&: errs());
201 });
202 report_fatal_error(reason: "Can't load module, abort.");
203 }
204 if (!Lazy)
205 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
206 return std::move(*ModuleOrErr);
207}
208
209static void
210crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
211 StringMap<lto::InputFile *> &ModuleMap,
212 const FunctionImporter::ImportMapTy &ImportList,
213 bool ClearDSOLocalOnDeclarations) {
214 auto Loader = [&](StringRef Identifier) {
215 auto &Input = ModuleMap[Identifier];
216 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
217 /*Lazy=*/true, /*IsImporting*/ true);
218 };
219
220 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
221 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
222 if (!Result) {
223 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
224 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
225 SourceMgr::DK_Error, EIB.message());
226 Err.print(ProgName: "ThinLTO", S&: errs());
227 });
228 report_fatal_error(reason: "importFunctions failed");
229 }
230 // Verify again after cross-importing.
231 verifyLoadedModule(TheModule);
232}
233
234static void optimizeModule(Module &TheModule, TargetMachine &TM,
235 unsigned OptLevel, bool Freestanding,
236 bool DebugPassManager, ModuleSummaryIndex *Index) {
237 std::optional<PGOOptions> PGOOpt;
238 LoopAnalysisManager LAM;
239 FunctionAnalysisManager FAM;
240 CGSCCAnalysisManager CGAM;
241 ModuleAnalysisManager MAM;
242
243 PassInstrumentationCallbacks PIC;
244 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
245 SI.registerCallbacks(PIC, MAM: &MAM);
246 PipelineTuningOptions PTO;
247 PTO.LoopVectorization = true;
248 PTO.SLPVectorization = true;
249 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
250
251 std::unique_ptr<TargetLibraryInfoImpl> TLII(
252 new TargetLibraryInfoImpl(TM.getTargetTriple(), TM.Options.VecLib));
253 if (Freestanding)
254 TLII->disableAllFunctions();
255 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
256
257 // Register all the basic analyses with the managers.
258 PB.registerModuleAnalyses(MAM);
259 PB.registerCGSCCAnalyses(CGAM);
260 PB.registerFunctionAnalyses(FAM);
261 PB.registerLoopAnalyses(LAM);
262 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
263
264 ModulePassManager MPM;
265
266 OptimizationLevel OL;
267
268 switch (OptLevel) {
269 default:
270 llvm_unreachable("Invalid optimization level");
271 case 0:
272 OL = OptimizationLevel::O0;
273 break;
274 case 1:
275 OL = OptimizationLevel::O1;
276 break;
277 case 2:
278 OL = OptimizationLevel::O2;
279 break;
280 case 3:
281 OL = OptimizationLevel::O3;
282 break;
283 }
284
285 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
286
287 MPM.run(IR&: TheModule, AM&: MAM);
288}
289
290static void
291addUsedSymbolToPreservedGUID(const lto::InputFile &File,
292 DenseSet<GlobalValue::GUID> &PreservedGUID) {
293 Triple TT(File.getTargetTriple());
294 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
295 for (const auto &Sym : File.symbols())
296 if (Sym.isUsed() || Sym.isLibcall(Libcalls))
297 PreservedGUID.insert(
298 V: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Sym.getIRName()));
299}
300
301// Convert the PreservedSymbols map from "Name" based to "GUID" based.
302static void computeGUIDPreservedSymbols(const lto::InputFile &File,
303 const StringSet<> &PreservedSymbols,
304 const Triple &TheTriple,
305 DenseSet<GlobalValue::GUID> &GUIDs) {
306 // Iterate the symbols in the input file and if the input has preserved symbol
307 // compute the GUID for the symbol.
308 for (const auto &Sym : File.symbols()) {
309 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
310 GUIDs.insert(V: GlobalValue::getGUIDAssumingExternalLinkage(
311 GlobalName: GlobalValue::getGlobalIdentifier(Name: Sym.getIRName(),
312 Linkage: GlobalValue::ExternalLinkage, FileName: "")));
313 }
314}
315
316static DenseSet<GlobalValue::GUID>
317computeGUIDPreservedSymbols(const lto::InputFile &File,
318 const StringSet<> &PreservedSymbols,
319 const Triple &TheTriple) {
320 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
321 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
322 GUIDs&: GUIDPreservedSymbols);
323 return GUIDPreservedSymbols;
324}
325
326static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
327 TargetMachine &TM) {
328 SmallVector<char, 128> OutputBuffer;
329
330 // CodeGen
331 {
332 raw_svector_ostream OS(OutputBuffer);
333 legacy::PassManager PM;
334
335 // Setup the codegen now.
336 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
337 /* DisableVerify */ true))
338 report_fatal_error(reason: "Failed to setup codegen");
339
340 // Run codegen now. resulting binary is in OutputBuffer.
341 PM.run(M&: TheModule);
342 }
343 return std::make_unique<SmallVectorMemoryBuffer>(
344 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
345}
346
347namespace {
348/// Manage caching for a single Module.
349class ModuleCacheEntry {
350 SmallString<128> EntryPath;
351
352public:
353 // Create a cache entry. This compute a unique hash for the Module considering
354 // the current list of export/import, and offer an interface to query to
355 // access the content in the cache.
356 ModuleCacheEntry(
357 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
358 const FunctionImporter::ImportMapTy &ImportList,
359 const FunctionImporter::ExportSetTy &ExportList,
360 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
361 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
362 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
363 if (CachePath.empty())
364 return;
365
366 if (!Index.modulePaths().count(Key: ModuleID))
367 // The module does not have an entry, it can't have a hash at all
368 return;
369
370 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
371 P: [](uint32_t V) { return V == 0; }))
372 // No hash entry, no caching!
373 return;
374
375 llvm::lto::Config Conf;
376 Conf.OptLevel = OptLevel;
377 Conf.Options = TMBuilder.Options;
378 Conf.CPU = TMBuilder.MCpu;
379 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
380 Conf.RelocModel = TMBuilder.RelocModel;
381 Conf.CGOptLevel = TMBuilder.CGOptLevel;
382 Conf.Freestanding = Freestanding;
383 std::string Key =
384 computeLTOCacheKey(Conf, Index, ModuleID, ImportList, ExportList,
385 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
386
387 // This choice of file name allows the cache to be pruned (see pruneCache()
388 // in include/llvm/Support/CachePruning.h).
389 sys::path::append(path&: EntryPath, a: CachePath, b: Twine("llvmcache-", Key));
390 }
391
392 // Access the path to this entry in the cache.
393 StringRef getEntryPath() { return EntryPath; }
394
395 // Try loading the buffer for this cache entry.
396 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
397 if (EntryPath.empty())
398 return std::error_code();
399 SmallString<64> ResultPath;
400 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
401 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
402 if (!FDOrErr)
403 return errorToErrorCode(Err: FDOrErr.takeError());
404 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
405 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
406 sys::fs::closeFile(F&: *FDOrErr);
407 return MBOrErr;
408 }
409
410 // Cache the Produced object file
411 void write(const MemoryBuffer &OutputBuffer) {
412 if (EntryPath.empty())
413 return;
414
415 if (auto Err = llvm::writeToOutput(
416 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
417 OS << OutputBuffer.getBuffer();
418 return llvm::Error::success();
419 }))
420 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
421 Vals&: EntryPath,
422 Vals: toString(E: std::move(Err)).c_str()));
423 }
424};
425} // end anonymous namespace
426
427static std::unique_ptr<MemoryBuffer>
428ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
429 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
430 const FunctionImporter::ImportMapTy &ImportList,
431 const FunctionImporter::ExportSetTy &ExportList,
432 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
433 const GVSummaryMapTy &DefinedGlobals,
434 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
435 bool DisableCodeGen, StringRef SaveTempsDir,
436 bool Freestanding, unsigned OptLevel, unsigned count,
437 bool DebugPassManager) {
438 // "Benchmark"-like optimization: single-source case
439 bool SingleModule = (ModuleMap.size() == 1);
440
441 // When linking an ELF shared object, dso_local should be dropped. We
442 // conservatively do this for -fpic.
443 bool ClearDSOLocalOnDeclarations =
444 TM.getTargetTriple().isOSBinFormatELF() &&
445 TM.getRelocationModel() != Reloc::Static &&
446 TheModule.getPIELevel() == PIELevel::Default;
447
448 if (!SingleModule) {
449 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
450
451 // Apply summary-based prevailing-symbol resolution decisions.
452 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
453
454 // Save temps: after promotion.
455 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
456 }
457
458 // Be friendly and don't nuke totally the module when the client didn't
459 // supply anything to preserve.
460 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
461 // Apply summary-based internalization decisions.
462 thinLTOInternalizeModule(TheModule, DefinedGlobals);
463 }
464
465 // Save internalized bitcode
466 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
467
468 if (!SingleModule)
469 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
470 ClearDSOLocalOnDeclarations);
471
472 // Do this after any importing so that imported code is updated.
473 // See comment at call to updateVCallVisibilityInIndex() for why
474 // WholeProgramVisibilityEnabledInLTO is false.
475 updatePublicTypeTestCalls(M&: TheModule,
476 /* WholeProgramVisibilityEnabledInLTO */ false);
477
478 // Save temps: after cross-module import.
479 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
480
481 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
482 Index: &Index);
483
484 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
485
486 if (DisableCodeGen) {
487 // Configured to stop before CodeGen, serialize the bitcode and return.
488 SmallVector<char, 128> OutputBuffer;
489 {
490 raw_svector_ostream OS(OutputBuffer);
491 ProfileSummaryInfo PSI(TheModule);
492 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
493 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
494 }
495 return std::make_unique<SmallVectorMemoryBuffer>(
496 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
497 }
498
499 return codegenModule(TheModule, TM);
500}
501
502/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
503/// for caching, and in the \p Index for application during the ThinLTO
504/// backends. This is needed for correctness for exported symbols (ensure
505/// at least one copy kept) and a compile-time optimization (to drop duplicate
506/// copies when possible).
507static void resolvePrevailingInIndex(
508 ModuleSummaryIndex &Index,
509 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
510 &ResolvedODR,
511 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
512 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
513 &PrevailingCopy) {
514
515 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
516 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
517 // Not in map means that there was only one copy, which must be prevailing.
518 if (Prevailing == PrevailingCopy.end())
519 return true;
520 return Prevailing->second == S;
521 };
522
523 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
524 GlobalValue::GUID GUID,
525 GlobalValue::LinkageTypes NewLinkage) {
526 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
527 };
528
529 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
530 lto::Config Conf;
531 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
532 GUIDPreservedSymbols);
533}
534
535// Initialize the TargetMachine builder for a given Triple
536static void initTMBuilder(TargetMachineBuilder &TMBuilder,
537 const Triple &TheTriple) {
538 if (TMBuilder.MCpu.empty())
539 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
540 TMBuilder.TheTriple = std::move(TheTriple);
541}
542
543void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
544 MemoryBufferRef Buffer(Data, Identifier);
545
546 auto InputOrError = lto::InputFile::create(Object: Buffer);
547 if (!InputOrError)
548 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
549 toString(E: InputOrError.takeError()));
550
551 auto TripleStr = (*InputOrError)->getTargetTriple();
552 Triple TheTriple(TripleStr);
553
554 if (Modules.empty())
555 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
556 else if (TMBuilder.TheTriple != TheTriple) {
557 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
558 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
559 "supported");
560 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
561 }
562
563 Modules.emplace_back(args: std::move(*InputOrError));
564}
565
566void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
567 PreservedSymbols.insert(key: Name);
568}
569
570void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
571 // FIXME: At the moment, we don't take advantage of this extra information,
572 // we're conservatively considering cross-references as preserved.
573 // CrossReferencedSymbols.insert(Name);
574 PreservedSymbols.insert(key: Name);
575}
576
577// TargetMachine factory
578std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
579 std::string ErrMsg;
580 const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error&: ErrMsg);
581 if (!TheTarget) {
582 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
583 }
584
585 // Use MAttr as the default set of features.
586 SubtargetFeatures Features(MAttr);
587 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
588 std::string FeatureStr = Features.getString();
589
590 std::unique_ptr<TargetMachine> TM(
591 TheTarget->createTargetMachine(TT: TheTriple, CPU: MCpu, Features: FeatureStr, Options,
592 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
593 assert(TM && "Cannot create target machine");
594
595 return TM;
596}
597
598/**
599 * Produce the combined summary index from all the bitcode files:
600 * "thin-link".
601 */
602std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
603 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
604 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
605 for (auto &Mod : Modules) {
606 auto &M = Mod->getSingleBitcodeModule();
607 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
608 // FIXME diagnose
609 logAllUnhandledErrors(
610 E: std::move(Err), OS&: errs(),
611 ErrorBanner: "error: can't create module summary index for buffer: ");
612 return nullptr;
613 }
614 }
615 return CombinedIndex;
616}
617
618namespace {
619struct IsExported {
620 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
621 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
622
623 IsExported(
624 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
625 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
626 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
627
628 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
629 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
630 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
631 GUIDPreservedSymbols.count(V: VI.getGUID());
632 }
633};
634
635struct IsPrevailing {
636 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
637 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
638 &PrevailingCopy)
639 : PrevailingCopy(PrevailingCopy) {}
640
641 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
642 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
643 // Not in map means that there was only one copy, which must be prevailing.
644 if (Prevailing == PrevailingCopy.end())
645 return true;
646 return Prevailing->second == S;
647 };
648};
649} // namespace
650
651static void computeDeadSymbolsInIndex(
652 ModuleSummaryIndex &Index,
653 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
654 // We have no symbols resolution available. And can't do any better now in the
655 // case where the prevailing symbol is in a native object. It can be refined
656 // with linker information in the future.
657 auto isPrevailing = [&](GlobalValue::GUID G) {
658 return PrevailingType::Unknown;
659 };
660 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
661 /* ImportEnabled = */ true);
662}
663
664/**
665 * Perform promotion and renaming of exported internal functions.
666 * Index is updated to reflect linkage changes from weak resolution.
667 */
668void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
669 const lto::InputFile &File) {
670 auto ModuleCount = Index.modulePaths().size();
671 auto ModuleIdentifier = TheModule.getModuleIdentifier();
672
673 // Collect for each module the list of function it defines (GUID -> Summary).
674 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
675 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
676
677 // Convert the preserved symbols set from string to GUID
678 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
679 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
680
681 // Add used symbol to the preserved symbols.
682 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
683
684 // Compute "dead" symbols, we don't want to import/export these!
685 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
686
687 // Compute prevailing symbols
688 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
689 computePrevailingCopies(Index, PrevailingCopy);
690
691 // Generate import/export list
692 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
693 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
694 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
695 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
696 ExportLists);
697
698 // Resolve prevailing symbols
699 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
700 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
701 PrevailingCopy);
702
703 thinLTOFinalizeInModule(TheModule,
704 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
705 /*PropagateAttrs=*/false);
706
707 // Promote the exported values in the index, so that they are promoted
708 // in the module.
709 thinLTOInternalizeAndPromoteInIndex(
710 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
711 isPrevailing: IsPrevailing(PrevailingCopy));
712
713 // FIXME Set ClearDSOLocalOnDeclarations.
714 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
715}
716
717/**
718 * Perform cross-module importing for the module identified by ModuleIdentifier.
719 */
720void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
721 ModuleSummaryIndex &Index,
722 const lto::InputFile &File) {
723 auto ModuleMap = generateModuleMap(Modules);
724 auto ModuleCount = Index.modulePaths().size();
725
726 // Collect for each module the list of function it defines (GUID -> Summary).
727 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
728 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
729
730 // Convert the preserved symbols set from string to GUID
731 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
732 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
733
734 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
735
736 // Compute "dead" symbols, we don't want to import/export these!
737 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
738
739 // Compute prevailing symbols
740 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
741 computePrevailingCopies(Index, PrevailingCopy);
742
743 // Generate import/export list
744 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
745 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
746 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
747 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
748 ExportLists);
749 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
750
751 // FIXME Set ClearDSOLocalOnDeclarations.
752 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
753 /*ClearDSOLocalOnDeclarations=*/false);
754}
755
756/**
757 * Compute the list of summaries needed for importing into module.
758 */
759void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
760 Module &TheModule, ModuleSummaryIndex &Index,
761 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
762 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
763 auto ModuleCount = Index.modulePaths().size();
764 auto ModuleIdentifier = TheModule.getModuleIdentifier();
765
766 // Collect for each module the list of function it defines (GUID -> Summary).
767 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
768 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
769
770 // Convert the preserved symbols set from string to GUID
771 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
772 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
773
774 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
775
776 // Compute "dead" symbols, we don't want to import/export these!
777 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
778
779 // Compute prevailing symbols
780 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
781 computePrevailingCopies(Index, PrevailingCopy);
782
783 // Generate import/export list
784 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
785 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
786 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
787 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
788 ExportLists);
789
790 llvm::gatherImportedSummariesForModule(
791 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
792 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
793}
794
795/**
796 * Emit the list of files needed for importing into module.
797 */
798void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
799 ModuleSummaryIndex &Index,
800 const lto::InputFile &File) {
801 auto ModuleCount = Index.modulePaths().size();
802 auto ModuleIdentifier = TheModule.getModuleIdentifier();
803
804 // Collect for each module the list of function it defines (GUID -> Summary).
805 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
806 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
807
808 // Convert the preserved symbols set from string to GUID
809 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
810 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
811
812 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
813
814 // Compute "dead" symbols, we don't want to import/export these!
815 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
816
817 // Compute prevailing symbols
818 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
819 computePrevailingCopies(Index, PrevailingCopy);
820
821 // Generate import/export list
822 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
823 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
824 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
825 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
826 ExportLists);
827
828 // 'EmitImportsFiles' emits the list of modules from which to import from, and
829 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
830 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportsFiles`.
831 GVSummaryPtrSet DecSummaries;
832 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
833 llvm::gatherImportedSummariesForModule(
834 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
835 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
836
837 if (Error EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
838 ModuleToSummariesForIndex))
839 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
840 " to save imports lists\n");
841}
842
843/**
844 * Perform internalization. Runs promote and internalization together.
845 * Index is updated to reflect linkage changes.
846 */
847void ThinLTOCodeGenerator::internalize(Module &TheModule,
848 ModuleSummaryIndex &Index,
849 const lto::InputFile &File) {
850 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
851 auto ModuleCount = Index.modulePaths().size();
852 auto ModuleIdentifier = TheModule.getModuleIdentifier();
853
854 // Convert the preserved symbols set from string to GUID
855 auto GUIDPreservedSymbols =
856 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
857
858 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
859
860 // Collect for each module the list of function it defines (GUID -> Summary).
861 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
862 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
863
864 // Compute "dead" symbols, we don't want to import/export these!
865 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
866
867 // Compute prevailing symbols
868 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
869 computePrevailingCopies(Index, PrevailingCopy);
870
871 // Generate import/export list
872 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
873 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
874 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
875 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
876 ExportLists);
877 auto &ExportList = ExportLists[ModuleIdentifier];
878
879 // Be friendly and don't nuke totally the module when the client didn't
880 // supply anything to preserve.
881 if (ExportList.empty() && GUIDPreservedSymbols.empty())
882 return;
883
884 // Resolve prevailing symbols
885 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
886 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
887 PrevailingCopy);
888
889 // Promote the exported values in the index, so that they are promoted
890 // in the module.
891 thinLTOInternalizeAndPromoteInIndex(
892 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
893 isPrevailing: IsPrevailing(PrevailingCopy));
894
895 // FIXME Set ClearDSOLocalOnDeclarations.
896 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
897
898 // Internalization
899 thinLTOFinalizeInModule(TheModule,
900 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
901 /*PropagateAttrs=*/false);
902
903 thinLTOInternalizeModule(TheModule,
904 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
905}
906
907/**
908 * Perform post-importing ThinLTO optimizations.
909 */
910void ThinLTOCodeGenerator::optimize(Module &TheModule) {
911 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
912
913 // Optimize now
914 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
915 DebugPassManager, Index: nullptr);
916}
917
918/// Write out the generated object file, either from CacheEntryPath or from
919/// OutputBuffer, preferring hard-link when possible.
920/// Returns the path to the generated file in SavedObjectsDirectoryPath.
921std::string
922ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
923 const MemoryBuffer &OutputBuffer) {
924 auto ArchName = TMBuilder.TheTriple.getArchName();
925 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
926 llvm::sys::path::append(path&: OutputPath,
927 a: Twine(count) + "." + ArchName + ".thinlto.o");
928 OutputPath.c_str(); // Ensure the string is null terminated.
929 if (sys::fs::exists(Path: OutputPath))
930 sys::fs::remove(path: OutputPath);
931
932 // We don't return a memory buffer to the linker, just a list of files.
933 if (!CacheEntryPath.empty()) {
934 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
935 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
936 if (!Err)
937 return std::string(OutputPath);
938 // Hard linking failed, try to copy.
939 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
940 if (!Err)
941 return std::string(OutputPath);
942 // Copy failed (could be because the CacheEntry was removed from the cache
943 // in the meantime by another process), fall back and try to write down the
944 // buffer to the output.
945 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
946 << "' to '" << OutputPath << "'\n";
947 }
948 // No cache entry, just write out the buffer.
949 std::error_code Err;
950 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
951 if (Err)
952 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
953 OS << OutputBuffer.getBuffer();
954 return std::string(OutputPath);
955}
956
957// Main entry point for the ThinLTO processing
958void ThinLTOCodeGenerator::run() {
959 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
960 llvm::scope_exit TimeTraceScopeExit([]() {
961 if (llvm::timeTraceProfilerEnabled())
962 llvm::timeTraceProfilerEnd();
963 });
964 // Prepare the resulting object vector
965 assert(ProducedBinaries.empty() && "The generator should not be reused");
966 if (SavedObjectsDirectoryPath.empty())
967 ProducedBinaries.resize(new_size: Modules.size());
968 else {
969 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
970 bool IsDir;
971 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
972 if (!IsDir)
973 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
974 ProducedBinaryFiles.resize(new_size: Modules.size());
975 }
976
977 if (CodeGenOnly) {
978 // Perform only parallel codegen and return.
979 DefaultThreadPool Pool;
980 int count = 0;
981 for (auto &Mod : Modules) {
982 Pool.async(F: [&](int count) {
983 LLVMContext Context;
984 Context.setDiscardValueNames(LTODiscardValueNames);
985
986 // Parse module now
987 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
988 /*IsImporting*/ false);
989
990 // CodeGen
991 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
992 if (SavedObjectsDirectoryPath.empty())
993 ProducedBinaries[count] = std::move(OutputBuffer);
994 else
995 ProducedBinaryFiles[count] =
996 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
997 }, ArgList: count++);
998 }
999
1000 return;
1001 }
1002
1003 // Sequential linking phase
1004 auto Index = linkCombinedIndex();
1005
1006 // Save temps: index.
1007 if (!SaveTempsDir.empty()) {
1008 auto SaveTempPath = SaveTempsDir + "index.bc";
1009 std::error_code EC;
1010 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1011 if (EC)
1012 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1013 " to save optimized bitcode\n");
1014 writeIndexToFile(Index: *Index, Out&: OS);
1015 }
1016
1017
1018 // Prepare the module map.
1019 auto ModuleMap = generateModuleMap(Modules);
1020 auto ModuleCount = Modules.size();
1021
1022 // Collect for each module the list of function it defines (GUID -> Summary).
1023 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1024 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1025
1026 // Convert the preserved symbols set from string to GUID, this is needed for
1027 // computing the caching hash and the internalization.
1028 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1029 for (const auto &M : Modules)
1030 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1031 GUIDs&: GUIDPreservedSymbols);
1032
1033 // Add used symbol from inputs to the preserved symbols.
1034 for (const auto &M : Modules)
1035 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1036
1037 // Compute "dead" symbols, we don't want to import/export these!
1038 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1039
1040 // Currently there is no support for enabling whole program visibility via a
1041 // linker option in the old LTO API, but this call allows it to be specified
1042 // via the internal option. Must be done before WPD below.
1043 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1044 Index->setWithWholeProgramVisibility();
1045
1046 // FIXME: This needs linker information via a TBD new interface
1047 updateVCallVisibilityInIndex(Index&: *Index,
1048 /*WholeProgramVisibilityEnabledInLTO=*/false,
1049 // FIXME: These need linker information via a
1050 // TBD new interface.
1051 /*DynamicExportSymbols=*/{},
1052 /*VisibleToRegularObjSymbols=*/{});
1053
1054 // Perform index-based WPD. This will return immediately if there are
1055 // no index entries in the typeIdMetadata map (e.g. if we are instead
1056 // performing IR-based WPD in hybrid regular/thin LTO mode).
1057 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1058 std::set<GlobalValue::GUID> ExportedGUIDs;
1059 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1060 GUIDPreservedSymbols.insert_range(R&: ExportedGUIDs);
1061
1062 // Compute prevailing symbols
1063 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1064 computePrevailingCopies(Index: *Index, PrevailingCopy);
1065
1066 // Collect the import/export lists for all modules from the call-graph in the
1067 // combined index.
1068 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
1069 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1070 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1071 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1072 ExportLists);
1073
1074 // We use a std::map here to be able to have a defined ordering when
1075 // producing a hash for the cache entry.
1076 // FIXME: we should be able to compute the caching hash for the entry based
1077 // on the index, and nuke this map.
1078 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1079
1080 // Resolve prevailing symbols, this has to be computed early because it
1081 // impacts the caching.
1082 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1083 PrevailingCopy);
1084
1085 // Use global summary-based analysis to identify symbols that can be
1086 // internalized (because they aren't exported or preserved as per callback).
1087 // Changes are made in the index, consumed in the ThinLTO backends.
1088 updateIndexWPDForExports(Summary&: *Index,
1089 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1090 LocalWPDTargetsMap);
1091 thinLTOInternalizeAndPromoteInIndex(
1092 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1093 isPrevailing: IsPrevailing(PrevailingCopy));
1094
1095 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1096
1097 // Make sure that every module has an entry in the ExportLists, ImportList,
1098 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1099 // below.
1100 for (auto &Module : Modules) {
1101 auto ModuleIdentifier = Module->getName();
1102 ExportLists[ModuleIdentifier];
1103 ImportLists[ModuleIdentifier];
1104 ResolvedODR[ModuleIdentifier];
1105 ModuleToDefinedGVSummaries[ModuleIdentifier];
1106 }
1107
1108 std::vector<BitcodeModule *> ModulesVec;
1109 ModulesVec.reserve(n: Modules.size());
1110 for (auto &Mod : Modules)
1111 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1112 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1113
1114 if (llvm::timeTraceProfilerEnabled())
1115 llvm::timeTraceProfilerEnd();
1116
1117 TimeTraceScopeExit.release();
1118
1119 // Parallel optimizer + codegen
1120 {
1121 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1122 for (auto IndexCount : ModulesOrdering) {
1123 auto &Mod = Modules[IndexCount];
1124 Pool.async(F: [&](int count) {
1125 auto ModuleIdentifier = Mod->getName();
1126 auto &ExportList = ExportLists[ModuleIdentifier];
1127
1128 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1129
1130 // The module may be cached, this helps handling it.
1131 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1132 ImportLists[ModuleIdentifier], ExportList,
1133 ResolvedODR[ModuleIdentifier],
1134 DefinedGVSummaries, OptLevel, Freestanding,
1135 TMBuilder);
1136 auto CacheEntryPath = CacheEntry.getEntryPath();
1137
1138 {
1139 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1140 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1141 << " '" << CacheEntryPath << "' for buffer "
1142 << count << " " << ModuleIdentifier << "\n");
1143
1144 if (ErrOrBuffer) {
1145 // Cache Hit!
1146 if (SavedObjectsDirectoryPath.empty())
1147 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1148 else
1149 ProducedBinaryFiles[count] = writeGeneratedObject(
1150 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1151 return;
1152 }
1153 }
1154
1155 LLVMContext Context;
1156 Context.setDiscardValueNames(LTODiscardValueNames);
1157 Context.enableDebugTypeODRUniquing();
1158 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1159 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1160 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1161 if (!DiagFileOrErr) {
1162 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1163 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1164 "remarks");
1165 }
1166
1167 // Parse module now
1168 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1169 /*IsImporting*/ false);
1170
1171 // Save temps: original file.
1172 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1173
1174 auto &ImportList = ImportLists[ModuleIdentifier];
1175 // Run the main process now, and generates a binary
1176 auto OutputBuffer = ProcessThinLTOModule(
1177 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1178 ExportList, GUIDPreservedSymbols,
1179 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1180 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1181 DebugPassManager);
1182
1183 // Commit to the cache (if enabled)
1184 CacheEntry.write(OutputBuffer: *OutputBuffer);
1185
1186 if (SavedObjectsDirectoryPath.empty()) {
1187 // We need to generated a memory buffer for the linker.
1188 if (!CacheEntryPath.empty()) {
1189 // When cache is enabled, reload from the cache if possible.
1190 // Releasing the buffer from the heap and reloading it from the
1191 // cache file with mmap helps us to lower memory pressure.
1192 // The freed memory can be used for the next input file.
1193 // The final binary link will read from the VFS cache (hopefully!)
1194 // or from disk (if the memory pressure was too high).
1195 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1196 if (auto EC = ReloadedBufferOrErr.getError()) {
1197 // On error, keep the preexisting buffer and print a diagnostic.
1198 errs() << "remark: can't reload cached file '" << CacheEntryPath
1199 << "': " << EC.message() << "\n";
1200 } else {
1201 OutputBuffer = std::move(*ReloadedBufferOrErr);
1202 }
1203 }
1204 ProducedBinaries[count] = std::move(OutputBuffer);
1205 return;
1206 }
1207 ProducedBinaryFiles[count] = writeGeneratedObject(
1208 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1209 }, ArgList&: IndexCount);
1210 }
1211 }
1212
1213 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1214
1215 // If statistics were requested, print them out now.
1216 if (llvm::AreStatisticsEnabled())
1217 llvm::PrintStatistics();
1218 reportAndResetTimings();
1219}
1220