1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/ModuleSummaryAnalysis.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/Bitcode/BitcodeWriterPass.h"
26#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/LLVMRemarkStreamer.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/Mangler.h"
33#include "llvm/IR/PassTimingInfo.h"
34#include "llvm/IR/Verifier.h"
35#include "llvm/IRReader/IRReader.h"
36#include "llvm/LTO/LTO.h"
37#include "llvm/MC/TargetRegistry.h"
38#include "llvm/Object/IRObjectFile.h"
39#include "llvm/Passes/PassBuilder.h"
40#include "llvm/Passes/StandardInstrumentations.h"
41#include "llvm/Remarks/HotnessThresholdParser.h"
42#include "llvm/Support/CachePruning.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/FormatVariadic.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/SHA1.h"
49#include "llvm/Support/SmallVectorMemoryBuffer.h"
50#include "llvm/Support/ThreadPool.h"
51#include "llvm/Support/Threading.h"
52#include "llvm/Support/ToolOutputFile.h"
53#include "llvm/Support/raw_ostream.h"
54#include "llvm/Target/TargetMachine.h"
55#include "llvm/TargetParser/SubtargetFeature.h"
56#include "llvm/Transforms/IPO/FunctionAttrs.h"
57#include "llvm/Transforms/IPO/FunctionImport.h"
58#include "llvm/Transforms/IPO/Internalize.h"
59#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
60#include "llvm/Transforms/Utils/FunctionImportUtils.h"
61
62#if !defined(_MSC_VER) && !defined(__MINGW32__)
63#include <unistd.h>
64#else
65#include <io.h>
66#endif
67
68using namespace llvm;
69using namespace ThinLTOCodeGeneratorImpl;
70
71#define DEBUG_TYPE "thinlto"
72
73namespace llvm {
74// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
75extern cl::opt<bool> LTODiscardValueNames;
76extern cl::opt<std::string> RemarksFilename;
77extern cl::opt<std::string> RemarksPasses;
78extern cl::opt<bool> RemarksWithHotness;
79extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
80 RemarksHotnessThreshold;
81extern cl::opt<std::string> RemarksFormat;
82extern cl::opt<bool> LTORunCSIRInstr;
83extern cl::opt<std::string> LTOCSIRProfile;
84extern cl::opt<std::string> SampleProfileFile;
85}
86
87// Default to using all available threads in the system, but using only one
88// thred per core, as indicated by the usage of
89// heavyweight_hardware_concurrency() below.
90static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
91
92// Simple helper to save temporary files for debug.
93static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
94 unsigned count, StringRef Suffix) {
95 if (TempDir.empty())
96 return;
97 // User asked to save temps, let dump the bitcode file after import.
98 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
99 std::error_code EC;
100 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
101 if (EC)
102 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
103 " to save optimized bitcode\n");
104 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
105}
106
107static const GlobalValueSummary *getFirstDefinitionForLinker(
108 ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
109 // If there is any strong definition anywhere, get it.
110 auto StrongDefForLinker = llvm::find_if(
111 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
112 auto Linkage = Summary->linkage();
113 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
114 !GlobalValue::isWeakForLinker(Linkage);
115 });
116 if (StrongDefForLinker != GVSummaryList.end())
117 return StrongDefForLinker->get();
118 // Get the first *linker visible* definition for this global in the summary
119 // list.
120 auto FirstDefForLinker = llvm::find_if(
121 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
122 auto Linkage = Summary->linkage();
123 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
124 });
125 // Extern templates can be emitted as available_externally.
126 if (FirstDefForLinker == GVSummaryList.end())
127 return nullptr;
128 return FirstDefForLinker->get();
129}
130
131// Populate map of GUID to the prevailing copy for any multiply defined
132// symbols. Currently assume first copy is prevailing, or any strong
133// definition. Can be refined with Linker information in the future.
134static void computePrevailingCopies(
135 const ModuleSummaryIndex &Index,
136 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
137 auto HasMultipleCopies =
138 [&](ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
139 return GVSummaryList.size() > 1;
140 };
141
142 for (auto &I : Index) {
143 if (HasMultipleCopies(I.second.getSummaryList()))
144 PrevailingCopy[I.first] =
145 getFirstDefinitionForLinker(GVSummaryList: I.second.getSummaryList());
146 }
147}
148
149static StringMap<lto::InputFile *>
150generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
151 StringMap<lto::InputFile *> ModuleMap;
152 for (auto &M : Modules) {
153 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
154 assert(!ModuleMap.contains(M->getName()) &&
155 "Expect unique Buffer Identifier");
156 ModuleMap[M->getName()] = M.get();
157 }
158 return ModuleMap;
159}
160
161static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
162 bool ClearDSOLocalOnDeclarations) {
163 renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations);
164}
165
166namespace {
167class ThinLTODiagnosticInfo : public DiagnosticInfo {
168 const Twine &Msg;
169public:
170 ThinLTODiagnosticInfo(const Twine &DiagMsg LLVM_LIFETIME_BOUND,
171 DiagnosticSeverity Severity = DS_Error)
172 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
173 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
174};
175}
176
177/// Verify the module and strip broken debug info.
178static void verifyLoadedModule(Module &TheModule) {
179 bool BrokenDebugInfo = false;
180 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
181 report_fatal_error(reason: "Broken module found, compilation aborted!");
182 if (BrokenDebugInfo) {
183 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
184 "Invalid debug info found, debug info will be stripped", DS_Warning));
185 StripDebugInfo(M&: TheModule);
186 }
187}
188
189static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
190 LLVMContext &Context,
191 bool Lazy,
192 bool IsImporting) {
193 auto &Mod = Input->getSingleBitcodeModule();
194 SMDiagnostic Err;
195 Expected<std::unique_ptr<Module>> ModuleOrErr =
196 Lazy ? Mod.getLazyModule(Context,
197 /* ShouldLazyLoadMetadata */ true, IsImporting)
198 : Mod.parseModule(Context);
199 if (!ModuleOrErr) {
200 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
201 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
202 SourceMgr::DK_Error, EIB.message());
203 Err.print(ProgName: "ThinLTO", S&: errs());
204 });
205 report_fatal_error(reason: "Can't load module, abort.");
206 }
207 if (!Lazy)
208 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
209 return std::move(*ModuleOrErr);
210}
211
212static void
213crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
214 StringMap<lto::InputFile *> &ModuleMap,
215 const FunctionImporter::ImportMapTy &ImportList,
216 bool ClearDSOLocalOnDeclarations) {
217 auto Loader = [&](StringRef Identifier) {
218 auto &Input = ModuleMap[Identifier];
219 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
220 /*Lazy=*/true, /*IsImporting*/ true);
221 };
222
223 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
224 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
225 if (!Result) {
226 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
227 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
228 SourceMgr::DK_Error, EIB.message());
229 Err.print(ProgName: "ThinLTO", S&: errs());
230 });
231 report_fatal_error(reason: "importFunctions failed");
232 }
233 // Verify again after cross-importing.
234 verifyLoadedModule(TheModule);
235}
236
237static void optimizeModule(Module &TheModule, TargetMachine &TM,
238 unsigned OptLevel, bool Freestanding,
239 bool DebugPassManager, ModuleSummaryIndex *Index) {
240 std::optional<PGOOptions> PGOOpt;
241 if (LTORunCSIRInstr) {
242 PGOOpt =
243 PGOOptions("", LTOCSIRProfile, "",
244 /*MemoryProfile=*/"", PGOOptions::IRUse,
245 PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default);
246 } else if (!LTOCSIRProfile.empty()) {
247 PGOOpt = PGOOptions(LTOCSIRProfile, "", "",
248 /*MemoryProfile=*/"", PGOOptions::IRUse,
249 PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default);
250 } else if (!SampleProfileFile.empty()) {
251 PGOOpt =
252 PGOOptions(SampleProfileFile, "", "",
253 /*MemoryProfile=*/"", PGOOptions::SampleUse,
254 PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default);
255 }
256 LoopAnalysisManager LAM;
257 FunctionAnalysisManager FAM;
258 CGSCCAnalysisManager CGAM;
259 ModuleAnalysisManager MAM;
260
261 PassInstrumentationCallbacks PIC;
262 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
263 SI.registerCallbacks(PIC, MAM: &MAM);
264 PipelineTuningOptions PTO;
265 PTO.LoopVectorization = true;
266 PTO.SLPVectorization = true;
267 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
268
269 std::unique_ptr<TargetLibraryInfoImpl> TLII(
270 new TargetLibraryInfoImpl(TM.getTargetTriple(), TM.Options.VecLib));
271 if (Freestanding)
272 TLII->disableAllFunctions();
273 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
274
275 // Register all the basic analyses with the managers.
276 PB.registerModuleAnalyses(MAM);
277 PB.registerCGSCCAnalyses(CGAM);
278 PB.registerFunctionAnalyses(FAM);
279 PB.registerLoopAnalyses(LAM);
280 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
281
282 ModulePassManager MPM;
283
284 OptimizationLevel OL;
285
286 switch (OptLevel) {
287 default:
288 llvm_unreachable("Invalid optimization level");
289 case 0:
290 OL = OptimizationLevel::O0;
291 break;
292 case 1:
293 OL = OptimizationLevel::O1;
294 break;
295 case 2:
296 OL = OptimizationLevel::O2;
297 break;
298 case 3:
299 OL = OptimizationLevel::O3;
300 break;
301 }
302
303 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
304
305 MPM.run(IR&: TheModule, AM&: MAM);
306}
307
308static void
309addUsedSymbolToPreservedGUID(const lto::InputFile &File,
310 DenseSet<GlobalValue::GUID> &PreservedGUID) {
311 Triple TT(File.getTargetTriple());
312 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
313 TargetLibraryInfoImpl TLII(TT);
314 TargetLibraryInfo TLI(TLII);
315 for (const auto &Sym : File.symbols())
316 if (Sym.isUsed() || Sym.isLibcall(TLI, Libcalls))
317 PreservedGUID.insert(
318 V: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Sym.getIRName()));
319}
320
321// Convert the PreservedSymbols map from "Name" based to "GUID" based.
322static void computeGUIDPreservedSymbols(const lto::InputFile &File,
323 const StringSet<> &PreservedSymbols,
324 const Triple &TheTriple,
325 DenseSet<GlobalValue::GUID> &GUIDs) {
326 // Iterate the symbols in the input file and if the input has preserved symbol
327 // compute the GUID for the symbol.
328 for (const auto &Sym : File.symbols()) {
329 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
330 GUIDs.insert(V: GlobalValue::getGUIDAssumingExternalLinkage(
331 GlobalName: GlobalValue::getGlobalIdentifier(Name: Sym.getIRName(),
332 Linkage: GlobalValue::ExternalLinkage, FileName: "")));
333 }
334}
335
336static DenseSet<GlobalValue::GUID>
337computeGUIDPreservedSymbols(const lto::InputFile &File,
338 const StringSet<> &PreservedSymbols,
339 const Triple &TheTriple) {
340 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
341 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
342 GUIDs&: GUIDPreservedSymbols);
343 return GUIDPreservedSymbols;
344}
345
346static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
347 TargetMachine &TM) {
348 SmallVector<char, 128> OutputBuffer;
349
350 // CodeGen
351 {
352 raw_svector_ostream OS(OutputBuffer);
353 legacy::PassManager PM;
354
355 // Setup the codegen now.
356 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
357 /* DisableVerify */ true))
358 report_fatal_error(reason: "Failed to setup codegen");
359
360 // Run codegen now. resulting binary is in OutputBuffer.
361 PM.run(M&: TheModule);
362 }
363 return std::make_unique<SmallVectorMemoryBuffer>(
364 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
365}
366
367namespace {
368/// Manage caching for a single Module.
369class ModuleCacheEntry {
370 SmallString<128> EntryPath;
371
372public:
373 // Create a cache entry. This compute a unique hash for the Module considering
374 // the current list of export/import, and offer an interface to query to
375 // access the content in the cache.
376 ModuleCacheEntry(
377 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
378 const FunctionImporter::ImportMapTy &ImportList,
379 const FunctionImporter::ExportSetTy &ExportList,
380 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
381 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
382 bool Freestanding, const TargetMachineBuilder &TMBuilder,
383 ArrayRef<std::string> MllvmArgs) {
384 if (CachePath.empty())
385 return;
386
387 if (!Index.modulePaths().count(Key: ModuleID))
388 // The module does not have an entry, it can't have a hash at all
389 return;
390
391 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
392 P: [](uint32_t V) { return V == 0; }))
393 // No hash entry, no caching!
394 return;
395
396 llvm::lto::Config Conf;
397 Conf.OptLevel = OptLevel;
398 Conf.Options = TMBuilder.Options;
399 Conf.CPU = TMBuilder.MCpu;
400 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
401 Conf.RelocModel = TMBuilder.RelocModel;
402 Conf.CGOptLevel = TMBuilder.CGOptLevel;
403 Conf.Freestanding = Freestanding;
404 append_range(C&: Conf.MllvmArgs, R&: MllvmArgs);
405 std::string Key =
406 computeLTOCacheKey(Conf, Index, ModuleID, ImportList, ExportList,
407 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
408
409 // This choice of file name allows the cache to be pruned (see pruneCache()
410 // in include/llvm/Support/CachePruning.h).
411 sys::path::append(path&: EntryPath, a: CachePath, b: Twine("llvmcache-", Key));
412 }
413
414 // Access the path to this entry in the cache.
415 StringRef getEntryPath() { return EntryPath; }
416
417 // Try loading the buffer for this cache entry.
418 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
419 if (EntryPath.empty())
420 return std::error_code();
421 SmallString<64> ResultPath;
422 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
423 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
424 if (!FDOrErr)
425 return errorToErrorCode(Err: FDOrErr.takeError());
426 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
427 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
428 sys::fs::closeFile(F&: *FDOrErr);
429 return MBOrErr;
430 }
431
432 // Cache the Produced object file
433 void write(const MemoryBuffer &OutputBuffer) {
434 if (EntryPath.empty())
435 return;
436
437 if (auto Err = llvm::writeToOutput(
438 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
439 OS << OutputBuffer.getBuffer();
440 return llvm::Error::success();
441 }))
442 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
443 Vals&: EntryPath,
444 Vals: toString(E: std::move(Err)).c_str()));
445 }
446};
447} // end anonymous namespace
448
449static std::unique_ptr<MemoryBuffer>
450ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
451 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
452 const FunctionImporter::ImportMapTy &ImportList,
453 const FunctionImporter::ExportSetTy &ExportList,
454 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
455 const GVSummaryMapTy &DefinedGlobals,
456 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
457 bool DisableCodeGen, StringRef SaveTempsDir,
458 bool Freestanding, unsigned OptLevel, unsigned count,
459 bool DebugPassManager) {
460 // "Benchmark"-like optimization: single-source case
461 bool SingleModule = (ModuleMap.size() == 1);
462
463 // When linking an ELF shared object, dso_local should be dropped. We
464 // conservatively do this for -fpic.
465 bool ClearDSOLocalOnDeclarations =
466 TM.getTargetTriple().isOSBinFormatELF() &&
467 TM.getRelocationModel() != Reloc::Static &&
468 TheModule.getPIELevel() == PIELevel::Default;
469
470 if (!SingleModule) {
471 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
472
473 // Apply summary-based prevailing-symbol resolution decisions.
474 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
475
476 // Save temps: after promotion.
477 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
478 }
479
480 // Be friendly and don't nuke totally the module when the client didn't
481 // supply anything to preserve.
482 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
483 // Apply summary-based internalization decisions.
484 thinLTOInternalizeModule(TheModule, DefinedGlobals);
485 }
486
487 // Save internalized bitcode
488 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
489
490 if (!SingleModule)
491 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
492 ClearDSOLocalOnDeclarations);
493
494 // Do this after any importing so that imported code is updated.
495 // See comment at call to updateVCallVisibilityInIndex() for why
496 // WholeProgramVisibilityEnabledInLTO is false.
497 updatePublicTypeTestCalls(M&: TheModule,
498 /* WholeProgramVisibilityEnabledInLTO */ false);
499
500 // Save temps: after cross-module import.
501 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
502
503 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
504 Index: &Index);
505
506 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
507
508 if (DisableCodeGen) {
509 // Configured to stop before CodeGen, serialize the bitcode and return.
510 SmallVector<char, 128> OutputBuffer;
511 {
512 raw_svector_ostream OS(OutputBuffer);
513 ProfileSummaryInfo PSI(TheModule);
514 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
515 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
516 }
517 return std::make_unique<SmallVectorMemoryBuffer>(
518 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
519 }
520
521 return codegenModule(TheModule, TM);
522}
523
524/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
525/// for caching, and in the \p Index for application during the ThinLTO
526/// backends. This is needed for correctness for exported symbols (ensure
527/// at least one copy kept) and a compile-time optimization (to drop duplicate
528/// copies when possible).
529static void resolvePrevailingInIndex(
530 ModuleSummaryIndex &Index,
531 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
532 &ResolvedODR,
533 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
534 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
535 &PrevailingCopy) {
536
537 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
538 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
539 // Not in map means that there was only one copy, which must be prevailing.
540 if (Prevailing == PrevailingCopy.end())
541 return true;
542 return Prevailing->second == S;
543 };
544
545 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
546 GlobalValue::GUID GUID,
547 GlobalValue::LinkageTypes NewLinkage) {
548 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
549 };
550
551 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
552 lto::Config Conf;
553 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
554 GUIDPreservedSymbols);
555}
556
557// Initialize the TargetMachine builder for a given Triple
558static void initTMBuilder(TargetMachineBuilder &TMBuilder,
559 const Triple &TheTriple) {
560 if (TMBuilder.MCpu.empty())
561 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
562 TMBuilder.TheTriple = std::move(TheTriple);
563}
564
565void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
566 MemoryBufferRef Buffer(Data, Identifier);
567
568 auto InputOrError = lto::InputFile::create(Object: Buffer);
569 if (!InputOrError)
570 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
571 toString(E: InputOrError.takeError()));
572
573 auto TripleStr = (*InputOrError)->getTargetTriple();
574 Triple TheTriple(TripleStr);
575
576 if (Modules.empty())
577 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
578 else if (TMBuilder.TheTriple != TheTriple) {
579 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
580 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
581 "supported");
582 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
583 }
584
585 Modules.emplace_back(args: std::move(*InputOrError));
586}
587
588void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
589 PreservedSymbols.insert(key: Name);
590}
591
592void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
593 // FIXME: At the moment, we don't take advantage of this extra information,
594 // we're conservatively considering cross-references as preserved.
595 // CrossReferencedSymbols.insert(Name);
596 PreservedSymbols.insert(key: Name);
597}
598
599// TargetMachine factory
600std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
601 std::string ErrMsg;
602 const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error&: ErrMsg);
603 if (!TheTarget) {
604 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
605 }
606
607 // Use MAttr as the default set of features.
608 SubtargetFeatures Features(MAttr);
609 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
610 std::string FeatureStr = Features.getString();
611
612 std::unique_ptr<TargetMachine> TM(
613 TheTarget->createTargetMachine(TT: TheTriple, CPU: MCpu, Features: FeatureStr, Options,
614 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
615 assert(TM && "Cannot create target machine");
616
617 return TM;
618}
619
620/**
621 * Produce the combined summary index from all the bitcode files:
622 * "thin-link".
623 */
624std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
625 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
626 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
627 for (auto &Mod : Modules) {
628 auto &M = Mod->getSingleBitcodeModule();
629 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
630 // FIXME diagnose
631 logAllUnhandledErrors(
632 E: std::move(Err), OS&: errs(),
633 ErrorBanner: "error: can't create module summary index for buffer: ");
634 return nullptr;
635 }
636 }
637 return CombinedIndex;
638}
639
640namespace {
641struct IsExported {
642 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
643 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
644
645 IsExported(
646 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
647 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
648 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
649
650 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
651 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
652 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
653 GUIDPreservedSymbols.count(V: VI.getGUID());
654 }
655};
656
657struct IsPrevailing {
658 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
659 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
660 &PrevailingCopy)
661 : PrevailingCopy(PrevailingCopy) {}
662
663 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
664 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
665 // Not in map means that there was only one copy, which must be prevailing.
666 if (Prevailing == PrevailingCopy.end())
667 return true;
668 return Prevailing->second == S;
669 };
670};
671} // namespace
672
673static void computeDeadSymbolsInIndex(
674 ModuleSummaryIndex &Index,
675 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
676 // We have no symbols resolution available. And can't do any better now in the
677 // case where the prevailing symbol is in a native object. It can be refined
678 // with linker information in the future.
679 auto isPrevailing = [&](GlobalValue::GUID G) {
680 return PrevailingType::Unknown;
681 };
682 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
683 /* ImportEnabled = */ true);
684}
685
686/**
687 * Perform promotion and renaming of exported internal functions.
688 * Index is updated to reflect linkage changes from weak resolution.
689 */
690void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
691 const lto::InputFile &File) {
692 auto ModuleCount = Index.modulePaths().size();
693 auto ModuleIdentifier = TheModule.getModuleIdentifier();
694
695 // Collect for each module the list of function it defines (GUID -> Summary).
696 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
697 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
698
699 // Convert the preserved symbols set from string to GUID
700 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
701 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
702
703 // Add used symbol to the preserved symbols.
704 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
705
706 // Compute "dead" symbols, we don't want to import/export these!
707 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
708
709 // Compute prevailing symbols
710 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
711 computePrevailingCopies(Index, PrevailingCopy);
712
713 // Generate import/export list
714 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
715 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
716 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
717 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
718 ExportLists);
719
720 // Resolve prevailing symbols
721 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
722 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
723 PrevailingCopy);
724
725 thinLTOFinalizeInModule(TheModule,
726 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
727 /*PropagateAttrs=*/false);
728
729 // Promote the exported values in the index, so that they are promoted
730 // in the module.
731 thinLTOInternalizeAndPromoteInIndex(
732 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
733 isPrevailing: IsPrevailing(PrevailingCopy));
734
735 // FIXME Set ClearDSOLocalOnDeclarations.
736 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
737}
738
739/**
740 * Perform cross-module importing for the module identified by ModuleIdentifier.
741 */
742void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
743 ModuleSummaryIndex &Index,
744 const lto::InputFile &File) {
745 auto ModuleMap = generateModuleMap(Modules);
746 auto ModuleCount = Index.modulePaths().size();
747
748 // Collect for each module the list of function it defines (GUID -> Summary).
749 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
750 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
751
752 // Convert the preserved symbols set from string to GUID
753 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
754 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
755
756 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
757
758 // Compute "dead" symbols, we don't want to import/export these!
759 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
760
761 // Compute prevailing symbols
762 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
763 computePrevailingCopies(Index, PrevailingCopy);
764
765 // Generate import/export list
766 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
767 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
768 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
769 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
770 ExportLists);
771 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
772
773 // FIXME Set ClearDSOLocalOnDeclarations.
774 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
775 /*ClearDSOLocalOnDeclarations=*/false);
776}
777
778/**
779 * Compute the list of summaries needed for importing into module.
780 */
781void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
782 Module &TheModule, ModuleSummaryIndex &Index,
783 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
784 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
785 auto ModuleCount = Index.modulePaths().size();
786 auto ModuleIdentifier = TheModule.getModuleIdentifier();
787
788 // Collect for each module the list of function it defines (GUID -> Summary).
789 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
790 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
791
792 // Convert the preserved symbols set from string to GUID
793 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
794 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
795
796 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
797
798 // Compute "dead" symbols, we don't want to import/export these!
799 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
800
801 // Compute prevailing symbols
802 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
803 computePrevailingCopies(Index, PrevailingCopy);
804
805 // Generate import/export list
806 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
807 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
808 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
809 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
810 ExportLists);
811
812 llvm::gatherImportedSummariesForModule(
813 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
814 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
815}
816
817/**
818 * Emit the list of files needed for importing into module.
819 */
820void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
821 ModuleSummaryIndex &Index,
822 const lto::InputFile &File) {
823 auto ModuleCount = Index.modulePaths().size();
824 auto ModuleIdentifier = TheModule.getModuleIdentifier();
825
826 // Collect for each module the list of function it defines (GUID -> Summary).
827 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
828 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
829
830 // Convert the preserved symbols set from string to GUID
831 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
832 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
833
834 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
835
836 // Compute "dead" symbols, we don't want to import/export these!
837 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
838
839 // Compute prevailing symbols
840 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
841 computePrevailingCopies(Index, PrevailingCopy);
842
843 // Generate import/export list
844 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
845 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
846 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
847 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
848 ExportLists);
849
850 // 'EmitImportsFiles' emits the list of modules from which to import from, and
851 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
852 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportsFiles`.
853 GVSummaryPtrSet DecSummaries;
854 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
855 llvm::gatherImportedSummariesForModule(
856 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
857 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
858
859 if (Error EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
860 ModuleToSummariesForIndex))
861 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
862 " to save imports lists\n");
863}
864
865/**
866 * Perform internalization. Runs promote and internalization together.
867 * Index is updated to reflect linkage changes.
868 */
869void ThinLTOCodeGenerator::internalize(Module &TheModule,
870 ModuleSummaryIndex &Index,
871 const lto::InputFile &File) {
872 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
873 auto ModuleCount = Index.modulePaths().size();
874 auto ModuleIdentifier = TheModule.getModuleIdentifier();
875
876 // Convert the preserved symbols set from string to GUID
877 auto GUIDPreservedSymbols =
878 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
879
880 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
881
882 // Collect for each module the list of function it defines (GUID -> Summary).
883 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
884 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
885
886 // Compute "dead" symbols, we don't want to import/export these!
887 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
888
889 // Compute prevailing symbols
890 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
891 computePrevailingCopies(Index, PrevailingCopy);
892
893 // Generate import/export list
894 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
895 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
896 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
897 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
898 ExportLists);
899 auto &ExportList = ExportLists[ModuleIdentifier];
900
901 // Be friendly and don't nuke totally the module when the client didn't
902 // supply anything to preserve.
903 if (ExportList.empty() && GUIDPreservedSymbols.empty())
904 return;
905
906 // Resolve prevailing symbols
907 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
908 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
909 PrevailingCopy);
910
911 // Promote the exported values in the index, so that they are promoted
912 // in the module.
913 thinLTOInternalizeAndPromoteInIndex(
914 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
915 isPrevailing: IsPrevailing(PrevailingCopy));
916
917 // FIXME Set ClearDSOLocalOnDeclarations.
918 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
919
920 // Internalization
921 thinLTOFinalizeInModule(TheModule,
922 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
923 /*PropagateAttrs=*/false);
924
925 thinLTOInternalizeModule(TheModule,
926 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
927}
928
929/**
930 * Perform post-importing ThinLTO optimizations.
931 */
932void ThinLTOCodeGenerator::optimize(Module &TheModule) {
933 initTMBuilder(TMBuilder, TheTriple: TheModule.getTargetTriple());
934
935 // Optimize now
936 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
937 DebugPassManager, Index: nullptr);
938}
939
940/// Write out the generated object file, either from CacheEntryPath or from
941/// OutputBuffer, preferring hard-link when possible.
942/// Returns the path to the generated file in SavedObjectsDirectoryPath.
943std::string
944ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
945 const MemoryBuffer &OutputBuffer) {
946 auto ArchName = TMBuilder.TheTriple.getArchName();
947 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
948 llvm::sys::path::append(path&: OutputPath,
949 a: Twine(count) + "." + ArchName + ".thinlto.o");
950 OutputPath.c_str(); // Ensure the string is null terminated.
951 if (sys::fs::exists(Path: OutputPath))
952 sys::fs::remove(path: OutputPath);
953
954 // We don't return a memory buffer to the linker, just a list of files.
955 if (!CacheEntryPath.empty()) {
956 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
957 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
958 if (!Err)
959 return std::string(OutputPath);
960 // Hard linking failed, try to copy.
961 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
962 if (!Err)
963 return std::string(OutputPath);
964 // Copy failed (could be because the CacheEntry was removed from the cache
965 // in the meantime by another process), fall back and try to write down the
966 // buffer to the output.
967 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
968 << "' to '" << OutputPath << "'\n";
969 }
970 // No cache entry, just write out the buffer.
971 std::error_code Err;
972 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
973 if (Err)
974 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
975 OS << OutputBuffer.getBuffer();
976 return std::string(OutputPath);
977}
978
979// Main entry point for the ThinLTO processing
980void ThinLTOCodeGenerator::run() {
981 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
982 llvm::scope_exit TimeTraceScopeExit([]() {
983 if (llvm::timeTraceProfilerEnabled())
984 llvm::timeTraceProfilerEnd();
985 });
986 // Prepare the resulting object vector
987 assert(ProducedBinaries.empty() && "The generator should not be reused");
988 if (SavedObjectsDirectoryPath.empty())
989 ProducedBinaries.resize(new_size: Modules.size());
990 else {
991 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
992 bool IsDir;
993 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
994 if (!IsDir)
995 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
996 ProducedBinaryFiles.resize(new_size: Modules.size());
997 }
998
999 if (CodeGenOnly) {
1000 // Perform only parallel codegen and return.
1001 DefaultThreadPool Pool;
1002 int count = 0;
1003 for (auto &Mod : Modules) {
1004 Pool.async(F: [&](int count) {
1005 LLVMContext Context;
1006 Context.setDiscardValueNames(LTODiscardValueNames);
1007
1008 // Parse module now
1009 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1010 /*IsImporting*/ false);
1011
1012 // CodeGen
1013 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1014 if (SavedObjectsDirectoryPath.empty())
1015 ProducedBinaries[count] = std::move(OutputBuffer);
1016 else
1017 ProducedBinaryFiles[count] =
1018 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1019 }, ArgList: count++);
1020 }
1021
1022 return;
1023 }
1024
1025 // Sequential linking phase
1026 auto Index = linkCombinedIndex();
1027
1028 // Save temps: index.
1029 if (!SaveTempsDir.empty()) {
1030 auto SaveTempPath = SaveTempsDir + "index.bc";
1031 std::error_code EC;
1032 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1033 if (EC)
1034 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1035 " to save optimized bitcode\n");
1036 writeIndexToFile(Index: *Index, Out&: OS);
1037 }
1038
1039
1040 // Prepare the module map.
1041 auto ModuleMap = generateModuleMap(Modules);
1042 auto ModuleCount = Modules.size();
1043
1044 // Collect for each module the list of function it defines (GUID -> Summary).
1045 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1046 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1047
1048 // Convert the preserved symbols set from string to GUID, this is needed for
1049 // computing the caching hash and the internalization.
1050 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1051 for (const auto &M : Modules)
1052 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1053 GUIDs&: GUIDPreservedSymbols);
1054
1055 // Add used symbol from inputs to the preserved symbols.
1056 for (const auto &M : Modules)
1057 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1058
1059 // Compute "dead" symbols, we don't want to import/export these!
1060 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1061
1062 // Currently there is no support for enabling whole program visibility via a
1063 // linker option in the old LTO API, but this call allows it to be specified
1064 // via the internal option. Must be done before WPD below.
1065 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1066 Index->setWithWholeProgramVisibility();
1067
1068 // FIXME: This needs linker information via a TBD new interface
1069 updateVCallVisibilityInIndex(Index&: *Index,
1070 /*WholeProgramVisibilityEnabledInLTO=*/false,
1071 // FIXME: These need linker information via a
1072 // TBD new interface.
1073 /*DynamicExportSymbols=*/{},
1074 /*VisibleToRegularObjSymbols=*/{});
1075
1076 // Perform index-based WPD. This will return immediately if there are
1077 // no index entries in the typeIdMetadata map (e.g. if we are instead
1078 // performing IR-based WPD in hybrid regular/thin LTO mode).
1079 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1080 std::set<GlobalValue::GUID> ExportedGUIDs;
1081 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1082 GUIDPreservedSymbols.insert_range(R&: ExportedGUIDs);
1083
1084 // Compute prevailing symbols
1085 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1086 computePrevailingCopies(Index: *Index, PrevailingCopy);
1087
1088 // Collect the import/export lists for all modules from the call-graph in the
1089 // combined index.
1090 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
1091 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1092 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1093 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1094 ExportLists);
1095
1096 // We use a std::map here to be able to have a defined ordering when
1097 // producing a hash for the cache entry.
1098 // FIXME: we should be able to compute the caching hash for the entry based
1099 // on the index, and nuke this map.
1100 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1101
1102 // Resolve prevailing symbols, this has to be computed early because it
1103 // impacts the caching.
1104 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1105 PrevailingCopy);
1106
1107 // Use global summary-based analysis to identify symbols that can be
1108 // internalized (because they aren't exported or preserved as per callback).
1109 // Changes are made in the index, consumed in the ThinLTO backends.
1110 updateIndexWPDForExports(Summary&: *Index,
1111 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1112 LocalWPDTargetsMap);
1113 thinLTOInternalizeAndPromoteInIndex(
1114 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1115 isPrevailing: IsPrevailing(PrevailingCopy));
1116
1117 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1118
1119 // Make sure that every module has an entry in the ExportLists, ImportList,
1120 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1121 // below.
1122 for (auto &Module : Modules) {
1123 auto ModuleIdentifier = Module->getName();
1124 ExportLists[ModuleIdentifier];
1125 ImportLists[ModuleIdentifier];
1126 ResolvedODR[ModuleIdentifier];
1127 ModuleToDefinedGVSummaries[ModuleIdentifier];
1128 }
1129
1130 std::vector<BitcodeModule *> ModulesVec;
1131 ModulesVec.reserve(n: Modules.size());
1132 for (auto &Mod : Modules)
1133 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1134 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1135
1136 if (llvm::timeTraceProfilerEnabled())
1137 llvm::timeTraceProfilerEnd();
1138
1139 TimeTraceScopeExit.release();
1140
1141 // Parallel optimizer + codegen
1142 {
1143 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1144 for (auto IndexCount : ModulesOrdering) {
1145 auto &Mod = Modules[IndexCount];
1146 Pool.async(F: [&](int count) {
1147 auto ModuleIdentifier = Mod->getName();
1148 auto &ExportList = ExportLists[ModuleIdentifier];
1149
1150 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1151
1152 // The module may be cached, this helps handling it.
1153 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1154 ImportLists[ModuleIdentifier], ExportList,
1155 ResolvedODR[ModuleIdentifier],
1156 DefinedGVSummaries, OptLevel, Freestanding,
1157 TMBuilder, MllvmArgs);
1158 auto CacheEntryPath = CacheEntry.getEntryPath();
1159
1160 {
1161 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1162 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1163 << " '" << CacheEntryPath << "' for buffer "
1164 << count << " " << ModuleIdentifier << "\n");
1165
1166 if (ErrOrBuffer) {
1167 // Cache Hit!
1168 if (SavedObjectsDirectoryPath.empty())
1169 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1170 else
1171 ProducedBinaryFiles[count] = writeGeneratedObject(
1172 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1173 return;
1174 }
1175 }
1176
1177 LLVMContext Context;
1178 Context.setDiscardValueNames(LTODiscardValueNames);
1179 Context.enableDebugTypeODRUniquing();
1180 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1181 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1182 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1183 if (!DiagFileOrErr) {
1184 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1185 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1186 "remarks");
1187 }
1188
1189 // Parse module now
1190 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1191 /*IsImporting*/ false);
1192
1193 // Save temps: original file.
1194 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1195
1196 auto &ImportList = ImportLists[ModuleIdentifier];
1197 // Run the main process now, and generates a binary
1198 auto OutputBuffer = ProcessThinLTOModule(
1199 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1200 ExportList, GUIDPreservedSymbols,
1201 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1202 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1203 DebugPassManager);
1204
1205 // Commit to the cache (if enabled)
1206 CacheEntry.write(OutputBuffer: *OutputBuffer);
1207
1208 if (SavedObjectsDirectoryPath.empty()) {
1209 // We need to generated a memory buffer for the linker.
1210 if (!CacheEntryPath.empty()) {
1211 // When cache is enabled, reload from the cache if possible.
1212 // Releasing the buffer from the heap and reloading it from the
1213 // cache file with mmap helps us to lower memory pressure.
1214 // The freed memory can be used for the next input file.
1215 // The final binary link will read from the VFS cache (hopefully!)
1216 // or from disk (if the memory pressure was too high).
1217 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1218 if (auto EC = ReloadedBufferOrErr.getError()) {
1219 // On error, keep the preexisting buffer and print a diagnostic.
1220 errs() << "remark: can't reload cached file '" << CacheEntryPath
1221 << "': " << EC.message() << "\n";
1222 } else {
1223 OutputBuffer = std::move(*ReloadedBufferOrErr);
1224 }
1225 }
1226 ProducedBinaries[count] = std::move(OutputBuffer);
1227 return;
1228 }
1229 ProducedBinaryFiles[count] = writeGeneratedObject(
1230 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1231 }, ArgList&: IndexCount);
1232 }
1233 }
1234
1235 Expected<bool> PrunedOrErr =
1236 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1237 if (!PrunedOrErr) {
1238 errs() << "Error: " << toString(E: PrunedOrErr.takeError()) << "\n";
1239 report_fatal_error(reason: "ThinLTO: failure to prune cache");
1240 }
1241
1242 // If statistics were requested, print them out now.
1243 if (llvm::AreStatisticsEnabled())
1244 llvm::PrintStatistics();
1245 reportAndResetTimings();
1246}
1247