1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Passes/RunCodeGen.h"
16#include "llvm/Support/CodeGen.h"
17#include "llvm/Support/CommandLine.h"
18
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/Analysis/ModuleSummaryAnalysis.h"
23#include "llvm/Analysis/ProfileSummaryInfo.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Bitcode/BitcodeReader.h"
26#include "llvm/Bitcode/BitcodeWriter.h"
27#include "llvm/Bitcode/BitcodeWriterPass.h"
28#include "llvm/CodeGen/CommandFlags.h"
29#include "llvm/Config/llvm-config.h"
30#include "llvm/IR/DebugInfo.h"
31#include "llvm/IR/DiagnosticPrinter.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/LLVMRemarkStreamer.h"
34#include "llvm/IR/LegacyPassManager.h"
35#include "llvm/IR/Mangler.h"
36#include "llvm/IR/PassTimingInfo.h"
37#include "llvm/IR/Verifier.h"
38#include "llvm/IRReader/IRReader.h"
39#include "llvm/LTO/LTO.h"
40#include "llvm/MC/TargetRegistry.h"
41#include "llvm/Object/IRObjectFile.h"
42#include "llvm/Passes/PassBuilder.h"
43#include "llvm/Passes/StandardInstrumentations.h"
44#include "llvm/Remarks/HotnessThresholdParser.h"
45#include "llvm/Support/CachePruning.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/Error.h"
48#include "llvm/Support/FileSystem.h"
49#include "llvm/Support/FormatVariadic.h"
50#include "llvm/Support/LibraryOptionsContext.h"
51#include "llvm/Support/Path.h"
52#include "llvm/Support/SHA1.h"
53#include "llvm/Support/SmallVectorMemoryBuffer.h"
54#include "llvm/Support/ThreadPool.h"
55#include "llvm/Support/Threading.h"
56#include "llvm/Support/ToolOutputFile.h"
57#include "llvm/Support/raw_ostream.h"
58#include "llvm/Target/TargetMachine.h"
59#include "llvm/TargetParser/SubtargetFeature.h"
60#include "llvm/Transforms/IPO/FunctionAttrs.h"
61#include "llvm/Transforms/IPO/FunctionImport.h"
62#include "llvm/Transforms/IPO/Internalize.h"
63#include "llvm/Transforms/IPO/SampleProfile.h"
64#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
65#include "llvm/Transforms/Utils/FunctionImportUtils.h"
66
67#if !defined(_MSC_VER) && !defined(__MINGW32__)
68#include <unistd.h>
69#else
70#include <io.h>
71#endif
72
73using namespace llvm;
74using namespace ThinLTOCodeGeneratorImpl;
75
76#define DEBUG_TYPE "thinlto"
77
78namespace llvm {
79// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
80extern cl::opt<bool> LTODiscardValueNames;
81extern cl::opt<std::string> RemarksFilename;
82extern cl::opt<std::string> RemarksPasses;
83extern cl::opt<bool> RemarksWithHotness;
84extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
85 RemarksHotnessThreshold;
86extern cl::opt<std::string> RemarksFormat;
87extern cl::opt<bool> LTORunCSIRInstr;
88extern cl::opt<std::string> LTOCSIRProfile;
89}
90
91// Default to using all available threads in the system, but using only one
92// thred per core, as indicated by the usage of
93// heavyweight_hardware_concurrency() below.
94static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
95
96// Simple helper to save temporary files for debug.
97static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
98 unsigned count, StringRef Suffix) {
99 if (TempDir.empty())
100 return;
101 // User asked to save temps, let dump the bitcode file after import.
102 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
103 std::error_code EC;
104 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
105 if (EC)
106 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
107 " to save optimized bitcode\n");
108 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
109}
110
111static const GlobalValueSummary *getFirstDefinitionForLinker(
112 ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
113 // If there is any strong definition anywhere, get it.
114 auto StrongDefForLinker = llvm::find_if(
115 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
116 auto Linkage = Summary->linkage();
117 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
118 !GlobalValue::isWeakForLinker(Linkage);
119 });
120 if (StrongDefForLinker != GVSummaryList.end())
121 return StrongDefForLinker->get();
122 // Get the first *linker visible* definition for this global in the summary
123 // list.
124 auto FirstDefForLinker = llvm::find_if(
125 Range&: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
126 auto Linkage = Summary->linkage();
127 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
128 });
129 // Extern templates can be emitted as available_externally.
130 if (FirstDefForLinker == GVSummaryList.end())
131 return nullptr;
132 return FirstDefForLinker->get();
133}
134
135// Populate map of GUID to the prevailing copy for any multiply defined
136// symbols. Currently assume first copy is prevailing, or any strong
137// definition. Can be refined with Linker information in the future.
138static void computePrevailingCopies(
139 const ModuleSummaryIndex &Index,
140 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
141 auto HasMultipleCopies =
142 [&](ArrayRef<std::unique_ptr<GlobalValueSummary>> GVSummaryList) {
143 return GVSummaryList.size() > 1;
144 };
145
146 for (auto &I : Index) {
147 if (HasMultipleCopies(I.second.getSummaryList()))
148 PrevailingCopy[I.first] =
149 getFirstDefinitionForLinker(GVSummaryList: I.second.getSummaryList());
150 }
151}
152
153static StringMap<lto::InputFile *>
154generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
155 StringMap<lto::InputFile *> ModuleMap;
156 for (auto &M : Modules) {
157 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
158 assert(!ModuleMap.contains(M->getName()) &&
159 "Expect unique Buffer Identifier");
160 ModuleMap[M->getName()] = M.get();
161 }
162 return ModuleMap;
163}
164
165static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
166 bool ClearDSOLocalOnDeclarations) {
167 renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations);
168}
169
170namespace {
171class ThinLTODiagnosticInfo : public DiagnosticInfo {
172 const Twine &Msg;
173public:
174 ThinLTODiagnosticInfo(const Twine &DiagMsg LLVM_LIFETIME_BOUND,
175 DiagnosticSeverity Severity = DS_Error)
176 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
177 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
178};
179}
180
181/// Verify the module and strip broken debug info.
182static void verifyLoadedModule(Module &TheModule) {
183 bool BrokenDebugInfo = false;
184 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
185 report_fatal_error(reason: "Broken module found, compilation aborted!");
186 if (BrokenDebugInfo) {
187 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
188 "Invalid debug info found, debug info will be stripped", DS_Warning));
189 StripDebugInfo(M&: TheModule);
190 }
191}
192
193static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
194 LLVMContext &Context,
195 bool Lazy,
196 bool IsImporting) {
197 auto &Mod = Input->getSingleBitcodeModule();
198 Expected<std::unique_ptr<Module>> ModuleOrErr =
199 Lazy ? Mod.getLazyModule(Context,
200 /* ShouldLazyLoadMetadata */ true, IsImporting)
201 : Mod.parseModule(Context);
202 if (!ModuleOrErr) {
203 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
204 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
205 SourceMgr::DK_Error, EIB.message());
206 Err.print(ProgName: "ThinLTO", S&: errs());
207 });
208 report_fatal_error(reason: "Can't load module, abort.");
209 }
210 if (!Lazy)
211 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
212 return std::move(*ModuleOrErr);
213}
214
215static void
216crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
217 StringMap<lto::InputFile *> &ModuleMap,
218 const FunctionImporter::ImportMapTy &ImportList,
219 bool ClearDSOLocalOnDeclarations) {
220 auto Loader = [&](StringRef Identifier) {
221 auto &Input = ModuleMap[Identifier];
222 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
223 /*Lazy=*/true, /*IsImporting*/ true);
224 };
225
226 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
227 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
228 if (!Result) {
229 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
230 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
231 SourceMgr::DK_Error, EIB.message());
232 Err.print(ProgName: "ThinLTO", S&: errs());
233 });
234 report_fatal_error(reason: "importFunctions failed");
235 }
236 // Verify again after cross-importing.
237 verifyLoadedModule(TheModule);
238}
239
240static void optimizeModule(Module &TheModule, TargetMachine &TM,
241 unsigned OptLevel, bool Freestanding,
242 bool DebugPassManager, ModuleSummaryIndex *Index) {
243 std::optional<PGOOptions> PGOOpt;
244 if (LTORunCSIRInstr) {
245 PGOOpt =
246 PGOOptions("", LTOCSIRProfile, "",
247 /*MemoryProfile=*/"", PGOOptions::IRUse,
248 PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default);
249 } else if (!LTOCSIRProfile.empty()) {
250 PGOOpt = PGOOptions(LTOCSIRProfile, "", "",
251 /*MemoryProfile=*/"", PGOOptions::IRUse,
252 PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default);
253 } else if (StringRef SampleProfileFile =
254 SampleProfileLoaderPass::getSampleProfileFile(
255 C: TheModule.getContext());
256 !SampleProfileFile.empty()) {
257 PGOOpt =
258 PGOOptions(SampleProfileFile.str(), "", "",
259 /*MemoryProfile=*/"", PGOOptions::SampleUse,
260 PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default);
261 }
262 LoopAnalysisManager LAM;
263 FunctionAnalysisManager FAM;
264 CGSCCAnalysisManager CGAM;
265 ModuleAnalysisManager MAM;
266
267 PassInstrumentationCallbacks PIC;
268 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
269 SI.registerCallbacks(PIC, MAM: &MAM);
270 PipelineTuningOptions PTO;
271 PTO.LoopVectorization = true;
272 PTO.SLPVectorization = true;
273 PassBuilder PB(TheModule.getContext(), &TM, PTO, PGOOpt, &PIC);
274
275 std::unique_ptr<TargetLibraryInfoImpl> TLII(
276 new TargetLibraryInfoImpl(TM.getTargetTriple(), TM.Options.VecLib));
277 if (Freestanding)
278 TLII->disableAllFunctions();
279 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
280
281 // Register all the basic analyses with the managers.
282 PB.registerModuleAnalyses(MAM);
283 PB.registerCGSCCAnalyses(CGAM);
284 PB.registerFunctionAnalyses(FAM);
285 PB.registerLoopAnalyses(LAM);
286 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
287
288 ModulePassManager MPM;
289
290 OptimizationLevel OL;
291
292 switch (OptLevel) {
293 default:
294 llvm_unreachable("Invalid optimization level");
295 case 0:
296 OL = OptimizationLevel::O0;
297 break;
298 case 1:
299 OL = OptimizationLevel::O1;
300 break;
301 case 2:
302 OL = OptimizationLevel::O2;
303 break;
304 case 3:
305 OL = OptimizationLevel::O3;
306 break;
307 }
308
309 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
310
311 MPM.run(IR&: TheModule, AM&: MAM);
312}
313
314static void
315addUsedSymbolToPreservedGUID(const lto::InputFile &File,
316 DenseSet<GlobalValue::GUID> &PreservedGUID) {
317 Triple TT(File.getTargetTriple());
318 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
319 TargetLibraryInfoImpl TLII(TT);
320 TargetLibraryInfo TLI(TLII);
321 for (const auto &Sym : File.symbols())
322 if (Sym.isUsed() || Sym.isLibcall(TLI, Libcalls))
323 PreservedGUID.insert(
324 V: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Sym.getIRName()));
325}
326
327// Convert the PreservedSymbols map from "Name" based to "GUID" based.
328static void computeGUIDPreservedSymbols(const lto::InputFile &File,
329 const StringSet<> &PreservedSymbols,
330 const Triple &TheTriple,
331 DenseSet<GlobalValue::GUID> &GUIDs) {
332 // Iterate the symbols in the input file and if the input has preserved symbol
333 // compute the GUID for the symbol.
334 for (const auto &Sym : File.symbols()) {
335 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
336 GUIDs.insert(V: GlobalValue::getGUIDAssumingExternalLinkage(
337 GlobalName: GlobalValue::getGlobalIdentifier(Name: Sym.getIRName(),
338 Linkage: GlobalValue::ExternalLinkage, FileName: "")));
339 }
340}
341
342static DenseSet<GlobalValue::GUID>
343computeGUIDPreservedSymbols(const lto::InputFile &File,
344 const StringSet<> &PreservedSymbols,
345 const Triple &TheTriple) {
346 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
347 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
348 GUIDs&: GUIDPreservedSymbols);
349 return GUIDPreservedSymbols;
350}
351
352static std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
353 TargetMachine &TM) {
354 SmallVector<char, 128> OutputBuffer;
355
356 // CodeGen
357 {
358 raw_svector_ostream OS(OutputBuffer);
359 if (runCodeGenPipeline(TM, M&: TheModule, OS, DwoOS: nullptr,
360 CGFT: CodeGenFileType::ObjectFile))
361 report_fatal_error(reason: "Failed to run codegen");
362 }
363 return std::make_unique<SmallVectorMemoryBuffer>(
364 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
365}
366
367namespace {
368/// Manage caching for a single Module.
369class ModuleCacheEntry {
370 SmallString<128> EntryPath;
371
372public:
373 // Create a cache entry. This compute a unique hash for the Module considering
374 // the current list of export/import, and offer an interface to query to
375 // access the content in the cache.
376 ModuleCacheEntry(
377 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
378 const FunctionImporter::ImportMapTy &ImportList,
379 const FunctionImporter::ExportSetTy &ExportList,
380 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
381 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
382 bool Freestanding, const TargetMachineBuilder &TMBuilder,
383 ArrayRef<std::string> MllvmArgs) {
384 if (CachePath.empty())
385 return;
386
387 if (!Index.modulePaths().count(Key: ModuleID))
388 // The module does not have an entry, it can't have a hash at all
389 return;
390
391 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
392 P: [](uint32_t V) { return V == 0; }))
393 // No hash entry, no caching!
394 return;
395
396 llvm::lto::Config Conf;
397 Conf.OptLevel = OptLevel;
398 Conf.Options = TMBuilder.Options;
399 Conf.CPU = TMBuilder.MCpu;
400 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
401 Conf.RelocModel = TMBuilder.RelocModel;
402 Conf.CGOptLevel = TMBuilder.CGOptLevel;
403 Conf.Freestanding = Freestanding;
404 append_range(C&: Conf.MllvmArgs, R&: MllvmArgs);
405 std::string Key =
406 computeLTOCacheKey(Conf, Index, ModuleID, ImportList, ExportList,
407 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
408
409 // This choice of file name allows the cache to be pruned (see pruneCache()
410 // in include/llvm/Support/CachePruning.h).
411 sys::path::append(path&: EntryPath, a: CachePath, b: Twine("llvmcache-", Key));
412 }
413
414 // Access the path to this entry in the cache.
415 StringRef getEntryPath() { return EntryPath; }
416
417 // Try loading the buffer for this cache entry.
418 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
419 if (EntryPath.empty())
420 return std::error_code();
421 SmallString<64> ResultPath;
422 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
423 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
424 if (!FDOrErr)
425 return errorToErrorCode(Err: FDOrErr.takeError());
426 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
427 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
428 sys::fs::closeFile(F&: *FDOrErr);
429 return MBOrErr;
430 }
431
432 // Cache the Produced object file
433 void write(const MemoryBuffer &OutputBuffer) {
434 if (EntryPath.empty())
435 return;
436
437 if (auto Err = llvm::writeToOutput(
438 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
439 OS << OutputBuffer.getBuffer();
440 return llvm::Error::success();
441 }))
442 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
443 Vals&: EntryPath,
444 Vals: toString(E: std::move(Err)).c_str()));
445 }
446};
447} // end anonymous namespace
448
449static std::unique_ptr<MemoryBuffer>
450ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
451 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
452 const FunctionImporter::ImportMapTy &ImportList,
453 const FunctionImporter::ExportSetTy &ExportList,
454 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
455 const GVSummaryMapTy &DefinedGlobals,
456 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
457 bool DisableCodeGen, StringRef SaveTempsDir,
458 bool Freestanding, unsigned OptLevel, unsigned count,
459 bool DebugPassManager) {
460 // "Benchmark"-like optimization: single-source case
461 bool SingleModule = (ModuleMap.size() == 1);
462
463 // When linking an ELF shared object, dso_local should be dropped. We
464 // conservatively do this for -fpic.
465 bool ClearDSOLocalOnDeclarations =
466 TheModule.getTargetTriple().isOSBinFormatELF() &&
467 TM.getRelocationModel() != Reloc::Static &&
468 TheModule.getPIELevel() == PIELevel::Default;
469
470 if (!SingleModule) {
471 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
472
473 // Apply summary-based prevailing-symbol resolution decisions.
474 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
475
476 // Save temps: after promotion.
477 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
478 }
479
480 // Be friendly and don't nuke totally the module when the client didn't
481 // supply anything to preserve.
482 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
483 // Apply summary-based internalization decisions.
484 thinLTOInternalizeModule(TheModule, DefinedGlobals);
485 }
486
487 // Save internalized bitcode
488 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
489
490 if (!SingleModule)
491 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
492 ClearDSOLocalOnDeclarations);
493
494 // Do this after any importing so that imported code is updated.
495 // See comment at call to updateVCallVisibilityInIndex() for why
496 // WholeProgramVisibilityEnabledInLTO is false.
497 updatePublicTypeTestCalls(M&: TheModule,
498 /* WholeProgramVisibilityEnabledInLTO */ false);
499
500 // Save temps: after cross-module import.
501 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
502
503 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
504 Index: &Index);
505
506 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
507
508 if (DisableCodeGen) {
509 // Configured to stop before CodeGen, serialize the bitcode and return.
510 SmallVector<char, 128> OutputBuffer;
511 {
512 raw_svector_ostream OS(OutputBuffer);
513 ProfileSummaryInfo PSI(TheModule);
514 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
515 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
516 }
517 return std::make_unique<SmallVectorMemoryBuffer>(
518 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
519 }
520
521 return codegenModule(TheModule, TM);
522}
523
524/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
525/// for caching, and in the \p Index for application during the ThinLTO
526/// backends. This is needed for correctness for exported symbols (ensure
527/// at least one copy kept) and a compile-time optimization (to drop duplicate
528/// copies when possible).
529static void resolvePrevailingInIndex(
530 ModuleSummaryIndex &Index,
531 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
532 &ResolvedODR,
533 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
534 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
535 &PrevailingCopy) {
536
537 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
538 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
539 // Not in map means that there was only one copy, which must be prevailing.
540 if (Prevailing == PrevailingCopy.end())
541 return true;
542 return Prevailing->second == S;
543 };
544
545 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
546 GlobalValue::GUID GUID,
547 GlobalValue::LinkageTypes NewLinkage) {
548 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
549 };
550
551 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
552 lto::Config Conf;
553 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
554 GUIDPreservedSymbols);
555}
556
557// Initialize the TargetMachine builder for a given Triple
558static void initTMBuilder(const cl::LibraryOptionsContext &LibraryOptions,
559 TargetMachineBuilder &TMBuilder,
560 const Triple &TheTriple) {
561 if (TMBuilder.MCpu.empty())
562 TMBuilder.MCpu = lto::getThinLTODefaultCPU(TheTriple);
563 TMBuilder.TheTriple = std::move(TheTriple);
564 // If data-sections is not explicitly set or unset, set data-sections by
565 // default to match the behaviour of lld and gold plugin.
566 if (!codegen::getExplicitDataSections(C: LibraryOptions))
567 TMBuilder.Options.DataSections = true;
568}
569
570void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
571 MemoryBufferRef Buffer(Data, Identifier);
572
573 auto InputOrError = lto::InputFile::create(Object: Buffer);
574 if (!InputOrError)
575 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
576 toString(E: InputOrError.takeError()));
577
578 auto TripleStr = (*InputOrError)->getTargetTriple();
579 Triple TheTriple(TripleStr);
580
581 if (Modules.empty())
582 initTMBuilder(LibraryOptions, TMBuilder, TheTriple: Triple(TheTriple));
583 else if (TMBuilder.TheTriple != TheTriple) {
584 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
585 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
586 "supported");
587 initTMBuilder(LibraryOptions, TMBuilder,
588 TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
589 }
590
591 Modules.emplace_back(args: std::move(*InputOrError));
592}
593
594void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
595 PreservedSymbols.insert(key: Name);
596}
597
598void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
599 // FIXME: At the moment, we don't take advantage of this extra information,
600 // we're conservatively considering cross-references as preserved.
601 // CrossReferencedSymbols.insert(Name);
602 PreservedSymbols.insert(key: Name);
603}
604
605// TargetMachine factory
606std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
607 std::string ErrMsg;
608 const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error&: ErrMsg);
609 if (!TheTarget) {
610 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
611 }
612
613 // Use MAttr as the default set of features.
614 SubtargetFeatures Features(MAttr);
615 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
616 std::string FeatureStr = Features.getString();
617
618 std::unique_ptr<TargetMachine> TM(
619 TheTarget->createTargetMachine(TT: TheTriple, CPU: MCpu, Features: FeatureStr, Options,
620 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
621 assert(TM && "Cannot create target machine");
622
623 return TM;
624}
625
626/**
627 * Produce the combined summary index from all the bitcode files:
628 * "thin-link".
629 */
630std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
631 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
632 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
633 for (auto &Mod : Modules) {
634 auto &M = Mod->getSingleBitcodeModule();
635 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
636 // FIXME diagnose
637 logAllUnhandledErrors(
638 E: std::move(Err), OS&: errs(),
639 ErrorBanner: "error: can't create module summary index for buffer: ");
640 return nullptr;
641 }
642 }
643 return CombinedIndex;
644}
645
646namespace {
647struct IsExported {
648 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
649 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
650
651 IsExported(
652 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
653 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
654 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
655
656 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
657 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
658 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
659 GUIDPreservedSymbols.count(V: VI.getGUID());
660 }
661};
662
663struct IsPrevailing {
664 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
665 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
666 &PrevailingCopy)
667 : PrevailingCopy(PrevailingCopy) {}
668
669 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
670 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
671 // Not in map means that there was only one copy, which must be prevailing.
672 if (Prevailing == PrevailingCopy.end())
673 return true;
674 return Prevailing->second == S;
675 };
676};
677} // namespace
678
679static void computeDeadSymbolsInIndex(
680 ModuleSummaryIndex &Index,
681 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
682 // We have no symbols resolution available. And can't do any better now in the
683 // case where the prevailing symbol is in a native object. It can be refined
684 // with linker information in the future.
685 auto isPrevailing = [&](GlobalValue::GUID G) {
686 return PrevailingType::Unknown;
687 };
688 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
689 /* ImportEnabled = */ true);
690}
691
692/**
693 * Perform promotion and renaming of exported internal functions.
694 * Index is updated to reflect linkage changes from weak resolution.
695 */
696void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
697 const lto::InputFile &File) {
698 auto ModuleCount = Index.modulePaths().size();
699 auto ModuleIdentifier = TheModule.getModuleIdentifier();
700
701 // Collect for each module the list of function it defines (GUID -> Summary).
702 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
703 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
704
705 // Convert the preserved symbols set from string to GUID
706 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
707 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
708
709 // Add used symbol to the preserved symbols.
710 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
711
712 // Compute "dead" symbols, we don't want to import/export these!
713 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
714
715 // Compute prevailing symbols
716 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
717 computePrevailingCopies(Index, PrevailingCopy);
718
719 // Generate import/export list
720 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
721 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
722 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
723 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
724 ExportLists);
725
726 // Resolve prevailing symbols
727 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
728 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
729 PrevailingCopy);
730
731 thinLTOFinalizeInModule(TheModule,
732 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
733 /*PropagateAttrs=*/false);
734
735 // Promote the exported values in the index, so that they are promoted
736 // in the module.
737 thinLTOInternalizeAndPromoteInIndex(
738 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
739 isPrevailing: IsPrevailing(PrevailingCopy));
740
741 // FIXME Set ClearDSOLocalOnDeclarations.
742 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
743}
744
745/**
746 * Perform cross-module importing for the module identified by ModuleIdentifier.
747 */
748void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
749 ModuleSummaryIndex &Index,
750 const lto::InputFile &File) {
751 auto ModuleMap = generateModuleMap(Modules);
752 auto ModuleCount = Index.modulePaths().size();
753
754 // Collect for each module the list of function it defines (GUID -> Summary).
755 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
756 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
757
758 // Convert the preserved symbols set from string to GUID
759 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
760 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
761
762 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
763
764 // Compute "dead" symbols, we don't want to import/export these!
765 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
766
767 // Compute prevailing symbols
768 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
769 computePrevailingCopies(Index, PrevailingCopy);
770
771 // Generate import/export list
772 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
773 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
774 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
775 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
776 ExportLists);
777 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
778
779 // FIXME Set ClearDSOLocalOnDeclarations.
780 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
781 /*ClearDSOLocalOnDeclarations=*/false);
782}
783
784/**
785 * Compute the list of summaries needed for importing into module.
786 */
787void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
788 Module &TheModule, ModuleSummaryIndex &Index,
789 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
790 GVSummaryPtrSet &DecSummaries, const lto::InputFile &File) {
791 auto ModuleCount = Index.modulePaths().size();
792 auto ModuleIdentifier = TheModule.getModuleIdentifier();
793
794 // Collect for each module the list of function it defines (GUID -> Summary).
795 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
796 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
797
798 // Convert the preserved symbols set from string to GUID
799 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
800 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
801
802 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
803
804 // Compute "dead" symbols, we don't want to import/export these!
805 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
806
807 // Compute prevailing symbols
808 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
809 computePrevailingCopies(Index, PrevailingCopy);
810
811 // Generate import/export list
812 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
813 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
814 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
815 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
816 ExportLists);
817
818 llvm::gatherImportedSummariesForModule(
819 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
820 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
821}
822
823/**
824 * Emit the list of files needed for importing into module.
825 */
826void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
827 ModuleSummaryIndex &Index,
828 const lto::InputFile &File) {
829 auto ModuleCount = Index.modulePaths().size();
830 auto ModuleIdentifier = TheModule.getModuleIdentifier();
831
832 // Collect for each module the list of function it defines (GUID -> Summary).
833 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
834 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
835
836 // Convert the preserved symbols set from string to GUID
837 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
838 File, PreservedSymbols, TheTriple: TheModule.getTargetTriple());
839
840 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
841
842 // Compute "dead" symbols, we don't want to import/export these!
843 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
844
845 // Compute prevailing symbols
846 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
847 computePrevailingCopies(Index, PrevailingCopy);
848
849 // Generate import/export list
850 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
851 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
852 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
853 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
854 ExportLists);
855
856 // 'EmitImportsFiles' emits the list of modules from which to import from, and
857 // the set of keys in `ModuleToSummariesForIndex` should be a superset of keys
858 // in `DecSummaries`, so no need to use `DecSummaries` in `EmitImportsFiles`.
859 GVSummaryPtrSet DecSummaries;
860 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
861 llvm::gatherImportedSummariesForModule(
862 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
863 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex, DecSummaries);
864
865 if (Error EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
866 ModuleToSummariesForIndex))
867 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
868 " to save imports lists\n");
869}
870
871/**
872 * Perform internalization. Runs promote and internalization together.
873 * Index is updated to reflect linkage changes.
874 */
875void ThinLTOCodeGenerator::internalize(Module &TheModule,
876 ModuleSummaryIndex &Index,
877 const lto::InputFile &File) {
878 initTMBuilder(LibraryOptions, TMBuilder, TheTriple: TheModule.getTargetTriple());
879 auto ModuleCount = Index.modulePaths().size();
880 auto ModuleIdentifier = TheModule.getModuleIdentifier();
881
882 // Convert the preserved symbols set from string to GUID
883 auto GUIDPreservedSymbols =
884 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
885
886 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
887
888 // Collect for each module the list of function it defines (GUID -> Summary).
889 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
890 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
891
892 // Compute "dead" symbols, we don't want to import/export these!
893 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
894
895 // Compute prevailing symbols
896 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
897 computePrevailingCopies(Index, PrevailingCopy);
898
899 // Generate import/export list
900 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
901 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
902 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
903 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
904 ExportLists);
905 auto &ExportList = ExportLists[ModuleIdentifier];
906
907 // Be friendly and don't nuke totally the module when the client didn't
908 // supply anything to preserve.
909 if (ExportList.empty() && GUIDPreservedSymbols.empty())
910 return;
911
912 // Resolve prevailing symbols
913 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
914 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
915 PrevailingCopy);
916
917 // Promote the exported values in the index, so that they are promoted
918 // in the module.
919 thinLTOInternalizeAndPromoteInIndex(
920 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
921 isPrevailing: IsPrevailing(PrevailingCopy));
922
923 // FIXME Set ClearDSOLocalOnDeclarations.
924 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
925
926 // Internalization
927 thinLTOFinalizeInModule(TheModule,
928 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
929 /*PropagateAttrs=*/false);
930
931 thinLTOInternalizeModule(TheModule,
932 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
933}
934
935/**
936 * Perform post-importing ThinLTO optimizations.
937 */
938void ThinLTOCodeGenerator::optimize(Module &TheModule) {
939 initTMBuilder(LibraryOptions, TMBuilder, TheTriple: TheModule.getTargetTriple());
940
941 // Optimize now
942 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
943 DebugPassManager, Index: nullptr);
944}
945
946/// Write out the generated object file, either from CacheEntryPath or from
947/// OutputBuffer, preferring hard-link when possible.
948/// Returns the path to the generated file in SavedObjectsDirectoryPath.
949std::string
950ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
951 const MemoryBuffer &OutputBuffer) {
952 auto ArchName = TMBuilder.TheTriple.getArchName();
953 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
954 llvm::sys::path::append(path&: OutputPath,
955 a: Twine(count) + "." + ArchName + ".thinlto.o");
956 OutputPath.c_str(); // Ensure the string is null terminated.
957 if (sys::fs::exists(Path: OutputPath))
958 sys::fs::remove(path: OutputPath);
959
960 // We don't return a memory buffer to the linker, just a list of files.
961 if (!CacheEntryPath.empty()) {
962 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
963 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
964 if (!Err)
965 return std::string(OutputPath);
966 // Hard linking failed, try to copy.
967 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
968 if (!Err)
969 return std::string(OutputPath);
970 // Copy failed (could be because the CacheEntry was removed from the cache
971 // in the meantime by another process), fall back and try to write down the
972 // buffer to the output.
973 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
974 << "' to '" << OutputPath << "'\n";
975 }
976 // No cache entry, just write out the buffer.
977 std::error_code Err;
978 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
979 if (Err)
980 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
981 OS << OutputBuffer.getBuffer();
982 return std::string(OutputPath);
983}
984
985// Main entry point for the ThinLTO processing
986void ThinLTOCodeGenerator::run() {
987 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
988 llvm::scope_exit TimeTraceScopeExit([]() {
989 if (llvm::timeTraceProfilerEnabled())
990 llvm::timeTraceProfilerEnd();
991 });
992 // Prepare the resulting object vector
993 assert(ProducedBinaries.empty() && "The generator should not be reused");
994 if (SavedObjectsDirectoryPath.empty())
995 ProducedBinaries.resize(new_size: Modules.size());
996 else {
997 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
998 bool IsDir;
999 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
1000 if (!IsDir)
1001 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
1002 ProducedBinaryFiles.resize(new_size: Modules.size());
1003 }
1004
1005 if (CodeGenOnly) {
1006 // Perform only parallel codegen and return.
1007 DefaultThreadPool Pool;
1008 int count = 0;
1009 for (auto &Mod : Modules) {
1010 Pool.async(F: [&](int count) {
1011 LLVMContext Context;
1012 Context.getLibraryOptions().assign(Other: LibraryOptions);
1013 Context.setDiscardValueNames(LTODiscardValueNames);
1014
1015 // Parse module now
1016 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1017 /*IsImporting*/ false);
1018
1019 // CodeGen
1020 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1021 if (SavedObjectsDirectoryPath.empty())
1022 ProducedBinaries[count] = std::move(OutputBuffer);
1023 else
1024 ProducedBinaryFiles[count] =
1025 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1026 }, ArgList: count++);
1027 }
1028
1029 return;
1030 }
1031
1032 // Sequential linking phase
1033 auto Index = linkCombinedIndex();
1034
1035 // Save temps: index.
1036 if (!SaveTempsDir.empty()) {
1037 auto SaveTempPath = SaveTempsDir + "index.bc";
1038 std::error_code EC;
1039 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1040 if (EC)
1041 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1042 " to save optimized bitcode\n");
1043 writeIndexToFile(Index: *Index, Out&: OS);
1044 }
1045
1046
1047 // Prepare the module map.
1048 auto ModuleMap = generateModuleMap(Modules);
1049 auto ModuleCount = Modules.size();
1050
1051 // Collect for each module the list of function it defines (GUID -> Summary).
1052 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1053 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1054
1055 // Convert the preserved symbols set from string to GUID, this is needed for
1056 // computing the caching hash and the internalization.
1057 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1058 for (const auto &M : Modules)
1059 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1060 GUIDs&: GUIDPreservedSymbols);
1061
1062 // Add used symbol from inputs to the preserved symbols.
1063 for (const auto &M : Modules)
1064 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1065
1066 // Compute "dead" symbols, we don't want to import/export these!
1067 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1068
1069 // Currently there is no support for enabling whole program visibility via a
1070 // linker option in the old LTO API, but this call allows it to be specified
1071 // via the internal option. Must be done before WPD below.
1072 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1073 Index->setWithWholeProgramVisibility();
1074
1075 // FIXME: This needs linker information via a TBD new interface
1076 updateVCallVisibilityInIndex(Index&: *Index,
1077 /*WholeProgramVisibilityEnabledInLTO=*/false,
1078 // FIXME: These need linker information via a
1079 // TBD new interface.
1080 /*DynamicExportSymbols=*/{},
1081 /*VisibleToRegularObjSymbols=*/{});
1082
1083 // Perform index-based WPD. This will return immediately if there are
1084 // no index entries in the typeIdMetadata map (e.g. if we are instead
1085 // performing IR-based WPD in hybrid regular/thin LTO mode).
1086 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1087 std::set<GlobalValue::GUID> ExportedGUIDs;
1088 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap,
1089 ExternallyVisibleSymbolNamesPtr: nullptr, Ctx: nullptr);
1090 GUIDPreservedSymbols.insert_range(R&: ExportedGUIDs);
1091
1092 // Compute prevailing symbols
1093 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1094 computePrevailingCopies(Index: *Index, PrevailingCopy);
1095
1096 // Collect the import/export lists for all modules from the call-graph in the
1097 // combined index.
1098 FunctionImporter::ImportListsTy ImportLists(ModuleCount);
1099 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1100 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1101 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1102 ExportLists);
1103
1104 // We use a std::map here to be able to have a defined ordering when
1105 // producing a hash for the cache entry.
1106 // FIXME: we should be able to compute the caching hash for the entry based
1107 // on the index, and nuke this map.
1108 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1109
1110 // Resolve prevailing symbols, this has to be computed early because it
1111 // impacts the caching.
1112 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1113 PrevailingCopy);
1114
1115 // Use global summary-based analysis to identify symbols that can be
1116 // internalized (because they aren't exported or preserved as per callback).
1117 // Changes are made in the index, consumed in the ThinLTO backends.
1118 updateIndexWPDForExports(Summary&: *Index,
1119 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1120 LocalWPDTargetsMap);
1121 thinLTOInternalizeAndPromoteInIndex(
1122 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1123 isPrevailing: IsPrevailing(PrevailingCopy));
1124
1125 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1126
1127 // Make sure that every module has an entry in the ExportLists, ImportList,
1128 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1129 // below.
1130 for (auto &Module : Modules) {
1131 auto ModuleIdentifier = Module->getName();
1132 ExportLists[ModuleIdentifier];
1133 ImportLists[ModuleIdentifier];
1134 ResolvedODR[ModuleIdentifier];
1135 ModuleToDefinedGVSummaries[ModuleIdentifier];
1136 }
1137
1138 std::vector<BitcodeModule *> ModulesVec;
1139 ModulesVec.reserve(n: Modules.size());
1140 for (auto &Mod : Modules)
1141 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1142 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1143
1144 if (llvm::timeTraceProfilerEnabled())
1145 llvm::timeTraceProfilerEnd();
1146
1147 TimeTraceScopeExit.release();
1148
1149 // Parallel optimizer + codegen
1150 {
1151 DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1152 for (auto IndexCount : ModulesOrdering) {
1153 auto &Mod = Modules[IndexCount];
1154 Pool.async(F: [&](int count) {
1155 auto ModuleIdentifier = Mod->getName();
1156 auto &ExportList = ExportLists[ModuleIdentifier];
1157
1158 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1159
1160 // The module may be cached, this helps handling it.
1161 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1162 ImportLists[ModuleIdentifier], ExportList,
1163 ResolvedODR[ModuleIdentifier],
1164 DefinedGVSummaries, OptLevel, Freestanding,
1165 TMBuilder, MllvmArgs);
1166 auto CacheEntryPath = CacheEntry.getEntryPath();
1167
1168 {
1169 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1170 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1171 << " '" << CacheEntryPath << "' for buffer "
1172 << count << " " << ModuleIdentifier << "\n");
1173
1174 if (ErrOrBuffer) {
1175 // Cache Hit!
1176 if (SavedObjectsDirectoryPath.empty())
1177 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1178 else
1179 ProducedBinaryFiles[count] = writeGeneratedObject(
1180 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1181 return;
1182 }
1183 }
1184
1185 LLVMContext Context;
1186 Context.getLibraryOptions().assign(Other: LibraryOptions);
1187 Context.setDiscardValueNames(LTODiscardValueNames);
1188 Context.enableDebugTypeODRUniquing();
1189 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1190 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1191 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1192 if (!DiagFileOrErr) {
1193 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1194 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1195 "remarks");
1196 }
1197
1198 // Parse module now
1199 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1200 /*IsImporting*/ false);
1201
1202 // Save temps: original file.
1203 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1204
1205 auto &ImportList = ImportLists[ModuleIdentifier];
1206 // Run the main process now, and generates a binary
1207 auto OutputBuffer = ProcessThinLTOModule(
1208 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1209 ExportList, GUIDPreservedSymbols,
1210 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1211 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1212 DebugPassManager);
1213
1214 // Commit to the cache (if enabled)
1215 CacheEntry.write(OutputBuffer: *OutputBuffer);
1216
1217 if (SavedObjectsDirectoryPath.empty()) {
1218 // We need to generated a memory buffer for the linker.
1219 if (!CacheEntryPath.empty()) {
1220 // When cache is enabled, reload from the cache if possible.
1221 // Releasing the buffer from the heap and reloading it from the
1222 // cache file with mmap helps us to lower memory pressure.
1223 // The freed memory can be used for the next input file.
1224 // The final binary link will read from the VFS cache (hopefully!)
1225 // or from disk (if the memory pressure was too high).
1226 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1227 if (auto EC = ReloadedBufferOrErr.getError()) {
1228 // On error, keep the preexisting buffer and print a diagnostic.
1229 errs() << "remark: can't reload cached file '" << CacheEntryPath
1230 << "': " << EC.message() << "\n";
1231 } else {
1232 OutputBuffer = std::move(*ReloadedBufferOrErr);
1233 }
1234 }
1235 ProducedBinaries[count] = std::move(OutputBuffer);
1236 return;
1237 }
1238 ProducedBinaryFiles[count] = writeGeneratedObject(
1239 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1240 }, ArgList&: IndexCount);
1241 }
1242 }
1243
1244 Expected<bool> PrunedOrErr =
1245 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1246 if (!PrunedOrErr) {
1247 errs() << "Error: " << toString(E: PrunedOrErr.takeError()) << "\n";
1248 report_fatal_error(reason: "ThinLTO: failure to prune cache");
1249 }
1250
1251 // If statistics were requested, print them out now.
1252 if (llvm::AreStatisticsEnabled())
1253 llvm::PrintStatistics();
1254 reportAndResetTimings();
1255}
1256