1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/StableHashing.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/StackSafetyAnalysis.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/CGData/CodeGenData.h"
26#include "llvm/CodeGen/Analysis.h"
27#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/AutoUpgrade.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/LLVMRemarkStreamer.h"
33#include "llvm/IR/LegacyPassManager.h"
34#include "llvm/IR/Mangler.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/IR/RuntimeLibcalls.h"
37#include "llvm/LTO/LTOBackend.h"
38#include "llvm/Linker/IRMover.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Object/IRObjectFile.h"
41#include "llvm/Support/Caching.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Compiler.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/MemoryBuffer.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/Process.h"
49#include "llvm/Support/SHA1.h"
50#include "llvm/Support/Signals.h"
51#include "llvm/Support/SourceMgr.h"
52#include "llvm/Support/ThreadPool.h"
53#include "llvm/Support/Threading.h"
54#include "llvm/Support/TimeProfiler.h"
55#include "llvm/Support/ToolOutputFile.h"
56#include "llvm/Support/VCSRevision.h"
57#include "llvm/Support/raw_ostream.h"
58#include "llvm/Target/TargetOptions.h"
59#include "llvm/Transforms/IPO.h"
60#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
61#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
62#include "llvm/Transforms/Utils/FunctionImportUtils.h"
63#include "llvm/Transforms/Utils/SplitModule.h"
64
65#include <optional>
66#include <set>
67
68using namespace llvm;
69using namespace lto;
70using namespace object;
71
72#define DEBUG_TYPE "lto"
73
74Error LTO::setupOptimizationRemarks() {
75 // Setup the remark streamer according to the provided configuration.
76 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
77 Context&: RegularLTO.Ctx, RemarksFilename: Conf.RemarksFilename, RemarksPasses: Conf.RemarksPasses,
78 RemarksFormat: Conf.RemarksFormat, RemarksWithHotness: Conf.RemarksWithHotness,
79 RemarksHotnessThreshold: Conf.RemarksHotnessThreshold);
80 if (!DiagFileOrErr)
81 return DiagFileOrErr.takeError();
82
83 DiagnosticOutputFile = std::move(*DiagFileOrErr);
84
85 // Create a dummy function to serve as a context for LTO-link remarks.
86 // This is required because OptimizationRemark requires a valid Function,
87 // and in ThinLTO we may not have any IR functions available during the
88 // thin link. Host it in a private module to avoid interfering with the LTO
89 // process.
90 if (!LinkerRemarkFunction) {
91 DummyModule = std::make_unique<Module>(args: "remark_dummy", args&: RegularLTO.Ctx);
92 LinkerRemarkFunction = Function::Create(
93 Ty: FunctionType::get(Result: Type::getVoidTy(C&: RegularLTO.Ctx), isVarArg: false),
94 Linkage: GlobalValue::ExternalLinkage, N: "thinlto_remark_dummy",
95 M: DummyModule.get());
96 }
97
98 return Error::success();
99}
100
101void LTO::emitRemark(OptimizationRemark &Remark) {
102 const Function &F = Remark.getFunction();
103 OptimizationRemarkEmitter ORE(const_cast<Function *>(&F));
104 ORE.emit(OptDiag&: Remark);
105}
106
107static cl::opt<bool>
108 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(Val: false), cl::Hidden,
109 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
110namespace llvm {
111extern cl::opt<bool> CodeGenDataThinLTOTwoRounds;
112extern cl::opt<bool> ForceImportAll;
113extern cl::opt<bool> AlwaysRenamePromotedLocals;
114} // end namespace llvm
115
116namespace llvm {
117/// Enable global value internalization in LTO.
118cl::opt<bool> EnableLTOInternalization(
119 "enable-lto-internalization", cl::init(Val: true), cl::Hidden,
120 cl::desc("Enable global value internalization in LTO"));
121
122static cl::opt<bool>
123 LTOKeepSymbolCopies("lto-keep-symbol-copies", cl::init(Val: false), cl::Hidden,
124 cl::desc("Keep copies of symbols in LTO indexing"));
125
126/// Indicate we are linking with an allocator that supports hot/cold operator
127/// new interfaces.
128extern cl::opt<bool> SupportsHotColdNew;
129
130/// Enable MemProf context disambiguation for thin link.
131extern cl::opt<bool> EnableMemProfContextDisambiguation;
132} // namespace llvm
133
134// Computes a unique hash for the Module considering the current list of
135// export/import and other global analysis results.
136// Returns the hash in its hexadecimal representation.
137std::string llvm::computeLTOCacheKey(
138 const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID,
139 const FunctionImporter::ImportMapTy &ImportList,
140 const FunctionImporter::ExportSetTy &ExportList,
141 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
142 const GVSummaryMapTy &DefinedGlobals,
143 const DenseSet<GlobalValue::GUID> &CfiFunctionDefs,
144 const DenseSet<GlobalValue::GUID> &CfiFunctionDecls) {
145 // Compute the unique hash for this entry.
146 // This is based on the current compiler version, the module itself, the
147 // export list, the hash for every single module in the import list, the
148 // list of ResolvedODR for the module, and the list of preserved symbols.
149 SHA1 Hasher;
150
151 // Start with the compiler revision
152 Hasher.update(LLVM_VERSION_STRING);
153#ifdef LLVM_REVISION
154 Hasher.update(LLVM_REVISION);
155#endif
156
157 // Include the parts of the LTO configuration that affect code generation.
158 auto AddString = [&](StringRef Str) {
159 Hasher.update(Str);
160 Hasher.update(Data: ArrayRef<uint8_t>{0});
161 };
162 auto AddUnsigned = [&](unsigned I) {
163 uint8_t Data[4];
164 support::endian::write32le(P: Data, V: I);
165 Hasher.update(Data);
166 };
167 auto AddUint64 = [&](uint64_t I) {
168 uint8_t Data[8];
169 support::endian::write64le(P: Data, V: I);
170 Hasher.update(Data);
171 };
172 auto AddUint8 = [&](const uint8_t I) {
173 Hasher.update(Data: ArrayRef<uint8_t>(&I, 1));
174 };
175 AddString(Conf.CPU);
176 // FIXME: Hash more of Options. For now all clients initialize Options from
177 // command-line flags (which is unsupported in production), but may set
178 // X86RelaxRelocations. The clang driver can also pass FunctionSections,
179 // DataSections and DebuggerTuning via command line flags.
180 AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations);
181 AddUnsigned(Conf.Options.FunctionSections);
182 AddUnsigned(Conf.Options.DataSections);
183 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
184 for (auto &A : Conf.MAttrs)
185 AddString(A);
186 if (Conf.RelocModel)
187 AddUnsigned(*Conf.RelocModel);
188 else
189 AddUnsigned(-1);
190 if (Conf.CodeModel)
191 AddUnsigned(*Conf.CodeModel);
192 else
193 AddUnsigned(-1);
194 for (const auto &S : Conf.MllvmArgs)
195 AddString(S);
196 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
197 AddUnsigned(static_cast<int>(Conf.CGFileType));
198 AddUnsigned(Conf.OptLevel);
199 AddUnsigned(Conf.Freestanding);
200 AddString(Conf.OptPipeline);
201 AddString(Conf.AAPipeline);
202 AddString(Conf.OverrideTriple);
203 AddString(Conf.DefaultTriple);
204 AddString(Conf.DwoDir);
205 AddUint8(Conf.Dtlto);
206
207 // Include the hash for the current module
208 auto ModHash = Index.getModuleHash(ModPath: ModuleID);
209 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
210
211 // TODO: `ExportList` is determined by `ImportList`. Since `ImportList` is
212 // used to compute cache key, we could omit hashing `ExportList` here.
213 std::vector<uint64_t> ExportsGUID;
214 ExportsGUID.reserve(n: ExportList.size());
215 for (const auto &VI : ExportList)
216 ExportsGUID.push_back(x: VI.getGUID());
217
218 // Sort the export list elements GUIDs.
219 llvm::sort(C&: ExportsGUID);
220 for (auto GUID : ExportsGUID)
221 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
222
223 // Order using module hash, to be both independent of module name and
224 // module order.
225 auto Comp = [&](const std::pair<StringRef, GlobalValue::GUID> &L,
226 const std::pair<StringRef, GlobalValue::GUID> &R) {
227 return std::make_pair(x: Index.getModule(ModPath: L.first)->second, y: L.second) <
228 std::make_pair(x: Index.getModule(ModPath: R.first)->second, y: R.second);
229 };
230 FunctionImporter::SortedImportList SortedImportList(ImportList, Comp);
231
232 // Count the number of imports for each source module.
233 DenseMap<StringRef, unsigned> ModuleToNumImports;
234 for (const auto &[FromModule, GUID, Type] : SortedImportList)
235 ++ModuleToNumImports[FromModule];
236
237 std::optional<StringRef> LastModule;
238 for (const auto &[FromModule, GUID, Type] : SortedImportList) {
239 if (LastModule != FromModule) {
240 // Include the hash for every module we import functions from. The set of
241 // imported symbols for each module may affect code generation and is
242 // sensitive to link order, so include that as well.
243 LastModule = FromModule;
244 auto ModHash = Index.getModule(ModPath: FromModule)->second;
245 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
246 AddUint64(ModuleToNumImports[FromModule]);
247 }
248 AddUint64(GUID);
249 AddUint8(Type);
250 }
251
252 // Include the hash for the resolved ODR.
253 for (auto &Entry : ResolvedODR) {
254 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
255 sizeof(GlobalValue::GUID)));
256 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
257 sizeof(GlobalValue::LinkageTypes)));
258 }
259
260 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
261 // defined in this module.
262 std::set<GlobalValue::GUID> UsedCfiDefs;
263 std::set<GlobalValue::GUID> UsedCfiDecls;
264
265 // Typeids used in this module.
266 std::set<GlobalValue::GUID> UsedTypeIds;
267
268 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
269 if (CfiFunctionDefs.contains(V: ValueGUID))
270 UsedCfiDefs.insert(x: ValueGUID);
271 if (CfiFunctionDecls.contains(V: ValueGUID))
272 UsedCfiDecls.insert(x: ValueGUID);
273 };
274
275 auto AddUsedThings = [&](GlobalValueSummary *GS) {
276 if (!GS) return;
277 AddUnsigned(GS->getVisibility());
278 AddUnsigned(GS->isLive());
279 AddUnsigned(GS->canAutoHide());
280 for (const ValueInfo &VI : GS->refs()) {
281 AddUnsigned(VI.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
282 AddUsedCfiGlobal(VI.getGUID());
283 }
284 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: GS)) {
285 AddUnsigned(GVS->maybeReadOnly());
286 AddUnsigned(GVS->maybeWriteOnly());
287 }
288 if (auto *FS = dyn_cast<FunctionSummary>(Val: GS)) {
289 for (auto &TT : FS->type_tests())
290 UsedTypeIds.insert(x: TT);
291 for (auto &TT : FS->type_test_assume_vcalls())
292 UsedTypeIds.insert(x: TT.GUID);
293 for (auto &TT : FS->type_checked_load_vcalls())
294 UsedTypeIds.insert(x: TT.GUID);
295 for (auto &TT : FS->type_test_assume_const_vcalls())
296 UsedTypeIds.insert(x: TT.VFunc.GUID);
297 for (auto &TT : FS->type_checked_load_const_vcalls())
298 UsedTypeIds.insert(x: TT.VFunc.GUID);
299 for (auto &ET : FS->calls()) {
300 AddUnsigned(ET.first.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
301 AddUsedCfiGlobal(ET.first.getGUID());
302 }
303 }
304 };
305
306 // Sort the defined globals by GUID to be independent of the insertion order,
307 // which may depend on the order that modules are added.
308 SmallVector<std::pair<GlobalValue::GUID, GlobalValueSummary *>>
309 SortedDefinedGlobals(DefinedGlobals.begin(), DefinedGlobals.end());
310 llvm::sort(C&: SortedDefinedGlobals, Comp: llvm::less_first());
311 for (auto &GS : SortedDefinedGlobals) {
312 // Include the hash for the linkage type to reflect internalization and weak
313 // resolution, and collect any used type identifier resolutions.
314 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
315 Hasher.update(
316 Data: ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
317 AddUsedCfiGlobal(GS.first);
318 AddUsedThings(GS.second);
319 }
320
321 // Imported functions may introduce new uses of type identifier resolutions,
322 // so we need to collect their used resolutions as well.
323 for (const auto &[FromModule, GUID, Type] : SortedImportList) {
324 GlobalValueSummary *S = Index.findSummaryInModule(ValueGUID: GUID, ModuleId: FromModule);
325 AddUsedThings(S);
326 // If this is an alias, we also care about any types/etc. that the aliasee
327 // may reference.
328 if (auto *AS = dyn_cast_or_null<AliasSummary>(Val: S))
329 AddUsedThings(AS->getBaseObject());
330 }
331
332 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
333 AddString(TId);
334
335 AddUnsigned(S.TTRes.TheKind);
336 AddUnsigned(S.TTRes.SizeM1BitWidth);
337
338 AddUint64(S.TTRes.AlignLog2);
339 AddUint64(S.TTRes.SizeM1);
340 AddUint64(S.TTRes.BitMask);
341 AddUint64(S.TTRes.InlineBits);
342
343 AddUint64(S.WPDRes.size());
344 for (auto &WPD : S.WPDRes) {
345 AddUnsigned(WPD.first);
346 AddUnsigned(WPD.second.TheKind);
347 AddString(WPD.second.SingleImplName);
348
349 AddUint64(WPD.second.ResByArg.size());
350 for (auto &ByArg : WPD.second.ResByArg) {
351 AddUint64(ByArg.first.size());
352 for (uint64_t Arg : ByArg.first)
353 AddUint64(Arg);
354 AddUnsigned(ByArg.second.TheKind);
355 AddUint64(ByArg.second.Info);
356 AddUnsigned(ByArg.second.Byte);
357 AddUnsigned(ByArg.second.Bit);
358 }
359 }
360 };
361
362 // Include the hash for all type identifiers used by this module.
363 for (GlobalValue::GUID TId : UsedTypeIds) {
364 auto TidIter = Index.typeIds().equal_range(x: TId);
365 for (const auto &I : make_range(p: TidIter))
366 AddTypeIdSummary(I.second.first, I.second.second);
367 }
368
369 AddUnsigned(UsedCfiDefs.size());
370 for (auto &V : UsedCfiDefs)
371 AddUint64(V);
372
373 AddUnsigned(UsedCfiDecls.size());
374 for (auto &V : UsedCfiDecls)
375 AddUint64(V);
376
377 if (!Conf.SampleProfile.empty()) {
378 auto FileOrErr = MemoryBuffer::getFile(Filename: Conf.SampleProfile);
379 if (FileOrErr) {
380 Hasher.update(Str: FileOrErr.get()->getBuffer());
381
382 if (!Conf.ProfileRemapping.empty()) {
383 FileOrErr = MemoryBuffer::getFile(Filename: Conf.ProfileRemapping);
384 if (FileOrErr)
385 Hasher.update(Str: FileOrErr.get()->getBuffer());
386 }
387 }
388 }
389
390 return toHex(Input: Hasher.result());
391}
392
393std::string llvm::recomputeLTOCacheKey(const std::string &Key,
394 StringRef ExtraID) {
395 SHA1 Hasher;
396
397 auto AddString = [&](StringRef Str) {
398 Hasher.update(Str);
399 Hasher.update(Data: ArrayRef<uint8_t>{0});
400 };
401 AddString(Key);
402 AddString(ExtraID);
403
404 return toHex(Input: Hasher.result());
405}
406
407static void thinLTOResolvePrevailingGUID(
408 const Config &C, ValueInfo VI,
409 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
410 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
411 isPrevailing,
412 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
413 recordNewLinkage,
414 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
415 GlobalValue::VisibilityTypes Visibility =
416 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
417 : GlobalValue::DefaultVisibility;
418 for (auto &S : VI.getSummaryList()) {
419 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
420 // Ignore local and appending linkage values since the linker
421 // doesn't resolve them.
422 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
423 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
424 continue;
425 // We need to emit only one of these. The prevailing module will keep it,
426 // but turned into a weak, while the others will drop it when possible.
427 // This is both a compile-time optimization and a correctness
428 // transformation. This is necessary for correctness when we have exported
429 // a reference - we need to convert the linkonce to weak to
430 // ensure a copy is kept to satisfy the exported reference.
431 // FIXME: We may want to split the compile time and correctness
432 // aspects into separate routines.
433 if (isPrevailing(VI.getGUID(), S.get())) {
434 assert(!S->wasPromoted() &&
435 "promoted symbols used to be internal linkage and shouldn't have "
436 "a prevailing variant");
437 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage)) {
438 S->setLinkage(GlobalValue::getWeakLinkage(
439 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
440 // The kept copy is eligible for auto-hiding (hidden visibility) if all
441 // copies were (i.e. they were all linkonce_odr global unnamed addr).
442 // If any copy is not (e.g. it was originally weak_odr), then the symbol
443 // must remain externally available (e.g. a weak_odr from an explicitly
444 // instantiated template). Additionally, if it is in the
445 // GUIDPreservedSymbols set, that means that it is visibile outside
446 // the summary (e.g. in a native object or a bitcode file without
447 // summary), and in that case we cannot hide it as it isn't possible to
448 // check all copies.
449 S->setCanAutoHide(VI.canAutoHide() &&
450 !GUIDPreservedSymbols.count(V: VI.getGUID()));
451 }
452 if (C.VisibilityScheme == Config::FromPrevailing)
453 Visibility = S->getVisibility();
454 }
455 // Alias and aliasee can't be turned into available_externally.
456 // When force-import-all is used, it indicates that object linking is not
457 // supported by the target. In this case, we can't change the linkage as
458 // well in case the global is converted to declaration.
459 // Also, if the symbol was promoted, it wouldn't have a prevailing variant,
460 // but also its linkage is set correctly (to External) already.
461 else if (!isa<AliasSummary>(Val: S.get()) &&
462 !GlobalInvolvedWithAlias.count(V: S.get()) && !ForceImportAll &&
463 !S->wasPromoted())
464 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
465
466 // For ELF, set visibility to the computed visibility from summaries. We
467 // don't track visibility from declarations so this may be more relaxed than
468 // the most constraining one.
469 if (C.VisibilityScheme == Config::ELF)
470 S->setVisibility(Visibility);
471
472 if (S->linkage() != OriginalLinkage)
473 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
474 }
475
476 if (C.VisibilityScheme == Config::FromPrevailing) {
477 for (auto &S : VI.getSummaryList()) {
478 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
479 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
480 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
481 continue;
482 S->setVisibility(Visibility);
483 }
484 }
485}
486
487/// Resolve linkage for prevailing symbols in the \p Index.
488//
489// We'd like to drop these functions if they are no longer referenced in the
490// current module. However there is a chance that another module is still
491// referencing them because of the import. We make sure we always emit at least
492// one copy.
493void llvm::thinLTOResolvePrevailingInIndex(
494 const Config &C, ModuleSummaryIndex &Index,
495 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
496 isPrevailing,
497 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
498 recordNewLinkage,
499 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
500 // We won't optimize the globals that are referenced by an alias for now
501 // Ideally we should turn the alias into a global and duplicate the definition
502 // when needed.
503 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
504 for (auto &I : Index)
505 for (auto &S : I.second.getSummaryList())
506 if (auto AS = dyn_cast<AliasSummary>(Val: S.get()))
507 GlobalInvolvedWithAlias.insert(V: &AS->getAliasee());
508
509 for (auto &I : Index)
510 thinLTOResolvePrevailingGUID(C, VI: Index.getValueInfo(R: I),
511 GlobalInvolvedWithAlias, isPrevailing,
512 recordNewLinkage, GUIDPreservedSymbols);
513}
514
515static void thinLTOInternalizeAndPromoteGUID(
516 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
517 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
518 isPrevailing,
519 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr) {
520 // Before performing index-based internalization and promotion for this GUID,
521 // the local flag should be consistent with the summary list linkage types.
522 VI.verifyLocal();
523
524 const bool SingleExternallyVisibleCopy =
525 VI.getSummaryList().size() == 1 &&
526 !GlobalValue::isLocalLinkage(Linkage: VI.getSummaryList().front()->linkage());
527
528 bool NameRecorded = false;
529 for (auto &S : VI.getSummaryList()) {
530 // First see if we need to promote an internal value because it is not
531 // exported.
532 if (isExported(S->modulePath(), VI)) {
533 if (GlobalValue::isLocalLinkage(Linkage: S->linkage())) {
534 // Only the first local GlobalValue in a list of summaries does not
535 // need renaming. In rare cases if there exist more than one summaries
536 // in the list, the rest of them must have renaming (through promotion)
537 // to avoid conflict.
538 if (ExternallyVisibleSymbolNamesPtr && !NameRecorded) {
539 NameRecorded = true;
540 if (ExternallyVisibleSymbolNamesPtr->insert(V: VI.name()).second)
541 S->setNoRenameOnPromotion(true);
542 }
543
544 S->promote();
545 }
546 continue;
547 }
548
549 // Otherwise, see if we can internalize.
550 if (!EnableLTOInternalization)
551 continue;
552
553 // Non-exported values with external linkage can be internalized.
554 if (GlobalValue::isExternalLinkage(Linkage: S->linkage())) {
555 S->setLinkage(GlobalValue::InternalLinkage);
556 continue;
557 }
558
559 // Non-exported function and variable definitions with a weak-for-linker
560 // linkage can be internalized in certain cases. The minimum legality
561 // requirements would be that they are not address taken to ensure that we
562 // don't break pointer equality checks, and that variables are either read-
563 // or write-only. For functions, this is the case if either all copies are
564 // [local_]unnamed_addr, or we can propagate reference edge attributes
565 // (which is how this is guaranteed for variables, when analyzing whether
566 // they are read or write-only).
567 //
568 // However, we only get to this code for weak-for-linkage values in one of
569 // two cases:
570 // 1) The prevailing copy is not in IR (it is in native code).
571 // 2) The prevailing copy in IR is not exported from its module.
572 // Additionally, at least for the new LTO API, case 2 will only happen if
573 // there is exactly one definition of the value (i.e. in exactly one
574 // module), as duplicate defs are result in the value being marked exported.
575 // Likely, users of the legacy LTO API are similar, however, currently there
576 // are llvm-lto based tests of the legacy LTO API that do not mark
577 // duplicate linkonce_odr copies as exported via the tool, so we need
578 // to handle that case below by checking the number of copies.
579 //
580 // Generally, we only want to internalize a weak-for-linker value in case
581 // 2, because in case 1 we cannot see how the value is used to know if it
582 // is read or write-only. We also don't want to bloat the binary with
583 // multiple internalized copies of non-prevailing linkonce/weak functions.
584 // Note if we don't internalize, we will convert non-prevailing copies to
585 // available_externally anyway, so that we drop them after inlining. The
586 // only reason to internalize such a function is if we indeed have a single
587 // copy, because internalizing it won't increase binary size, and enables
588 // use of inliner heuristics that are more aggressive in the face of a
589 // single call to a static (local). For variables, internalizing a read or
590 // write only variable can enable more aggressive optimization. However, we
591 // already perform this elsewhere in the ThinLTO backend handling for
592 // read or write-only variables (processGlobalForThinLTO).
593 //
594 // Therefore, only internalize linkonce/weak if there is a single copy, that
595 // is prevailing in this IR module. We can do so aggressively, without
596 // requiring the address to be insignificant, or that a variable be read or
597 // write-only.
598 if (!GlobalValue::isWeakForLinker(Linkage: S->linkage()) ||
599 GlobalValue::isExternalWeakLinkage(Linkage: S->linkage()))
600 continue;
601
602 // We may have a single summary copy that is externally visible but not
603 // prevailing if the prevailing copy is in a native object.
604 if (SingleExternallyVisibleCopy && isPrevailing(VI.getGUID(), S.get()))
605 S->setLinkage(GlobalValue::InternalLinkage);
606 }
607}
608
609// Update the linkages in the given \p Index to mark exported values
610// as external and non-exported values as internal.
611void llvm::thinLTOInternalizeAndPromoteInIndex(
612 ModuleSummaryIndex &Index,
613 function_ref<bool(StringRef, ValueInfo)> isExported,
614 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
615 isPrevailing,
616 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr) {
617 assert(!Index.withInternalizeAndPromote());
618
619 for (auto &I : Index)
620 thinLTOInternalizeAndPromoteGUID(VI: Index.getValueInfo(R: I), isExported,
621 isPrevailing,
622 ExternallyVisibleSymbolNamesPtr);
623 Index.setWithInternalizeAndPromote();
624}
625
626// Requires a destructor for std::vector<InputModule>.
627InputFile::~InputFile() = default;
628
629Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
630 std::unique_ptr<InputFile> File(new InputFile);
631
632 Expected<IRSymtabFile> FOrErr = readIRSymtab(MBRef: Object);
633 if (!FOrErr)
634 return FOrErr.takeError();
635
636 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
637 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
638 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
639 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
640 File->ComdatTable = FOrErr->TheReader.getComdatTable();
641 File->MbRef =
642 Object; // Save a memory buffer reference to an input file object.
643
644 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
645 size_t Begin = File->Symbols.size();
646 for (const irsymtab::Reader::SymbolRef &Sym :
647 FOrErr->TheReader.module_symbols(I))
648 // Skip symbols that are irrelevant to LTO. Note that this condition needs
649 // to match the one in Skip() in LTO::addRegularLTO().
650 if (Sym.isGlobal() && !Sym.isFormatSpecific())
651 File->Symbols.push_back(x: Sym);
652 File->ModuleSymIndices.push_back(x: {Begin, File->Symbols.size()});
653 }
654
655 File->Mods = FOrErr->Mods;
656 File->Strtab = std::move(FOrErr->Strtab);
657 return std::move(File);
658}
659
660bool InputFile::Symbol::isLibcall(
661 const TargetLibraryInfo &TLI,
662 const RTLIB::RuntimeLibcallsInfo &Libcalls) const {
663 if (TLI.has(F: TLI.getLibFunc(funcName: IRName)))
664 return true;
665 return Libcalls.getSupportedLibcallImpl(FuncName: IRName) != RTLIB::Unsupported;
666}
667
668StringRef InputFile::getName() const {
669 return Mods[0].getModuleIdentifier();
670}
671
672BitcodeModule &InputFile::getSingleBitcodeModule() {
673 assert(Mods.size() == 1 && "Expect only one bitcode module");
674 return Mods[0];
675}
676
677BitcodeModule &InputFile::getPrimaryBitcodeModule() { return Mods[0]; }
678
679LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
680 const Config &Conf)
681 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
682 Ctx(Conf), CombinedModule(std::make_unique<Module>(args: "ld-temp.o", args&: Ctx)),
683 Mover(std::make_unique<IRMover>(args&: *CombinedModule)) {}
684
685LTO::ThinLTOState::ThinLTOState(ThinBackend BackendParam)
686 : Backend(std::move(BackendParam)), CombinedIndex(/*HaveGVs*/ false) {
687 if (!Backend.isValid())
688 Backend =
689 createInProcessThinBackend(Parallelism: llvm::heavyweight_hardware_concurrency());
690}
691
692LTO::LTO(Config Conf, ThinBackend Backend,
693 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
694 : Conf(std::move(Conf)),
695 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
696 ThinLTO(std::move(Backend)),
697 GlobalResolutions(
698 std::make_unique<DenseMap<StringRef, GlobalResolution>>()),
699 LTOMode(LTOMode) {
700 if (Conf.KeepSymbolNameCopies || LTOKeepSymbolCopies) {
701 Alloc = std::make_unique<BumpPtrAllocator>();
702 GlobalResolutionSymbolSaver = std::make_unique<llvm::StringSaver>(args&: *Alloc);
703 }
704}
705
706// Requires a destructor for MapVector<BitcodeModule>.
707LTO::~LTO() = default;
708
709void LTO::cleanup() {
710 DummyModule.reset();
711 LinkerRemarkFunction = nullptr;
712 consumeError(Err: finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile)));
713}
714
715// Add the symbols in the given module to the GlobalResolutions map, and resolve
716// their partitions.
717void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
718 ArrayRef<SymbolResolution> Res,
719 unsigned Partition, bool InSummary,
720 const Triple &TT) {
721 llvm::TimeTraceScope timeScope("LTO add module to global resolution");
722 auto *ResI = Res.begin();
723 auto *ResE = Res.end();
724 (void)ResE;
725 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
726 TargetLibraryInfoImpl TLII(TT);
727 TargetLibraryInfo TLI(TLII);
728 for (const InputFile::Symbol &Sym : Syms) {
729 assert(ResI != ResE);
730 SymbolResolution Res = *ResI++;
731
732 StringRef SymbolName = Sym.getName();
733 // Keep copies of symbols if the client of LTO says so.
734 if (GlobalResolutionSymbolSaver && !GlobalResolutions->contains(Val: SymbolName))
735 SymbolName = GlobalResolutionSymbolSaver->save(S: SymbolName);
736
737 auto &GlobalRes = (*GlobalResolutions)[SymbolName];
738 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
739 if (Res.Prevailing) {
740 assert(!GlobalRes.Prevailing &&
741 "Multiple prevailing defs are not allowed");
742 GlobalRes.Prevailing = true;
743 GlobalRes.IRName = std::string(Sym.getIRName());
744 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
745 // Sometimes it can be two copies of symbol in a module and prevailing
746 // symbol can have no IR name. That might happen if symbol is defined in
747 // module level inline asm block. In case we have multiple modules with
748 // the same symbol we want to use IR name of the prevailing symbol.
749 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
750 // we can later use it to check if there is any prevailing copy in IR.
751 GlobalRes.IRName = std::string(Sym.getIRName());
752 }
753
754 // In rare occasion, the symbol used to initialize GlobalRes has a different
755 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
756 // symbol is referenced through its mangled name, say @"\01_symbol" while
757 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
758 // In that case, we have the same actual Symbol that can get two different
759 // GUID, leading to some invalid internalization. Workaround this by marking
760 // the GlobalRes external.
761
762 // FIXME: instead of this check, it would be desirable to compute GUIDs
763 // based on mangled name, but this requires an access to the Target Triple
764 // and would be relatively invasive on the codebase.
765 // FIXME: use the GUID member of GlobalRes.
766 if (GlobalRes.IRName != Sym.getIRName()) {
767 GlobalRes.Partition = GlobalResolution::External;
768 GlobalRes.VisibleOutsideSummary = true;
769 }
770
771 bool IsLibcall = Sym.isLibcall(TLI, Libcalls);
772
773 // Set the partition to external if we know it is re-defined by the linker
774 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
775 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
776 // already recorded as being referenced from a different partition.
777 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
778 IsLibcall ||
779 (GlobalRes.Partition != GlobalResolution::Unknown &&
780 GlobalRes.Partition != Partition)) {
781 GlobalRes.Partition = GlobalResolution::External;
782 } else
783 // First recorded reference, save the current partition.
784 GlobalRes.Partition = Partition;
785
786 // Flag as visible outside of summary if visible from a regular object or
787 // from a module that does not have a summary.
788 GlobalRes.VisibleOutsideSummary |=
789 (Res.VisibleToRegularObj || Sym.isUsed() || IsLibcall || !InSummary);
790
791 GlobalRes.ExportDynamic |= Res.ExportDynamic;
792 }
793}
794
795void LTO::releaseGlobalResolutionsMemory() {
796 // Release GlobalResolutions dense-map itself.
797 GlobalResolutions.reset();
798 // Release the string saver memory.
799 GlobalResolutionSymbolSaver.reset();
800 Alloc.reset();
801}
802
803static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
804 ArrayRef<SymbolResolution> Res) {
805 StringRef Path = Input->getName();
806 OS << Path << '\n';
807 auto ResI = Res.begin();
808 for (const InputFile::Symbol &Sym : Input->symbols()) {
809 assert(ResI != Res.end());
810 SymbolResolution Res = *ResI++;
811
812 OS << "-r=" << Path << ',' << Sym.getName() << ',';
813 if (Res.Prevailing)
814 OS << 'p';
815 if (Res.FinalDefinitionInLinkageUnit)
816 OS << 'l';
817 if (Res.VisibleToRegularObj)
818 OS << 'x';
819 if (Res.LinkerRedefined)
820 OS << 'r';
821 OS << '\n';
822 }
823 OS.flush();
824 assert(ResI == Res.end());
825}
826
827Error LTO::add(std::unique_ptr<InputFile> InputPtr,
828 ArrayRef<SymbolResolution> Res) {
829 llvm::TimeTraceScope timeScope("LTO add input", InputPtr->getName());
830 assert(!CalledGetMaxTasks);
831
832 Expected<std::shared_ptr<InputFile>> InputOrErr =
833 addInput(InputPtr: std::move(InputPtr));
834 if (!InputOrErr)
835 return InputOrErr.takeError();
836 InputFile *Input = (*InputOrErr).get();
837
838 if (Conf.ResolutionFile)
839 writeToResolutionFile(OS&: *Conf.ResolutionFile, Input, Res);
840
841 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
842 Triple InputTriple(Input->getTargetTriple());
843 RegularLTO.CombinedModule->setTargetTriple(InputTriple);
844 if (InputTriple.isOSBinFormatELF())
845 Conf.VisibilityScheme = Config::ELF;
846 }
847
848 ArrayRef<SymbolResolution> InputRes = Res;
849 for (unsigned I = 0; I != Input->Mods.size(); ++I) {
850 if (auto Err = addModule(Input&: *Input, InputRes, ModI: I, Res).moveInto(Value&: Res))
851 return Err;
852 }
853
854 assert(Res.empty());
855 return Error::success();
856}
857
858void LTO::setBitcodeLibFuncs(ArrayRef<StringRef> BitcodeLibFuncs) {
859 assert(this->BitcodeLibFuncs.empty() &&
860 "bitcode libfuncs were set twice; maybe accidentally clobbered?");
861 this->BitcodeLibFuncs.append(in_start: BitcodeLibFuncs.begin(), in_end: BitcodeLibFuncs.end());
862}
863
864Expected<ArrayRef<SymbolResolution>>
865LTO::addModule(InputFile &Input, ArrayRef<SymbolResolution> InputRes,
866 unsigned ModI, ArrayRef<SymbolResolution> Res) {
867 llvm::TimeTraceScope timeScope("LTO add module", Input.getName());
868 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
869 if (!LTOInfo)
870 return LTOInfo.takeError();
871
872 if (EnableSplitLTOUnit) {
873 // If only some modules were split, flag this in the index so that
874 // we can skip or error on optimizations that need consistently split
875 // modules (whole program devirt and lower type tests).
876 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
877 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
878 } else
879 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
880
881 BitcodeModule BM = Input.Mods[ModI];
882
883 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
884 !LTOInfo->UnifiedLTO)
885 return make_error<StringError>(
886 Args: "unified LTO compilation must use "
887 "compatible bitcode modules (use -funified-lto)",
888 Args: inconvertibleErrorCode());
889
890 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
891 LTOMode = LTOK_UnifiedThin;
892
893 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
894 // If any of the modules inside of a input bitcode file was compiled with
895 // ThinLTO, we assume that the whole input file also was compiled with
896 // ThinLTO.
897 Input.IsThinLTO |= IsThinLTO;
898
899 auto ModSyms = Input.module_symbols(I: ModI);
900 addModuleToGlobalRes(Syms: ModSyms, Res,
901 Partition: IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
902 InSummary: LTOInfo->HasSummary, TT: Triple(Input.getTargetTriple()));
903
904 if (IsThinLTO)
905 return addThinLTO(BM, Syms: ModSyms, Res);
906
907 RegularLTO.EmptyCombinedModule = false;
908 auto ModOrErr = addRegularLTO(Input, InputRes, BM, Syms: ModSyms, Res);
909 if (!ModOrErr)
910 return ModOrErr.takeError();
911 Res = ModOrErr->second;
912
913 if (!LTOInfo->HasSummary) {
914 if (Error Err = linkRegularLTO(Mod: std::move(ModOrErr->first),
915 /*LivenessFromIndex=*/false))
916 return Err;
917 return Res;
918 }
919
920 // Regular LTO module summaries are added to a dummy module that represents
921 // the combined regular LTO module.
922 if (Error Err = BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: ""))
923 return Err;
924 RegularLTO.ModsWithSummaries.push_back(x: std::move(ModOrErr->first));
925 return Res;
926}
927
928// Checks whether the given global value is in a non-prevailing comdat
929// (comdat containing values the linker indicated were not prevailing,
930// which we then dropped to available_externally), and if so, removes
931// it from the comdat. This is called for all global values to ensure the
932// comdat is empty rather than leaving an incomplete comdat. It is needed for
933// regular LTO modules, in case we are in a mixed-LTO mode (both regular
934// and thin LTO modules) compilation. Since the regular LTO module will be
935// linked first in the final native link, we want to make sure the linker
936// doesn't select any of these incomplete comdats that would be left
937// in the regular LTO module without this cleanup.
938static void
939handleNonPrevailingComdat(GlobalValue &GV,
940 std::set<const Comdat *> &NonPrevailingComdats) {
941 Comdat *C = GV.getComdat();
942 if (!C)
943 return;
944
945 if (!NonPrevailingComdats.count(x: C))
946 return;
947
948 // Additionally need to drop all global values from the comdat to
949 // available_externally, to satisfy the COMDAT requirement that all members
950 // are discarded as a unit. The non-local linkage global values avoid
951 // duplicate definition linker errors.
952 GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
953
954 if (auto GO = dyn_cast<GlobalObject>(Val: &GV))
955 GO->setComdat(nullptr);
956}
957
958// Add a regular LTO object to the link.
959// The resulting module needs to be linked into the combined LTO module with
960// linkRegularLTO.
961Expected<
962 std::pair<LTO::RegularLTOState::AddedModule, ArrayRef<SymbolResolution>>>
963LTO::addRegularLTO(InputFile &Input, ArrayRef<SymbolResolution> InputRes,
964 BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
965 ArrayRef<SymbolResolution> Res) {
966 llvm::TimeTraceScope timeScope("LTO add regular LTO");
967 RegularLTOState::AddedModule Mod;
968 Expected<std::unique_ptr<Module>> MOrErr =
969 BM.getLazyModule(Context&: RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
970 /*IsImporting*/ false);
971 if (!MOrErr)
972 return MOrErr.takeError();
973 Module &M = **MOrErr;
974 Mod.M = std::move(*MOrErr);
975
976 if (Error Err = M.materializeMetadata())
977 return std::move(Err);
978
979 if (LTOMode == LTOK_UnifiedRegular) {
980 // cfi.functions metadata is intended to be used with ThinLTO and may
981 // trigger invalid IR transformations if they are present when doing regular
982 // LTO, so delete it.
983 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata(Name: "cfi.functions"))
984 M.eraseNamedMetadata(NMD: CfiFunctionsMD);
985 } else if (NamedMDNode *AliasesMD = M.getNamedMetadata(Name: "aliases")) {
986 // Delete aliases entries for non-prevailing symbols on the ThinLTO side of
987 // this input file.
988 DenseSet<StringRef> Prevailing;
989 for (auto [I, R] : zip(t: Input.symbols(), u&: InputRes))
990 if (R.Prevailing && !I.getIRName().empty())
991 Prevailing.insert(V: I.getIRName());
992 std::vector<MDNode *> AliasGroups;
993 for (MDNode *AliasGroup : AliasesMD->operands()) {
994 std::vector<Metadata *> Aliases;
995 for (Metadata *Alias : AliasGroup->operands()) {
996 if (isa<MDString>(Val: Alias) &&
997 Prevailing.count(V: cast<MDString>(Val: Alias)->getString()))
998 Aliases.push_back(x: Alias);
999 }
1000 if (Aliases.size() > 1)
1001 AliasGroups.push_back(x: MDTuple::get(Context&: RegularLTO.Ctx, MDs: Aliases));
1002 }
1003 AliasesMD->clearOperands();
1004 for (MDNode *G : AliasGroups)
1005 AliasesMD->addOperand(M: G);
1006 }
1007
1008 UpgradeDebugInfo(M);
1009
1010 ModuleSymbolTable SymTab;
1011 SymTab.addModule(M: &M);
1012
1013 for (GlobalVariable &GV : M.globals())
1014 if (GV.hasAppendingLinkage())
1015 Mod.Keep.push_back(x: &GV);
1016
1017 DenseSet<GlobalObject *> AliasedGlobals;
1018 for (auto &GA : M.aliases())
1019 if (GlobalObject *GO = GA.getAliaseeObject())
1020 AliasedGlobals.insert(V: GO);
1021
1022 // In this function we need IR GlobalValues matching the symbols in Syms
1023 // (which is not backed by a module), so we need to enumerate them in the same
1024 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
1025 // matches the order of an irsymtab, but when we read the irsymtab in
1026 // InputFile::create we omit some symbols that are irrelevant to LTO. The
1027 // Skip() function skips the same symbols from the module as InputFile does
1028 // from the symbol table.
1029 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
1030 auto Skip = [&]() {
1031 while (MsymI != MsymE) {
1032 auto Flags = SymTab.getSymbolFlags(S: *MsymI);
1033 if ((Flags & object::BasicSymbolRef::SF_Global) &&
1034 !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
1035 return;
1036 ++MsymI;
1037 }
1038 };
1039 Skip();
1040
1041 std::set<const Comdat *> NonPrevailingComdats;
1042 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
1043 for (const InputFile::Symbol &Sym : Syms) {
1044 assert(!Res.empty());
1045 const SymbolResolution &R = Res.consume_front();
1046
1047 assert(MsymI != MsymE);
1048 ModuleSymbolTable::Symbol Msym = *MsymI++;
1049 Skip();
1050
1051 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Val&: Msym)) {
1052 if (R.Prevailing) {
1053 if (Sym.isUndefined())
1054 continue;
1055 Mod.Keep.push_back(x: GV);
1056 // For symbols re-defined with linker -wrap and -defsym options,
1057 // set the linkage to weak to inhibit IPO. The linkage will be
1058 // restored by the linker.
1059 if (R.LinkerRedefined)
1060 GV->setLinkage(GlobalValue::WeakAnyLinkage);
1061
1062 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
1063 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage))
1064 GV->setLinkage(GlobalValue::getWeakLinkage(
1065 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
1066 } else if (isa<GlobalObject>(Val: GV) &&
1067 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
1068 GV->hasAvailableExternallyLinkage()) &&
1069 !AliasedGlobals.count(V: cast<GlobalObject>(Val: GV))) {
1070 // Any of the above three types of linkage indicates that the
1071 // chosen prevailing symbol will have the same semantics as this copy of
1072 // the symbol, so we may be able to link it with available_externally
1073 // linkage. We will decide later whether to do that when we link this
1074 // module (in linkRegularLTO), based on whether it is undefined.
1075 Mod.Keep.push_back(x: GV);
1076 GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
1077 if (GV->hasComdat())
1078 NonPrevailingComdats.insert(x: GV->getComdat());
1079 cast<GlobalObject>(Val: GV)->setComdat(nullptr);
1080 }
1081
1082 // Set the 'local' flag based on the linker resolution for this symbol.
1083 if (R.FinalDefinitionInLinkageUnit) {
1084 GV->setDSOLocal(true);
1085 if (GV->hasDLLImportStorageClass())
1086 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
1087 DefaultStorageClass);
1088 }
1089 } else if (auto *AS =
1090 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Val&: Msym)) {
1091 // Collect non-prevailing symbols.
1092 if (!R.Prevailing)
1093 NonPrevailingAsmSymbols.insert(V: AS->first);
1094 } else {
1095 llvm_unreachable("unknown symbol type");
1096 }
1097
1098 // Common resolution: collect the maximum size/alignment over all commons.
1099 // We also record if we see an instance of a common as prevailing, so that
1100 // if none is prevailing we can ignore it later.
1101 if (Sym.isCommon()) {
1102 // FIXME: We should figure out what to do about commons defined by asm.
1103 // For now they aren't reported correctly by ModuleSymbolTable.
1104 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
1105 CommonRes.Size = std::max(a: CommonRes.Size, b: Sym.getCommonSize());
1106 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
1107 CommonRes.Alignment =
1108 std::max(a: Align(SymAlignValue), b: CommonRes.Alignment);
1109 }
1110 CommonRes.Prevailing |= R.Prevailing;
1111 }
1112 }
1113
1114 if (!M.getComdatSymbolTable().empty())
1115 for (GlobalValue &GV : M.global_values())
1116 handleNonPrevailingComdat(GV, NonPrevailingComdats);
1117
1118 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
1119 // block.
1120 if (M.hasModuleInlineAsm()) {
1121 std::string NewIA = ".lto_discard";
1122 if (!NonPrevailingAsmSymbols.empty()) {
1123 // Don't dicard a symbol if there is a live .symver for it.
1124 ModuleSymbolTable::CollectAsmSymvers(
1125 M, AsmSymver: [&](StringRef Name, StringRef Alias) {
1126 if (!NonPrevailingAsmSymbols.count(V: Alias))
1127 NonPrevailingAsmSymbols.erase(V: Name);
1128 });
1129 NewIA += " " + llvm::join(R&: NonPrevailingAsmSymbols, Separator: ", ");
1130 }
1131 NewIA += "\n";
1132 M.prependModuleInlineAsm(Fragment: NewIA);
1133 }
1134
1135 assert(MsymI == MsymE);
1136 return std::make_pair(x: std::move(Mod), y&: Res);
1137}
1138
1139Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
1140 bool LivenessFromIndex) {
1141 llvm::TimeTraceScope timeScope("LTO link regular LTO");
1142 std::vector<GlobalValue *> Keep;
1143 for (GlobalValue *GV : Mod.Keep) {
1144 if (LivenessFromIndex) {
1145 const auto GUID = GV->getGUIDOrFallback();
1146 if (!ThinLTO.CombinedIndex.isGUIDLive(GUID)) {
1147 if (Function *F = dyn_cast<Function>(Val: GV)) {
1148 if (DiagnosticOutputFile) {
1149 if (Error Err = F->materialize())
1150 return Err;
1151 auto R = OptimizationRemark(DEBUG_TYPE, "deadfunction", F);
1152 R << ore::NV("Function", F) << " not added to the combined module ";
1153 emitRemark(Remark&: R);
1154 }
1155 }
1156 continue;
1157 }
1158 }
1159
1160 if (!GV->hasAvailableExternallyLinkage()) {
1161 Keep.push_back(x: GV);
1162 continue;
1163 }
1164
1165 // Only link available_externally definitions if we don't already have a
1166 // definition.
1167 GlobalValue *CombinedGV =
1168 RegularLTO.CombinedModule->getNamedValue(Name: GV->getName());
1169 if (CombinedGV && !CombinedGV->isDeclaration())
1170 continue;
1171
1172 Keep.push_back(x: GV);
1173 }
1174
1175 return RegularLTO.Mover->move(Src: std::move(Mod.M), ValuesToLink: Keep, AddLazyFor: nullptr,
1176 /* IsPerformingImport */ false);
1177}
1178
1179// Add a ThinLTO module to the link.
1180Expected<ArrayRef<SymbolResolution>>
1181LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
1182 ArrayRef<SymbolResolution> Res) {
1183 llvm::TimeTraceScope timeScope("LTO add thin LTO");
1184 const auto BMID = BM.getModuleIdentifier();
1185 ArrayRef<SymbolResolution> ResTmp = Res;
1186 DenseSet<StringRef> Prevailing;
1187 for (const InputFile::Symbol &Sym : Syms) {
1188 assert(!ResTmp.empty());
1189 const SymbolResolution &R = ResTmp.consume_front();
1190 if (!Sym.getIRName().empty() && R.Prevailing)
1191 Prevailing.insert(V: Sym.getIRName());
1192 }
1193
1194 // Track the GUIDs stored in the bitcode GUID table.
1195 StringMap<GlobalValue::GUID> IRSpecifiedGUIDs;
1196 if (Error Err = BM.readSummary(
1197 CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: BMID,
1198 IsPrevailing: [&](StringRef Name) { return (Prevailing.count(V: Name) > 0); },
1199 OnValueInfo: [&](ValueInfo VI) {
1200 auto IT = IRSpecifiedGUIDs.insert(KV: {VI.name(), VI.getGUID()});
1201 (void)IT;
1202 assert(IT.second);
1203 if (auto GRIt = GlobalResolutions->find(Val: VI.name());
1204 GRIt != GlobalResolutions->end() &&
1205 Prevailing.count(V: VI.name())) {
1206 GRIt->second.setGUID(VI.getGUID());
1207 }
1208 }))
1209 return Err;
1210 LLVM_DEBUG(dbgs() << "Module " << BMID << "\n");
1211
1212 for (const InputFile::Symbol &Sym : Syms) {
1213 assert(!Res.empty());
1214 const SymbolResolution &R = Res.consume_front();
1215 auto GUIDIter = IRSpecifiedGUIDs.find(Key: Sym.getIRName());
1216 // The bitcode GUID table might not be present if this is an old bitcode
1217 // file. For backwards-compatibility, just compute the GUID now in that
1218 // case.
1219 auto GUID =
1220 GUIDIter == IRSpecifiedGUIDs.end()
1221 ? GlobalValue::getGUIDAssumingExternalLinkage(
1222 GlobalName: GlobalValue::getGlobalIdentifier(
1223 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: ""))
1224 : GUIDIter->second;
1225 if (!Sym.getIRName().empty() &&
1226 (R.Prevailing || R.FinalDefinitionInLinkageUnit)) {
1227 if (R.Prevailing) {
1228 ThinLTO.setPrevailingModuleForGUID(GUID, Module: BMID);
1229 // For linker redefined symbols (via --wrap or --defsym) we want to
1230 // switch the linkage to `weak` to prevent IPOs from happening.
1231 // Find the summary in the module for this very GV and record the new
1232 // linkage so that we can switch it when we import the GV.
1233 if (R.LinkerRedefined)
1234 if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(ValueGUID: GUID, ModuleId: BMID))
1235 S->setLinkage(GlobalValue::WeakAnyLinkage);
1236 }
1237
1238 // If the linker resolved the symbol to a local definition then mark it
1239 // as local in the summary for the module we are adding.
1240 if (R.FinalDefinitionInLinkageUnit) {
1241 if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(ValueGUID: GUID, ModuleId: BMID)) {
1242 S->setDSOLocal(true);
1243 }
1244 }
1245 }
1246 }
1247
1248 if (!ThinLTO.ModuleMap.insert(KV: {BMID, BM}).second)
1249 return make_error<StringError>(
1250 Args: "Expected at most one ThinLTO module per bitcode file",
1251 Args: inconvertibleErrorCode());
1252
1253 if (!Conf.ThinLTOModulesToCompile.empty()) {
1254 if (!ThinLTO.ModulesToCompile)
1255 ThinLTO.ModulesToCompile = ModuleMapType();
1256 // This is a fuzzy name matching where only modules with name containing the
1257 // specified switch values are going to be compiled.
1258 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1259 if (BMID.contains(Other: Name)) {
1260 ThinLTO.ModulesToCompile->insert(KV: {BMID, BM});
1261 LLVM_DEBUG(dbgs() << "[ThinLTO] Selecting " << BMID << " to compile\n");
1262 break;
1263 }
1264 }
1265 }
1266
1267 return Res;
1268}
1269
1270unsigned LTO::getMaxTasks() const {
1271 CalledGetMaxTasks = true;
1272 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1273 : ThinLTO.ModuleMap.size();
1274 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1275}
1276
1277// If only some of the modules were split, we cannot correctly handle
1278// code that contains type tests or type checked loads.
1279Error LTO::checkPartiallySplit() {
1280 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1281 return Error::success();
1282
1283 const Module *Combined = RegularLTO.CombinedModule.get();
1284 Function *TypeTestFunc =
1285 Intrinsic::getDeclarationIfExists(M: Combined, id: Intrinsic::type_test);
1286 Function *TypeCheckedLoadFunc =
1287 Intrinsic::getDeclarationIfExists(M: Combined, id: Intrinsic::type_checked_load);
1288 Function *TypeCheckedLoadRelativeFunc = Intrinsic::getDeclarationIfExists(
1289 M: Combined, id: Intrinsic::type_checked_load_relative);
1290
1291 // First check if there are type tests / type checked loads in the
1292 // merged regular LTO module IR.
1293 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1294 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1295 (TypeCheckedLoadRelativeFunc &&
1296 !TypeCheckedLoadRelativeFunc->use_empty()))
1297 return make_error<StringError>(
1298 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1299 Args: inconvertibleErrorCode());
1300
1301 // Otherwise check if there are any recorded in the combined summary from the
1302 // ThinLTO modules.
1303 for (auto &P : ThinLTO.CombinedIndex) {
1304 for (auto &S : P.second.getSummaryList()) {
1305 auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
1306 if (!FS)
1307 continue;
1308 if (!FS->type_test_assume_vcalls().empty() ||
1309 !FS->type_checked_load_vcalls().empty() ||
1310 !FS->type_test_assume_const_vcalls().empty() ||
1311 !FS->type_checked_load_const_vcalls().empty() ||
1312 !FS->type_tests().empty())
1313 return make_error<StringError>(
1314 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1315 Args: inconvertibleErrorCode());
1316 }
1317 }
1318 return Error::success();
1319}
1320
1321Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
1322 // Call the base class cleanup() explicitly since run() may be invoked on a
1323 // derived LTO object.
1324 llvm::scope_exit CleanUp([this]() { LTO::cleanup(); });
1325
1326 // Compute "dead" symbols, we don't want to import/export these!
1327 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1328 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1329 for (auto &Res : *GlobalResolutions) {
1330 // Normally resolution have IR name of symbol. We can do nothing here
1331 // otherwise. See comments in GlobalResolution struct for more details.
1332 if (Res.second.IRName.empty())
1333 continue;
1334
1335 GlobalValue::GUID GUID = Res.second.getGUID();
1336
1337 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1338 GUIDPreservedSymbols.insert(V: GUID);
1339
1340 if (Res.second.ExportDynamic)
1341 DynamicExportSymbols.insert(V: GUID);
1342
1343 GUIDPrevailingResolutions[GUID] =
1344 Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
1345 }
1346
1347 auto isPrevailing = [&](GlobalValue::GUID G) {
1348 auto It = GUIDPrevailingResolutions.find(Val: G);
1349 if (It == GUIDPrevailingResolutions.end())
1350 return PrevailingType::Unknown;
1351 return It->second;
1352 };
1353 computeDeadSymbolsWithConstProp(Index&: ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1354 isPrevailing, ImportEnabled: Conf.OptLevel > 0);
1355
1356 // Setup output file to emit statistics.
1357 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1358 if (!StatsFileOrErr)
1359 return StatsFileOrErr.takeError();
1360 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1361
1362 if (Error Err = setupOptimizationRemarks())
1363 return Err;
1364
1365 // TODO: Ideally this would be controlled automatically by detecting that we
1366 // are linking with an allocator that supports these interfaces, rather than
1367 // an internal option (which would still be needed for tests, however). For
1368 // example, if the library exported a symbol like __malloc_hot_cold the linker
1369 // could recognize that and set a flag in the lto::Config.
1370 if (SupportsHotColdNew)
1371 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1372
1373 Error Result = runRegularLTO(AddStream);
1374 if (!Result)
1375 // This will reset the GlobalResolutions optional once done with it to
1376 // reduce peak memory before importing.
1377 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1378
1379 if (StatsFile)
1380 PrintStatisticsJSON(OS&: StatsFile->os());
1381
1382 return Result;
1383}
1384
1385Error LTO::runRegularLTO(AddStreamFn AddStream) {
1386 llvm::TimeTraceScope timeScope("Run regular LTO");
1387 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1388
1389 // Finalize linking of regular LTO modules containing summaries now that
1390 // we have computed liveness information.
1391 {
1392 llvm::TimeTraceScope timeScope("Link regular LTO");
1393 for (auto &M : RegularLTO.ModsWithSummaries)
1394 if (Error Err = linkRegularLTO(Mod: std::move(M), /*LivenessFromIndex=*/true))
1395 return Err;
1396 }
1397
1398 // Ensure we don't have inconsistently split LTO units with type tests.
1399 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1400 // this path both cases but eventually this should be split into two and
1401 // do the ThinLTO checks in `runThinLTO`.
1402 if (Error Err = checkPartiallySplit())
1403 return Err;
1404
1405 // Make sure commons have the right size/alignment: we kept the largest from
1406 // all the prevailing when adding the inputs, and we apply it here.
1407 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1408 for (auto &I : RegularLTO.Commons) {
1409 if (!I.second.Prevailing)
1410 // Don't do anything if no instance of this common was prevailing.
1411 continue;
1412 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(Name: I.first);
1413 if (OldGV && OldGV->getGlobalSize(DL) == I.second.Size) {
1414 // Don't create a new global if the type is already correct, just make
1415 // sure the alignment is correct.
1416 OldGV->setAlignment(I.second.Alignment);
1417 continue;
1418 }
1419 ArrayType *Ty =
1420 ArrayType::get(ElementType: Type::getInt8Ty(C&: RegularLTO.Ctx), NumElements: I.second.Size);
1421 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1422 GlobalValue::CommonLinkage,
1423 ConstantAggregateZero::get(Ty), "");
1424 GV->setAlignment(I.second.Alignment);
1425 if (OldGV) {
1426 OldGV->replaceAllUsesWith(V: GV);
1427 GV->takeName(V: OldGV);
1428 OldGV->eraseFromParent();
1429 } else {
1430 GV->setName(I.first);
1431 }
1432 }
1433
1434 bool WholeProgramVisibilityEnabledInLTO =
1435 Conf.HasWholeProgramVisibility &&
1436 // If validation is enabled, upgrade visibility only when all vtables
1437 // have typeinfos.
1438 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
1439
1440 // This returns true when the name is local or not defined. Locals are
1441 // expected to be handled separately.
1442 auto IsVisibleToRegularObj = [&](StringRef name) {
1443 auto It = GlobalResolutions->find(Val: name);
1444 return (It == GlobalResolutions->end() ||
1445 It->second.VisibleOutsideSummary || !It->second.Prevailing);
1446 };
1447
1448 // If allowed, upgrade public vcall visibility metadata to linkage unit
1449 // visibility before whole program devirtualization in the optimizer.
1450 updateVCallVisibilityInModule(
1451 M&: *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1452 DynamicExportSymbols, ValidateAllVtablesHaveTypeInfos: Conf.ValidateAllVtablesHaveTypeInfos,
1453 IsVisibleToRegularObj);
1454 updatePublicTypeTestCalls(M&: *RegularLTO.CombinedModule,
1455 WholeProgramVisibilityEnabledInLTO);
1456
1457 if (Conf.PreOptModuleHook &&
1458 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1459 return Error::success();
1460
1461 if (!Conf.CodeGenOnly) {
1462 for (const auto &R : *GlobalResolutions) {
1463 GlobalValue *GV =
1464 RegularLTO.CombinedModule->getNamedValue(Name: R.second.IRName);
1465 if (!R.second.isPrevailingIRSymbol())
1466 continue;
1467 if (R.second.Partition != 0 &&
1468 R.second.Partition != GlobalResolution::External)
1469 continue;
1470
1471 // Ignore symbols defined in other partitions.
1472 // Also skip declarations, which are not allowed to have internal linkage.
1473 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1474 continue;
1475
1476 // Symbols that are marked DLLImport or DLLExport should not be
1477 // internalized, as they are either externally visible or referencing
1478 // external symbols. Symbols that have AvailableExternally or Appending
1479 // linkage might be used by future passes and should be kept as is.
1480 // These linkages are seen in Unified regular LTO, because the process
1481 // of creating split LTO units introduces symbols with that linkage into
1482 // one of the created modules. Normally, only the ThinLTO backend would
1483 // compile this module, but Unified Regular LTO processes both
1484 // modules created by the splitting process as regular LTO modules.
1485 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1486 ((GV->getDLLStorageClass() != GlobalValue::DefaultStorageClass) ||
1487 GV->hasAvailableExternallyLinkage() || GV->hasAppendingLinkage()))
1488 continue;
1489
1490 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1491 : GlobalValue::UnnamedAddr::None);
1492 if (EnableLTOInternalization && R.second.Partition == 0)
1493 GV->setLinkage(GlobalValue::InternalLinkage);
1494 }
1495
1496 if (Conf.PostInternalizeModuleHook &&
1497 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1498 return Error::success();
1499 }
1500
1501 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1502 if (Error Err = backend(
1503 C: Conf, AddStream, ParallelCodeGenParallelismLevel: RegularLTO.ParallelCodeGenParallelismLevel,
1504 M&: *RegularLTO.CombinedModule, CombinedIndex&: ThinLTO.CombinedIndex, BitcodeLibFuncs))
1505 return Err;
1506 }
1507
1508 return Error::success();
1509}
1510
1511SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
1512 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
1513 SmallVector<const char *> LibcallSymbols;
1514 LibcallSymbols.reserve(N: Libcalls.getNumAvailableLibcallImpls());
1515
1516 for (RTLIB::LibcallImpl Impl : RTLIB::libcall_impls()) {
1517 if (Libcalls.isAvailable(Impl))
1518 LibcallSymbols.push_back(Elt: Libcalls.getLibcallImplName(CallImpl: Impl).data());
1519 }
1520
1521 return LibcallSymbols;
1522}
1523
1524SmallVector<StringRef> LTO::getLibFuncSymbols(const Triple &TT,
1525 StringSaver &Saver) {
1526 auto TLII = std::make_unique<TargetLibraryInfoImpl>(args: TT);
1527 TargetLibraryInfo TLI(*TLII);
1528 SmallVector<StringRef> LibFuncSymbols;
1529 LibFuncSymbols.reserve(N: LibFunc::NumLibFuncs);
1530 for (unsigned I = LibFunc::Begin_LibFunc; I != LibFunc::End_LibFunc; ++I) {
1531 LibFunc F = static_cast<LibFunc>(I);
1532 if (TLI.has(F))
1533 LibFuncSymbols.push_back(Elt: Saver.save(S: TLI.getName(F)).data());
1534 }
1535 return LibFuncSymbols;
1536}
1537
1538Error ThinBackendProc::emitFiles(
1539 const FunctionImporter::ImportMapTy &ImportList, unsigned Task,
1540 llvm::StringRef ModulePath, const std::string &NewModulePath) const {
1541 return emitFiles(ImportList, Task, ModulePath, NewModulePath,
1542 SummaryPath: NewModulePath + ".thinlto.bc");
1543}
1544
1545Error ThinBackendProc::emitFiles(
1546 const FunctionImporter::ImportMapTy &ImportList, unsigned Task,
1547 llvm::StringRef ModulePath, const std::string &NewModulePath,
1548 StringRef SummaryPath) const {
1549 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
1550 GVSummaryPtrSet DeclarationSummaries;
1551
1552 std::error_code EC;
1553 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1554 ImportList, ModuleToSummariesForIndex,
1555 DecSummaries&: DeclarationSummaries);
1556 // Resolve the output stream (either file-backed or callback-provided) for the
1557 // index file.
1558 std::unique_ptr<raw_pwrite_stream> OS;
1559 if (Conf.GetSummaryIndexOutputStream) {
1560 OS = Conf.GetSummaryIndexOutputStream(Task);
1561 assert(OS && "GetSummaryIndexOutputStream returned null");
1562 } else {
1563 auto FileOS = std::make_unique<raw_fd_ostream>(args&: SummaryPath, args&: EC,
1564 args: sys::fs::OpenFlags::OF_None);
1565 if (EC)
1566 return createFileError(F: "cannot open " + Twine(SummaryPath), EC);
1567 OS = std::move(FileOS);
1568 }
1569
1570 writeIndexToFile(Index: CombinedIndex, Out&: *OS, ModuleToSummariesForIndex: &ModuleToSummariesForIndex,
1571 DecSummaries: &DeclarationSummaries);
1572
1573 // Emit imports files if requested, using callback if provided.
1574 if (Conf.GetImportsListOutputArray) {
1575 std::vector<std::string> &ImportsListRef =
1576 Conf.GetImportsListOutputArray(Task);
1577 processImportsFiles(
1578 ModulePath, ModuleToSummariesForIndex,
1579 F: [&](StringRef M) { ImportsListRef.push_back(x: M.str()); });
1580 } else if (ShouldEmitImportsFiles) {
1581 if (Error E = EmitImportsFiles(ModulePath, OutputFilename: NewModulePath + ".imports",
1582 ModuleToSummariesForIndex))
1583 return E;
1584 }
1585 return Error::success();
1586}
1587
1588namespace {
1589/// Base class for ThinLTO backends that perform code generation and insert the
1590/// generated files back into the link.
1591class CGThinBackend : public ThinBackendProc {
1592protected:
1593 DenseSet<GlobalValue::GUID> CfiFunctionDefs;
1594 DenseSet<GlobalValue::GUID> CfiFunctionDecls;
1595 bool ShouldEmitIndexFiles;
1596
1597public:
1598 CGThinBackend(
1599 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1600 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1601 lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles,
1602 bool ShouldEmitImportsFiles, ThreadPoolStrategy ThinLTOParallelism)
1603 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1604 OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),
1605 ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1606 auto &Defs = CombinedIndex.cfiFunctionDefs();
1607 CfiFunctionDefs.insert_range(R: Defs.getExportedThinLTOGUIDs());
1608 auto &Decls = CombinedIndex.cfiFunctionDecls();
1609 CfiFunctionDecls.insert_range(R: Decls.getExportedThinLTOGUIDs());
1610 }
1611};
1612
1613/// This backend performs code generation by scheduling a job to run on
1614/// an in-process thread when invoked for each task.
1615class InProcessThinBackend : public CGThinBackend {
1616protected:
1617 // Callback used to add generated native object files to the link by code
1618 // generating directly into the returned output stream.
1619 AddStreamFn AddStream;
1620 FileCache Cache;
1621 ArrayRef<StringRef> BitcodeLibFuncs;
1622
1623public:
1624 InProcessThinBackend(
1625 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1626 ThreadPoolStrategy ThinLTOParallelism,
1627 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1628 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1629 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
1630 ArrayRef<StringRef> BitcodeLibFuncs)
1631 : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite,
1632 ShouldEmitIndexFiles, ShouldEmitImportsFiles,
1633 ThinLTOParallelism),
1634 AddStream(std::move(AddStream)), Cache(std::move(Cache)),
1635 BitcodeLibFuncs(BitcodeLibFuncs) {}
1636
1637 virtual Error runThinLTOBackendThread(
1638 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1639 ModuleSummaryIndex &CombinedIndex,
1640 const FunctionImporter::ImportMapTy &ImportList,
1641 const FunctionImporter::ExportSetTy &ExportList,
1642 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1643 const GVSummaryMapTy &DefinedGlobals,
1644 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1645 auto ModuleID = BM.getModuleIdentifier();
1646 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (in-process)",
1647 ModuleID);
1648 auto RunThinBackend = [&](AddStreamFn AddStream) {
1649 LTOLLVMContext BackendContext(Conf);
1650 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext);
1651 if (!MOrErr)
1652 return MOrErr.takeError();
1653
1654 return thinBackend(C: Conf, Task, AddStream, M&: **MOrErr, CombinedIndex,
1655 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1656 CodeGenOnly: Conf.CodeGenOnly, BitcodeLibFuncs);
1657 };
1658 if (ShouldEmitIndexFiles) {
1659 if (auto E = emitFiles(ImportList, Task, ModulePath: ModuleID, NewModulePath: ModuleID.str()))
1660 return E;
1661 }
1662
1663 if (!Cache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1664 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1665 P: [](uint32_t V) { return V == 0; }))
1666 // Cache disabled or no entry for this module in the combined index or
1667 // no module hash.
1668 return RunThinBackend(AddStream);
1669
1670 // The module may be cached, this helps handling it.
1671 std::string Key = computeLTOCacheKey(
1672 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1673 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1674 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1675 if (Error Err = CacheAddStreamOrErr.takeError())
1676 return Err;
1677 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1678 if (CacheAddStream)
1679 return RunThinBackend(CacheAddStream);
1680
1681 return Error::success();
1682 }
1683
1684 Error start(
1685 unsigned Task, BitcodeModule BM,
1686 const FunctionImporter::ImportMapTy &ImportList,
1687 const FunctionImporter::ExportSetTy &ExportList,
1688 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1689 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1690 StringRef ModulePath = BM.getModuleIdentifier();
1691 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1692 const GVSummaryMapTy &DefinedGlobals =
1693 ModuleToDefinedGVSummaries.find(Val: ModulePath)->second;
1694 BackendThreadPool.async(
1695 F: [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1696 const FunctionImporter::ImportMapTy &ImportList,
1697 const FunctionImporter::ExportSetTy &ExportList,
1698 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1699 &ResolvedODR,
1700 const GVSummaryMapTy &DefinedGlobals,
1701 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1702 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1703 timeTraceProfilerInitialize(TimeTraceGranularity: Conf.TimeTraceGranularity,
1704 ProcName: "thin backend");
1705 Error E = runThinLTOBackendThread(
1706 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1707 ResolvedODR, DefinedGlobals, ModuleMap);
1708 if (E) {
1709 std::unique_lock<std::mutex> L(ErrMu);
1710 if (Err)
1711 Err = joinErrors(E1: std::move(*Err), E2: std::move(E));
1712 else
1713 Err = std::move(E);
1714 }
1715 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1716 timeTraceProfilerFinishThread();
1717 },
1718 ArgList&: BM, ArgList: std::ref(t&: CombinedIndex), ArgList: std::ref(t: ImportList), ArgList: std::ref(t: ExportList),
1719 ArgList: std::ref(t: ResolvedODR), ArgList: std::ref(t: DefinedGlobals), ArgList: std::ref(t&: ModuleMap));
1720
1721 if (OnWrite)
1722 OnWrite(std::string(ModulePath));
1723 return Error::success();
1724 }
1725};
1726
1727/// This backend is utilized in the first round of a two-codegen round process.
1728/// It first saves optimized bitcode files to disk before the codegen process
1729/// begins. After codegen, it stores the resulting object files in a scratch
1730/// buffer. Note the codegen data stored in the scratch buffer will be extracted
1731/// and merged in the subsequent step.
1732class FirstRoundThinBackend : public InProcessThinBackend {
1733 AddStreamFn IRAddStream;
1734 FileCache IRCache;
1735
1736public:
1737 FirstRoundThinBackend(
1738 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1739 ThreadPoolStrategy ThinLTOParallelism,
1740 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1741 AddStreamFn CGAddStream, FileCache CGCache,
1742 ArrayRef<StringRef> BitcodeLibFuncs, AddStreamFn IRAddStream,
1743 FileCache IRCache)
1744 : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
1745 ModuleToDefinedGVSummaries, std::move(CGAddStream),
1746 std::move(CGCache), /*OnWrite=*/nullptr,
1747 /*ShouldEmitIndexFiles=*/false,
1748 /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
1749 IRAddStream(std::move(IRAddStream)), IRCache(std::move(IRCache)) {}
1750
1751 Error runThinLTOBackendThread(
1752 AddStreamFn CGAddStream, FileCache CGCache, unsigned Task,
1753 BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1754 const FunctionImporter::ImportMapTy &ImportList,
1755 const FunctionImporter::ExportSetTy &ExportList,
1756 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1757 const GVSummaryMapTy &DefinedGlobals,
1758 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1759 auto ModuleID = BM.getModuleIdentifier();
1760 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (first round)",
1761 ModuleID);
1762 auto RunThinBackend = [&](AddStreamFn CGAddStream,
1763 AddStreamFn IRAddStream) {
1764 LTOLLVMContext BackendContext(Conf);
1765 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext);
1766 if (!MOrErr)
1767 return MOrErr.takeError();
1768
1769 return thinBackend(C: Conf, Task, AddStream: CGAddStream, M&: **MOrErr, CombinedIndex,
1770 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1771 CodeGenOnly: Conf.CodeGenOnly, BitcodeLibFuncs, IRAddStream);
1772 };
1773 // Like InProcessThinBackend, we produce index files as needed for
1774 // FirstRoundThinBackend. However, these files are not generated for
1775 // SecondRoundThinBackend.
1776 if (ShouldEmitIndexFiles) {
1777 if (auto E = emitFiles(ImportList, Task, ModulePath: ModuleID, NewModulePath: ModuleID.str()))
1778 return E;
1779 }
1780
1781 assert((CGCache.isValid() == IRCache.isValid()) &&
1782 "Both caches for CG and IR should have matching availability");
1783 if (!CGCache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1784 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1785 P: [](uint32_t V) { return V == 0; }))
1786 // Cache disabled or no entry for this module in the combined index or
1787 // no module hash.
1788 return RunThinBackend(CGAddStream, IRAddStream);
1789
1790 // Get CGKey for caching object in CGCache.
1791 std::string CGKey = computeLTOCacheKey(
1792 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1793 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1794 Expected<AddStreamFn> CacheCGAddStreamOrErr =
1795 CGCache(Task, CGKey, ModuleID);
1796 if (Error Err = CacheCGAddStreamOrErr.takeError())
1797 return Err;
1798 AddStreamFn &CacheCGAddStream = *CacheCGAddStreamOrErr;
1799
1800 // Get IRKey for caching (optimized) IR in IRCache with an extra ID.
1801 std::string IRKey = recomputeLTOCacheKey(Key: CGKey, /*ExtraID=*/"IR");
1802 Expected<AddStreamFn> CacheIRAddStreamOrErr =
1803 IRCache(Task, IRKey, ModuleID);
1804 if (Error Err = CacheIRAddStreamOrErr.takeError())
1805 return Err;
1806 AddStreamFn &CacheIRAddStream = *CacheIRAddStreamOrErr;
1807
1808 // Ideally, both CG and IR caching should be synchronized. However, in
1809 // practice, their availability may differ due to different expiration
1810 // times. Therefore, if either cache is missing, the backend process is
1811 // triggered.
1812 if (CacheCGAddStream || CacheIRAddStream) {
1813 LLVM_DEBUG(dbgs() << "[FirstRound] Cache Miss for "
1814 << BM.getModuleIdentifier() << "\n");
1815 return RunThinBackend(CacheCGAddStream ? CacheCGAddStream : CGAddStream,
1816 CacheIRAddStream ? CacheIRAddStream : IRAddStream);
1817 }
1818
1819 return Error::success();
1820 }
1821};
1822
1823/// This backend operates in the second round of a two-codegen round process.
1824/// It starts by reading the optimized bitcode files that were saved during the
1825/// first round. The backend then executes the codegen only to further optimize
1826/// the code, utilizing the codegen data merged from the first round. Finally,
1827/// it writes the resulting object files as usual.
1828class SecondRoundThinBackend : public InProcessThinBackend {
1829 std::unique_ptr<SmallVector<StringRef>> IRFiles;
1830 stable_hash CombinedCGDataHash;
1831
1832public:
1833 SecondRoundThinBackend(
1834 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1835 ThreadPoolStrategy ThinLTOParallelism,
1836 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1837 AddStreamFn AddStream, FileCache Cache,
1838 ArrayRef<StringRef> BitcodeLibFuncs,
1839 std::unique_ptr<SmallVector<StringRef>> IRFiles,
1840 stable_hash CombinedCGDataHash)
1841 : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
1842 ModuleToDefinedGVSummaries, std::move(AddStream),
1843 std::move(Cache),
1844 /*OnWrite=*/nullptr,
1845 /*ShouldEmitIndexFiles=*/false,
1846 /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
1847 IRFiles(std::move(IRFiles)), CombinedCGDataHash(CombinedCGDataHash) {}
1848
1849 Error runThinLTOBackendThread(
1850 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1851 ModuleSummaryIndex &CombinedIndex,
1852 const FunctionImporter::ImportMapTy &ImportList,
1853 const FunctionImporter::ExportSetTy &ExportList,
1854 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1855 const GVSummaryMapTy &DefinedGlobals,
1856 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1857 auto ModuleID = BM.getModuleIdentifier();
1858 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (second round)",
1859 ModuleID);
1860 auto RunThinBackend = [&](AddStreamFn AddStream) {
1861 LTOLLVMContext BackendContext(Conf);
1862 std::unique_ptr<Module> LoadedModule =
1863 cgdata::loadModuleForTwoRounds(OrigModule&: BM, Task, Context&: BackendContext, IRFiles: *IRFiles);
1864
1865 return thinBackend(C: Conf, Task, AddStream, M&: *LoadedModule, CombinedIndex,
1866 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1867 /*CodeGenOnly=*/true, BitcodeLibFuncs);
1868 };
1869 if (!Cache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1870 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1871 P: [](uint32_t V) { return V == 0; }))
1872 // Cache disabled or no entry for this module in the combined index or
1873 // no module hash.
1874 return RunThinBackend(AddStream);
1875
1876 // Get Key for caching the final object file in Cache with the combined
1877 // CGData hash.
1878 std::string Key = computeLTOCacheKey(
1879 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1880 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1881 Key = recomputeLTOCacheKey(Key,
1882 /*ExtraID=*/std::to_string(val: CombinedCGDataHash));
1883 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1884 if (Error Err = CacheAddStreamOrErr.takeError())
1885 return Err;
1886 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1887
1888 if (CacheAddStream) {
1889 LLVM_DEBUG(dbgs() << "[SecondRound] Cache Miss for "
1890 << BM.getModuleIdentifier() << "\n");
1891 return RunThinBackend(CacheAddStream);
1892 }
1893
1894 return Error::success();
1895 }
1896};
1897} // end anonymous namespace
1898
1899ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,
1900 lto::IndexWriteCallback OnWrite,
1901 bool ShouldEmitIndexFiles,
1902 bool ShouldEmitImportsFiles) {
1903 auto Func =
1904 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1905 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1906 AddStreamFn AddStream, FileCache Cache,
1907 ArrayRef<StringRef> BitcodeLibFuncs) {
1908 return std::make_unique<InProcessThinBackend>(
1909 args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries,
1910 args&: AddStream, args&: Cache, args: OnWrite, args: ShouldEmitIndexFiles,
1911 args: ShouldEmitImportsFiles, args&: BitcodeLibFuncs);
1912 };
1913 return ThinBackend(Func, Parallelism);
1914}
1915
1916StringLiteral lto::getThinLTODefaultCPU(const Triple &TheTriple) {
1917 if (!TheTriple.isOSDarwin())
1918 return "";
1919 if (TheTriple.getArch() == Triple::x86_64)
1920 return "core2";
1921 if (TheTriple.getArch() == Triple::x86)
1922 return "yonah";
1923 if (TheTriple.isArm64e())
1924 return "apple-a12";
1925 if (TheTriple.getArch() == Triple::aarch64 ||
1926 TheTriple.getArch() == Triple::aarch64_32)
1927 return "cyclone";
1928 return "";
1929}
1930
1931// Given the original \p Path to an output file, replace any path
1932// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1933// resulting directory if it does not yet exist.
1934std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
1935 StringRef NewPrefix) {
1936 if (OldPrefix.empty() && NewPrefix.empty())
1937 return std::string(Path);
1938 SmallString<128> NewPath(Path);
1939 llvm::sys::path::replace_path_prefix(Path&: NewPath, OldPrefix, NewPrefix);
1940 StringRef ParentPath = llvm::sys::path::parent_path(path: NewPath.str());
1941 if (!ParentPath.empty()) {
1942 // Make sure the new directory exists, creating it if necessary.
1943 if (std::error_code EC = llvm::sys::fs::create_directories(path: ParentPath))
1944 llvm::errs() << "warning: could not create directory '" << ParentPath
1945 << "': " << EC.message() << '\n';
1946 }
1947 return std::string(NewPath);
1948}
1949
1950namespace {
1951class WriteIndexesThinBackend : public ThinBackendProc {
1952 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1953 raw_fd_ostream *LinkedObjectsFile;
1954 DenseSet<GlobalValue::GUID> CfiFunctionDefs;
1955 DenseSet<GlobalValue::GUID> CfiFunctionDecls;
1956
1957public:
1958 WriteIndexesThinBackend(
1959 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1960 ThreadPoolStrategy ThinLTOParallelism,
1961 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1962 std::string OldPrefix, std::string NewPrefix,
1963 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1964 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1965 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1966 OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),
1967 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1968 NativeObjectPrefix(NativeObjectPrefix),
1969 LinkedObjectsFile(LinkedObjectsFile) {
1970 auto Defs = CombinedIndex.cfiFunctionDefs().getExportedThinLTOGUIDs();
1971 CfiFunctionDefs.insert(I: Defs.begin(), E: Defs.end());
1972 auto Decls = CombinedIndex.cfiFunctionDecls().getExportedThinLTOGUIDs();
1973 CfiFunctionDecls.insert(I: Decls.begin(), E: Decls.end());
1974 }
1975
1976 Error start(
1977 unsigned Task, BitcodeModule BM,
1978 const FunctionImporter::ImportMapTy &ImportList,
1979 const FunctionImporter::ExportSetTy &ExportList,
1980 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1981 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1982 StringRef ModulePath = BM.getModuleIdentifier();
1983
1984 // The contents of this file may be used as input to a native link, and must
1985 // therefore contain the processed modules in a determinstic order that
1986 // match the order they are provided on the command line. For that reason,
1987 // we cannot include this in the asynchronously executed lambda below.
1988 if (LinkedObjectsFile) {
1989 std::string ObjectPrefix =
1990 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1991 std::string LinkedObjectsFilePath =
1992 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix: ObjectPrefix);
1993 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1994 }
1995
1996 BackendThreadPool.async(
1997 F: [this](unsigned Task, const StringRef ModulePath,
1998 const FunctionImporter::ImportMapTy &ImportList,
1999 const FunctionImporter::ExportSetTy &ExportList,
2000 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
2001 &ResolvedODR,
2002 const std::string &OldPrefix, const std::string &NewPrefix) {
2003 std::string NewModulePath =
2004 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix);
2005 auto E = emitFiles(ImportList, Task, ModulePath, NewModulePath);
2006 if (E) {
2007 std::unique_lock<std::mutex> L(ErrMu);
2008 if (Err)
2009 Err = joinErrors(E1: std::move(*Err), E2: std::move(E));
2010 else
2011 Err = std::move(E);
2012 }
2013 assert(ModuleToDefinedGVSummaries.count(ModulePath));
2014 const GVSummaryMapTy &DefinedGlobals =
2015 ModuleToDefinedGVSummaries.find(Val: ModulePath)->second;
2016
2017 // DTLTO needs the per-module LTO cache key to probe the cache.
2018 if (Conf.GetCacheKeyOutputString) {
2019 std::string &CacheKey = Conf.GetCacheKeyOutputString(Task);
2020 CacheKey = computeLTOCacheKey(
2021 Conf, Index: CombinedIndex, ModuleID: ModulePath, ImportList, ExportList,
2022 ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
2023 }
2024 },
2025 ArgList&: Task, ArgList&: ModulePath, ArgList: ImportList, ArgList: ExportList, ArgList: ResolvedODR, ArgList&: OldPrefix,
2026 ArgList&: NewPrefix);
2027
2028 if (OnWrite)
2029 OnWrite(std::string(ModulePath));
2030 return Error::success();
2031 }
2032
2033 bool isSensitiveToInputOrder() override {
2034 // The order which modules are written to LinkedObjectsFile should be
2035 // deterministic and match the order they are passed on the command line.
2036 return true;
2037 }
2038};
2039} // end anonymous namespace
2040
2041ThinBackend lto::createWriteIndexesThinBackend(
2042 ThreadPoolStrategy Parallelism, std::string OldPrefix,
2043 std::string NewPrefix, std::string NativeObjectPrefix,
2044 bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile,
2045 IndexWriteCallback OnWrite) {
2046 auto Func =
2047 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
2048 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
2049 AddStreamFn AddStream, FileCache Cache,
2050 ArrayRef<StringRef> BitcodeLibFuncs) {
2051 return std::make_unique<WriteIndexesThinBackend>(
2052 args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries,
2053 args: OldPrefix, args: NewPrefix, args: NativeObjectPrefix, args: ShouldEmitImportsFiles,
2054 args: LinkedObjectsFile, args: OnWrite);
2055 };
2056 return ThinBackend(Func, Parallelism);
2057}
2058
2059Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
2060 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
2061 llvm::TimeTraceScope timeScope("Run ThinLTO");
2062 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
2063 ThinLTO.CombinedIndex.releaseTemporaryMemory();
2064 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
2065 llvm::scope_exit TimeTraceScopeExit([]() {
2066 if (llvm::timeTraceProfilerEnabled())
2067 llvm::timeTraceProfilerEnd();
2068 });
2069 if (ThinLTO.ModuleMap.empty())
2070 return Error::success();
2071
2072 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
2073 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
2074 return Error::success();
2075 }
2076
2077 if (Conf.CombinedIndexHook &&
2078 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
2079 return Error::success();
2080
2081 // Collect for each module the list of function it defines (GUID ->
2082 // Summary).
2083 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
2084 ThinLTO.ModuleMap.size());
2085 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
2086 ModuleToDefinedGVSummaries);
2087 // Create entries for any modules that didn't have any GV summaries
2088 // (either they didn't have any GVs to start with, or we suppressed
2089 // generation of the summaries because they e.g. had inline assembly
2090 // uses that couldn't be promoted/renamed on export). This is so
2091 // InProcessThinBackend::start can still launch a backend thread, which
2092 // is passed the map of summaries for the module, without any special
2093 // handling for this case.
2094 for (auto &Mod : ThinLTO.ModuleMap)
2095 if (!ModuleToDefinedGVSummaries.count(Val: Mod.first))
2096 ModuleToDefinedGVSummaries.try_emplace(Key: Mod.first);
2097
2098 FunctionImporter::ImportListsTy ImportLists(ThinLTO.ModuleMap.size());
2099 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(
2100 ThinLTO.ModuleMap.size());
2101 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
2102
2103 if (DumpThinCGSCCs)
2104 ThinLTO.CombinedIndex.dumpSCCs(OS&: outs());
2105
2106 std::set<GlobalValue::GUID> ExportedGUIDs;
2107
2108 bool WholeProgramVisibilityEnabledInLTO =
2109 Conf.HasWholeProgramVisibility &&
2110 // If validation is enabled, upgrade visibility only when all vtables
2111 // have typeinfos.
2112 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
2113 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
2114 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
2115
2116 // If we're validating, get the vtable symbols that should not be
2117 // upgraded because they correspond to typeIDs outside of index-based
2118 // WPD info.
2119 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
2120 if (WholeProgramVisibilityEnabledInLTO &&
2121 Conf.ValidateAllVtablesHaveTypeInfos) {
2122 // This returns true when the name is local or not defined. Locals are
2123 // expected to be handled separately.
2124 auto IsVisibleToRegularObj = [&](StringRef name) {
2125 auto It = GlobalResolutions->find(Val: name);
2126 return (It == GlobalResolutions->end() ||
2127 It->second.VisibleOutsideSummary || !It->second.Prevailing);
2128 };
2129
2130 getVisibleToRegularObjVtableGUIDs(Index&: ThinLTO.CombinedIndex,
2131 VisibleToRegularObjSymbols,
2132 IsVisibleToRegularObj);
2133 }
2134
2135 // If allowed, upgrade public vcall visibility to linkage unit visibility in
2136 // the summaries before whole program devirtualization below.
2137 updateVCallVisibilityInIndex(
2138 Index&: ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
2139 DynamicExportSymbols, VisibleToRegularObjSymbols);
2140
2141 // Perform index-based WPD. This will return immediately if there are
2142 // no index entries in the typeIdMetadata map (e.g. if we are instead
2143 // performing IR-based WPD in hybrid regular/thin LTO mode).
2144 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
2145 DenseSet<StringRef> ExternallyVisibleSymbolNames;
2146
2147 // Used by the promotion-time renaming logic. When non-null, this set
2148 // identifies symbols that should not be renamed during promotion.
2149 // It is non-null only when whole-program visibility is enabled and
2150 // renaming is not forced. Otherwise, the default renaming behavior applies.
2151 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr =
2152 (WholeProgramVisibilityEnabledInLTO && !AlwaysRenamePromotedLocals)
2153 ? &ExternallyVisibleSymbolNames
2154 : nullptr;
2155 runWholeProgramDevirtOnIndex(Summary&: ThinLTO.CombinedIndex, ExportedGUIDs,
2156 LocalWPDTargetsMap,
2157 ExternallyVisibleSymbolNamesPtr);
2158
2159 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
2160 return ThinLTO.isPrevailingModuleForGUID(GUID, Module: S->modulePath());
2161 };
2162 if (EnableMemProfContextDisambiguation) {
2163 MemProfContextDisambiguation ContextDisambiguation;
2164 ContextDisambiguation.run(
2165 Index&: ThinLTO.CombinedIndex, isPrevailing, Ctx&: RegularLTO.Ctx,
2166 EmitRemark: [&](StringRef PassName, StringRef RemarkName, const Twine &Msg) {
2167 auto R = OptimizationRemark(PassName.data(), RemarkName,
2168 LinkerRemarkFunction);
2169 R << Msg.str();
2170 emitRemark(Remark&: R);
2171 });
2172 }
2173
2174 // Figure out which symbols need to be internalized. This also needs to happen
2175 // at -O0 because summary-based DCE is implemented using internalization, and
2176 // we must apply DCE consistently with the full LTO module in order to avoid
2177 // undefined references during the final link.
2178 for (auto &Res : *GlobalResolutions) {
2179 // If the symbol does not have external references or it is not prevailing,
2180 // then not need to mark it as exported from a ThinLTO partition.
2181 if (Res.second.Partition != GlobalResolution::External ||
2182 !Res.second.isPrevailingIRSymbol())
2183 continue;
2184 auto GUID = Res.second.getGUID();
2185 // Mark exported unless index-based analysis determined it to be dead.
2186 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
2187 ExportedGUIDs.insert(x: GUID);
2188 }
2189
2190 // Reset the GlobalResolutions to deallocate the associated memory, as there
2191 // are no further accesses. We specifically want to do this before computing
2192 // cross module importing, which adds to peak memory via the computed import
2193 // and export lists.
2194 releaseGlobalResolutionsMemory();
2195
2196 if (Conf.OptLevel > 0)
2197 ComputeCrossModuleImport(Index: ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
2198 isPrevailing, ImportLists, ExportLists);
2199
2200 // Any functions referenced by the jump table in the regular LTO object must
2201 // be exported.
2202 auto Defs = ThinLTO.CombinedIndex.cfiFunctionDefs().getExportedThinLTOGUIDs();
2203 ExportedGUIDs.insert(first: Defs.begin(), last: Defs.end());
2204 auto Decls =
2205 ThinLTO.CombinedIndex.cfiFunctionDecls().getExportedThinLTOGUIDs();
2206 ExportedGUIDs.insert(first: Decls.begin(), last: Decls.end());
2207
2208 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
2209 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
2210 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
2211 ExportedGUIDs.count(x: VI.getGUID());
2212 };
2213
2214 // Update local devirtualized targets that were exported by cross-module
2215 // importing or by other devirtualizations marked in the ExportedGUIDs set.
2216 updateIndexWPDForExports(Summary&: ThinLTO.CombinedIndex, isExported,
2217 LocalWPDTargetsMap, ExternallyVisibleSymbolNamesPtr);
2218
2219 if (ExternallyVisibleSymbolNamesPtr) {
2220 // Add to ExternallyVisibleSymbolNames the set of unique names used by all
2221 // externally visible symbols in the index.
2222 for (auto &I : ThinLTO.CombinedIndex) {
2223 ValueInfo VI = ThinLTO.CombinedIndex.getValueInfo(R: I);
2224 for (const auto &Summary : VI.getSummaryList()) {
2225 const GlobalValueSummary *Base = Summary->getBaseObject();
2226 if (GlobalValue::isLocalLinkage(Linkage: Base->linkage()))
2227 continue;
2228
2229 ExternallyVisibleSymbolNamesPtr->insert(V: VI.name());
2230 break;
2231 }
2232 }
2233 }
2234
2235 thinLTOInternalizeAndPromoteInIndex(Index&: ThinLTO.CombinedIndex, isExported,
2236 isPrevailing,
2237 ExternallyVisibleSymbolNamesPtr);
2238
2239 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
2240 GlobalValue::GUID GUID,
2241 GlobalValue::LinkageTypes NewLinkage) {
2242 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
2243 };
2244 thinLTOResolvePrevailingInIndex(C: Conf, Index&: ThinLTO.CombinedIndex, isPrevailing,
2245 recordNewLinkage, GUIDPreservedSymbols);
2246
2247 thinLTOPropagateFunctionAttrs(Index&: ThinLTO.CombinedIndex, isPrevailing);
2248
2249 generateParamAccessSummary(Index&: ThinLTO.CombinedIndex);
2250
2251 if (llvm::timeTraceProfilerEnabled())
2252 llvm::timeTraceProfilerEnd();
2253
2254 TimeTraceScopeExit.release();
2255
2256 auto &ModuleMap =
2257 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
2258
2259 auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error {
2260 auto ProcessOneModule = [&](int I) -> Error {
2261 auto &Mod = *(ModuleMap.begin() + I);
2262 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
2263 // combined module and parallel code generation partitions.
2264 return BackendProcess->start(
2265 Task: RegularLTO.ParallelCodeGenParallelismLevel + I, BM: Mod.second,
2266 ImportList: ImportLists[Mod.first], ExportList: ExportLists[Mod.first],
2267 ResolvedODR: ResolvedODR[Mod.first], ModuleMap&: ThinLTO.ModuleMap);
2268 };
2269
2270 BackendProcess->setup(ThinLTONumTasks: ModuleMap.size(),
2271 ThinLTOTaskOffset: RegularLTO.ParallelCodeGenParallelismLevel,
2272 Triple: RegularLTO.CombinedModule->getTargetTriple());
2273
2274 if (BackendProcess->getThreadCount() == 1 ||
2275 BackendProcess->isSensitiveToInputOrder()) {
2276 // Process the modules in the order they were provided on the
2277 // command-line. It is important for this codepath to be used for
2278 // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists
2279 // ThinLTO objects in the same order as the inputs, which otherwise would
2280 // affect the final link order.
2281 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
2282 if (Error E = ProcessOneModule(I))
2283 return E;
2284 } else {
2285 // When executing in parallel, process largest bitsize modules first to
2286 // improve parallelism, and avoid starving the thread pool near the end.
2287 // This saves about 15 sec on a 36-core machine while link `clang.exe`
2288 // (out of 100 sec).
2289 std::vector<BitcodeModule *> ModulesVec;
2290 ModulesVec.reserve(n: ModuleMap.size());
2291 for (auto &Mod : ModuleMap)
2292 ModulesVec.push_back(x: &Mod.second);
2293 for (int I : generateModulesOrdering(R: ModulesVec))
2294 if (Error E = ProcessOneModule(I))
2295 return E;
2296 }
2297 return BackendProcess->wait();
2298 };
2299
2300 if (!CodeGenDataThinLTOTwoRounds) {
2301 std::unique_ptr<ThinBackendProc> BackendProc =
2302 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
2303 AddStream, Cache, BitcodeLibFuncs);
2304 return RunBackends(BackendProc.get());
2305 }
2306
2307 // Perform two rounds of code generation for ThinLTO:
2308 // 1. First round: Perform optimization and code generation, outputting to
2309 // temporary scratch objects.
2310 // 2. Merge code generation data extracted from the temporary scratch objects.
2311 // 3. Second round: Execute code generation again using the merged data.
2312 LLVM_DEBUG(dbgs() << "[TwoRounds] Initializing ThinLTO two-codegen rounds\n");
2313
2314 unsigned MaxTasks = getMaxTasks();
2315 auto Parallelism = ThinLTO.Backend.getParallelism();
2316 // Set up two additional streams and caches for storing temporary scratch
2317 // objects and optimized IRs, using the same cache directory as the original.
2318 cgdata::StreamCacheData CG(MaxTasks, Cache, "CG"), IR(MaxTasks, Cache, "IR");
2319
2320 // First round: Execute optimization and code generation, outputting to
2321 // temporary scratch objects. Serialize the optimized IRs before initiating
2322 // code generation.
2323 LLVM_DEBUG(dbgs() << "[TwoRounds] Running the first round of codegen\n");
2324 auto FirstRoundLTO = std::make_unique<FirstRoundThinBackend>(
2325 args&: Conf, args&: ThinLTO.CombinedIndex, args&: Parallelism, args&: ModuleToDefinedGVSummaries,
2326 args&: CG.AddStream, args&: CG.Cache, args&: BitcodeLibFuncs, args&: IR.AddStream, args&: IR.Cache);
2327 if (Error E = RunBackends(FirstRoundLTO.get()))
2328 return E;
2329
2330 LLVM_DEBUG(dbgs() << "[TwoRounds] Merging codegen data\n");
2331 auto CombinedHashOrErr = cgdata::mergeCodeGenData(ObjectFiles: *CG.getResult());
2332 if (Error E = CombinedHashOrErr.takeError())
2333 return E;
2334 auto CombinedHash = *CombinedHashOrErr;
2335 LLVM_DEBUG(dbgs() << "[TwoRounds] CGData hash: " << CombinedHash << "\n");
2336
2337 // Second round: Read the optimized IRs and execute code generation using the
2338 // merged data.
2339 LLVM_DEBUG(dbgs() << "[TwoRounds] Running the second round of codegen\n");
2340 auto SecondRoundLTO = std::make_unique<SecondRoundThinBackend>(
2341 args&: Conf, args&: ThinLTO.CombinedIndex, args&: Parallelism, args&: ModuleToDefinedGVSummaries,
2342 args&: AddStream, args&: Cache, args&: BitcodeLibFuncs, args: IR.getResult(), args&: CombinedHash);
2343 return RunBackends(SecondRoundLTO.get());
2344}
2345
2346Expected<LLVMRemarkFileHandle> lto::setupLLVMOptimizationRemarks(
2347 LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
2348 StringRef RemarksFormat, bool RemarksWithHotness,
2349 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
2350 std::string Filename = std::string(RemarksFilename);
2351 // For ThinLTO, file.opt.<format> becomes
2352 // file.opt.<format>.thin.<num>.<format>.
2353 if (!Filename.empty() && Count != -1)
2354 Filename =
2355 (Twine(Filename) + ".thin." + llvm::utostr(X: Count) + "." + RemarksFormat)
2356 .str();
2357
2358 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
2359 Context, RemarksFilename: Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,
2360 RemarksHotnessThreshold);
2361 if (Error E = ResultOrErr.takeError())
2362 return std::move(E);
2363
2364 if (*ResultOrErr)
2365 (*ResultOrErr)->keep();
2366
2367 return ResultOrErr;
2368}
2369
2370Expected<std::unique_ptr<ToolOutputFile>>
2371lto::setupStatsFile(StringRef StatsFilename) {
2372 // Setup output file to emit statistics.
2373 if (StatsFilename.empty())
2374 return nullptr;
2375
2376 llvm::EnableStatistics(DoPrintOnExit: false);
2377 std::error_code EC;
2378 auto StatsFile =
2379 std::make_unique<ToolOutputFile>(args&: StatsFilename, args&: EC, args: sys::fs::OF_None);
2380 if (EC)
2381 return errorCodeToError(EC);
2382
2383 StatsFile->keep();
2384 return std::move(StatsFile);
2385}
2386
2387// Compute the ordering we will process the inputs: the rough heuristic here
2388// is to sort them per size so that the largest module get schedule as soon as
2389// possible. This is purely a compile-time optimization.
2390std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
2391 auto Seq = llvm::seq<int>(Begin: 0, End: R.size());
2392 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
2393 llvm::sort(C&: ModulesOrdering, Comp: [&](int LeftIndex, int RightIndex) {
2394 auto LSize = R[LeftIndex]->getBuffer().size();
2395 auto RSize = R[RightIndex]->getBuffer().size();
2396 return LSize > RSize;
2397 });
2398 return ModulesOrdering;
2399}
2400