1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/StableHashing.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/StackSafetyAnalysis.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/Bitcode/BitcodeWriter.h"
25#include "llvm/CGData/CodeGenData.h"
26#include "llvm/CodeGen/Analysis.h"
27#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/AutoUpgrade.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/LLVMRemarkStreamer.h"
33#include "llvm/IR/LegacyPassManager.h"
34#include "llvm/IR/Mangler.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/IR/RuntimeLibcalls.h"
37#include "llvm/LTO/LTOBackend.h"
38#include "llvm/Linker/IRMover.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Object/IRObjectFile.h"
41#include "llvm/Support/Caching.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Compiler.h"
44#include "llvm/Support/Error.h"
45#include "llvm/Support/FileSystem.h"
46#include "llvm/Support/JSON.h"
47#include "llvm/Support/MemoryBuffer.h"
48#include "llvm/Support/Path.h"
49#include "llvm/Support/Process.h"
50#include "llvm/Support/SHA1.h"
51#include "llvm/Support/Signals.h"
52#include "llvm/Support/SourceMgr.h"
53#include "llvm/Support/ThreadPool.h"
54#include "llvm/Support/Threading.h"
55#include "llvm/Support/TimeProfiler.h"
56#include "llvm/Support/ToolOutputFile.h"
57#include "llvm/Support/VCSRevision.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/Target/TargetOptions.h"
60#include "llvm/Transforms/IPO.h"
61#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
62#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
63#include "llvm/Transforms/Utils/FunctionImportUtils.h"
64#include "llvm/Transforms/Utils/SplitModule.h"
65
66#include <optional>
67#include <set>
68
69using namespace llvm;
70using namespace lto;
71using namespace object;
72
73#define DEBUG_TYPE "lto"
74
75Error LTO::setupOptimizationRemarks() {
76 // Setup the remark streamer according to the provided configuration.
77 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
78 Context&: RegularLTO.Ctx, RemarksFilename: Conf.RemarksFilename, RemarksPasses: Conf.RemarksPasses,
79 RemarksFormat: Conf.RemarksFormat, RemarksWithHotness: Conf.RemarksWithHotness,
80 RemarksHotnessThreshold: Conf.RemarksHotnessThreshold);
81 if (!DiagFileOrErr)
82 return DiagFileOrErr.takeError();
83
84 DiagnosticOutputFile = std::move(*DiagFileOrErr);
85
86 // Create a dummy function to serve as a context for LTO-link remarks.
87 // This is required because OptimizationRemark requires a valid Function,
88 // and in ThinLTO we may not have any IR functions available during the
89 // thin link. Host it in a private module to avoid interfering with the LTO
90 // process.
91 if (!LinkerRemarkFunction) {
92 DummyModule = std::make_unique<Module>(args: "remark_dummy", args&: RegularLTO.Ctx);
93 LinkerRemarkFunction = Function::Create(
94 Ty: FunctionType::get(Result: Type::getVoidTy(C&: RegularLTO.Ctx), isVarArg: false),
95 Linkage: GlobalValue::ExternalLinkage, N: "thinlto_remark_dummy",
96 M: DummyModule.get());
97 }
98
99 return Error::success();
100}
101
102void LTO::emitRemark(OptimizationRemark &Remark) {
103 const Function &F = Remark.getFunction();
104 OptimizationRemarkEmitter ORE(const_cast<Function *>(&F));
105 ORE.emit(OptDiag&: Remark);
106}
107
108static cl::opt<bool>
109 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(Val: false), cl::Hidden,
110 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
111namespace llvm {
112extern cl::opt<bool> CodeGenDataThinLTOTwoRounds;
113extern cl::opt<bool> ForceImportAll;
114extern cl::opt<bool> AlwaysRenamePromotedLocals;
115} // end namespace llvm
116
117namespace llvm {
118/// Enable global value internalization in LTO.
119cl::opt<bool> EnableLTOInternalization(
120 "enable-lto-internalization", cl::init(Val: true), cl::Hidden,
121 cl::desc("Enable global value internalization in LTO"));
122
123static cl::opt<bool>
124 LTOKeepSymbolCopies("lto-keep-symbol-copies", cl::init(Val: false), cl::Hidden,
125 cl::desc("Keep copies of symbols in LTO indexing"));
126
127/// Indicate we are linking with an allocator that supports hot/cold operator
128/// new interfaces.
129extern cl::opt<bool> SupportsHotColdNew;
130
131/// Enable MemProf context disambiguation for thin link.
132extern cl::opt<bool> EnableMemProfContextDisambiguation;
133} // namespace llvm
134
135// Computes a unique hash for the Module considering the current list of
136// export/import and other global analysis results.
137// Returns the hash in its hexadecimal representation.
138std::string llvm::computeLTOCacheKey(
139 const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID,
140 const FunctionImporter::ImportMapTy &ImportList,
141 const FunctionImporter::ExportSetTy &ExportList,
142 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
143 const GVSummaryMapTy &DefinedGlobals,
144 const DenseSet<GlobalValue::GUID> &CfiFunctionDefs,
145 const DenseSet<GlobalValue::GUID> &CfiFunctionDecls) {
146 // Compute the unique hash for this entry.
147 // This is based on the current compiler version, the module itself, the
148 // export list, the hash for every single module in the import list, the
149 // list of ResolvedODR for the module, and the list of preserved symbols.
150 SHA1 Hasher;
151
152 // Start with the compiler revision
153 Hasher.update(LLVM_VERSION_STRING);
154#ifdef LLVM_REVISION
155 Hasher.update(LLVM_REVISION);
156#endif
157
158 // Include the parts of the LTO configuration that affect code generation.
159 auto AddString = [&](StringRef Str) {
160 Hasher.update(Str);
161 Hasher.update(Data: ArrayRef<uint8_t>{0});
162 };
163 auto AddUnsigned = [&](unsigned I) {
164 uint8_t Data[4];
165 support::endian::write32le(P: Data, V: I);
166 Hasher.update(Data);
167 };
168 auto AddUint64 = [&](uint64_t I) {
169 uint8_t Data[8];
170 support::endian::write64le(P: Data, V: I);
171 Hasher.update(Data);
172 };
173 auto AddUint8 = [&](const uint8_t I) {
174 Hasher.update(Data: ArrayRef<uint8_t>(&I, 1));
175 };
176 AddString(Conf.CPU);
177 // FIXME: Hash more of Options. For now all clients initialize Options from
178 // command-line flags (which is unsupported in production), but may set
179 // X86RelaxRelocations. The clang driver can also pass FunctionSections,
180 // DataSections and DebuggerTuning via command line flags.
181 AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations);
182 AddUnsigned(Conf.Options.FunctionSections);
183 AddUnsigned(Conf.Options.DataSections);
184 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
185 for (auto &A : Conf.MAttrs)
186 AddString(A);
187 if (Conf.RelocModel)
188 AddUnsigned(*Conf.RelocModel);
189 else
190 AddUnsigned(-1);
191 if (Conf.CodeModel)
192 AddUnsigned(*Conf.CodeModel);
193 else
194 AddUnsigned(-1);
195 for (const auto &S : Conf.MllvmArgs)
196 AddString(S);
197 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
198 AddUnsigned(static_cast<int>(Conf.CGFileType));
199 AddUnsigned(Conf.OptLevel);
200 AddUnsigned(Conf.Freestanding);
201 AddString(Conf.OptPipeline);
202 AddString(Conf.AAPipeline);
203 AddString(Conf.OverrideTriple);
204 AddString(Conf.DefaultTriple);
205 AddString(Conf.DwoDir);
206 AddUint8(Conf.Dtlto);
207
208 // Include the hash for the current module
209 auto ModHash = Index.getModuleHash(ModPath: ModuleID);
210 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
211
212 // TODO: `ExportList` is determined by `ImportList`. Since `ImportList` is
213 // used to compute cache key, we could omit hashing `ExportList` here.
214 std::vector<uint64_t> ExportsGUID;
215 ExportsGUID.reserve(n: ExportList.size());
216 for (const auto &VI : ExportList)
217 ExportsGUID.push_back(x: VI.getGUID());
218
219 // Sort the export list elements GUIDs.
220 llvm::sort(C&: ExportsGUID);
221 for (auto GUID : ExportsGUID)
222 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
223
224 // Order using module hash, to be both independent of module name and
225 // module order.
226 auto Comp = [&](const std::pair<StringRef, GlobalValue::GUID> &L,
227 const std::pair<StringRef, GlobalValue::GUID> &R) {
228 return std::make_pair(x: Index.getModule(ModPath: L.first)->second, y: L.second) <
229 std::make_pair(x: Index.getModule(ModPath: R.first)->second, y: R.second);
230 };
231 FunctionImporter::SortedImportList SortedImportList(ImportList, Comp);
232
233 // Count the number of imports for each source module.
234 DenseMap<StringRef, unsigned> ModuleToNumImports;
235 for (const auto &[FromModule, GUID, Type] : SortedImportList)
236 ++ModuleToNumImports[FromModule];
237
238 std::optional<StringRef> LastModule;
239 for (const auto &[FromModule, GUID, Type] : SortedImportList) {
240 if (LastModule != FromModule) {
241 // Include the hash for every module we import functions from. The set of
242 // imported symbols for each module may affect code generation and is
243 // sensitive to link order, so include that as well.
244 LastModule = FromModule;
245 auto ModHash = Index.getModule(ModPath: FromModule)->second;
246 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
247 AddUint64(ModuleToNumImports[FromModule]);
248 }
249 AddUint64(GUID);
250 AddUint8(Type);
251 }
252
253 // Include the hash for the resolved ODR.
254 for (auto &Entry : ResolvedODR) {
255 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
256 sizeof(GlobalValue::GUID)));
257 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
258 sizeof(GlobalValue::LinkageTypes)));
259 }
260
261 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
262 // defined in this module.
263 std::set<GlobalValue::GUID> UsedCfiDefs;
264 std::set<GlobalValue::GUID> UsedCfiDecls;
265
266 // Typeids used in this module.
267 std::set<GlobalValue::GUID> UsedTypeIds;
268
269 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
270 if (CfiFunctionDefs.contains(V: ValueGUID))
271 UsedCfiDefs.insert(x: ValueGUID);
272 if (CfiFunctionDecls.contains(V: ValueGUID))
273 UsedCfiDecls.insert(x: ValueGUID);
274 };
275
276 auto AddUsedThings = [&](GlobalValueSummary *GS) {
277 if (!GS) return;
278 AddUnsigned(GS->getVisibility());
279 AddUnsigned(GS->isLive());
280 AddUnsigned(GS->canAutoHide());
281 for (const ValueInfo &VI : GS->refs()) {
282 AddUnsigned(VI.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
283 AddUsedCfiGlobal(VI.getGUID());
284 }
285 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: GS)) {
286 AddUnsigned(GVS->maybeReadOnly());
287 AddUnsigned(GVS->maybeWriteOnly());
288 }
289 if (auto *FS = dyn_cast<FunctionSummary>(Val: GS)) {
290 for (auto &TT : FS->type_tests())
291 UsedTypeIds.insert(x: TT);
292 for (auto &TT : FS->type_test_assume_vcalls())
293 UsedTypeIds.insert(x: TT.GUID);
294 for (auto &TT : FS->type_checked_load_vcalls())
295 UsedTypeIds.insert(x: TT.GUID);
296 for (auto &TT : FS->type_test_assume_const_vcalls())
297 UsedTypeIds.insert(x: TT.VFunc.GUID);
298 for (auto &TT : FS->type_checked_load_const_vcalls())
299 UsedTypeIds.insert(x: TT.VFunc.GUID);
300 for (auto &ET : FS->calls()) {
301 AddUnsigned(ET.first.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
302 AddUsedCfiGlobal(ET.first.getGUID());
303 }
304 }
305 };
306
307 // Sort the defined globals by GUID to be independent of the insertion order,
308 // which may depend on the order that modules are added.
309 SmallVector<std::pair<GlobalValue::GUID, GlobalValueSummary *>>
310 SortedDefinedGlobals(DefinedGlobals.begin(), DefinedGlobals.end());
311 llvm::sort(C&: SortedDefinedGlobals, Comp: llvm::less_first());
312 for (auto &GS : SortedDefinedGlobals) {
313 // Include the hash for the linkage type to reflect internalization and weak
314 // resolution, and collect any used type identifier resolutions.
315 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
316 Hasher.update(
317 Data: ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
318 AddUsedCfiGlobal(GS.first);
319 AddUsedThings(GS.second);
320 }
321
322 // Imported functions may introduce new uses of type identifier resolutions,
323 // so we need to collect their used resolutions as well.
324 for (const auto &[FromModule, GUID, Type] : SortedImportList) {
325 GlobalValueSummary *S = Index.findSummaryInModule(ValueGUID: GUID, ModuleId: FromModule);
326 AddUsedThings(S);
327 // If this is an alias, we also care about any types/etc. that the aliasee
328 // may reference.
329 if (auto *AS = dyn_cast_or_null<AliasSummary>(Val: S))
330 AddUsedThings(AS->getBaseObject());
331 }
332
333 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
334 AddString(TId);
335
336 AddUnsigned(S.TTRes.TheKind);
337 AddUnsigned(S.TTRes.SizeM1BitWidth);
338
339 AddUint64(S.TTRes.AlignLog2);
340 AddUint64(S.TTRes.SizeM1);
341 AddUint64(S.TTRes.BitMask);
342 AddUint64(S.TTRes.InlineBits);
343
344 AddUint64(S.WPDRes.size());
345 for (auto &WPD : S.WPDRes) {
346 AddUnsigned(WPD.first);
347 AddUnsigned(WPD.second.TheKind);
348 AddString(WPD.second.SingleImplName);
349
350 AddUint64(WPD.second.ResByArg.size());
351 for (auto &ByArg : WPD.second.ResByArg) {
352 AddUint64(ByArg.first.size());
353 for (uint64_t Arg : ByArg.first)
354 AddUint64(Arg);
355 AddUnsigned(ByArg.second.TheKind);
356 AddUint64(ByArg.second.Info);
357 AddUnsigned(ByArg.second.Byte);
358 AddUnsigned(ByArg.second.Bit);
359 }
360 }
361 };
362
363 // Include the hash for all type identifiers used by this module.
364 for (GlobalValue::GUID TId : UsedTypeIds) {
365 auto TidIter = Index.typeIds().equal_range(x: TId);
366 for (const auto &I : make_range(p: TidIter))
367 AddTypeIdSummary(I.second.first, I.second.second);
368 }
369
370 AddUnsigned(UsedCfiDefs.size());
371 for (auto &V : UsedCfiDefs)
372 AddUint64(V);
373
374 AddUnsigned(UsedCfiDecls.size());
375 for (auto &V : UsedCfiDecls)
376 AddUint64(V);
377
378 if (!Conf.SampleProfile.empty()) {
379 auto FileOrErr = MemoryBuffer::getFile(Filename: Conf.SampleProfile);
380 if (FileOrErr) {
381 Hasher.update(Str: FileOrErr.get()->getBuffer());
382
383 if (!Conf.ProfileRemapping.empty()) {
384 FileOrErr = MemoryBuffer::getFile(Filename: Conf.ProfileRemapping);
385 if (FileOrErr)
386 Hasher.update(Str: FileOrErr.get()->getBuffer());
387 }
388 }
389 }
390
391 return toHex(Input: Hasher.result());
392}
393
394std::string llvm::recomputeLTOCacheKey(const std::string &Key,
395 StringRef ExtraID) {
396 SHA1 Hasher;
397
398 auto AddString = [&](StringRef Str) {
399 Hasher.update(Str);
400 Hasher.update(Data: ArrayRef<uint8_t>{0});
401 };
402 AddString(Key);
403 AddString(ExtraID);
404
405 return toHex(Input: Hasher.result());
406}
407
408static void thinLTOResolvePrevailingGUID(
409 const Config &C, ValueInfo VI,
410 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
411 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
412 isPrevailing,
413 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
414 recordNewLinkage,
415 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
416 GlobalValue::VisibilityTypes Visibility =
417 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
418 : GlobalValue::DefaultVisibility;
419 for (auto &S : VI.getSummaryList()) {
420 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
421 // Ignore local and appending linkage values since the linker
422 // doesn't resolve them.
423 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
424 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
425 continue;
426 // We need to emit only one of these. The prevailing module will keep it,
427 // but turned into a weak, while the others will drop it when possible.
428 // This is both a compile-time optimization and a correctness
429 // transformation. This is necessary for correctness when we have exported
430 // a reference - we need to convert the linkonce to weak to
431 // ensure a copy is kept to satisfy the exported reference.
432 // FIXME: We may want to split the compile time and correctness
433 // aspects into separate routines.
434 if (isPrevailing(VI.getGUID(), S.get())) {
435 assert(!S->wasPromoted() &&
436 "promoted symbols used to be internal linkage and shouldn't have "
437 "a prevailing variant");
438 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage)) {
439 S->setLinkage(GlobalValue::getWeakLinkage(
440 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
441 // The kept copy is eligible for auto-hiding (hidden visibility) if all
442 // copies were (i.e. they were all linkonce_odr global unnamed addr).
443 // If any copy is not (e.g. it was originally weak_odr), then the symbol
444 // must remain externally available (e.g. a weak_odr from an explicitly
445 // instantiated template). Additionally, if it is in the
446 // GUIDPreservedSymbols set, that means that it is visibile outside
447 // the summary (e.g. in a native object or a bitcode file without
448 // summary), and in that case we cannot hide it as it isn't possible to
449 // check all copies.
450 S->setCanAutoHide(VI.canAutoHide() &&
451 !GUIDPreservedSymbols.count(V: VI.getGUID()));
452 }
453 if (C.VisibilityScheme == Config::FromPrevailing)
454 Visibility = S->getVisibility();
455 }
456 // Alias and aliasee can't be turned into available_externally.
457 // When force-import-all is used, it indicates that object linking is not
458 // supported by the target. In this case, we can't change the linkage as
459 // well in case the global is converted to declaration.
460 // Also, if the symbol was promoted, it wouldn't have a prevailing variant,
461 // but also its linkage is set correctly (to External) already.
462 else if (!isa<AliasSummary>(Val: S.get()) &&
463 !GlobalInvolvedWithAlias.count(V: S.get()) && !ForceImportAll &&
464 !S->wasPromoted())
465 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
466
467 // For ELF, set visibility to the computed visibility from summaries. We
468 // don't track visibility from declarations so this may be more relaxed than
469 // the most constraining one.
470 if (C.VisibilityScheme == Config::ELF)
471 S->setVisibility(Visibility);
472
473 if (S->linkage() != OriginalLinkage)
474 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
475 }
476
477 if (C.VisibilityScheme == Config::FromPrevailing) {
478 for (auto &S : VI.getSummaryList()) {
479 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
480 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
481 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
482 continue;
483 S->setVisibility(Visibility);
484 }
485 }
486}
487
488/// Resolve linkage for prevailing symbols in the \p Index.
489//
490// We'd like to drop these functions if they are no longer referenced in the
491// current module. However there is a chance that another module is still
492// referencing them because of the import. We make sure we always emit at least
493// one copy.
494void llvm::thinLTOResolvePrevailingInIndex(
495 const Config &C, ModuleSummaryIndex &Index,
496 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
497 isPrevailing,
498 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
499 recordNewLinkage,
500 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
501 // We won't optimize the globals that are referenced by an alias for now
502 // Ideally we should turn the alias into a global and duplicate the definition
503 // when needed.
504 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
505 for (auto &I : Index)
506 for (auto &S : I.second.getSummaryList())
507 if (auto AS = dyn_cast<AliasSummary>(Val: S.get()))
508 GlobalInvolvedWithAlias.insert(V: &AS->getAliasee());
509
510 for (auto &I : Index)
511 thinLTOResolvePrevailingGUID(C, VI: Index.getValueInfo(R: I),
512 GlobalInvolvedWithAlias, isPrevailing,
513 recordNewLinkage, GUIDPreservedSymbols);
514}
515
516static void thinLTOInternalizeAndPromoteGUID(
517 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
518 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
519 isPrevailing,
520 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr) {
521 // Before performing index-based internalization and promotion for this GUID,
522 // the local flag should be consistent with the summary list linkage types.
523 VI.verifyLocal();
524
525 const bool SingleExternallyVisibleCopy =
526 VI.getSummaryList().size() == 1 &&
527 !GlobalValue::isLocalLinkage(Linkage: VI.getSummaryList().front()->linkage());
528
529 bool NameRecorded = false;
530 for (auto &S : VI.getSummaryList()) {
531 // First see if we need to promote an internal value because it is not
532 // exported.
533 if (isExported(S->modulePath(), VI)) {
534 if (GlobalValue::isLocalLinkage(Linkage: S->linkage())) {
535 // Only the first local GlobalValue in a list of summaries does not
536 // need renaming. In rare cases if there exist more than one summaries
537 // in the list, the rest of them must have renaming (through promotion)
538 // to avoid conflict.
539 if (ExternallyVisibleSymbolNamesPtr && !NameRecorded) {
540 NameRecorded = true;
541 if (ExternallyVisibleSymbolNamesPtr->insert(V: VI.name()).second)
542 S->setNoRenameOnPromotion(true);
543 }
544
545 S->promote();
546 }
547 continue;
548 }
549
550 // Otherwise, see if we can internalize.
551 if (!EnableLTOInternalization)
552 continue;
553
554 // Non-exported values with external linkage can be internalized.
555 if (GlobalValue::isExternalLinkage(Linkage: S->linkage())) {
556 S->setLinkage(GlobalValue::InternalLinkage);
557 continue;
558 }
559
560 // Non-exported function and variable definitions with a weak-for-linker
561 // linkage can be internalized in certain cases. The minimum legality
562 // requirements would be that they are not address taken to ensure that we
563 // don't break pointer equality checks, and that variables are either read-
564 // or write-only. For functions, this is the case if either all copies are
565 // [local_]unnamed_addr, or we can propagate reference edge attributes
566 // (which is how this is guaranteed for variables, when analyzing whether
567 // they are read or write-only).
568 //
569 // However, we only get to this code for weak-for-linkage values in one of
570 // two cases:
571 // 1) The prevailing copy is not in IR (it is in native code).
572 // 2) The prevailing copy in IR is not exported from its module.
573 // Additionally, at least for the new LTO API, case 2 will only happen if
574 // there is exactly one definition of the value (i.e. in exactly one
575 // module), as duplicate defs are result in the value being marked exported.
576 // Likely, users of the legacy LTO API are similar, however, currently there
577 // are llvm-lto based tests of the legacy LTO API that do not mark
578 // duplicate linkonce_odr copies as exported via the tool, so we need
579 // to handle that case below by checking the number of copies.
580 //
581 // Generally, we only want to internalize a weak-for-linker value in case
582 // 2, because in case 1 we cannot see how the value is used to know if it
583 // is read or write-only. We also don't want to bloat the binary with
584 // multiple internalized copies of non-prevailing linkonce/weak functions.
585 // Note if we don't internalize, we will convert non-prevailing copies to
586 // available_externally anyway, so that we drop them after inlining. The
587 // only reason to internalize such a function is if we indeed have a single
588 // copy, because internalizing it won't increase binary size, and enables
589 // use of inliner heuristics that are more aggressive in the face of a
590 // single call to a static (local). For variables, internalizing a read or
591 // write only variable can enable more aggressive optimization. However, we
592 // already perform this elsewhere in the ThinLTO backend handling for
593 // read or write-only variables (processGlobalForThinLTO).
594 //
595 // Therefore, only internalize linkonce/weak if there is a single copy, that
596 // is prevailing in this IR module. We can do so aggressively, without
597 // requiring the address to be insignificant, or that a variable be read or
598 // write-only.
599 if (!GlobalValue::isWeakForLinker(Linkage: S->linkage()) ||
600 GlobalValue::isExternalWeakLinkage(Linkage: S->linkage()))
601 continue;
602
603 // We may have a single summary copy that is externally visible but not
604 // prevailing if the prevailing copy is in a native object.
605 if (SingleExternallyVisibleCopy && isPrevailing(VI.getGUID(), S.get()))
606 S->setLinkage(GlobalValue::InternalLinkage);
607 }
608}
609
610// Update the linkages in the given \p Index to mark exported values
611// as external and non-exported values as internal.
612void llvm::thinLTOInternalizeAndPromoteInIndex(
613 ModuleSummaryIndex &Index,
614 function_ref<bool(StringRef, ValueInfo)> isExported,
615 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
616 isPrevailing,
617 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr) {
618 assert(!Index.withInternalizeAndPromote());
619
620 for (auto &I : Index)
621 thinLTOInternalizeAndPromoteGUID(VI: Index.getValueInfo(R: I), isExported,
622 isPrevailing,
623 ExternallyVisibleSymbolNamesPtr);
624 Index.setWithInternalizeAndPromote();
625}
626
627// Requires a destructor for std::vector<InputModule>.
628InputFile::~InputFile() = default;
629
630Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
631 std::unique_ptr<InputFile> File(new InputFile);
632
633 Expected<IRSymtabFile> FOrErr = readIRSymtab(MBRef: Object);
634 if (!FOrErr)
635 return FOrErr.takeError();
636
637 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
638 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
639 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
640 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
641 File->ComdatTable = FOrErr->TheReader.getComdatTable();
642 File->MbRef =
643 Object; // Save a memory buffer reference to an input file object.
644
645 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
646 size_t Begin = File->Symbols.size();
647 for (const irsymtab::Reader::SymbolRef &Sym :
648 FOrErr->TheReader.module_symbols(I))
649 // Skip symbols that are irrelevant to LTO. Note that this condition needs
650 // to match the one in Skip() in LTO::addRegularLTO().
651 if (Sym.isGlobal() && !Sym.isFormatSpecific())
652 File->Symbols.push_back(x: Sym);
653 File->ModuleSymIndices.push_back(x: {Begin, File->Symbols.size()});
654 }
655
656 File->Mods = FOrErr->Mods;
657 File->Strtab = std::move(FOrErr->Strtab);
658 return std::move(File);
659}
660
661bool InputFile::Symbol::isLibcall(
662 const TargetLibraryInfo &TLI,
663 const RTLIB::RuntimeLibcallsInfo &Libcalls) const {
664 LibFunc F;
665 if (TLI.getLibFunc(funcName: IRName, F) && TLI.has(F))
666 return true;
667 return Libcalls.getSupportedLibcallImpl(FuncName: IRName) != RTLIB::Unsupported;
668}
669
670StringRef InputFile::getName() const {
671 return Mods[0].getModuleIdentifier();
672}
673
674BitcodeModule &InputFile::getSingleBitcodeModule() {
675 assert(Mods.size() == 1 && "Expect only one bitcode module");
676 return Mods[0];
677}
678
679BitcodeModule &InputFile::getPrimaryBitcodeModule() { return Mods[0]; }
680
681LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
682 const Config &Conf)
683 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
684 Ctx(Conf), CombinedModule(std::make_unique<Module>(args: "ld-temp.o", args&: Ctx)),
685 Mover(std::make_unique<IRMover>(args&: *CombinedModule)) {}
686
687LTO::ThinLTOState::ThinLTOState(ThinBackend BackendParam)
688 : Backend(std::move(BackendParam)), CombinedIndex(/*HaveGVs*/ false) {
689 if (!Backend.isValid())
690 Backend =
691 createInProcessThinBackend(Parallelism: llvm::heavyweight_hardware_concurrency());
692}
693
694LTO::LTO(Config Conf, ThinBackend Backend,
695 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
696 : Conf(std::move(Conf)),
697 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
698 ThinLTO(std::move(Backend)),
699 GlobalResolutions(
700 std::make_unique<DenseMap<StringRef, GlobalResolution>>()),
701 LTOMode(LTOMode) {
702 if (Conf.KeepSymbolNameCopies || LTOKeepSymbolCopies) {
703 Alloc = std::make_unique<BumpPtrAllocator>();
704 GlobalResolutionSymbolSaver = std::make_unique<llvm::StringSaver>(args&: *Alloc);
705 }
706}
707
708// Requires a destructor for MapVector<BitcodeModule>.
709LTO::~LTO() = default;
710
711void LTO::cleanup() {
712 DummyModule.reset();
713 LinkerRemarkFunction = nullptr;
714 consumeError(Err: finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile)));
715}
716
717// Add the symbols in the given module to the GlobalResolutions map, and resolve
718// their partitions.
719void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
720 ArrayRef<SymbolResolution> Res,
721 unsigned Partition, bool InSummary,
722 const Triple &TT) {
723 llvm::TimeTraceScope timeScope("LTO add module to global resolution");
724 auto *ResI = Res.begin();
725 auto *ResE = Res.end();
726 (void)ResE;
727 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
728 TargetLibraryInfoImpl TLII(TT);
729 TargetLibraryInfo TLI(TLII);
730 for (const InputFile::Symbol &Sym : Syms) {
731 assert(ResI != ResE);
732 SymbolResolution Res = *ResI++;
733
734 StringRef SymbolName = Sym.getName();
735 // Keep copies of symbols if the client of LTO says so.
736 if (GlobalResolutionSymbolSaver && !GlobalResolutions->contains(Val: SymbolName))
737 SymbolName = GlobalResolutionSymbolSaver->save(S: SymbolName);
738
739 auto &GlobalRes = (*GlobalResolutions)[SymbolName];
740 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
741 if (Res.Prevailing) {
742 assert(!GlobalRes.Prevailing &&
743 "Multiple prevailing defs are not allowed");
744 GlobalRes.Prevailing = true;
745 GlobalRes.IRName = std::string(Sym.getIRName());
746 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
747 // Sometimes it can be two copies of symbol in a module and prevailing
748 // symbol can have no IR name. That might happen if symbol is defined in
749 // module level inline asm block. In case we have multiple modules with
750 // the same symbol we want to use IR name of the prevailing symbol.
751 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
752 // we can later use it to check if there is any prevailing copy in IR.
753 GlobalRes.IRName = std::string(Sym.getIRName());
754 }
755
756 // In rare occasion, the symbol used to initialize GlobalRes has a different
757 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
758 // symbol is referenced through its mangled name, say @"\01_symbol" while
759 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
760 // In that case, we have the same actual Symbol that can get two different
761 // GUID, leading to some invalid internalization. Workaround this by marking
762 // the GlobalRes external.
763
764 // FIXME: instead of this check, it would be desirable to compute GUIDs
765 // based on mangled name, but this requires an access to the Target Triple
766 // and would be relatively invasive on the codebase.
767 // FIXME: use the GUID member of GlobalRes.
768 if (GlobalRes.IRName != Sym.getIRName()) {
769 GlobalRes.Partition = GlobalResolution::External;
770 GlobalRes.VisibleOutsideSummary = true;
771 }
772
773 bool IsLibcall = Sym.isLibcall(TLI, Libcalls);
774
775 // Set the partition to external if we know it is re-defined by the linker
776 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
777 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
778 // already recorded as being referenced from a different partition.
779 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
780 IsLibcall ||
781 (GlobalRes.Partition != GlobalResolution::Unknown &&
782 GlobalRes.Partition != Partition)) {
783 GlobalRes.Partition = GlobalResolution::External;
784 } else
785 // First recorded reference, save the current partition.
786 GlobalRes.Partition = Partition;
787
788 // Flag as visible outside of summary if visible from a regular object or
789 // from a module that does not have a summary.
790 GlobalRes.VisibleOutsideSummary |=
791 (Res.VisibleToRegularObj || Sym.isUsed() || IsLibcall || !InSummary);
792
793 GlobalRes.ExportDynamic |= Res.ExportDynamic;
794 }
795}
796
797void LTO::releaseGlobalResolutionsMemory() {
798 // Release GlobalResolutions dense-map itself.
799 GlobalResolutions.reset();
800 // Release the string saver memory.
801 GlobalResolutionSymbolSaver.reset();
802 Alloc.reset();
803}
804
805static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
806 ArrayRef<SymbolResolution> Res) {
807 StringRef Path = Input->getName();
808 OS << Path << '\n';
809 auto ResI = Res.begin();
810 for (const InputFile::Symbol &Sym : Input->symbols()) {
811 assert(ResI != Res.end());
812 SymbolResolution Res = *ResI++;
813
814 OS << "-r=" << Path << ',' << Sym.getName() << ',';
815 if (Res.Prevailing)
816 OS << 'p';
817 if (Res.FinalDefinitionInLinkageUnit)
818 OS << 'l';
819 if (Res.VisibleToRegularObj)
820 OS << 'x';
821 if (Res.LinkerRedefined)
822 OS << 'r';
823 OS << '\n';
824 }
825 OS.flush();
826 assert(ResI == Res.end());
827}
828
829Error LTO::add(std::unique_ptr<InputFile> InputPtr,
830 ArrayRef<SymbolResolution> Res) {
831 llvm::TimeTraceScope timeScope("LTO add input", InputPtr->getName());
832 assert(!CalledGetMaxTasks);
833
834 Expected<std::shared_ptr<InputFile>> InputOrErr =
835 addInput(InputPtr: std::move(InputPtr));
836 if (!InputOrErr)
837 return InputOrErr.takeError();
838 InputFile *Input = (*InputOrErr).get();
839
840 if (Conf.ResolutionFile)
841 writeToResolutionFile(OS&: *Conf.ResolutionFile, Input, Res);
842
843 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
844 Triple InputTriple(Input->getTargetTriple());
845 RegularLTO.CombinedModule->setTargetTriple(InputTriple);
846 if (InputTriple.isOSBinFormatELF())
847 Conf.VisibilityScheme = Config::ELF;
848 }
849
850 ArrayRef<SymbolResolution> InputRes = Res;
851 for (unsigned I = 0; I != Input->Mods.size(); ++I) {
852 if (auto Err = addModule(Input&: *Input, InputRes, ModI: I, Res).moveInto(Value&: Res))
853 return Err;
854 }
855
856 assert(Res.empty());
857 return Error::success();
858}
859
860void LTO::setBitcodeLibFuncs(ArrayRef<StringRef> BitcodeLibFuncs) {
861 assert(this->BitcodeLibFuncs.empty() &&
862 "bitcode libfuncs were set twice; maybe accidentally clobbered?");
863 this->BitcodeLibFuncs.append(in_start: BitcodeLibFuncs.begin(), in_end: BitcodeLibFuncs.end());
864}
865
866Expected<ArrayRef<SymbolResolution>>
867LTO::addModule(InputFile &Input, ArrayRef<SymbolResolution> InputRes,
868 unsigned ModI, ArrayRef<SymbolResolution> Res) {
869 llvm::TimeTraceScope timeScope("LTO add module", Input.getName());
870 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
871 if (!LTOInfo)
872 return LTOInfo.takeError();
873
874 if (EnableSplitLTOUnit) {
875 // If only some modules were split, flag this in the index so that
876 // we can skip or error on optimizations that need consistently split
877 // modules (whole program devirt and lower type tests).
878 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
879 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
880 } else
881 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
882
883 BitcodeModule BM = Input.Mods[ModI];
884
885 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
886 !LTOInfo->UnifiedLTO)
887 return make_error<StringError>(
888 Args: "unified LTO compilation must use "
889 "compatible bitcode modules (use -funified-lto)",
890 Args: inconvertibleErrorCode());
891
892 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
893 LTOMode = LTOK_UnifiedThin;
894
895 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
896 // If any of the modules inside of a input bitcode file was compiled with
897 // ThinLTO, we assume that the whole input file also was compiled with
898 // ThinLTO.
899 Input.IsThinLTO |= IsThinLTO;
900
901 auto ModSyms = Input.module_symbols(I: ModI);
902 addModuleToGlobalRes(Syms: ModSyms, Res,
903 Partition: IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
904 InSummary: LTOInfo->HasSummary, TT: Triple(Input.getTargetTriple()));
905
906 if (IsThinLTO)
907 return addThinLTO(BM, Syms: ModSyms, Res);
908
909 RegularLTO.EmptyCombinedModule = false;
910 auto ModOrErr = addRegularLTO(Input, InputRes, BM, Syms: ModSyms, Res);
911 if (!ModOrErr)
912 return ModOrErr.takeError();
913 Res = ModOrErr->second;
914
915 if (!LTOInfo->HasSummary) {
916 if (Error Err = linkRegularLTO(Mod: std::move(ModOrErr->first),
917 /*LivenessFromIndex=*/false))
918 return Err;
919 return Res;
920 }
921
922 // Regular LTO module summaries are added to a dummy module that represents
923 // the combined regular LTO module.
924 if (Error Err = BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: ""))
925 return Err;
926 RegularLTO.ModsWithSummaries.push_back(x: std::move(ModOrErr->first));
927 return Res;
928}
929
930// Checks whether the given global value is in a non-prevailing comdat
931// (comdat containing values the linker indicated were not prevailing,
932// which we then dropped to available_externally), and if so, removes
933// it from the comdat. This is called for all global values to ensure the
934// comdat is empty rather than leaving an incomplete comdat. It is needed for
935// regular LTO modules, in case we are in a mixed-LTO mode (both regular
936// and thin LTO modules) compilation. Since the regular LTO module will be
937// linked first in the final native link, we want to make sure the linker
938// doesn't select any of these incomplete comdats that would be left
939// in the regular LTO module without this cleanup.
940static void
941handleNonPrevailingComdat(GlobalValue &GV,
942 std::set<const Comdat *> &NonPrevailingComdats) {
943 Comdat *C = GV.getComdat();
944 if (!C)
945 return;
946
947 if (!NonPrevailingComdats.count(x: C))
948 return;
949
950 // Additionally need to drop all global values from the comdat to
951 // available_externally, to satisfy the COMDAT requirement that all members
952 // are discarded as a unit. The non-local linkage global values avoid
953 // duplicate definition linker errors.
954 GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
955
956 if (auto GO = dyn_cast<GlobalObject>(Val: &GV))
957 GO->setComdat(nullptr);
958}
959
960// Add a regular LTO object to the link.
961// The resulting module needs to be linked into the combined LTO module with
962// linkRegularLTO.
963Expected<
964 std::pair<LTO::RegularLTOState::AddedModule, ArrayRef<SymbolResolution>>>
965LTO::addRegularLTO(InputFile &Input, ArrayRef<SymbolResolution> InputRes,
966 BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
967 ArrayRef<SymbolResolution> Res) {
968 llvm::TimeTraceScope timeScope("LTO add regular LTO");
969 RegularLTOState::AddedModule Mod;
970 Expected<std::unique_ptr<Module>> MOrErr =
971 BM.getLazyModule(Context&: RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
972 /*IsImporting*/ false);
973 if (!MOrErr)
974 return MOrErr.takeError();
975 Module &M = **MOrErr;
976 Mod.M = std::move(*MOrErr);
977
978 if (Error Err = M.materializeMetadata())
979 return std::move(Err);
980
981 if (LTOMode == LTOK_UnifiedRegular) {
982 // cfi.functions metadata is intended to be used with ThinLTO and may
983 // trigger invalid IR transformations if they are present when doing regular
984 // LTO, so delete it.
985 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata(Name: "cfi.functions"))
986 M.eraseNamedMetadata(NMD: CfiFunctionsMD);
987 } else if (NamedMDNode *AliasesMD = M.getNamedMetadata(Name: "aliases")) {
988 // Delete aliases entries for non-prevailing symbols on the ThinLTO side of
989 // this input file.
990 DenseSet<StringRef> Prevailing;
991 for (auto [I, R] : zip(t: Input.symbols(), u&: InputRes))
992 if (R.Prevailing && !I.getIRName().empty())
993 Prevailing.insert(V: I.getIRName());
994 std::vector<MDNode *> AliasGroups;
995 for (MDNode *AliasGroup : AliasesMD->operands()) {
996 std::vector<Metadata *> Aliases;
997 for (Metadata *Alias : AliasGroup->operands()) {
998 if (isa<MDString>(Val: Alias) &&
999 Prevailing.count(V: cast<MDString>(Val: Alias)->getString()))
1000 Aliases.push_back(x: Alias);
1001 }
1002 if (Aliases.size() > 1)
1003 AliasGroups.push_back(x: MDTuple::get(Context&: RegularLTO.Ctx, MDs: Aliases));
1004 }
1005 AliasesMD->clearOperands();
1006 for (MDNode *G : AliasGroups)
1007 AliasesMD->addOperand(M: G);
1008 }
1009
1010 UpgradeDebugInfo(M);
1011
1012 ModuleSymbolTable SymTab;
1013 SymTab.addModule(M: &M);
1014
1015 for (GlobalVariable &GV : M.globals())
1016 if (GV.hasAppendingLinkage())
1017 Mod.Keep.push_back(x: &GV);
1018
1019 DenseSet<GlobalObject *> AliasedGlobals;
1020 for (auto &GA : M.aliases())
1021 if (GlobalObject *GO = GA.getAliaseeObject())
1022 AliasedGlobals.insert(V: GO);
1023
1024 // In this function we need IR GlobalValues matching the symbols in Syms
1025 // (which is not backed by a module), so we need to enumerate them in the same
1026 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
1027 // matches the order of an irsymtab, but when we read the irsymtab in
1028 // InputFile::create we omit some symbols that are irrelevant to LTO. The
1029 // Skip() function skips the same symbols from the module as InputFile does
1030 // from the symbol table.
1031 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
1032 auto Skip = [&]() {
1033 while (MsymI != MsymE) {
1034 auto Flags = SymTab.getSymbolFlags(S: *MsymI);
1035 if ((Flags & object::BasicSymbolRef::SF_Global) &&
1036 !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
1037 return;
1038 ++MsymI;
1039 }
1040 };
1041 Skip();
1042
1043 std::set<const Comdat *> NonPrevailingComdats;
1044 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
1045 for (const InputFile::Symbol &Sym : Syms) {
1046 assert(!Res.empty());
1047 const SymbolResolution &R = Res.consume_front();
1048
1049 assert(MsymI != MsymE);
1050 ModuleSymbolTable::Symbol Msym = *MsymI++;
1051 Skip();
1052
1053 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Val&: Msym)) {
1054 if (R.Prevailing) {
1055 if (Sym.isUndefined())
1056 continue;
1057 Mod.Keep.push_back(x: GV);
1058 // For symbols re-defined with linker -wrap and -defsym options,
1059 // set the linkage to weak to inhibit IPO. The linkage will be
1060 // restored by the linker.
1061 if (R.LinkerRedefined)
1062 GV->setLinkage(GlobalValue::WeakAnyLinkage);
1063
1064 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
1065 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage))
1066 GV->setLinkage(GlobalValue::getWeakLinkage(
1067 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
1068 } else if (isa<GlobalObject>(Val: GV) &&
1069 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
1070 GV->hasAvailableExternallyLinkage()) &&
1071 !AliasedGlobals.count(V: cast<GlobalObject>(Val: GV))) {
1072 // Any of the above three types of linkage indicates that the
1073 // chosen prevailing symbol will have the same semantics as this copy of
1074 // the symbol, so we may be able to link it with available_externally
1075 // linkage. We will decide later whether to do that when we link this
1076 // module (in linkRegularLTO), based on whether it is undefined.
1077 Mod.Keep.push_back(x: GV);
1078 GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
1079 if (GV->hasComdat())
1080 NonPrevailingComdats.insert(x: GV->getComdat());
1081 cast<GlobalObject>(Val: GV)->setComdat(nullptr);
1082 }
1083
1084 // Set the 'local' flag based on the linker resolution for this symbol.
1085 if (R.FinalDefinitionInLinkageUnit) {
1086 GV->setDSOLocal(true);
1087 if (GV->hasDLLImportStorageClass())
1088 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
1089 DefaultStorageClass);
1090 }
1091 } else if (auto *AS =
1092 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Val&: Msym)) {
1093 // Collect non-prevailing symbols.
1094 if (!R.Prevailing)
1095 NonPrevailingAsmSymbols.insert(V: AS->first);
1096 } else {
1097 llvm_unreachable("unknown symbol type");
1098 }
1099
1100 // Common resolution: collect the maximum size/alignment over all commons.
1101 // We also record if we see an instance of a common as prevailing, so that
1102 // if none is prevailing we can ignore it later.
1103 if (Sym.isCommon()) {
1104 // FIXME: We should figure out what to do about commons defined by asm.
1105 // For now they aren't reported correctly by ModuleSymbolTable.
1106 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
1107 CommonRes.Size = std::max(a: CommonRes.Size, b: Sym.getCommonSize());
1108 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
1109 CommonRes.Alignment =
1110 std::max(a: Align(SymAlignValue), b: CommonRes.Alignment);
1111 }
1112 CommonRes.Prevailing |= R.Prevailing;
1113 }
1114 }
1115
1116 if (!M.getComdatSymbolTable().empty())
1117 for (GlobalValue &GV : M.global_values())
1118 handleNonPrevailingComdat(GV, NonPrevailingComdats);
1119
1120 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
1121 // block.
1122 if (M.hasModuleInlineAsm()) {
1123 std::string NewIA = ".lto_discard";
1124 if (!NonPrevailingAsmSymbols.empty()) {
1125 // Don't dicard a symbol if there is a live .symver for it.
1126 ModuleSymbolTable::CollectAsmSymvers(
1127 M, AsmSymver: [&](StringRef Name, StringRef Alias) {
1128 if (!NonPrevailingAsmSymbols.count(V: Alias))
1129 NonPrevailingAsmSymbols.erase(V: Name);
1130 });
1131 NewIA += " " + llvm::join(R&: NonPrevailingAsmSymbols, Separator: ", ");
1132 }
1133 NewIA += "\n";
1134 M.prependModuleInlineAsm(Fragment: NewIA);
1135 }
1136
1137 assert(MsymI == MsymE);
1138 return std::make_pair(x: std::move(Mod), y&: Res);
1139}
1140
1141Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
1142 bool LivenessFromIndex) {
1143 llvm::TimeTraceScope timeScope("LTO link regular LTO");
1144 std::vector<GlobalValue *> Keep;
1145 for (GlobalValue *GV : Mod.Keep) {
1146 if (LivenessFromIndex) {
1147 const auto GUID = GV->getGUIDOrFallback();
1148 if (!ThinLTO.CombinedIndex.isGUIDLive(GUID)) {
1149 if (Function *F = dyn_cast<Function>(Val: GV)) {
1150 if (DiagnosticOutputFile) {
1151 if (Error Err = F->materialize())
1152 return Err;
1153 auto R = OptimizationRemark(DEBUG_TYPE, "deadfunction", F);
1154 R << ore::NV("Function", F) << " not added to the combined module ";
1155 emitRemark(Remark&: R);
1156 }
1157 }
1158 continue;
1159 }
1160 }
1161
1162 if (!GV->hasAvailableExternallyLinkage()) {
1163 Keep.push_back(x: GV);
1164 continue;
1165 }
1166
1167 // Only link available_externally definitions if we don't already have a
1168 // definition.
1169 GlobalValue *CombinedGV =
1170 RegularLTO.CombinedModule->getNamedValue(Name: GV->getName());
1171 if (CombinedGV && !CombinedGV->isDeclaration())
1172 continue;
1173
1174 Keep.push_back(x: GV);
1175 }
1176
1177 return RegularLTO.Mover->move(Src: std::move(Mod.M), ValuesToLink: Keep, AddLazyFor: nullptr,
1178 /* IsPerformingImport */ false);
1179}
1180
1181// Add a ThinLTO module to the link.
1182Expected<ArrayRef<SymbolResolution>>
1183LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
1184 ArrayRef<SymbolResolution> Res) {
1185 llvm::TimeTraceScope timeScope("LTO add thin LTO");
1186 const auto BMID = BM.getModuleIdentifier();
1187 ArrayRef<SymbolResolution> ResTmp = Res;
1188 DenseSet<StringRef> Prevailing;
1189 for (const InputFile::Symbol &Sym : Syms) {
1190 assert(!ResTmp.empty());
1191 const SymbolResolution &R = ResTmp.consume_front();
1192 if (!Sym.getIRName().empty() && R.Prevailing)
1193 Prevailing.insert(V: Sym.getIRName());
1194 }
1195
1196 // Track the GUIDs stored in the bitcode GUID table.
1197 StringMap<GlobalValue::GUID> IRSpecifiedGUIDs;
1198 if (Error Err = BM.readSummary(
1199 CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: BMID,
1200 IsPrevailing: [&](StringRef Name) { return (Prevailing.count(V: Name) > 0); },
1201 OnValueInfo: [&](ValueInfo VI) {
1202 auto IT = IRSpecifiedGUIDs.insert(KV: {VI.name(), VI.getGUID()});
1203 (void)IT;
1204 assert(IT.second);
1205 if (auto GRIt = GlobalResolutions->find(Val: VI.name());
1206 GRIt != GlobalResolutions->end() &&
1207 Prevailing.count(V: VI.name())) {
1208 GRIt->second.setGUID(VI.getGUID());
1209 }
1210 }))
1211 return Err;
1212 LLVM_DEBUG(dbgs() << "Module " << BMID << "\n");
1213
1214 for (const InputFile::Symbol &Sym : Syms) {
1215 assert(!Res.empty());
1216 const SymbolResolution &R = Res.consume_front();
1217 auto GUIDIter = IRSpecifiedGUIDs.find(Key: Sym.getIRName());
1218 // The bitcode GUID table might not be present if this is an old bitcode
1219 // file. For backwards-compatibility, just compute the GUID now in that
1220 // case.
1221 auto GUID =
1222 GUIDIter == IRSpecifiedGUIDs.end()
1223 ? GlobalValue::getGUIDAssumingExternalLinkage(
1224 GlobalName: GlobalValue::getGlobalIdentifier(
1225 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: ""))
1226 : GUIDIter->second;
1227 if (!Sym.getIRName().empty() &&
1228 (R.Prevailing || R.FinalDefinitionInLinkageUnit)) {
1229 if (R.Prevailing) {
1230 ThinLTO.setPrevailingModuleForGUID(GUID, Module: BMID);
1231 // For linker redefined symbols (via --wrap or --defsym) we want to
1232 // switch the linkage to `weak` to prevent IPOs from happening.
1233 // Find the summary in the module for this very GV and record the new
1234 // linkage so that we can switch it when we import the GV.
1235 if (R.LinkerRedefined)
1236 if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(ValueGUID: GUID, ModuleId: BMID))
1237 S->setLinkage(GlobalValue::WeakAnyLinkage);
1238 }
1239
1240 // If the linker resolved the symbol to a local definition then mark it
1241 // as local in the summary for the module we are adding.
1242 if (R.FinalDefinitionInLinkageUnit) {
1243 if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(ValueGUID: GUID, ModuleId: BMID)) {
1244 S->setDSOLocal(true);
1245 }
1246 }
1247 }
1248 }
1249
1250 if (!ThinLTO.ModuleMap.insert(KV: {BMID, BM}).second)
1251 return make_error<StringError>(
1252 Args: "Expected at most one ThinLTO module per bitcode file",
1253 Args: inconvertibleErrorCode());
1254
1255 if (!Conf.ThinLTOModulesToCompile.empty()) {
1256 if (!ThinLTO.ModulesToCompile)
1257 ThinLTO.ModulesToCompile = ModuleMapType();
1258 // This is a fuzzy name matching where only modules with name containing the
1259 // specified switch values are going to be compiled.
1260 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1261 if (BMID.contains(Other: Name)) {
1262 ThinLTO.ModulesToCompile->insert(KV: {BMID, BM});
1263 LLVM_DEBUG(dbgs() << "[ThinLTO] Selecting " << BMID << " to compile\n");
1264 break;
1265 }
1266 }
1267 }
1268
1269 return Res;
1270}
1271
1272unsigned LTO::getMaxTasks() const {
1273 CalledGetMaxTasks = true;
1274 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1275 : ThinLTO.ModuleMap.size();
1276 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1277}
1278
1279// If only some of the modules were split, we cannot correctly handle
1280// code that contains type tests or type checked loads.
1281Error LTO::checkPartiallySplit() {
1282 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1283 return Error::success();
1284
1285 const Module *Combined = RegularLTO.CombinedModule.get();
1286 Function *TypeTestFunc =
1287 Intrinsic::getDeclarationIfExists(M: Combined, id: Intrinsic::type_test);
1288 Function *TypeCheckedLoadFunc =
1289 Intrinsic::getDeclarationIfExists(M: Combined, id: Intrinsic::type_checked_load);
1290 Function *TypeCheckedLoadRelativeFunc = Intrinsic::getDeclarationIfExists(
1291 M: Combined, id: Intrinsic::type_checked_load_relative);
1292
1293 // First check if there are type tests / type checked loads in the
1294 // merged regular LTO module IR.
1295 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1296 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1297 (TypeCheckedLoadRelativeFunc &&
1298 !TypeCheckedLoadRelativeFunc->use_empty()))
1299 return make_error<StringError>(
1300 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1301 Args: inconvertibleErrorCode());
1302
1303 // Otherwise check if there are any recorded in the combined summary from the
1304 // ThinLTO modules.
1305 for (auto &P : ThinLTO.CombinedIndex) {
1306 for (auto &S : P.second.getSummaryList()) {
1307 auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
1308 if (!FS)
1309 continue;
1310 if (!FS->type_test_assume_vcalls().empty() ||
1311 !FS->type_checked_load_vcalls().empty() ||
1312 !FS->type_test_assume_const_vcalls().empty() ||
1313 !FS->type_checked_load_const_vcalls().empty() ||
1314 !FS->type_tests().empty())
1315 return make_error<StringError>(
1316 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1317 Args: inconvertibleErrorCode());
1318 }
1319 }
1320 return Error::success();
1321}
1322
1323Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
1324 // Call the base class cleanup() explicitly since run() may be invoked on a
1325 // derived LTO object.
1326 llvm::scope_exit CleanUp([this]() { LTO::cleanup(); });
1327
1328 // Compute "dead" symbols, we don't want to import/export these!
1329 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1330 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1331 for (auto &Res : *GlobalResolutions) {
1332 // Normally resolution have IR name of symbol. We can do nothing here
1333 // otherwise. See comments in GlobalResolution struct for more details.
1334 if (Res.second.IRName.empty())
1335 continue;
1336
1337 GlobalValue::GUID GUID = Res.second.getGUID();
1338
1339 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1340 GUIDPreservedSymbols.insert(V: GUID);
1341
1342 if (Res.second.ExportDynamic)
1343 DynamicExportSymbols.insert(V: GUID);
1344
1345 GUIDPrevailingResolutions[GUID] =
1346 Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
1347 }
1348
1349 auto isPrevailing = [&](GlobalValue::GUID G) {
1350 auto It = GUIDPrevailingResolutions.find(Val: G);
1351 if (It == GUIDPrevailingResolutions.end())
1352 return PrevailingType::Unknown;
1353 return It->second;
1354 };
1355 computeDeadSymbolsWithConstProp(Index&: ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1356 isPrevailing, ImportEnabled: Conf.OptLevel > 0);
1357
1358 // Setup output file to emit statistics.
1359 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1360 if (!StatsFileOrErr)
1361 return StatsFileOrErr.takeError();
1362 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1363
1364 if (Error Err = setupOptimizationRemarks())
1365 return Err;
1366
1367 // TODO: Ideally this would be controlled automatically by detecting that we
1368 // are linking with an allocator that supports these interfaces, rather than
1369 // an internal option (which would still be needed for tests, however). For
1370 // example, if the library exported a symbol like __malloc_hot_cold the linker
1371 // could recognize that and set a flag in the lto::Config.
1372 if (SupportsHotColdNew)
1373 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1374
1375 Error Result = runRegularLTO(AddStream);
1376 if (!Result)
1377 // This will reset the GlobalResolutions optional once done with it to
1378 // reduce peak memory before importing.
1379 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1380
1381 if (StatsFile)
1382 PrintStatisticsJSON(OS&: StatsFile->os());
1383
1384 return Result;
1385}
1386
1387Error LTO::runRegularLTO(AddStreamFn AddStream) {
1388 llvm::TimeTraceScope timeScope("Run regular LTO");
1389 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1390
1391 // Finalize linking of regular LTO modules containing summaries now that
1392 // we have computed liveness information.
1393 {
1394 llvm::TimeTraceScope timeScope("Link regular LTO");
1395 for (auto &M : RegularLTO.ModsWithSummaries)
1396 if (Error Err = linkRegularLTO(Mod: std::move(M), /*LivenessFromIndex=*/true))
1397 return Err;
1398 }
1399
1400 // Ensure we don't have inconsistently split LTO units with type tests.
1401 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1402 // this path both cases but eventually this should be split into two and
1403 // do the ThinLTO checks in `runThinLTO`.
1404 if (Error Err = checkPartiallySplit())
1405 return Err;
1406
1407 // Make sure commons have the right size/alignment: we kept the largest from
1408 // all the prevailing when adding the inputs, and we apply it here.
1409 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1410 for (auto &I : RegularLTO.Commons) {
1411 if (!I.second.Prevailing)
1412 // Don't do anything if no instance of this common was prevailing.
1413 continue;
1414 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(Name: I.first);
1415 if (OldGV && OldGV->getGlobalSize(DL) == I.second.Size) {
1416 // Don't create a new global if the type is already correct, just make
1417 // sure the alignment is correct.
1418 OldGV->setAlignment(I.second.Alignment);
1419 continue;
1420 }
1421 ArrayType *Ty =
1422 ArrayType::get(ElementType: Type::getInt8Ty(C&: RegularLTO.Ctx), NumElements: I.second.Size);
1423 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1424 GlobalValue::CommonLinkage,
1425 ConstantAggregateZero::get(Ty), "");
1426 GV->setAlignment(I.second.Alignment);
1427 if (OldGV) {
1428 OldGV->replaceAllUsesWith(V: GV);
1429 GV->takeName(V: OldGV);
1430 OldGV->eraseFromParent();
1431 } else {
1432 GV->setName(I.first);
1433 }
1434 }
1435
1436 bool WholeProgramVisibilityEnabledInLTO =
1437 Conf.HasWholeProgramVisibility &&
1438 // If validation is enabled, upgrade visibility only when all vtables
1439 // have typeinfos.
1440 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
1441
1442 // This returns true when the name is local or not defined. Locals are
1443 // expected to be handled separately.
1444 auto IsVisibleToRegularObj = [&](StringRef name) {
1445 auto It = GlobalResolutions->find(Val: name);
1446 return (It == GlobalResolutions->end() ||
1447 It->second.VisibleOutsideSummary || !It->second.Prevailing);
1448 };
1449
1450 // If allowed, upgrade public vcall visibility metadata to linkage unit
1451 // visibility before whole program devirtualization in the optimizer.
1452 updateVCallVisibilityInModule(
1453 M&: *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1454 DynamicExportSymbols, ValidateAllVtablesHaveTypeInfos: Conf.ValidateAllVtablesHaveTypeInfos,
1455 IsVisibleToRegularObj);
1456 updatePublicTypeTestCalls(M&: *RegularLTO.CombinedModule,
1457 WholeProgramVisibilityEnabledInLTO);
1458
1459 if (Conf.PreOptModuleHook &&
1460 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1461 return Error::success();
1462
1463 if (!Conf.CodeGenOnly) {
1464 for (const auto &R : *GlobalResolutions) {
1465 GlobalValue *GV =
1466 RegularLTO.CombinedModule->getNamedValue(Name: R.second.IRName);
1467 if (!R.second.isPrevailingIRSymbol())
1468 continue;
1469 if (R.second.Partition != 0 &&
1470 R.second.Partition != GlobalResolution::External)
1471 continue;
1472
1473 // Ignore symbols defined in other partitions.
1474 // Also skip declarations, which are not allowed to have internal linkage.
1475 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1476 continue;
1477
1478 // Symbols that are marked DLLImport or DLLExport should not be
1479 // internalized, as they are either externally visible or referencing
1480 // external symbols. Symbols that have AvailableExternally or Appending
1481 // linkage might be used by future passes and should be kept as is.
1482 // These linkages are seen in Unified regular LTO, because the process
1483 // of creating split LTO units introduces symbols with that linkage into
1484 // one of the created modules. Normally, only the ThinLTO backend would
1485 // compile this module, but Unified Regular LTO processes both
1486 // modules created by the splitting process as regular LTO modules.
1487 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1488 ((GV->getDLLStorageClass() != GlobalValue::DefaultStorageClass) ||
1489 GV->hasAvailableExternallyLinkage() || GV->hasAppendingLinkage()))
1490 continue;
1491
1492 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1493 : GlobalValue::UnnamedAddr::None);
1494 if (EnableLTOInternalization && R.second.Partition == 0)
1495 GV->setLinkage(GlobalValue::InternalLinkage);
1496 }
1497
1498 if (Conf.PostInternalizeModuleHook &&
1499 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1500 return Error::success();
1501 }
1502
1503 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1504 if (Error Err = backend(
1505 C: Conf, AddStream, ParallelCodeGenParallelismLevel: RegularLTO.ParallelCodeGenParallelismLevel,
1506 M&: *RegularLTO.CombinedModule, CombinedIndex&: ThinLTO.CombinedIndex, BitcodeLibFuncs))
1507 return Err;
1508 }
1509
1510 return Error::success();
1511}
1512
1513SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
1514 RTLIB::RuntimeLibcallsInfo Libcalls(TT);
1515 SmallVector<const char *> LibcallSymbols;
1516 LibcallSymbols.reserve(N: Libcalls.getNumAvailableLibcallImpls());
1517
1518 for (RTLIB::LibcallImpl Impl : RTLIB::libcall_impls()) {
1519 if (Libcalls.isAvailable(Impl))
1520 LibcallSymbols.push_back(Elt: Libcalls.getLibcallImplName(CallImpl: Impl).data());
1521 }
1522
1523 return LibcallSymbols;
1524}
1525
1526SmallVector<StringRef> LTO::getLibFuncSymbols(const Triple &TT,
1527 StringSaver &Saver) {
1528 auto TLII = std::make_unique<TargetLibraryInfoImpl>(args: TT);
1529 TargetLibraryInfo TLI(*TLII);
1530 SmallVector<StringRef> LibFuncSymbols;
1531 LibFuncSymbols.reserve(N: LibFunc::NumLibFuncs);
1532 for (unsigned I = LibFunc::Begin_LibFunc; I != LibFunc::End_LibFunc; ++I) {
1533 LibFunc F = static_cast<LibFunc>(I);
1534 if (TLI.has(F))
1535 LibFuncSymbols.push_back(Elt: Saver.save(S: TLI.getName(F)).data());
1536 }
1537 return LibFuncSymbols;
1538}
1539
1540Error ThinBackendProc::emitFiles(
1541 const FunctionImporter::ImportMapTy &ImportList, unsigned Task,
1542 llvm::StringRef ModulePath, const std::string &NewModulePath) const {
1543 return emitFiles(ImportList, Task, ModulePath, NewModulePath,
1544 SummaryPath: NewModulePath + ".thinlto.bc");
1545}
1546
1547Error ThinBackendProc::emitFiles(
1548 const FunctionImporter::ImportMapTy &ImportList, unsigned Task,
1549 llvm::StringRef ModulePath, const std::string &NewModulePath,
1550 StringRef SummaryPath) const {
1551 ModuleToSummariesForIndexTy ModuleToSummariesForIndex;
1552 GVSummaryPtrSet DeclarationSummaries;
1553
1554 std::error_code EC;
1555 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1556 ImportList, ModuleToSummariesForIndex,
1557 DecSummaries&: DeclarationSummaries);
1558 // Resolve the output stream (either file-backed or callback-provided) for the
1559 // index file.
1560 std::unique_ptr<raw_pwrite_stream> OS;
1561 if (Conf.GetSummaryIndexOutputStream) {
1562 OS = Conf.GetSummaryIndexOutputStream(Task);
1563 assert(OS && "GetSummaryIndexOutputStream returned null");
1564 } else {
1565 auto FileOS = std::make_unique<raw_fd_ostream>(args&: SummaryPath, args&: EC,
1566 args: sys::fs::OpenFlags::OF_None);
1567 if (EC)
1568 return createFileError(F: "cannot open " + Twine(SummaryPath), EC);
1569 OS = std::move(FileOS);
1570 }
1571
1572 writeIndexToFile(Index: CombinedIndex, Out&: *OS, ModuleToSummariesForIndex: &ModuleToSummariesForIndex,
1573 DecSummaries: &DeclarationSummaries);
1574
1575 // Emit imports files if requested, using callback if provided.
1576 if (Conf.GetImportsListOutputArray) {
1577 std::vector<std::string> &ImportsListRef =
1578 Conf.GetImportsListOutputArray(Task);
1579 processImportsFiles(
1580 ModulePath, ModuleToSummariesForIndex,
1581 F: [&](StringRef M) { ImportsListRef.push_back(x: M.str()); });
1582 } else if (ShouldEmitImportsFiles) {
1583 if (Error E = EmitImportsFiles(ModulePath, OutputFilename: NewModulePath + ".imports",
1584 ModuleToSummariesForIndex))
1585 return E;
1586 }
1587 return Error::success();
1588}
1589
1590namespace {
1591/// Base class for ThinLTO backends that perform code generation and insert the
1592/// generated files back into the link.
1593class CGThinBackend : public ThinBackendProc {
1594protected:
1595 DenseSet<GlobalValue::GUID> CfiFunctionDefs;
1596 DenseSet<GlobalValue::GUID> CfiFunctionDecls;
1597 bool ShouldEmitIndexFiles;
1598
1599public:
1600 CGThinBackend(
1601 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1602 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1603 lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles,
1604 bool ShouldEmitImportsFiles, ThreadPoolStrategy ThinLTOParallelism)
1605 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1606 OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),
1607 ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1608 auto &Defs = CombinedIndex.cfiFunctionDefs();
1609 CfiFunctionDefs.insert_range(R: Defs.getExportedThinLTOGUIDs());
1610 auto &Decls = CombinedIndex.cfiFunctionDecls();
1611 CfiFunctionDecls.insert_range(R: Decls.getExportedThinLTOGUIDs());
1612 }
1613};
1614
1615/// This backend performs code generation by scheduling a job to run on
1616/// an in-process thread when invoked for each task.
1617class InProcessThinBackend : public CGThinBackend {
1618protected:
1619 // Callback used to add generated native object files to the link by code
1620 // generating directly into the returned output stream.
1621 AddStreamFn AddStream;
1622 FileCache Cache;
1623 ArrayRef<StringRef> BitcodeLibFuncs;
1624
1625public:
1626 InProcessThinBackend(
1627 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1628 ThreadPoolStrategy ThinLTOParallelism,
1629 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1630 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1631 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
1632 ArrayRef<StringRef> BitcodeLibFuncs)
1633 : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite,
1634 ShouldEmitIndexFiles, ShouldEmitImportsFiles,
1635 ThinLTOParallelism),
1636 AddStream(std::move(AddStream)), Cache(std::move(Cache)),
1637 BitcodeLibFuncs(BitcodeLibFuncs) {}
1638
1639 virtual Error runThinLTOBackendThread(
1640 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1641 ModuleSummaryIndex &CombinedIndex,
1642 const FunctionImporter::ImportMapTy &ImportList,
1643 const FunctionImporter::ExportSetTy &ExportList,
1644 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1645 const GVSummaryMapTy &DefinedGlobals,
1646 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1647 auto ModuleID = BM.getModuleIdentifier();
1648 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (in-process)",
1649 ModuleID);
1650 auto RunThinBackend = [&](AddStreamFn AddStream) {
1651 LTOLLVMContext BackendContext(Conf);
1652 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext);
1653 if (!MOrErr)
1654 return MOrErr.takeError();
1655
1656 return thinBackend(C: Conf, Task, AddStream, M&: **MOrErr, CombinedIndex,
1657 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1658 CodeGenOnly: Conf.CodeGenOnly, BitcodeLibFuncs);
1659 };
1660 if (ShouldEmitIndexFiles) {
1661 if (auto E = emitFiles(ImportList, Task, ModulePath: ModuleID, NewModulePath: ModuleID.str()))
1662 return E;
1663 }
1664
1665 if (!Cache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1666 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1667 P: [](uint32_t V) { return V == 0; }))
1668 // Cache disabled or no entry for this module in the combined index or
1669 // no module hash.
1670 return RunThinBackend(AddStream);
1671
1672 // The module may be cached, this helps handling it.
1673 std::string Key = computeLTOCacheKey(
1674 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1675 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1676 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1677 if (Error Err = CacheAddStreamOrErr.takeError())
1678 return Err;
1679 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1680 if (CacheAddStream)
1681 return RunThinBackend(CacheAddStream);
1682
1683 return Error::success();
1684 }
1685
1686 Error start(
1687 unsigned Task, BitcodeModule BM,
1688 const FunctionImporter::ImportMapTy &ImportList,
1689 const FunctionImporter::ExportSetTy &ExportList,
1690 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1691 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1692 StringRef ModulePath = BM.getModuleIdentifier();
1693 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1694 const GVSummaryMapTy &DefinedGlobals =
1695 ModuleToDefinedGVSummaries.find(Val: ModulePath)->second;
1696 BackendThreadPool.async(
1697 F: [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1698 const FunctionImporter::ImportMapTy &ImportList,
1699 const FunctionImporter::ExportSetTy &ExportList,
1700 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1701 &ResolvedODR,
1702 const GVSummaryMapTy &DefinedGlobals,
1703 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1704 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1705 timeTraceProfilerInitialize(TimeTraceGranularity: Conf.TimeTraceGranularity,
1706 ProcName: "thin backend");
1707 Error E = runThinLTOBackendThread(
1708 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1709 ResolvedODR, DefinedGlobals, ModuleMap);
1710 if (E) {
1711 std::unique_lock<std::mutex> L(ErrMu);
1712 if (Err)
1713 Err = joinErrors(E1: std::move(*Err), E2: std::move(E));
1714 else
1715 Err = std::move(E);
1716 }
1717 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1718 timeTraceProfilerFinishThread();
1719 },
1720 ArgList&: BM, ArgList: std::ref(t&: CombinedIndex), ArgList: std::ref(t: ImportList), ArgList: std::ref(t: ExportList),
1721 ArgList: std::ref(t: ResolvedODR), ArgList: std::ref(t: DefinedGlobals), ArgList: std::ref(t&: ModuleMap));
1722
1723 if (OnWrite)
1724 OnWrite(std::string(ModulePath));
1725 return Error::success();
1726 }
1727};
1728
1729/// This backend is utilized in the first round of a two-codegen round process.
1730/// It first saves optimized bitcode files to disk before the codegen process
1731/// begins. After codegen, it stores the resulting object files in a scratch
1732/// buffer. Note the codegen data stored in the scratch buffer will be extracted
1733/// and merged in the subsequent step.
1734class FirstRoundThinBackend : public InProcessThinBackend {
1735 AddStreamFn IRAddStream;
1736 FileCache IRCache;
1737
1738public:
1739 FirstRoundThinBackend(
1740 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1741 ThreadPoolStrategy ThinLTOParallelism,
1742 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1743 AddStreamFn CGAddStream, FileCache CGCache,
1744 ArrayRef<StringRef> BitcodeLibFuncs, AddStreamFn IRAddStream,
1745 FileCache IRCache)
1746 : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
1747 ModuleToDefinedGVSummaries, std::move(CGAddStream),
1748 std::move(CGCache), /*OnWrite=*/nullptr,
1749 /*ShouldEmitIndexFiles=*/false,
1750 /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
1751 IRAddStream(std::move(IRAddStream)), IRCache(std::move(IRCache)) {}
1752
1753 Error runThinLTOBackendThread(
1754 AddStreamFn CGAddStream, FileCache CGCache, unsigned Task,
1755 BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1756 const FunctionImporter::ImportMapTy &ImportList,
1757 const FunctionImporter::ExportSetTy &ExportList,
1758 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1759 const GVSummaryMapTy &DefinedGlobals,
1760 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1761 auto ModuleID = BM.getModuleIdentifier();
1762 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (first round)",
1763 ModuleID);
1764 auto RunThinBackend = [&](AddStreamFn CGAddStream,
1765 AddStreamFn IRAddStream) {
1766 LTOLLVMContext BackendContext(Conf);
1767 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext);
1768 if (!MOrErr)
1769 return MOrErr.takeError();
1770
1771 return thinBackend(C: Conf, Task, AddStream: CGAddStream, M&: **MOrErr, CombinedIndex,
1772 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1773 CodeGenOnly: Conf.CodeGenOnly, BitcodeLibFuncs, IRAddStream);
1774 };
1775 // Like InProcessThinBackend, we produce index files as needed for
1776 // FirstRoundThinBackend. However, these files are not generated for
1777 // SecondRoundThinBackend.
1778 if (ShouldEmitIndexFiles) {
1779 if (auto E = emitFiles(ImportList, Task, ModulePath: ModuleID, NewModulePath: ModuleID.str()))
1780 return E;
1781 }
1782
1783 assert((CGCache.isValid() == IRCache.isValid()) &&
1784 "Both caches for CG and IR should have matching availability");
1785 if (!CGCache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1786 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1787 P: [](uint32_t V) { return V == 0; }))
1788 // Cache disabled or no entry for this module in the combined index or
1789 // no module hash.
1790 return RunThinBackend(CGAddStream, IRAddStream);
1791
1792 // Get CGKey for caching object in CGCache.
1793 std::string CGKey = computeLTOCacheKey(
1794 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1795 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1796 Expected<AddStreamFn> CacheCGAddStreamOrErr =
1797 CGCache(Task, CGKey, ModuleID);
1798 if (Error Err = CacheCGAddStreamOrErr.takeError())
1799 return Err;
1800 AddStreamFn &CacheCGAddStream = *CacheCGAddStreamOrErr;
1801
1802 // Get IRKey for caching (optimized) IR in IRCache with an extra ID.
1803 std::string IRKey = recomputeLTOCacheKey(Key: CGKey, /*ExtraID=*/"IR");
1804 Expected<AddStreamFn> CacheIRAddStreamOrErr =
1805 IRCache(Task, IRKey, ModuleID);
1806 if (Error Err = CacheIRAddStreamOrErr.takeError())
1807 return Err;
1808 AddStreamFn &CacheIRAddStream = *CacheIRAddStreamOrErr;
1809
1810 // Ideally, both CG and IR caching should be synchronized. However, in
1811 // practice, their availability may differ due to different expiration
1812 // times. Therefore, if either cache is missing, the backend process is
1813 // triggered.
1814 if (CacheCGAddStream || CacheIRAddStream) {
1815 LLVM_DEBUG(dbgs() << "[FirstRound] Cache Miss for "
1816 << BM.getModuleIdentifier() << "\n");
1817 return RunThinBackend(CacheCGAddStream ? CacheCGAddStream : CGAddStream,
1818 CacheIRAddStream ? CacheIRAddStream : IRAddStream);
1819 }
1820
1821 return Error::success();
1822 }
1823};
1824
1825/// This backend operates in the second round of a two-codegen round process.
1826/// It starts by reading the optimized bitcode files that were saved during the
1827/// first round. The backend then executes the codegen only to further optimize
1828/// the code, utilizing the codegen data merged from the first round. Finally,
1829/// it writes the resulting object files as usual.
1830class SecondRoundThinBackend : public InProcessThinBackend {
1831 std::unique_ptr<SmallVector<StringRef>> IRFiles;
1832 stable_hash CombinedCGDataHash;
1833
1834public:
1835 SecondRoundThinBackend(
1836 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1837 ThreadPoolStrategy ThinLTOParallelism,
1838 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1839 AddStreamFn AddStream, FileCache Cache,
1840 ArrayRef<StringRef> BitcodeLibFuncs,
1841 std::unique_ptr<SmallVector<StringRef>> IRFiles,
1842 stable_hash CombinedCGDataHash)
1843 : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
1844 ModuleToDefinedGVSummaries, std::move(AddStream),
1845 std::move(Cache),
1846 /*OnWrite=*/nullptr,
1847 /*ShouldEmitIndexFiles=*/false,
1848 /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
1849 IRFiles(std::move(IRFiles)), CombinedCGDataHash(CombinedCGDataHash) {}
1850
1851 Error runThinLTOBackendThread(
1852 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1853 ModuleSummaryIndex &CombinedIndex,
1854 const FunctionImporter::ImportMapTy &ImportList,
1855 const FunctionImporter::ExportSetTy &ExportList,
1856 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1857 const GVSummaryMapTy &DefinedGlobals,
1858 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1859 auto ModuleID = BM.getModuleIdentifier();
1860 llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (second round)",
1861 ModuleID);
1862 auto RunThinBackend = [&](AddStreamFn AddStream) {
1863 LTOLLVMContext BackendContext(Conf);
1864 std::unique_ptr<Module> LoadedModule =
1865 cgdata::loadModuleForTwoRounds(OrigModule&: BM, Task, Context&: BackendContext, IRFiles: *IRFiles);
1866
1867 return thinBackend(C: Conf, Task, AddStream, M&: *LoadedModule, CombinedIndex,
1868 ImportList, DefinedGlobals, ModuleMap: &ModuleMap,
1869 /*CodeGenOnly=*/true, BitcodeLibFuncs);
1870 };
1871 if (!Cache.isValid() || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1872 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1873 P: [](uint32_t V) { return V == 0; }))
1874 // Cache disabled or no entry for this module in the combined index or
1875 // no module hash.
1876 return RunThinBackend(AddStream);
1877
1878 // Get Key for caching the final object file in Cache with the combined
1879 // CGData hash.
1880 std::string Key = computeLTOCacheKey(
1881 Conf, Index: CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
1882 DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
1883 Key = recomputeLTOCacheKey(Key,
1884 /*ExtraID=*/std::to_string(val: CombinedCGDataHash));
1885 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1886 if (Error Err = CacheAddStreamOrErr.takeError())
1887 return Err;
1888 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1889
1890 if (CacheAddStream) {
1891 LLVM_DEBUG(dbgs() << "[SecondRound] Cache Miss for "
1892 << BM.getModuleIdentifier() << "\n");
1893 return RunThinBackend(CacheAddStream);
1894 }
1895
1896 return Error::success();
1897 }
1898};
1899} // end anonymous namespace
1900
1901ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,
1902 lto::IndexWriteCallback OnWrite,
1903 bool ShouldEmitIndexFiles,
1904 bool ShouldEmitImportsFiles) {
1905 auto Func =
1906 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1907 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1908 AddStreamFn AddStream, FileCache Cache,
1909 ArrayRef<StringRef> BitcodeLibFuncs) {
1910 return std::make_unique<InProcessThinBackend>(
1911 args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries,
1912 args&: AddStream, args&: Cache, args: OnWrite, args: ShouldEmitIndexFiles,
1913 args: ShouldEmitImportsFiles, args&: BitcodeLibFuncs);
1914 };
1915 return ThinBackend(Func, Parallelism);
1916}
1917
1918StringLiteral lto::getThinLTODefaultCPU(const Triple &TheTriple) {
1919 if (!TheTriple.isOSDarwin())
1920 return "";
1921 if (TheTriple.getArch() == Triple::x86_64)
1922 return "core2";
1923 if (TheTriple.getArch() == Triple::x86)
1924 return "yonah";
1925 if (TheTriple.isArm64e())
1926 return "apple-a12";
1927 if (TheTriple.getArch() == Triple::aarch64 ||
1928 TheTriple.getArch() == Triple::aarch64_32)
1929 return "cyclone";
1930 return "";
1931}
1932
1933// Given the original \p Path to an output file, replace any path
1934// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1935// resulting directory if it does not yet exist.
1936std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
1937 StringRef NewPrefix) {
1938 if (OldPrefix.empty() && NewPrefix.empty())
1939 return std::string(Path);
1940 SmallString<128> NewPath(Path);
1941 llvm::sys::path::replace_path_prefix(Path&: NewPath, OldPrefix, NewPrefix);
1942 StringRef ParentPath = llvm::sys::path::parent_path(path: NewPath.str());
1943 if (!ParentPath.empty()) {
1944 // Make sure the new directory exists, creating it if necessary.
1945 if (std::error_code EC = llvm::sys::fs::create_directories(path: ParentPath))
1946 llvm::errs() << "warning: could not create directory '" << ParentPath
1947 << "': " << EC.message() << '\n';
1948 }
1949 return std::string(NewPath);
1950}
1951
1952namespace {
1953class WriteIndexesThinBackend : public ThinBackendProc {
1954 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1955 raw_fd_ostream *LinkedObjectsFile;
1956 DenseSet<GlobalValue::GUID> CfiFunctionDefs;
1957 DenseSet<GlobalValue::GUID> CfiFunctionDecls;
1958
1959public:
1960 WriteIndexesThinBackend(
1961 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1962 ThreadPoolStrategy ThinLTOParallelism,
1963 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1964 std::string OldPrefix, std::string NewPrefix,
1965 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1966 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1967 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1968 OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),
1969 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1970 NativeObjectPrefix(NativeObjectPrefix),
1971 LinkedObjectsFile(LinkedObjectsFile) {
1972 auto Defs = CombinedIndex.cfiFunctionDefs().getExportedThinLTOGUIDs();
1973 CfiFunctionDefs.insert(I: Defs.begin(), E: Defs.end());
1974 auto Decls = CombinedIndex.cfiFunctionDecls().getExportedThinLTOGUIDs();
1975 CfiFunctionDecls.insert(I: Decls.begin(), E: Decls.end());
1976 }
1977
1978 Error start(
1979 unsigned Task, BitcodeModule BM,
1980 const FunctionImporter::ImportMapTy &ImportList,
1981 const FunctionImporter::ExportSetTy &ExportList,
1982 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1983 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1984 StringRef ModulePath = BM.getModuleIdentifier();
1985
1986 // The contents of this file may be used as input to a native link, and must
1987 // therefore contain the processed modules in a determinstic order that
1988 // match the order they are provided on the command line. For that reason,
1989 // we cannot include this in the asynchronously executed lambda below.
1990 if (LinkedObjectsFile) {
1991 std::string ObjectPrefix =
1992 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1993 std::string LinkedObjectsFilePath =
1994 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix: ObjectPrefix);
1995 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1996 }
1997
1998 BackendThreadPool.async(
1999 F: [this](unsigned Task, const StringRef ModulePath,
2000 const FunctionImporter::ImportMapTy &ImportList,
2001 const FunctionImporter::ExportSetTy &ExportList,
2002 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
2003 &ResolvedODR,
2004 const std::string &OldPrefix, const std::string &NewPrefix) {
2005 std::string NewModulePath =
2006 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix);
2007 auto E = emitFiles(ImportList, Task, ModulePath, NewModulePath);
2008 if (E) {
2009 std::unique_lock<std::mutex> L(ErrMu);
2010 if (Err)
2011 Err = joinErrors(E1: std::move(*Err), E2: std::move(E));
2012 else
2013 Err = std::move(E);
2014 }
2015 assert(ModuleToDefinedGVSummaries.count(ModulePath));
2016 const GVSummaryMapTy &DefinedGlobals =
2017 ModuleToDefinedGVSummaries.find(Val: ModulePath)->second;
2018
2019 // DTLTO needs the per-module LTO cache key to probe the cache.
2020 if (Conf.GetCacheKeyOutputString) {
2021 std::string &CacheKey = Conf.GetCacheKeyOutputString(Task);
2022 CacheKey = computeLTOCacheKey(
2023 Conf, Index: CombinedIndex, ModuleID: ModulePath, ImportList, ExportList,
2024 ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
2025 }
2026 },
2027 ArgList&: Task, ArgList&: ModulePath, ArgList: ImportList, ArgList: ExportList, ArgList: ResolvedODR, ArgList&: OldPrefix,
2028 ArgList&: NewPrefix);
2029
2030 if (OnWrite)
2031 OnWrite(std::string(ModulePath));
2032 return Error::success();
2033 }
2034
2035 bool isSensitiveToInputOrder() override {
2036 // The order which modules are written to LinkedObjectsFile should be
2037 // deterministic and match the order they are passed on the command line.
2038 return true;
2039 }
2040};
2041} // end anonymous namespace
2042
2043ThinBackend lto::createWriteIndexesThinBackend(
2044 ThreadPoolStrategy Parallelism, std::string OldPrefix,
2045 std::string NewPrefix, std::string NativeObjectPrefix,
2046 bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile,
2047 IndexWriteCallback OnWrite) {
2048 auto Func =
2049 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
2050 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
2051 AddStreamFn AddStream, FileCache Cache,
2052 ArrayRef<StringRef> BitcodeLibFuncs) {
2053 return std::make_unique<WriteIndexesThinBackend>(
2054 args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries,
2055 args: OldPrefix, args: NewPrefix, args: NativeObjectPrefix, args: ShouldEmitImportsFiles,
2056 args: LinkedObjectsFile, args: OnWrite);
2057 };
2058 return ThinBackend(Func, Parallelism);
2059}
2060
2061Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
2062 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
2063 llvm::TimeTraceScope timeScope("Run ThinLTO");
2064 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
2065 ThinLTO.CombinedIndex.releaseTemporaryMemory();
2066 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
2067 llvm::scope_exit TimeTraceScopeExit([]() {
2068 if (llvm::timeTraceProfilerEnabled())
2069 llvm::timeTraceProfilerEnd();
2070 });
2071 if (ThinLTO.ModuleMap.empty())
2072 return Error::success();
2073
2074 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
2075 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
2076 return Error::success();
2077 }
2078
2079 if (Conf.CombinedIndexHook &&
2080 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
2081 return Error::success();
2082
2083 // Collect for each module the list of function it defines (GUID ->
2084 // Summary).
2085 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
2086 ThinLTO.ModuleMap.size());
2087 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
2088 ModuleToDefinedGVSummaries);
2089 // Create entries for any modules that didn't have any GV summaries
2090 // (either they didn't have any GVs to start with, or we suppressed
2091 // generation of the summaries because they e.g. had inline assembly
2092 // uses that couldn't be promoted/renamed on export). This is so
2093 // InProcessThinBackend::start can still launch a backend thread, which
2094 // is passed the map of summaries for the module, without any special
2095 // handling for this case.
2096 for (auto &Mod : ThinLTO.ModuleMap)
2097 if (!ModuleToDefinedGVSummaries.count(Val: Mod.first))
2098 ModuleToDefinedGVSummaries.try_emplace(Key: Mod.first);
2099
2100 FunctionImporter::ImportListsTy ImportLists(ThinLTO.ModuleMap.size());
2101 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(
2102 ThinLTO.ModuleMap.size());
2103 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
2104
2105 if (DumpThinCGSCCs)
2106 ThinLTO.CombinedIndex.dumpSCCs(OS&: outs());
2107
2108 std::set<GlobalValue::GUID> ExportedGUIDs;
2109
2110 bool WholeProgramVisibilityEnabledInLTO =
2111 Conf.HasWholeProgramVisibility &&
2112 // If validation is enabled, upgrade visibility only when all vtables
2113 // have typeinfos.
2114 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
2115 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
2116 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
2117
2118 // If we're validating, get the vtable symbols that should not be
2119 // upgraded because they correspond to typeIDs outside of index-based
2120 // WPD info.
2121 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
2122 if (WholeProgramVisibilityEnabledInLTO &&
2123 Conf.ValidateAllVtablesHaveTypeInfos) {
2124 // This returns true when the name is local or not defined. Locals are
2125 // expected to be handled separately.
2126 auto IsVisibleToRegularObj = [&](StringRef name) {
2127 auto It = GlobalResolutions->find(Val: name);
2128 return (It == GlobalResolutions->end() ||
2129 It->second.VisibleOutsideSummary || !It->second.Prevailing);
2130 };
2131
2132 getVisibleToRegularObjVtableGUIDs(Index&: ThinLTO.CombinedIndex,
2133 VisibleToRegularObjSymbols,
2134 IsVisibleToRegularObj);
2135 }
2136
2137 // If allowed, upgrade public vcall visibility to linkage unit visibility in
2138 // the summaries before whole program devirtualization below.
2139 updateVCallVisibilityInIndex(
2140 Index&: ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
2141 DynamicExportSymbols, VisibleToRegularObjSymbols);
2142
2143 // Perform index-based WPD. This will return immediately if there are
2144 // no index entries in the typeIdMetadata map (e.g. if we are instead
2145 // performing IR-based WPD in hybrid regular/thin LTO mode).
2146 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
2147 DenseSet<StringRef> ExternallyVisibleSymbolNames;
2148
2149 // Used by the promotion-time renaming logic. When non-null, this set
2150 // identifies symbols that should not be renamed during promotion.
2151 // It is non-null only when whole-program visibility is enabled and
2152 // renaming is not forced. Otherwise, the default renaming behavior applies.
2153 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr =
2154 (WholeProgramVisibilityEnabledInLTO && !AlwaysRenamePromotedLocals)
2155 ? &ExternallyVisibleSymbolNames
2156 : nullptr;
2157 runWholeProgramDevirtOnIndex(Summary&: ThinLTO.CombinedIndex, ExportedGUIDs,
2158 LocalWPDTargetsMap,
2159 ExternallyVisibleSymbolNamesPtr);
2160
2161 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
2162 return ThinLTO.isPrevailingModuleForGUID(GUID, Module: S->modulePath());
2163 };
2164 if (EnableMemProfContextDisambiguation) {
2165 MemProfContextDisambiguation ContextDisambiguation;
2166 ContextDisambiguation.run(
2167 Index&: ThinLTO.CombinedIndex, isPrevailing, Ctx&: RegularLTO.Ctx,
2168 EmitRemark: [&](StringRef PassName, StringRef RemarkName, const Twine &Msg) {
2169 auto R = OptimizationRemark(PassName.data(), RemarkName,
2170 LinkerRemarkFunction);
2171 R << Msg.str();
2172 emitRemark(Remark&: R);
2173 });
2174 }
2175
2176 // Figure out which symbols need to be internalized. This also needs to happen
2177 // at -O0 because summary-based DCE is implemented using internalization, and
2178 // we must apply DCE consistently with the full LTO module in order to avoid
2179 // undefined references during the final link.
2180 for (auto &Res : *GlobalResolutions) {
2181 // If the symbol does not have external references or it is not prevailing,
2182 // then not need to mark it as exported from a ThinLTO partition.
2183 if (Res.second.Partition != GlobalResolution::External ||
2184 !Res.second.isPrevailingIRSymbol())
2185 continue;
2186 auto GUID = Res.second.getGUID();
2187 // Mark exported unless index-based analysis determined it to be dead.
2188 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
2189 ExportedGUIDs.insert(x: GUID);
2190 }
2191
2192 // Reset the GlobalResolutions to deallocate the associated memory, as there
2193 // are no further accesses. We specifically want to do this before computing
2194 // cross module importing, which adds to peak memory via the computed import
2195 // and export lists.
2196 releaseGlobalResolutionsMemory();
2197
2198 if (Conf.OptLevel > 0)
2199 ComputeCrossModuleImport(Index: ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
2200 isPrevailing, ImportLists, ExportLists);
2201
2202 // Any functions referenced by the jump table in the regular LTO object must
2203 // be exported.
2204 auto Defs = ThinLTO.CombinedIndex.cfiFunctionDefs().getExportedThinLTOGUIDs();
2205 ExportedGUIDs.insert(first: Defs.begin(), last: Defs.end());
2206 auto Decls =
2207 ThinLTO.CombinedIndex.cfiFunctionDecls().getExportedThinLTOGUIDs();
2208 ExportedGUIDs.insert(first: Decls.begin(), last: Decls.end());
2209
2210 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
2211 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
2212 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
2213 ExportedGUIDs.count(x: VI.getGUID());
2214 };
2215
2216 // Update local devirtualized targets that were exported by cross-module
2217 // importing or by other devirtualizations marked in the ExportedGUIDs set.
2218 updateIndexWPDForExports(Summary&: ThinLTO.CombinedIndex, isExported,
2219 LocalWPDTargetsMap, ExternallyVisibleSymbolNamesPtr);
2220
2221 if (ExternallyVisibleSymbolNamesPtr) {
2222 // Add to ExternallyVisibleSymbolNames the set of unique names used by all
2223 // externally visible symbols in the index.
2224 for (auto &I : ThinLTO.CombinedIndex) {
2225 ValueInfo VI = ThinLTO.CombinedIndex.getValueInfo(R: I);
2226 for (const auto &Summary : VI.getSummaryList()) {
2227 const GlobalValueSummary *Base = Summary->getBaseObject();
2228 if (GlobalValue::isLocalLinkage(Linkage: Base->linkage()))
2229 continue;
2230
2231 ExternallyVisibleSymbolNamesPtr->insert(V: VI.name());
2232 break;
2233 }
2234 }
2235 }
2236
2237 thinLTOInternalizeAndPromoteInIndex(Index&: ThinLTO.CombinedIndex, isExported,
2238 isPrevailing,
2239 ExternallyVisibleSymbolNamesPtr);
2240
2241 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
2242 GlobalValue::GUID GUID,
2243 GlobalValue::LinkageTypes NewLinkage) {
2244 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
2245 };
2246 thinLTOResolvePrevailingInIndex(C: Conf, Index&: ThinLTO.CombinedIndex, isPrevailing,
2247 recordNewLinkage, GUIDPreservedSymbols);
2248
2249 thinLTOPropagateFunctionAttrs(Index&: ThinLTO.CombinedIndex, isPrevailing);
2250
2251 generateParamAccessSummary(Index&: ThinLTO.CombinedIndex);
2252
2253 if (llvm::timeTraceProfilerEnabled())
2254 llvm::timeTraceProfilerEnd();
2255
2256 TimeTraceScopeExit.release();
2257
2258 auto &ModuleMap =
2259 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
2260
2261 auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error {
2262 auto ProcessOneModule = [&](int I) -> Error {
2263 auto &Mod = *(ModuleMap.begin() + I);
2264 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
2265 // combined module and parallel code generation partitions.
2266 return BackendProcess->start(
2267 Task: RegularLTO.ParallelCodeGenParallelismLevel + I, BM: Mod.second,
2268 ImportList: ImportLists[Mod.first], ExportList: ExportLists[Mod.first],
2269 ResolvedODR: ResolvedODR[Mod.first], ModuleMap&: ThinLTO.ModuleMap);
2270 };
2271
2272 BackendProcess->setup(ThinLTONumTasks: ModuleMap.size(),
2273 ThinLTOTaskOffset: RegularLTO.ParallelCodeGenParallelismLevel,
2274 Triple: RegularLTO.CombinedModule->getTargetTriple());
2275
2276 if (BackendProcess->getThreadCount() == 1 ||
2277 BackendProcess->isSensitiveToInputOrder()) {
2278 // Process the modules in the order they were provided on the
2279 // command-line. It is important for this codepath to be used for
2280 // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists
2281 // ThinLTO objects in the same order as the inputs, which otherwise would
2282 // affect the final link order.
2283 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
2284 if (Error E = ProcessOneModule(I))
2285 return E;
2286 } else {
2287 // When executing in parallel, process largest bitsize modules first to
2288 // improve parallelism, and avoid starving the thread pool near the end.
2289 // This saves about 15 sec on a 36-core machine while link `clang.exe`
2290 // (out of 100 sec).
2291 std::vector<BitcodeModule *> ModulesVec;
2292 ModulesVec.reserve(n: ModuleMap.size());
2293 for (auto &Mod : ModuleMap)
2294 ModulesVec.push_back(x: &Mod.second);
2295 for (int I : generateModulesOrdering(R: ModulesVec))
2296 if (Error E = ProcessOneModule(I))
2297 return E;
2298 }
2299 return BackendProcess->wait();
2300 };
2301
2302 if (!CodeGenDataThinLTOTwoRounds) {
2303 std::unique_ptr<ThinBackendProc> BackendProc =
2304 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
2305 AddStream, Cache, BitcodeLibFuncs);
2306 return RunBackends(BackendProc.get());
2307 }
2308
2309 // Perform two rounds of code generation for ThinLTO:
2310 // 1. First round: Perform optimization and code generation, outputting to
2311 // temporary scratch objects.
2312 // 2. Merge code generation data extracted from the temporary scratch objects.
2313 // 3. Second round: Execute code generation again using the merged data.
2314 LLVM_DEBUG(dbgs() << "[TwoRounds] Initializing ThinLTO two-codegen rounds\n");
2315
2316 unsigned MaxTasks = getMaxTasks();
2317 auto Parallelism = ThinLTO.Backend.getParallelism();
2318 // Set up two additional streams and caches for storing temporary scratch
2319 // objects and optimized IRs, using the same cache directory as the original.
2320 cgdata::StreamCacheData CG(MaxTasks, Cache, "CG"), IR(MaxTasks, Cache, "IR");
2321
2322 // First round: Execute optimization and code generation, outputting to
2323 // temporary scratch objects. Serialize the optimized IRs before initiating
2324 // code generation.
2325 LLVM_DEBUG(dbgs() << "[TwoRounds] Running the first round of codegen\n");
2326 auto FirstRoundLTO = std::make_unique<FirstRoundThinBackend>(
2327 args&: Conf, args&: ThinLTO.CombinedIndex, args&: Parallelism, args&: ModuleToDefinedGVSummaries,
2328 args&: CG.AddStream, args&: CG.Cache, args&: BitcodeLibFuncs, args&: IR.AddStream, args&: IR.Cache);
2329 if (Error E = RunBackends(FirstRoundLTO.get()))
2330 return E;
2331
2332 LLVM_DEBUG(dbgs() << "[TwoRounds] Merging codegen data\n");
2333 auto CombinedHashOrErr = cgdata::mergeCodeGenData(ObjectFiles: *CG.getResult());
2334 if (Error E = CombinedHashOrErr.takeError())
2335 return E;
2336 auto CombinedHash = *CombinedHashOrErr;
2337 LLVM_DEBUG(dbgs() << "[TwoRounds] CGData hash: " << CombinedHash << "\n");
2338
2339 // Second round: Read the optimized IRs and execute code generation using the
2340 // merged data.
2341 LLVM_DEBUG(dbgs() << "[TwoRounds] Running the second round of codegen\n");
2342 auto SecondRoundLTO = std::make_unique<SecondRoundThinBackend>(
2343 args&: Conf, args&: ThinLTO.CombinedIndex, args&: Parallelism, args&: ModuleToDefinedGVSummaries,
2344 args&: AddStream, args&: Cache, args&: BitcodeLibFuncs, args: IR.getResult(), args&: CombinedHash);
2345 return RunBackends(SecondRoundLTO.get());
2346}
2347
2348Expected<LLVMRemarkFileHandle> lto::setupLLVMOptimizationRemarks(
2349 LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
2350 StringRef RemarksFormat, bool RemarksWithHotness,
2351 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
2352 std::string Filename = std::string(RemarksFilename);
2353 // For ThinLTO, file.opt.<format> becomes
2354 // file.opt.<format>.thin.<num>.<format>.
2355 if (!Filename.empty() && Count != -1)
2356 Filename =
2357 (Twine(Filename) + ".thin." + llvm::utostr(X: Count) + "." + RemarksFormat)
2358 .str();
2359
2360 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
2361 Context, RemarksFilename: Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,
2362 RemarksHotnessThreshold);
2363 if (Error E = ResultOrErr.takeError())
2364 return std::move(E);
2365
2366 if (*ResultOrErr)
2367 (*ResultOrErr)->keep();
2368
2369 return ResultOrErr;
2370}
2371
2372Expected<std::unique_ptr<ToolOutputFile>>
2373lto::setupStatsFile(StringRef StatsFilename) {
2374 // Setup output file to emit statistics.
2375 if (StatsFilename.empty())
2376 return nullptr;
2377
2378 llvm::EnableStatistics(DoPrintOnExit: false);
2379 std::error_code EC;
2380 auto StatsFile =
2381 std::make_unique<ToolOutputFile>(args&: StatsFilename, args&: EC, args: sys::fs::OF_None);
2382 if (EC)
2383 return errorCodeToError(EC);
2384
2385 StatsFile->keep();
2386 return std::move(StatsFile);
2387}
2388
2389// Compute the ordering we will process the inputs: the rough heuristic here
2390// is to sort them per size so that the largest module get schedule as soon as
2391// possible. This is purely a compile-time optimization.
2392std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
2393 auto Seq = llvm::seq<int>(Begin: 0, End: R.size());
2394 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
2395 llvm::sort(C&: ModulesOrdering, Comp: [&](int LeftIndex, int RightIndex) {
2396 auto LSize = R[LeftIndex]->getBuffer().size();
2397 auto RSize = R[RightIndex]->getBuffer().size();
2398 return LSize > RSize;
2399 });
2400 return ModulesOrdering;
2401}
2402