1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Bitcode/BitcodeReader.h"
21#include "llvm/IR/AutoUpgrade.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/GlobalObject.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/ModuleSummaryIndex.h"
30#include "llvm/IRReader/IRReader.h"
31#include "llvm/Linker/IRMover.h"
32#include "llvm/ProfileData/PGOCtxProfReader.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Errc.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Support/ErrorHandling.h"
39#include "llvm/Support/FileSystem.h"
40#include "llvm/Support/JSON.h"
41#include "llvm/Support/Path.h"
42#include "llvm/Support/SourceMgr.h"
43#include "llvm/Support/TimeProfiler.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/IPO/Internalize.h"
46#include "llvm/Transforms/Utils/Cloning.h"
47#include "llvm/Transforms/Utils/FunctionImportUtils.h"
48#include "llvm/Transforms/Utils/ValueMapper.h"
49#include <cassert>
50#include <memory>
51#include <string>
52#include <system_error>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "function-import"
59
60STATISTIC(NumImportedFunctionsThinLink,
61 "Number of functions thin link decided to import");
62STATISTIC(NumImportedHotFunctionsThinLink,
63 "Number of hot functions thin link decided to import");
64STATISTIC(NumImportedCriticalFunctionsThinLink,
65 "Number of critical functions thin link decided to import");
66STATISTIC(NumImportedGlobalVarsThinLink,
67 "Number of global variables thin link decided to import");
68STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
69STATISTIC(NumImportedGlobalVars,
70 "Number of global variables imported in backend");
71STATISTIC(NumImportedModules, "Number of modules imported from");
72STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
73STATISTIC(NumLiveSymbols, "Number of live symbols in index");
74
75namespace llvm {
76cl::opt<bool>
77 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
78 cl::desc("Import functions with noinline attribute"));
79
80/// Limit on instruction count of imported functions.
81static cl::opt<unsigned> ImportInstrLimit(
82 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
83 cl::desc("Only import functions with less than N instructions"));
84
85static cl::opt<int> ImportCutoff(
86 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
87 cl::desc("Only import first N functions if N>=0 (default -1)"));
88
89static cl::opt<float>
90 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
91 cl::Hidden, cl::value_desc("x"),
92 cl::desc("As we import functions, multiply the "
93 "`import-instr-limit` threshold by this factor "
94 "before processing newly imported functions"));
95
96static cl::opt<float> ImportHotInstrFactor(
97 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
98 cl::value_desc("x"),
99 cl::desc("As we import functions called from hot callsite, multiply the "
100 "`import-instr-limit` threshold by this factor "
101 "before processing newly imported functions"));
102
103static cl::opt<float> ImportHotMultiplier(
104 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
105 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
106
107static cl::opt<float> ImportCriticalMultiplier(
108 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
109 cl::value_desc("x"),
110 cl::desc(
111 "Multiply the `import-instr-limit` threshold for critical callsites"));
112
113// FIXME: This multiplier was not really tuned up.
114static cl::opt<float> ImportColdMultiplier(
115 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
116 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
117
118static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
119 cl::desc("Print imported functions"));
120
121static cl::opt<bool> PrintImportFailures(
122 "print-import-failures", cl::init(Val: false), cl::Hidden,
123 cl::desc("Print information for functions rejected for importing"));
124
125static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
126 cl::desc("Compute dead symbols"));
127
128static cl::opt<bool> EnableImportMetadata(
129 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
130 cl::desc("Enable import metadata like 'thinlto_src_module' and "
131 "'thinlto_src_file'"));
132
133/// Summary file to use for function importing when using -function-import from
134/// the command line.
135static cl::opt<std::string>
136 SummaryFile("summary-file",
137 cl::desc("The summary file to use for function importing."));
138
139/// Used when testing importing from distributed indexes via opt
140// -function-import.
141static cl::opt<bool>
142 ImportAllIndex("import-all-index",
143 cl::desc("Import all external functions in index."));
144
145/// This is a test-only option.
146/// If this option is enabled, the ThinLTO indexing step will import each
147/// function declaration as a fallback. In a real build this may increase ram
148/// usage of the indexing step unnecessarily.
149/// TODO: Implement selective import (based on combined summary analysis) to
150/// ensure the imported function has a use case in the postlink pipeline.
151static cl::opt<bool> ImportDeclaration(
152 "import-declaration", cl::init(Val: false), cl::Hidden,
153 cl::desc("If true, import function declaration as fallback if the function "
154 "definition is not imported."));
155
156/// Pass a workload description file - an example of workload would be the
157/// functions executed to satisfy a RPC request. A workload is defined by a root
158/// function and the list of functions that are (frequently) needed to satisfy
159/// it. The module that defines the root will have all those functions imported.
160/// The file contains a JSON dictionary. The keys are root functions, the values
161/// are lists of functions to import in the module defining the root. It is
162/// assumed -funique-internal-linkage-names was used, thus ensuring function
163/// names are unique even for local linkage ones.
164static cl::opt<std::string> WorkloadDefinitions(
165 "thinlto-workload-def",
166 cl::desc("Pass a workload definition. This is a file containing a JSON "
167 "dictionary. The keys are root functions, the values are lists of "
168 "functions to import in the module defining the root. It is "
169 "assumed -funique-internal-linkage-names was used, to ensure "
170 "local linkage functions have unique names. For example: \n"
171 "{\n"
172 " \"rootFunction_1\": [\"function_to_import_1\", "
173 "\"function_to_import_2\"], \n"
174 " \"rootFunction_2\": [\"function_to_import_3\", "
175 "\"function_to_import_4\"] \n"
176 "}"),
177 cl::Hidden);
178
179extern cl::opt<std::string> UseCtxProfile;
180
181static cl::opt<bool> CtxprofMoveRootsToOwnModule(
182 "thinlto-move-ctxprof-trees",
183 cl::desc("Move contextual profiling roots and the graphs under them in "
184 "their own module."),
185 cl::Hidden, cl::init(Val: false));
186
187extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
188
189extern cl::opt<bool> EnableMemProfContextDisambiguation;
190} // end namespace llvm
191
192// Load lazily a module from \p FileName in \p Context.
193static std::unique_ptr<Module> loadFile(const std::string &FileName,
194 LLVMContext &Context) {
195 SMDiagnostic Err;
196 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
197 // Metadata isn't loaded until functions are imported, to minimize
198 // the memory overhead.
199 std::unique_ptr<Module> Result =
200 getLazyIRFileModule(Filename: FileName, Err, Context,
201 /* ShouldLazyLoadMetadata = */ true);
202 if (!Result) {
203 Err.print(ProgName: "function-import", S&: errs());
204 report_fatal_error(reason: "Abort");
205 }
206
207 return Result;
208}
209
210static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
211 size_t NumDefs,
212 StringRef ImporterModule) {
213 // We can import a local when there is one definition.
214 if (NumDefs == 1)
215 return false;
216 // In other cases, make sure we import the copy in the caller's module if the
217 // referenced value has local linkage. The only time a local variable can
218 // share an entry in the index is if there is a local with the same name in
219 // another module that had the same source file name (in a different
220 // directory), where each was compiled in their own directory so there was not
221 // distinguishing path.
222 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
223 RefSummary->modulePath() != ImporterModule;
224}
225
226/// Given a list of possible callee implementation for a call site, qualify the
227/// legality of importing each. The return is a range of pairs. Each pair
228/// corresponds to a candidate. The first value is the ImportFailureReason for
229/// that candidate, the second is the candidate.
230static auto qualifyCalleeCandidates(
231 const ModuleSummaryIndex &Index,
232 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
233 StringRef CallerModulePath) {
234 return llvm::map_range(
235 C&: CalleeSummaryList,
236 F: [&Index, CalleeSummaryList,
237 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
238 -> std::pair<FunctionImporter::ImportFailureReason,
239 const GlobalValueSummary *> {
240 auto *GVSummary = SummaryPtr.get();
241 if (!Index.isGlobalValueLive(GVS: GVSummary))
242 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
243
244 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
245 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
246 GVSummary};
247
248 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
249
250 // Ignore any callees that aren't actually functions. This could happen
251 // in the case of GUID hash collisions. It could also happen in theory
252 // for SamplePGO profiles collected on old versions of the code after
253 // renaming, since we synthesize edges to any inlined callees appearing
254 // in the profile.
255 if (!Summary)
256 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
257
258 // If this is a local function, make sure we import the copy in the
259 // caller's module. The only time a local function can share an entry in
260 // the index is if there is a local with the same name in another module
261 // that had the same source file name (in a different directory), where
262 // each was compiled in their own directory so there was not
263 // distinguishing path.
264 // If the local function is from another module, it must be a reference
265 // due to indirect call profile data since a function pointer can point
266 // to a local in another module. Do the import from another module if
267 // there is only one entry in the list or when all files in the program
268 // are compiled with full path - in both cases the local function has
269 // unique PGO name and GUID.
270 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
271 ImporterModule: CallerModulePath))
272 return {
273 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
274 GVSummary};
275
276 // Skip if it isn't legal to import (e.g. may reference unpromotable
277 // locals).
278 if (Summary->notEligibleToImport())
279 return {FunctionImporter::ImportFailureReason::NotEligible,
280 GVSummary};
281
282 return {FunctionImporter::ImportFailureReason::None, GVSummary};
283 });
284}
285
286/// Given a list of possible callee implementation for a call site, select one
287/// that fits the \p Threshold for function definition import. If none are
288/// found, the Reason will give the last reason for the failure (last, in the
289/// order of CalleeSummaryList entries). While looking for a callee definition,
290/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
291/// candidate; other modules may want to know the function summary or
292/// declaration even if a definition is not needed.
293///
294/// FIXME: select "best" instead of first that fits. But what is "best"?
295/// - The smallest: more likely to be inlined.
296/// - The one with the least outgoing edges (already well optimized).
297/// - One from a module already being imported from in order to reduce the
298/// number of source modules parsed/linked.
299/// - One that has PGO data attached.
300/// - [insert you fancy metric here]
301static const GlobalValueSummary *
302selectCallee(const ModuleSummaryIndex &Index,
303 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
304 unsigned Threshold, StringRef CallerModulePath,
305 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
306 FunctionImporter::ImportFailureReason &Reason) {
307 // Records the last summary with reason noinline or too-large.
308 TooLargeOrNoInlineSummary = nullptr;
309 auto QualifiedCandidates =
310 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
311 for (auto QualifiedValue : QualifiedCandidates) {
312 Reason = QualifiedValue.first;
313 // Skip a summary if its import is not (proved to be) legal.
314 if (Reason != FunctionImporter::ImportFailureReason::None)
315 continue;
316 auto *Summary =
317 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
318
319 // Don't bother importing the definition if the chance of inlining it is
320 // not high enough (except under `--force-import-all`).
321 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
322 !ForceImportAll) {
323 TooLargeOrNoInlineSummary = Summary;
324 Reason = FunctionImporter::ImportFailureReason::TooLarge;
325 continue;
326 }
327
328 // Don't bother importing the definition if we can't inline it anyway.
329 if (Summary->fflags().NoInline && !ForceImportAll) {
330 TooLargeOrNoInlineSummary = Summary;
331 Reason = FunctionImporter::ImportFailureReason::NoInline;
332 continue;
333 }
334
335 return Summary;
336 }
337 return nullptr;
338}
339
340namespace {
341
342using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
343
344} // anonymous namespace
345
346FunctionImporter::ImportMapTy::AddDefinitionStatus
347FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
348 GlobalValue::GUID GUID) {
349 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
350 if (!Imports.insert(V: Def).second)
351 // Already there.
352 return AddDefinitionStatus::NoChange;
353
354 // Remove Decl in case it's there. Note that a definition takes precedence
355 // over a declaration for a given GUID.
356 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
357 : AddDefinitionStatus::Inserted;
358}
359
360void FunctionImporter::ImportMapTy::maybeAddDeclaration(
361 StringRef FromModule, GlobalValue::GUID GUID) {
362 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
363 // Insert Decl only if Def is not present. Note that a definition takes
364 // precedence over a declaration for a given GUID.
365 if (!Imports.contains(V: Def))
366 Imports.insert(V: Decl);
367}
368
369SmallVector<StringRef, 0>
370FunctionImporter::ImportMapTy::getSourceModules() const {
371 SetVector<StringRef> ModuleSet;
372 for (const auto &[SrcMod, GUID, ImportType] : *this)
373 ModuleSet.insert(X: SrcMod);
374 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
375 llvm::sort(C&: Modules);
376 return Modules;
377}
378
379std::optional<GlobalValueSummary::ImportKind>
380FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
381 GlobalValue::GUID GUID) const {
382 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
383 auto [Def, Decl] = *IDPair;
384 if (Imports.contains(V: Def))
385 return GlobalValueSummary::Definition;
386 if (Imports.contains(V: Decl))
387 return GlobalValueSummary::Declaration;
388 }
389 return std::nullopt;
390}
391
392/// Import globals referenced by a function or other globals that are being
393/// imported, if importing such global is possible.
394class GlobalsImporter final {
395 const ModuleSummaryIndex &Index;
396 const GVSummaryMapTy &DefinedGVSummaries;
397 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
398 IsPrevailing;
399 FunctionImporter::ImportMapTy &ImportList;
400 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
401
402 bool shouldImportGlobal(const ValueInfo &VI) {
403 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
404 if (GVS == DefinedGVSummaries.end())
405 return true;
406 // We should not skip import if the module contains a non-prevailing
407 // definition with interposable linkage type. This is required for
408 // correctness in the situation where there is a prevailing def available
409 // for import and marked read-only. In this case, the non-prevailing def
410 // will be converted to a declaration, while the prevailing one becomes
411 // internal, thus no definitions will be available for linking. In order to
412 // prevent undefined symbol link error, the prevailing definition must be
413 // imported.
414 // FIXME: Consider adding a check that the suitable prevailing definition
415 // exists and marked read-only.
416 if (VI.getSummaryList().size() > 1 &&
417 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
418 !IsPrevailing(VI.getGUID(), GVS->second))
419 return true;
420
421 return false;
422 }
423
424 void
425 onImportingSummaryImpl(const GlobalValueSummary &Summary,
426 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
427 for (const auto &VI : Summary.refs()) {
428 if (!shouldImportGlobal(VI)) {
429 LLVM_DEBUG(
430 dbgs() << "Ref ignored! Target already in destination module.\n");
431 continue;
432 }
433
434 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
435
436 for (const auto &RefSummary : VI.getSummaryList()) {
437 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
438 // Stop looking if this is not a global variable, e.g. a function.
439 // Functions could be referenced by global vars - e.g. a vtable; but we
440 // don't currently imagine a reason those would be imported here, rather
441 // than as part of the logic deciding which functions to import (i.e.
442 // based on profile information). Should we decide to handle them here,
443 // we can refactor accordingly at that time.
444 // Note that it is safe to stop looking because the one case where we
445 // might have to import (a read/write-only global variable) cannot occur
446 // if this GUID has a non-variable summary. The only case where we even
447 // might find another summary in the list that is a variable is in the
448 // case of same-named locals in different modules not compiled with
449 // enough path, and during attribute propagation we will mark all
450 // summaries for a GUID (ValueInfo) as non read/write-only if any is not
451 // a global variable.
452 if (!GVS)
453 break;
454 bool CanImportDecl = false;
455 if (shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
456 ImporterModule: Summary.modulePath()) ||
457 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
458 CanImportDecl)) {
459 if (ImportDeclaration && CanImportDecl)
460 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
461 GUID: VI.getGUID());
462
463 continue;
464 }
465
466 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
467 // Otherwise, definition should take precedence over declaration.
468 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
469 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
470 break;
471
472 // Only update stat and exports if we haven't already imported this
473 // variable.
474 NumImportedGlobalVarsThinLink++;
475 // Any references made by this variable will be marked exported
476 // later, in ComputeCrossModuleImport, after import decisions are
477 // complete, which is more efficient than adding them here.
478 if (ExportLists)
479 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
480
481 // If variable is not writeonly we attempt to recursively analyze
482 // its references in order to import referenced constants.
483 if (!Index.isWriteOnly(GVS))
484 Worklist.emplace_back(Args&: GVS);
485 break;
486 }
487 }
488 }
489
490public:
491 GlobalsImporter(
492 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
493 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
494 IsPrevailing,
495 FunctionImporter::ImportMapTy &ImportList,
496 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
497 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
498 IsPrevailing(IsPrevailing), ImportList(ImportList),
499 ExportLists(ExportLists) {}
500
501 void onImportingSummary(const GlobalValueSummary &Summary) {
502 SmallVector<const GlobalVarSummary *, 128> Worklist;
503 onImportingSummaryImpl(Summary, Worklist);
504 while (!Worklist.empty())
505 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
506 }
507};
508
509static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
510
511/// Determine the list of imports and exports for each module.
512class ModuleImportsManager {
513 void computeImportForFunction(
514 const FunctionSummary &Summary, unsigned Threshold,
515 const GVSummaryMapTy &DefinedGVSummaries,
516 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
517 FunctionImporter::ImportMapTy &ImportList,
518 FunctionImporter::ImportThresholdsTy &ImportThresholds);
519
520protected:
521 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
522 IsPrevailing;
523 const ModuleSummaryIndex &Index;
524 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
525
526 ModuleImportsManager(
527 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
528 IsPrevailing,
529 const ModuleSummaryIndex &Index,
530 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
531 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
532 virtual bool canImport(ValueInfo VI) { return true; }
533
534public:
535 virtual ~ModuleImportsManager() = default;
536
537 /// Given the list of globals defined in a module, compute the list of imports
538 /// as well as the list of "exports", i.e. the list of symbols referenced from
539 /// another module (that may require promotion).
540 virtual void
541 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
542 StringRef ModName,
543 FunctionImporter::ImportMapTy &ImportList);
544
545 static std::unique_ptr<ModuleImportsManager>
546 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
547 IsPrevailing,
548 const ModuleSummaryIndex &Index,
549 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
550 nullptr);
551};
552
553/// A ModuleImportsManager that operates based on a workload definition (see
554/// -thinlto-workload-def). For modules that do not define workload roots, it
555/// applies the base ModuleImportsManager import policy.
556class WorkloadImportsManager : public ModuleImportsManager {
557 // Keep a module name -> value infos to import association. We use it to
558 // determine if a module's import list should be done by the base
559 // ModuleImportsManager or by us.
560 StringMap<DenseSet<ValueInfo>> Workloads;
561 // Track the roots to avoid importing them due to other callers. We want there
562 // to be only one variant), for which we optimize according to the contextual
563 // profile. "Variants" refers to copies due to importing - we want there to be
564 // just one instance of this function.
565 DenseSet<ValueInfo> Roots;
566
567 void
568 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
569 StringRef ModName,
570 FunctionImporter::ImportMapTy &ImportList) override {
571 StringRef Filename = ModName;
572 if (CtxprofMoveRootsToOwnModule) {
573 Filename = sys::path::filename(path: ModName);
574 // Drop the file extension.
575 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
576 }
577 auto SetIter = Workloads.find(Key: Filename);
578
579 if (SetIter == Workloads.end()) {
580 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
581 << " does not contain the root of any context.\n");
582 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
583 ModName, ImportList);
584 }
585 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
586 << " contains the root(s) of context(s).\n");
587
588 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
589 ExportLists);
590 auto &ValueInfos = SetIter->second;
591 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
592 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
593 if (It != DefinedGVSummaries.end() &&
594 IsPrevailing(VI.getGUID(), It->second)) {
595 LLVM_DEBUG(
596 dbgs() << "[Workload] " << VI.name()
597 << " has the prevailing variant already in the module "
598 << ModName << ". No need to import\n");
599 continue;
600 }
601 auto Candidates =
602 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
603
604 const GlobalValueSummary *GVS = nullptr;
605 auto PotentialCandidates = llvm::map_range(
606 C: llvm::make_filter_range(
607 Range&: Candidates,
608 Pred: [&](const auto &Candidate) {
609 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
610 << " from " << Candidate.second->modulePath()
611 << " ImportFailureReason: "
612 << getFailureName(Candidate.first) << "\n");
613 return Candidate.first ==
614 FunctionImporter::ImportFailureReason::None;
615 }),
616 F: [](const auto &Candidate) { return Candidate.second; });
617 if (PotentialCandidates.empty()) {
618 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
619 << " because can't find eligible Callee. Guid is: "
620 << VI.getGUID() << "\n");
621 continue;
622 }
623 /// We will prefer importing the prevailing candidate, if not, we'll
624 /// still pick the first available candidate. The reason we want to make
625 /// sure we do import the prevailing candidate is because the goal of
626 /// workload-awareness is to enable optimizations specializing the call
627 /// graph of that workload. Suppose a function is already defined in the
628 /// module, but it's not the prevailing variant. Suppose also we do not
629 /// inline it (in fact, if it were interposable, we can't inline it),
630 /// but we could specialize it to the workload in other ways. However,
631 /// the linker would drop it in the favor of the prevailing copy.
632 /// Instead, by importing the prevailing variant (assuming also the use
633 /// of `-avail-extern-to-local`), we keep the specialization. We could
634 /// alteranatively make the non-prevailing variant local, but the
635 /// prevailing one is also the one for which we would have previously
636 /// collected profiles, making it preferrable.
637 auto PrevailingCandidates = llvm::make_filter_range(
638 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
639 return IsPrevailing(VI.getGUID(), Candidate);
640 });
641 if (PrevailingCandidates.empty()) {
642 GVS = *PotentialCandidates.begin();
643 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
644 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
645 LLVM_DEBUG(
646 dbgs()
647 << "[Workload] Found multiple non-prevailing candidates for "
648 << VI.name()
649 << ". This is unexpected. Are module paths passed to the "
650 "compiler unique for the modules passed to the linker?");
651 // We could in theory have multiple (interposable) copies of a symbol
652 // when there is no prevailing candidate, if say the prevailing copy was
653 // in a native object being linked in. However, we should in theory be
654 // marking all of these non-prevailing IR copies dead in that case, in
655 // which case they won't be candidates.
656 assert(GVS->isLive());
657 } else {
658 assert(llvm::hasSingleElement(PrevailingCandidates));
659 GVS = *PrevailingCandidates.begin();
660 }
661
662 auto ExportingModule = GVS->modulePath();
663 // We checked that for the prevailing case, but if we happen to have for
664 // example an internal that's defined in this module, it'd have no
665 // PrevailingCandidates.
666 if (ExportingModule == ModName) {
667 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
668 << " because its defining module is the same as the "
669 "current module\n");
670 continue;
671 }
672 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
673 << ExportingModule << " : " << VI.getGUID() << "\n");
674 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
675 GVI.onImportingSummary(Summary: *GVS);
676 if (ExportLists)
677 (*ExportLists)[ExportingModule].insert(V: VI);
678 }
679 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
680 }
681
682 void loadFromJson() {
683 // Since the workload def uses names, we need a quick lookup
684 // name->ValueInfo.
685 StringMap<ValueInfo> NameToValueInfo;
686 StringSet<> AmbiguousNames;
687 for (auto &I : Index) {
688 ValueInfo VI = Index.getValueInfo(R: I);
689 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
690 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
691 }
692 auto DbgReportIfAmbiguous = [&](StringRef Name) {
693 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
694 dbgs() << "[Workload] Function name " << Name
695 << " present in the workload definition is ambiguous. Consider "
696 "compiling with -funique-internal-linkage-names.";
697 });
698 };
699 std::error_code EC;
700 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
701 if (std::error_code EC = BufferOrErr.getError()) {
702 report_fatal_error(reason: "Failed to open context file");
703 return;
704 }
705 auto Buffer = std::move(BufferOrErr.get());
706 std::map<std::string, std::vector<std::string>> WorkloadDefs;
707 json::Path::Root NullRoot;
708 // The JSON is supposed to contain a dictionary matching the type of
709 // WorkloadDefs. For example:
710 // {
711 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
712 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
713 // }
714 auto Parsed = json::parse(JSON: Buffer->getBuffer());
715 if (!Parsed)
716 report_fatal_error(Err: Parsed.takeError());
717 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
718 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
719 for (const auto &Workload : WorkloadDefs) {
720 const auto &Root = Workload.first;
721 DbgReportIfAmbiguous(Root);
722 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
723 const auto &AllCallees = Workload.second;
724 auto RootIt = NameToValueInfo.find(Key: Root);
725 if (RootIt == NameToValueInfo.end()) {
726 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
727 << " not found in this linkage unit.\n");
728 continue;
729 }
730 auto RootVI = RootIt->second;
731 if (RootVI.getSummaryList().size() != 1) {
732 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
733 << " should have exactly one summary, but has "
734 << RootVI.getSummaryList().size() << ". Skipping.\n");
735 continue;
736 }
737 StringRef RootDefiningModule =
738 RootVI.getSummaryList().front()->modulePath();
739 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
740 << " is : " << RootDefiningModule << "\n");
741 auto &Set = Workloads[RootDefiningModule];
742 for (const auto &Callee : AllCallees) {
743 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
744 DbgReportIfAmbiguous(Callee);
745 auto ElemIt = NameToValueInfo.find(Key: Callee);
746 if (ElemIt == NameToValueInfo.end()) {
747 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
748 continue;
749 }
750 Set.insert(V: ElemIt->second);
751 }
752 }
753 }
754
755 void loadFromCtxProf() {
756 std::error_code EC;
757 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
758 if (std::error_code EC = BufferOrErr.getError()) {
759 report_fatal_error(reason: "Failed to open contextual profile file");
760 return;
761 }
762 auto Buffer = std::move(BufferOrErr.get());
763
764 PGOCtxProfileReader Reader(Buffer->getBuffer());
765 auto Ctx = Reader.loadProfiles();
766 if (!Ctx) {
767 report_fatal_error(reason: "Failed to parse contextual profiles");
768 return;
769 }
770 const auto &CtxMap = Ctx->Contexts;
771 SetVector<GlobalValue::GUID> ContainedGUIDs;
772 for (const auto &[RootGuid, Root] : CtxMap) {
773 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
774 // subsequent roots, but clear its contents.
775 ContainedGUIDs.clear();
776
777 auto RootVI = Index.getValueInfo(GUID: RootGuid);
778 if (!RootVI) {
779 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
780 << " not found in this linkage unit.\n");
781 continue;
782 }
783 if (RootVI.getSummaryList().size() != 1) {
784 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
785 << " should have exactly one summary, but has "
786 << RootVI.getSummaryList().size() << ". Skipping.\n");
787 continue;
788 }
789 std::string RootDefiningModule =
790 RootVI.getSummaryList().front()->modulePath().str();
791 if (CtxprofMoveRootsToOwnModule) {
792 RootDefiningModule = std::to_string(val: RootGuid);
793 LLVM_DEBUG(
794 dbgs() << "[Workload] Moving " << RootGuid
795 << " to a module with the filename without extension : "
796 << RootDefiningModule << "\n");
797 } else {
798 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
799 << " is : " << RootDefiningModule << "\n");
800 }
801 auto &Set = Workloads[RootDefiningModule];
802 Root.getContainedGuids(Guids&: ContainedGUIDs);
803 Roots.insert(V: RootVI);
804 for (auto Guid : ContainedGUIDs)
805 if (auto VI = Index.getValueInfo(GUID: Guid))
806 Set.insert(V: VI);
807 }
808 }
809
810 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
811
812public:
813 WorkloadImportsManager(
814 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
815 IsPrevailing,
816 const ModuleSummaryIndex &Index,
817 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
818 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
819 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
820 report_fatal_error(
821 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
822 return;
823 }
824 if (!UseCtxProfile.empty())
825 loadFromCtxProf();
826 else
827 loadFromJson();
828 LLVM_DEBUG({
829 for (const auto &[Root, Set] : Workloads) {
830 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
831 << " distinct callees.\n";
832 for (const auto &VI : Set) {
833 dbgs() << "[Workload] Root: " << Root
834 << " Would include: " << VI.getGUID() << "\n";
835 }
836 }
837 });
838 }
839};
840
841std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
842 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
843 IsPrevailing,
844 const ModuleSummaryIndex &Index,
845 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
846 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
847 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
848 return std::unique_ptr<ModuleImportsManager>(
849 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
850 }
851 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
852 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
853 args&: ExportLists);
854}
855
856static const char *
857getFailureName(FunctionImporter::ImportFailureReason Reason) {
858 switch (Reason) {
859 case FunctionImporter::ImportFailureReason::None:
860 return "None";
861 case FunctionImporter::ImportFailureReason::GlobalVar:
862 return "GlobalVar";
863 case FunctionImporter::ImportFailureReason::NotLive:
864 return "NotLive";
865 case FunctionImporter::ImportFailureReason::TooLarge:
866 return "TooLarge";
867 case FunctionImporter::ImportFailureReason::InterposableLinkage:
868 return "InterposableLinkage";
869 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
870 return "LocalLinkageNotInModule";
871 case FunctionImporter::ImportFailureReason::NotEligible:
872 return "NotEligible";
873 case FunctionImporter::ImportFailureReason::NoInline:
874 return "NoInline";
875 }
876 llvm_unreachable("invalid reason");
877}
878
879/// Compute the list of functions to import for a given caller. Mark these
880/// imported functions and the symbols they reference in their source module as
881/// exported from their source module.
882void ModuleImportsManager::computeImportForFunction(
883 const FunctionSummary &Summary, const unsigned Threshold,
884 const GVSummaryMapTy &DefinedGVSummaries,
885 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
886 FunctionImporter::ImportMapTy &ImportList,
887 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
888 GVImporter.onImportingSummary(Summary);
889 static int ImportCount = 0;
890 for (const auto &Edge : Summary.calls()) {
891 ValueInfo VI = Edge.first;
892 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
893 << "\n");
894
895 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
896 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
897 << " reached.\n");
898 continue;
899 }
900
901 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
902 // FIXME: Consider not skipping import if the module contains
903 // a non-prevailing def with interposable linkage. The prevailing copy
904 // can safely be imported (see shouldImportGlobal()).
905 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
906 continue;
907 }
908
909 if (!canImport(VI)) {
910 LLVM_DEBUG(
911 dbgs() << "Skipping over " << VI.getGUID()
912 << " because its import is handled in a different module.");
913 assert(VI.getSummaryList().size() == 1 &&
914 "The root was expected to be an external symbol");
915 continue;
916 }
917
918 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
919 if (Hotness == CalleeInfo::HotnessType::Hot)
920 return ImportHotMultiplier;
921 if (Hotness == CalleeInfo::HotnessType::Cold)
922 return ImportColdMultiplier;
923 if (Hotness == CalleeInfo::HotnessType::Critical)
924 return ImportCriticalMultiplier;
925 return 1.0;
926 };
927
928 const auto NewThreshold =
929 Threshold * GetBonusMultiplier(Edge.second.getHotness());
930
931 auto IT = ImportThresholds.insert(KV: std::make_pair(
932 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
933 bool PreviouslyVisited = !IT.second;
934 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
935 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
936 auto &FailureInfo = std::get<2>(t&: IT.first->second);
937
938 bool IsHotCallsite =
939 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
940 bool IsCriticalCallsite =
941 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
942
943 const FunctionSummary *ResolvedCalleeSummary = nullptr;
944 if (CalleeSummary) {
945 assert(PreviouslyVisited);
946 // Since the traversal of the call graph is DFS, we can revisit a function
947 // a second time with a higher threshold. In this case, it is added back
948 // to the worklist with the new threshold (so that its own callee chains
949 // can be considered with the higher threshold).
950 if (NewThreshold <= ProcessedThreshold) {
951 LLVM_DEBUG(
952 dbgs() << "ignored! Target was already imported with Threshold "
953 << ProcessedThreshold << "\n");
954 continue;
955 }
956 // Update with new larger threshold.
957 ProcessedThreshold = NewThreshold;
958 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
959 } else {
960 // If we already rejected importing a callee at the same or higher
961 // threshold, don't waste time calling selectCallee.
962 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
963 LLVM_DEBUG(
964 dbgs() << "ignored! Target was already rejected with Threshold "
965 << ProcessedThreshold << "\n");
966 if (PrintImportFailures) {
967 assert(FailureInfo &&
968 "Expected FailureInfo for previously rejected candidate");
969 FailureInfo->Attempts++;
970 }
971 continue;
972 }
973
974 FunctionImporter::ImportFailureReason Reason{};
975
976 // `SummaryForDeclImport` is an summary eligible for declaration import.
977 const GlobalValueSummary *SummaryForDeclImport = nullptr;
978 CalleeSummary =
979 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
980 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
981 if (!CalleeSummary) {
982 // There isn't a callee for definition import but one for declaration
983 // import.
984 if (ImportDeclaration && SummaryForDeclImport) {
985 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
986
987 // Note `ExportLists` only keeps track of exports due to imported
988 // definitions.
989 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
990 }
991 // Update with new larger threshold if this was a retry (otherwise
992 // we would have already inserted with NewThreshold above). Also
993 // update failure info if requested.
994 if (PreviouslyVisited) {
995 ProcessedThreshold = NewThreshold;
996 if (PrintImportFailures) {
997 assert(FailureInfo &&
998 "Expected FailureInfo for previously rejected candidate");
999 FailureInfo->Reason = Reason;
1000 FailureInfo->Attempts++;
1001 FailureInfo->MaxHotness =
1002 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
1003 }
1004 } else if (PrintImportFailures) {
1005 assert(!FailureInfo &&
1006 "Expected no FailureInfo for newly rejected candidate");
1007 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
1008 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
1009 }
1010 if (ForceImportAll) {
1011 std::string Msg = std::string("Failed to import function ") +
1012 VI.name().str() + " due to " +
1013 getFailureName(Reason);
1014 auto Error = make_error<StringError>(
1015 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1016 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1017 ErrorBanner: "Error importing module: ");
1018 break;
1019 } else {
1020 LLVM_DEBUG(dbgs()
1021 << "ignored! No qualifying callee with summary found.\n");
1022 continue;
1023 }
1024 }
1025
1026 // "Resolve" the summary
1027 CalleeSummary = CalleeSummary->getBaseObject();
1028 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1029
1030 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1031 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1032 "selectCallee() didn't honor the threshold");
1033
1034 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1035
1036 // Try emplace the definition entry, and update stats based on insertion
1037 // status.
1038 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1039 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1040 NumImportedFunctionsThinLink++;
1041 if (IsHotCallsite)
1042 NumImportedHotFunctionsThinLink++;
1043 if (IsCriticalCallsite)
1044 NumImportedCriticalFunctionsThinLink++;
1045 }
1046
1047 // Any calls/references made by this function will be marked exported
1048 // later, in ComputeCrossModuleImport, after import decisions are
1049 // complete, which is more efficient than adding them here.
1050 if (ExportLists)
1051 (*ExportLists)[ExportModulePath].insert(V: VI);
1052 }
1053
1054 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1055 // Adjust the threshold for next level of imported functions.
1056 // The threshold is different for hot callsites because we can then
1057 // inline chains of hot calls.
1058 if (IsHotCallsite)
1059 return Threshold * ImportHotInstrFactor;
1060 return Threshold * ImportInstrFactor;
1061 };
1062
1063 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1064
1065 ImportCount++;
1066
1067 // Insert the newly imported function to the worklist.
1068 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1069 }
1070}
1071
1072void ModuleImportsManager::computeImportForModule(
1073 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1074 FunctionImporter::ImportMapTy &ImportList) {
1075 // Worklist contains the list of function imported in this module, for which
1076 // we will analyse the callees and may import further down the callgraph.
1077 SmallVector<EdgeInfo, 128> Worklist;
1078 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1079 ExportLists);
1080 FunctionImporter::ImportThresholdsTy ImportThresholds;
1081
1082 // Populate the worklist with the import for the functions in the current
1083 // module
1084 for (const auto &GVSummary : DefinedGVSummaries) {
1085#ifndef NDEBUG
1086 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1087 // so this map look up (and possibly others) can be avoided.
1088 auto VI = Index.getValueInfo(GVSummary.first);
1089#endif
1090 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1091 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1092 continue;
1093 }
1094 auto *FuncSummary =
1095 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1096 if (!FuncSummary)
1097 // Skip import for global variables
1098 continue;
1099 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1100 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1101 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1102 }
1103
1104 // Process the newly imported functions and add callees to the worklist.
1105 while (!Worklist.empty()) {
1106 auto GVInfo = Worklist.pop_back_val();
1107 auto *Summary = std::get<0>(t&: GVInfo);
1108 auto Threshold = std::get<1>(t&: GVInfo);
1109
1110 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary))
1111 computeImportForFunction(Summary: *FS, Threshold, DefinedGVSummaries, Worklist,
1112 GVImporter&: GVI, ImportList, ImportThresholds);
1113 }
1114
1115 // Print stats about functions considered but rejected for importing
1116 // when requested.
1117 if (PrintImportFailures) {
1118 dbgs() << "Missed imports into module " << ModName << "\n";
1119 for (auto &I : ImportThresholds) {
1120 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1121 auto &CalleeSummary = std::get<1>(t&: I.second);
1122 auto &FailureInfo = std::get<2>(t&: I.second);
1123 if (CalleeSummary)
1124 continue; // We are going to import.
1125 assert(FailureInfo);
1126 FunctionSummary *FS = nullptr;
1127 if (!FailureInfo->VI.getSummaryList().empty())
1128 FS = dyn_cast<FunctionSummary>(
1129 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1130 dbgs() << FailureInfo->VI
1131 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1132 << ", Threshold = " << ProcessedThreshold
1133 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1134 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1135 << ", Attempts = " << FailureInfo->Attempts << "\n";
1136 }
1137 }
1138}
1139
1140#ifndef NDEBUG
1141static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1142 auto SL = VI.getSummaryList();
1143 return SL.empty()
1144 ? false
1145 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1146}
1147
1148static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1149 GlobalValue::GUID G) {
1150 if (const auto &VI = Index.getValueInfo(G))
1151 return isGlobalVarSummary(Index, VI);
1152 return false;
1153}
1154
1155// Return the number of global variable summaries in ExportSet.
1156static unsigned
1157numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1158 FunctionImporter::ExportSetTy &ExportSet) {
1159 unsigned NumGVS = 0;
1160 for (auto &VI : ExportSet)
1161 if (isGlobalVarSummary(Index, VI.getGUID()))
1162 ++NumGVS;
1163 return NumGVS;
1164}
1165
1166struct ImportStatistics {
1167 unsigned NumGVS = 0;
1168 unsigned DefinedFS = 0;
1169 unsigned Count = 0;
1170};
1171
1172// Compute import statistics for each source module in ImportList.
1173static DenseMap<StringRef, ImportStatistics>
1174collectImportStatistics(const ModuleSummaryIndex &Index,
1175 const FunctionImporter::ImportMapTy &ImportList) {
1176 DenseMap<StringRef, ImportStatistics> Histogram;
1177
1178 for (const auto &[FromModule, GUID, Type] : ImportList) {
1179 ImportStatistics &Entry = Histogram[FromModule];
1180 ++Entry.Count;
1181 if (isGlobalVarSummary(Index, GUID))
1182 ++Entry.NumGVS;
1183 else if (Type == GlobalValueSummary::Definition)
1184 ++Entry.DefinedFS;
1185 }
1186 return Histogram;
1187}
1188#endif
1189
1190#ifndef NDEBUG
1191static bool checkVariableImport(
1192 const ModuleSummaryIndex &Index,
1193 FunctionImporter::ImportListsTy &ImportLists,
1194 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1195 DenseSet<GlobalValue::GUID> FlattenedImports;
1196
1197 for (const auto &ImportPerModule : ImportLists)
1198 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1199 FlattenedImports.insert(GUID);
1200
1201 // Checks that all GUIDs of read/writeonly vars we see in export lists
1202 // are also in the import lists. Otherwise we my face linker undefs,
1203 // because readonly and writeonly vars are internalized in their
1204 // source modules. The exception would be if it has a linkage type indicating
1205 // that there may have been a copy existing in the importing module (e.g.
1206 // linkonce_odr). In that case we cannot accurately do this checking.
1207 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1208 const ValueInfo &VI) {
1209 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1210 Index.findSummaryInModule(VI, ModulePath));
1211 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1212 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1213 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1214 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1215 };
1216
1217 for (auto &ExportPerModule : ExportLists)
1218 for (auto &VI : ExportPerModule.second)
1219 if (!FlattenedImports.count(VI.getGUID()) &&
1220 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1221 return false;
1222
1223 return true;
1224}
1225#endif
1226
1227/// Compute all the import and export for every module using the Index.
1228void llvm::ComputeCrossModuleImport(
1229 const ModuleSummaryIndex &Index,
1230 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1231 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1232 isPrevailing,
1233 FunctionImporter::ImportListsTy &ImportLists,
1234 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1235 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1236 // For each module that has function defined, compute the import/export lists.
1237 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1238 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1239 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1240 << DefinedGVSummaries.first << "'\n");
1241 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1242 ModName: DefinedGVSummaries.first, ImportList);
1243 }
1244
1245 // When computing imports we only added the variables and functions being
1246 // imported to the export list. We also need to mark any references and calls
1247 // they make as exported as well. We do this here, as it is more efficient
1248 // since we may import the same values multiple times into different modules
1249 // during the import computation.
1250 for (auto &ELI : ExportLists) {
1251 // `NewExports` tracks the VI that gets exported because the full definition
1252 // of its user/referencer gets exported.
1253 FunctionImporter::ExportSetTy NewExports;
1254 const auto &DefinedGVSummaries =
1255 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1256 for (auto &EI : ELI.second) {
1257 // Find the copy defined in the exporting module so that we can mark the
1258 // values it references in that specific definition as exported.
1259 // Below we will add all references and called values, without regard to
1260 // whether they are also defined in this module. We subsequently prune the
1261 // list to only include those defined in the exporting module, see comment
1262 // there as to why.
1263 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1264 // Anything marked exported during the import computation must have been
1265 // defined in the exporting module.
1266 assert(DS != DefinedGVSummaries.end());
1267 auto *S = DS->getSecond();
1268 S = S->getBaseObject();
1269 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1270 // Export referenced functions and variables. We don't export/promote
1271 // objects referenced by writeonly variable initializer, because
1272 // we convert such variables initializers to "zeroinitializer".
1273 // See processGlobalForThinLTO.
1274 if (!Index.isWriteOnly(GVS))
1275 NewExports.insert_range(R: GVS->refs());
1276 } else {
1277 auto *FS = cast<FunctionSummary>(Val: S);
1278 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1279 NewExports.insert_range(R: FS->refs());
1280 }
1281 }
1282 // Prune list computed above to only include values defined in the
1283 // exporting module. We do this after the above insertion since we may hit
1284 // the same ref/call target multiple times in above loop, and it is more
1285 // efficient to avoid a set lookup each time.
1286 for (auto EI = NewExports.begin(); EI != NewExports.end();) {
1287 if (!DefinedGVSummaries.count(Val: EI->getGUID()))
1288 NewExports.erase(I: EI++);
1289 else
1290 ++EI;
1291 }
1292 ELI.second.insert_range(R&: NewExports);
1293 }
1294
1295 assert(checkVariableImport(Index, ImportLists, ExportLists));
1296#ifndef NDEBUG
1297 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1298 << " modules:\n");
1299 for (const auto &ModuleImports : ImportLists) {
1300 auto ModName = ModuleImports.first;
1301 auto &Exports = ExportLists[ModName];
1302 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1303 DenseMap<StringRef, ImportStatistics> Histogram =
1304 collectImportStatistics(Index, ModuleImports.second);
1305 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1306 << Exports.size() - NumGVS << " functions and " << NumGVS
1307 << " vars. Imports from " << Histogram.size()
1308 << " modules.\n");
1309 for (const auto &[SrcModName, Stats] : Histogram) {
1310 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1311 << " function definitions and "
1312 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1313 << " function declarations imported from " << SrcModName
1314 << "\n");
1315 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1316 << " global vars imported from " << SrcModName << "\n");
1317 }
1318 }
1319#endif
1320}
1321
1322#ifndef NDEBUG
1323static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1324 StringRef ModulePath,
1325 FunctionImporter::ImportMapTy &ImportList) {
1326 DenseMap<StringRef, ImportStatistics> Histogram =
1327 collectImportStatistics(Index, ImportList);
1328 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1329 << Histogram.size() << " modules.\n");
1330 for (const auto &[SrcModName, Stats] : Histogram) {
1331 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1332 << " function definitions and "
1333 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1334 << " function declarations imported from " << SrcModName
1335 << "\n");
1336 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1337 << SrcModName << "\n");
1338 }
1339}
1340#endif
1341
1342/// Compute all the imports for the given module using the Index.
1343///
1344/// \p isPrevailing is a callback that will be called with a global value's GUID
1345/// and summary and should return whether the module corresponding to the
1346/// summary contains the linker-prevailing copy of that value.
1347///
1348/// \p ImportList will be populated with a map that can be passed to
1349/// FunctionImporter::importFunctions() above (see description there).
1350static void ComputeCrossModuleImportForModuleForTest(
1351 StringRef ModulePath,
1352 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1353 isPrevailing,
1354 const ModuleSummaryIndex &Index,
1355 FunctionImporter::ImportMapTy &ImportList) {
1356 // Collect the list of functions this module defines.
1357 // GUID -> Summary
1358 GVSummaryMapTy FunctionSummaryMap;
1359 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1360
1361 // Compute the import list for this module.
1362 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1363 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1364 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1365
1366#ifndef NDEBUG
1367 dumpImportListForModule(Index, ModulePath, ImportList);
1368#endif
1369}
1370
1371/// Mark all external summaries in \p Index for import into the given module.
1372/// Used for testing the case of distributed builds using a distributed index.
1373///
1374/// \p ImportList will be populated with a map that can be passed to
1375/// FunctionImporter::importFunctions() above (see description there).
1376static void ComputeCrossModuleImportForModuleFromIndexForTest(
1377 StringRef ModulePath, const ModuleSummaryIndex &Index,
1378 FunctionImporter::ImportMapTy &ImportList) {
1379 for (const auto &GlobalList : Index) {
1380 // Ignore entries for undefined references.
1381 if (GlobalList.second.getSummaryList().empty())
1382 continue;
1383
1384 auto GUID = GlobalList.first;
1385 assert(GlobalList.second.getSummaryList().size() == 1 &&
1386 "Expected individual combined index to have one summary per GUID");
1387 auto &Summary = GlobalList.second.getSummaryList()[0];
1388 // Skip the summaries for the importing module. These are included to
1389 // e.g. record required linkage changes.
1390 if (Summary->modulePath() == ModulePath)
1391 continue;
1392 // Add an entry to provoke importing by thinBackend.
1393 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1394 }
1395#ifndef NDEBUG
1396 dumpImportListForModule(Index, ModulePath, ImportList);
1397#endif
1398}
1399
1400// For SamplePGO, the indirect call targets for local functions will
1401// have its original name annotated in profile. We try to find the
1402// corresponding PGOFuncName as the GUID, and fix up the edges
1403// accordingly.
1404void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1405 FunctionSummary *FS) {
1406 for (auto &EI : FS->mutableCalls()) {
1407 if (!EI.first.getSummaryList().empty())
1408 continue;
1409 auto GUID = Index.getGUIDFromOriginalID(OriginalID: EI.first.getGUID());
1410 if (GUID == 0)
1411 continue;
1412 // Update the edge to point directly to the correct GUID.
1413 auto VI = Index.getValueInfo(GUID);
1414 if (llvm::any_of(
1415 Range: VI.getSummaryList(),
1416 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1417 // The mapping from OriginalId to GUID may return a GUID
1418 // that corresponds to a static variable. Filter it out here.
1419 // This can happen when
1420 // 1) There is a call to a library function which is not defined
1421 // in the index.
1422 // 2) There is a static variable with the OriginalGUID identical
1423 // to the GUID of the library function in 1);
1424 // When this happens the static variable in 2) will be found,
1425 // which needs to be filtered out.
1426 return SummaryPtr->getSummaryKind() ==
1427 GlobalValueSummary::GlobalVarKind;
1428 }))
1429 continue;
1430 EI.first = VI;
1431 }
1432}
1433
1434void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1435 for (const auto &Entry : Index) {
1436 for (const auto &S : Entry.second.getSummaryList()) {
1437 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1438 updateValueInfoForIndirectCalls(Index, FS);
1439 }
1440 }
1441}
1442
1443void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1444 ModuleSummaryIndex &Index,
1445 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1446 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1447 assert(!Index.withGlobalValueDeadStripping());
1448 if (!ComputeDead ||
1449 // Don't do anything when nothing is live, this is friendly with tests.
1450 GUIDPreservedSymbols.empty()) {
1451 // Still need to update indirect calls.
1452 updateIndirectCalls(Index);
1453 return;
1454 }
1455 unsigned LiveSymbols = 0;
1456 SmallVector<ValueInfo, 128> Worklist;
1457 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1458 for (auto GUID : GUIDPreservedSymbols) {
1459 ValueInfo VI = Index.getValueInfo(GUID);
1460 if (!VI)
1461 continue;
1462 for (const auto &S : VI.getSummaryList())
1463 S->setLive(true);
1464 }
1465
1466 // Add values flagged in the index as live roots to the worklist.
1467 for (const auto &Entry : Index) {
1468 auto VI = Index.getValueInfo(R: Entry);
1469 for (const auto &S : Entry.second.getSummaryList()) {
1470 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1471 updateValueInfoForIndirectCalls(Index, FS);
1472 if (S->isLive()) {
1473 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1474 Worklist.push_back(Elt: VI);
1475 ++LiveSymbols;
1476 break;
1477 }
1478 }
1479 }
1480
1481 // Make value live and add it to the worklist if it was not live before.
1482 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1483 // FIXME: If we knew which edges were created for indirect call profiles,
1484 // we could skip them here. Any that are live should be reached via
1485 // other edges, e.g. reference edges. Otherwise, using a profile collected
1486 // on a slightly different binary might provoke preserving, importing
1487 // and ultimately promoting calls to functions not linked into this
1488 // binary, which increases the binary size unnecessarily. Note that
1489 // if this code changes, the importer needs to change so that edges
1490 // to functions marked dead are skipped.
1491
1492 if (llvm::any_of(Range: VI.getSummaryList(),
1493 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1494 return S->isLive();
1495 }))
1496 return;
1497
1498 // We only keep live symbols that are known to be non-prevailing if any are
1499 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1500 // later in the EliminateAvailableExternally pass and setting them to
1501 // not-live could break downstreams users of liveness information (PR36483)
1502 // or limit optimization opportunities.
1503 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1504 bool KeepAliveLinkage = false;
1505 bool Interposable = false;
1506 for (const auto &S : VI.getSummaryList()) {
1507 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1508 S->linkage() == GlobalValue::WeakODRLinkage ||
1509 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1510 KeepAliveLinkage = true;
1511 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1512 Interposable = true;
1513 }
1514
1515 if (!IsAliasee) {
1516 if (!KeepAliveLinkage)
1517 return;
1518
1519 if (Interposable)
1520 report_fatal_error(
1521 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1522 "symbol");
1523 }
1524 }
1525
1526 for (const auto &S : VI.getSummaryList())
1527 S->setLive(true);
1528 ++LiveSymbols;
1529 Worklist.push_back(Elt: VI);
1530 };
1531
1532 while (!Worklist.empty()) {
1533 auto VI = Worklist.pop_back_val();
1534 for (const auto &Summary : VI.getSummaryList()) {
1535 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1536 // If this is an alias, visit the aliasee VI to ensure that all copies
1537 // are marked live and it is added to the worklist for further
1538 // processing of its references.
1539 visit(AS->getAliaseeVI(), true);
1540 continue;
1541 }
1542 for (auto Ref : Summary->refs())
1543 visit(Ref, false);
1544 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1545 for (auto Call : FS->calls())
1546 visit(Call.first, false);
1547 }
1548 }
1549 Index.setWithGlobalValueDeadStripping();
1550
1551 unsigned DeadSymbols = Index.size() - LiveSymbols;
1552 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1553 << " symbols Dead \n");
1554 NumDeadSymbols += DeadSymbols;
1555 NumLiveSymbols += LiveSymbols;
1556}
1557
1558// Compute dead symbols and propagate constants in combined index.
1559void llvm::computeDeadSymbolsWithConstProp(
1560 ModuleSummaryIndex &Index,
1561 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1562 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1563 bool ImportEnabled) {
1564 llvm::TimeTraceScope timeScope("Drop dead symbols and propagate attributes");
1565 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1566 isPrevailing);
1567 if (ImportEnabled)
1568 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1569}
1570
1571/// Compute the set of summaries needed for a ThinLTO backend compilation of
1572/// \p ModulePath.
1573void llvm::gatherImportedSummariesForModule(
1574 StringRef ModulePath,
1575 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1576 const FunctionImporter::ImportMapTy &ImportList,
1577 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1578 GVSummaryPtrSet &DecSummaries) {
1579 // Include all summaries from the importing module.
1580 ModuleToSummariesForIndex[std::string(ModulePath)] =
1581 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1582
1583 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1584 // lets us look up the map without allocating an instance of std::string when
1585 // the key-value pair exists in the map.
1586 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1587 // C++26 when it becomes available for our codebase.
1588 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1589 StringRef Key) -> GVSummaryMapTy & {
1590 auto It = Map.find(x: Key);
1591 if (It == Map.end())
1592 std::tie(args&: It, args: std::ignore) =
1593 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1594 return It->second;
1595 };
1596
1597 // Include summaries for imports.
1598 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1599 auto &SummariesForIndex =
1600 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1601
1602 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1603 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1604 assert(DS != DefinedGVSummaries.end() &&
1605 "Expected a defined summary for imported global value");
1606 if (ImportType == GlobalValueSummary::Declaration)
1607 DecSummaries.insert(x: DS->second);
1608
1609 SummariesForIndex[GUID] = DS->second;
1610 }
1611}
1612
1613/// Emit the files \p ModulePath will import from into \p OutputFilename.
1614Error llvm::EmitImportsFiles(
1615 StringRef ModulePath, StringRef OutputFilename,
1616 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1617 std::error_code EC;
1618 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1619 if (EC)
1620 return createFileError(F: "cannot open " + OutputFilename,
1621 E: errorCodeToError(EC));
1622 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1623 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1624 return Error::success();
1625}
1626
1627/// Invoke callback \p F on the file paths from which \p ModulePath
1628/// will import.
1629void llvm::processImportsFiles(
1630 StringRef ModulePath,
1631 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1632 function_ref<void(const std::string &)> F) {
1633 for (const auto &ILI : ModuleToSummariesForIndex)
1634 // The ModuleToSummariesForIndex map includes an entry for the current
1635 // Module (needed for writing out the index files). We don't want to
1636 // include it in the imports file, however, so filter it out.
1637 if (ILI.first != ModulePath)
1638 F(ILI.first);
1639}
1640
1641bool llvm::convertToDeclaration(GlobalValue &GV) {
1642 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1643 << "\n");
1644 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1645 F->deleteBody();
1646 F->clearMetadata();
1647 F->setComdat(nullptr);
1648 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1649 V->setInitializer(nullptr);
1650 V->setLinkage(GlobalValue::ExternalLinkage);
1651 V->clearMetadata();
1652 V->setComdat(nullptr);
1653 } else {
1654 GlobalValue *NewGV;
1655 if (GV.getValueType()->isFunctionTy())
1656 NewGV =
1657 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1658 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1659 N: "", M: GV.getParent());
1660 else
1661 NewGV =
1662 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1663 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1664 /*init*/ nullptr, "",
1665 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1666 GV.getType()->getAddressSpace());
1667 NewGV->takeName(V: &GV);
1668 GV.replaceAllUsesWith(V: NewGV);
1669 return false;
1670 }
1671 if (!GV.isImplicitDSOLocal())
1672 GV.setDSOLocal(false);
1673 return true;
1674}
1675
1676void llvm::thinLTOFinalizeInModule(Module &TheModule,
1677 const GVSummaryMapTy &DefinedGlobals,
1678 bool PropagateAttrs) {
1679 llvm::TimeTraceScope timeScope("ThinLTO finalize in module");
1680 DenseSet<Comdat *> NonPrevailingComdats;
1681 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1682 // See if the global summary analysis computed a new resolved linkage.
1683 const auto &GS = DefinedGlobals.find(Val: GV.getGUID());
1684 if (GS == DefinedGlobals.end())
1685 return;
1686
1687 if (Propagate)
1688 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1689 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1690 // TODO: propagate ReadNone and ReadOnly.
1691 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1692 F->setDoesNotAccessMemory();
1693
1694 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1695 F->setOnlyReadsMemory();
1696
1697 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1698 F->setDoesNotRecurse();
1699
1700 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1701 F->setDoesNotThrow();
1702 }
1703 }
1704
1705 auto NewLinkage = GS->second->linkage();
1706 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1707 // Don't internalize anything here, because the code below
1708 // lacks necessary correctness checks. Leave this job to
1709 // LLVM 'internalize' pass.
1710 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1711 // In case it was dead and already converted to declaration.
1712 GV.isDeclaration())
1713 return;
1714
1715 // Set the potentially more constraining visibility computed from summaries.
1716 // The DefaultVisibility condition is because older GlobalValueSummary does
1717 // not record DefaultVisibility and we don't want to change protected/hidden
1718 // to default.
1719 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1720 GV.setVisibility(GS->second->getVisibility());
1721
1722 if (NewLinkage == GV.getLinkage())
1723 return;
1724
1725 // Check for a non-prevailing def that has interposable linkage
1726 // (e.g. non-odr weak or linkonce). In that case we can't simply
1727 // convert to available_externally, since it would lose the
1728 // interposable property and possibly get inlined. Simply drop
1729 // the definition in that case.
1730 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1731 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1732 if (!convertToDeclaration(GV))
1733 // FIXME: Change this to collect replaced GVs and later erase
1734 // them from the parent module once thinLTOResolvePrevailingGUID is
1735 // changed to enable this for aliases.
1736 llvm_unreachable("Expected GV to be converted");
1737 } else {
1738 // If all copies of the original symbol had global unnamed addr and
1739 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1740 // and are constants, then it should be an auto hide symbol. In that case
1741 // the thin link would have marked it as CanAutoHide. Add hidden
1742 // visibility to the symbol to preserve the property.
1743 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1744 GS->second->canAutoHide()) {
1745 assert(GV.canBeOmittedFromSymbolTable());
1746 GV.setVisibility(GlobalValue::HiddenVisibility);
1747 }
1748
1749 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1750 << "` from " << GV.getLinkage() << " to " << NewLinkage
1751 << "\n");
1752 GV.setLinkage(NewLinkage);
1753 }
1754 // Remove declarations from comdats, including available_externally
1755 // as this is a declaration for the linker, and will be dropped eventually.
1756 // It is illegal for comdats to contain declarations.
1757 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1758 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1759 if (GO->getComdat()->getName() == GO->getName())
1760 NonPrevailingComdats.insert(V: GO->getComdat());
1761 GO->setComdat(nullptr);
1762 }
1763 };
1764
1765 // Process functions and global now
1766 for (auto &GV : TheModule)
1767 FinalizeInModule(GV, PropagateAttrs);
1768 for (auto &GV : TheModule.globals())
1769 FinalizeInModule(GV);
1770 for (auto &GV : TheModule.aliases())
1771 FinalizeInModule(GV);
1772
1773 // For a non-prevailing comdat, all its members must be available_externally.
1774 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1775 // local linkage GlobalValues.
1776 if (NonPrevailingComdats.empty())
1777 return;
1778 for (auto &GO : TheModule.global_objects()) {
1779 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1780 GO.setComdat(nullptr);
1781 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1782 }
1783 }
1784 bool Changed;
1785 do {
1786 Changed = false;
1787 // If an alias references a GlobalValue in a non-prevailing comdat, change
1788 // it to available_externally. For simplicity we only handle GlobalValue and
1789 // ConstantExpr with a base object. ConstantExpr without a base object is
1790 // unlikely used in a COMDAT.
1791 for (auto &GA : TheModule.aliases()) {
1792 if (GA.hasAvailableExternallyLinkage())
1793 continue;
1794 GlobalObject *Obj = GA.getAliaseeObject();
1795 assert(Obj && "aliasee without an base object is unimplemented");
1796 if (Obj->hasAvailableExternallyLinkage()) {
1797 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1798 Changed = true;
1799 }
1800 }
1801 } while (Changed);
1802}
1803
1804/// Run internalization on \p TheModule based on symmary analysis.
1805void llvm::thinLTOInternalizeModule(Module &TheModule,
1806 const GVSummaryMapTy &DefinedGlobals) {
1807 llvm::TimeTraceScope timeScope("ThinLTO internalize module");
1808 // Declare a callback for the internalize pass that will ask for every
1809 // candidate GlobalValue if it can be internalized or not.
1810 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1811 // It may be the case that GV is on a chain of an ifunc, its alias and
1812 // subsequent aliases. In this case, the summary for the value is not
1813 // available.
1814 if (isa<GlobalIFunc>(Val: &GV) ||
1815 (isa<GlobalAlias>(Val: &GV) &&
1816 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1817 return true;
1818
1819 // Lookup the linkage recorded in the summaries during global analysis.
1820 auto GS = DefinedGlobals.find(Val: GV.getGUID());
1821 if (GS == DefinedGlobals.end()) {
1822 // Must have been promoted (possibly conservatively). Find original
1823 // name so that we can access the correct summary and see if it can
1824 // be internalized again.
1825 // FIXME: Eventually we should control promotion instead of promoting
1826 // and internalizing again.
1827 StringRef OrigName =
1828 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1829 std::string OrigId = GlobalValue::getGlobalIdentifier(
1830 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1831 FileName: TheModule.getSourceFileName());
1832 GS = DefinedGlobals.find(
1833 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1834 if (GS == DefinedGlobals.end()) {
1835 // Also check the original non-promoted non-globalized name. In some
1836 // cases a preempted weak value is linked in as a local copy because
1837 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1838 // In that case, since it was originally not a local value, it was
1839 // recorded in the index using the original name.
1840 // FIXME: This may not be needed once PR27866 is fixed.
1841 GS = DefinedGlobals.find(
1842 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1843 assert(GS != DefinedGlobals.end());
1844 }
1845 }
1846 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1847 };
1848
1849 // FIXME: See if we can just internalize directly here via linkage changes
1850 // based on the index, rather than invoking internalizeModule.
1851 internalizeModule(TheModule, MustPreserveGV);
1852}
1853
1854/// Make alias a clone of its aliasee.
1855static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1856 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1857
1858 ValueToValueMapTy VMap;
1859 Function *NewFn = CloneFunction(F: Fn, VMap);
1860 // Clone should use the original alias's linkage, visibility and name, and we
1861 // ensure all uses of alias instead use the new clone (casted if necessary).
1862 NewFn->setLinkage(GA->getLinkage());
1863 NewFn->setVisibility(GA->getVisibility());
1864 GA->replaceAllUsesWith(V: NewFn);
1865 NewFn->takeName(V: GA);
1866 return NewFn;
1867}
1868
1869// Internalize values that we marked with specific attribute
1870// in processGlobalForThinLTO.
1871static void internalizeGVsAfterImport(Module &M) {
1872 for (auto &GV : M.globals())
1873 // Skip GVs which have been converted to declarations
1874 // by dropDeadSymbols.
1875 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1876 GV.setLinkage(GlobalValue::InternalLinkage);
1877 GV.setVisibility(GlobalValue::DefaultVisibility);
1878 }
1879}
1880
1881// Automatically import functions in Module \p DestModule based on the summaries
1882// index.
1883Expected<bool> FunctionImporter::importFunctions(
1884 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1885 LLVM_DEBUG(dbgs() << "Starting import for Module "
1886 << DestModule.getModuleIdentifier() << "\n");
1887 unsigned ImportedCount = 0, ImportedGVCount = 0;
1888 // Before carrying out any imports, see if this module defines functions in
1889 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1890 // The function will be imported elsewhere, as extenal linkage, and the
1891 // destination doesn't yet have its definition.
1892 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1893 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1894 for (auto &F : DestModule)
1895 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUID()))
1896 F.deleteBody();
1897
1898 IRMover Mover(DestModule);
1899
1900 // Do the actual import of functions now, one Module at a time
1901 for (const auto &ModName : ImportList.getSourceModules()) {
1902 llvm::TimeTraceScope timeScope("Import", ModName);
1903 // Get the module for the import
1904 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1905 if (!SrcModuleOrErr)
1906 return SrcModuleOrErr.takeError();
1907 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1908 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1909 "Context mismatch");
1910
1911 // If modules were created with lazy metadata loading, materialize it
1912 // now, before linking it (otherwise this will be a noop).
1913 if (Error Err = SrcModule->materializeMetadata())
1914 return std::move(Err);
1915
1916 // Find the globals to import
1917 SetVector<GlobalValue *> GlobalsToImport;
1918 {
1919 llvm::TimeTraceScope functionsScope("Functions");
1920 for (Function &F : *SrcModule) {
1921 if (!F.hasName())
1922 continue;
1923 auto GUID = F.getGUID();
1924 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1925 bool ImportDefinition =
1926 MaybeImportType == GlobalValueSummary::Definition;
1927
1928 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1929 << " importing function"
1930 << (ImportDefinition
1931 ? " definition "
1932 : (MaybeImportType ? " declaration " : " "))
1933 << GUID << " " << F.getName() << " from "
1934 << SrcModule->getSourceFileName() << "\n");
1935 if (ImportDefinition) {
1936 if (Error Err = F.materialize())
1937 return std::move(Err);
1938 // MemProf should match function's definition and summary,
1939 // 'thinlto_src_module' is needed.
1940 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
1941 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
1942 // statistics and debugging.
1943 F.setMetadata(
1944 Kind: "thinlto_src_module",
1945 Node: MDNode::get(Context&: DestModule.getContext(),
1946 MDs: {MDString::get(Context&: DestModule.getContext(),
1947 Str: SrcModule->getModuleIdentifier())}));
1948 F.setMetadata(
1949 Kind: "thinlto_src_file",
1950 Node: MDNode::get(Context&: DestModule.getContext(),
1951 MDs: {MDString::get(Context&: DestModule.getContext(),
1952 Str: SrcModule->getSourceFileName())}));
1953 }
1954 GlobalsToImport.insert(X: &F);
1955 }
1956 }
1957 }
1958 {
1959 llvm::TimeTraceScope globalsScope("Globals");
1960 for (GlobalVariable &GV : SrcModule->globals()) {
1961 if (!GV.hasName())
1962 continue;
1963 auto GUID = GV.getGUID();
1964 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1965 bool ImportDefinition =
1966 MaybeImportType == GlobalValueSummary::Definition;
1967
1968 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1969 << " importing global"
1970 << (ImportDefinition
1971 ? " definition "
1972 : (MaybeImportType ? " declaration " : " "))
1973 << GUID << " " << GV.getName() << " from "
1974 << SrcModule->getSourceFileName() << "\n");
1975 if (ImportDefinition) {
1976 if (Error Err = GV.materialize())
1977 return std::move(Err);
1978 ImportedGVCount += GlobalsToImport.insert(X: &GV);
1979 }
1980 }
1981 }
1982 {
1983 llvm::TimeTraceScope aliasesScope("Aliases");
1984 for (GlobalAlias &GA : SrcModule->aliases()) {
1985 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
1986 continue;
1987 auto GUID = GA.getGUID();
1988 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1989 bool ImportDefinition =
1990 MaybeImportType == GlobalValueSummary::Definition;
1991
1992 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1993 << " importing alias"
1994 << (ImportDefinition
1995 ? " definition "
1996 : (MaybeImportType ? " declaration " : " "))
1997 << GUID << " " << GA.getName() << " from "
1998 << SrcModule->getSourceFileName() << "\n");
1999 if (ImportDefinition) {
2000 if (Error Err = GA.materialize())
2001 return std::move(Err);
2002 // Import alias as a copy of its aliasee.
2003 GlobalObject *GO = GA.getAliaseeObject();
2004 if (Error Err = GO->materialize())
2005 return std::move(Err);
2006 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
2007 LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID()
2008 << " " << GO->getName() << " from "
2009 << SrcModule->getSourceFileName() << "\n");
2010 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2011 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2012 // statistics and debugging.
2013 Fn->setMetadata(
2014 Kind: "thinlto_src_module",
2015 Node: MDNode::get(Context&: DestModule.getContext(),
2016 MDs: {MDString::get(Context&: DestModule.getContext(),
2017 Str: SrcModule->getModuleIdentifier())}));
2018 Fn->setMetadata(
2019 Kind: "thinlto_src_file",
2020 Node: MDNode::get(Context&: DestModule.getContext(),
2021 MDs: {MDString::get(Context&: DestModule.getContext(),
2022 Str: SrcModule->getSourceFileName())}));
2023 }
2024 GlobalsToImport.insert(X: Fn);
2025 }
2026 }
2027 }
2028
2029 // Upgrade debug info after we're done materializing all the globals and we
2030 // have loaded all the required metadata!
2031 UpgradeDebugInfo(M&: *SrcModule);
2032
2033 // Set the partial sample profile ratio in the profile summary module flag
2034 // of the imported source module, if applicable, so that the profile summary
2035 // module flag will match with that of the destination module when it's
2036 // imported.
2037 SrcModule->setPartialSampleProfileRatio(Index);
2038
2039 // Link in the specified functions.
2040 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2041 GlobalsToImport: &GlobalsToImport);
2042
2043 if (PrintImports) {
2044 for (const auto *GV : GlobalsToImport)
2045 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2046 << " from " << SrcModule->getSourceFileName() << "\n";
2047 }
2048
2049 if (Error Err = Mover.move(Src: std::move(SrcModule),
2050 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2051 /*IsPerformingImport=*/true))
2052 return createStringError(EC: errc::invalid_argument,
2053 S: Twine("Function Import: link error: ") +
2054 toString(E: std::move(Err)));
2055
2056 ImportedCount += GlobalsToImport.size();
2057 NumImportedModules++;
2058 }
2059
2060 internalizeGVsAfterImport(M&: DestModule);
2061
2062 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2063 NumImportedGlobalVars += ImportedGVCount;
2064
2065 // TODO: Print counters for definitions and declarations in the debugging log.
2066 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2067 << " functions for Module "
2068 << DestModule.getModuleIdentifier() << "\n");
2069 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2070 << " global variables for Module "
2071 << DestModule.getModuleIdentifier() << "\n");
2072 return ImportedCount;
2073}
2074
2075static bool doImportingForModuleForTest(
2076 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2077 isPrevailing) {
2078 if (SummaryFile.empty())
2079 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2080 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2081 getModuleSummaryIndexForFile(Path: SummaryFile);
2082 if (!IndexPtrOrErr) {
2083 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2084 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2085 return false;
2086 }
2087 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2088
2089 // First step is collecting the import list.
2090 FunctionImporter::ImportIDTable ImportIDs;
2091 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2092 // If requested, simply import all functions in the index. This is used
2093 // when testing distributed backend handling via the opt tool, when
2094 // we have distributed indexes containing exactly the summaries to import.
2095 if (ImportAllIndex)
2096 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2097 Index: *Index, ImportList);
2098 else
2099 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2100 isPrevailing, Index: *Index, ImportList);
2101
2102 // Conservatively mark all internal values as promoted. This interface is
2103 // only used when doing importing via the function importing pass. The pass
2104 // is only enabled when testing importing via the 'opt' tool, which does
2105 // not do the ThinLink that would normally determine what values to promote.
2106 for (auto &I : *Index) {
2107 for (auto &S : I.second.getSummaryList()) {
2108 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2109 S->setLinkage(GlobalValue::ExternalLinkage);
2110 }
2111 }
2112
2113 // Next we need to promote to global scope and rename any local values that
2114 // are potentially exported to other modules.
2115 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2116 /*GlobalsToImport=*/nullptr);
2117
2118 // Perform the import now.
2119 auto ModuleLoader = [&M](StringRef Identifier) {
2120 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2121 };
2122 FunctionImporter Importer(*Index, ModuleLoader,
2123 /*ClearDSOLocalOnDeclarations=*/false);
2124 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2125
2126 // FIXME: Probably need to propagate Errors through the pass manager.
2127 if (!Result) {
2128 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2129 ErrorBanner: "Error importing module: ");
2130 return true;
2131 }
2132
2133 return true;
2134}
2135
2136PreservedAnalyses FunctionImportPass::run(Module &M,
2137 ModuleAnalysisManager &AM) {
2138 // This is only used for testing the function import pass via opt, where we
2139 // don't have prevailing information from the LTO context available, so just
2140 // conservatively assume everything is prevailing (which is fine for the very
2141 // limited use of prevailing checking in this pass).
2142 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2143 return true;
2144 };
2145 if (!doImportingForModuleForTest(M, isPrevailing))
2146 return PreservedAnalyses::all();
2147
2148 return PreservedAnalyses::none();
2149}
2150