1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Bitcode/BitcodeReader.h"
21#include "llvm/IR/AutoUpgrade.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/GlobalObject.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/ModuleSummaryIndex.h"
30#include "llvm/IRReader/IRReader.h"
31#include "llvm/Linker/IRMover.h"
32#include "llvm/ProfileData/PGOCtxProfReader.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Errc.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Support/ErrorHandling.h"
39#include "llvm/Support/FileSystem.h"
40#include "llvm/Support/JSON.h"
41#include "llvm/Support/Path.h"
42#include "llvm/Support/SourceMgr.h"
43#include "llvm/Support/TimeProfiler.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/IPO/Internalize.h"
46#include "llvm/Transforms/Utils/Cloning.h"
47#include "llvm/Transforms/Utils/FunctionImportUtils.h"
48#include "llvm/Transforms/Utils/ValueMapper.h"
49#include <cassert>
50#include <memory>
51#include <string>
52#include <system_error>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "function-import"
59
60STATISTIC(NumImportedFunctionsThinLink,
61 "Number of functions thin link decided to import");
62STATISTIC(NumImportedHotFunctionsThinLink,
63 "Number of hot functions thin link decided to import");
64STATISTIC(NumImportedCriticalFunctionsThinLink,
65 "Number of critical functions thin link decided to import");
66STATISTIC(NumImportedGlobalVarsThinLink,
67 "Number of global variables thin link decided to import");
68STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
69STATISTIC(NumImportedGlobalVars,
70 "Number of global variables imported in backend");
71STATISTIC(NumImportedModules, "Number of modules imported from");
72STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
73STATISTIC(NumLiveSymbols, "Number of live symbols in index");
74
75namespace llvm {
76cl::opt<bool>
77 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
78 cl::desc("Import functions with noinline attribute"));
79
80/// Limit on instruction count of imported functions.
81static cl::opt<unsigned> ImportInstrLimit(
82 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
83 cl::desc("Only import functions with less than N instructions"));
84
85static cl::opt<int> ImportCutoff(
86 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
87 cl::desc("Only import first N functions if N>=0 (default -1)"));
88
89static cl::opt<float>
90 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
91 cl::Hidden, cl::value_desc("x"),
92 cl::desc("As we import functions, multiply the "
93 "`import-instr-limit` threshold by this factor "
94 "before processing newly imported functions"));
95
96static cl::opt<float> ImportHotInstrFactor(
97 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
98 cl::value_desc("x"),
99 cl::desc("As we import functions called from hot callsite, multiply the "
100 "`import-instr-limit` threshold by this factor "
101 "before processing newly imported functions"));
102
103static cl::opt<float> ImportHotMultiplier(
104 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
105 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
106
107static cl::opt<float> ImportCriticalMultiplier(
108 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
109 cl::value_desc("x"),
110 cl::desc(
111 "Multiply the `import-instr-limit` threshold for critical callsites"));
112
113// FIXME: This multiplier was not really tuned up.
114static cl::opt<float> ImportColdMultiplier(
115 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
116 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
117
118static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
119 cl::desc("Print imported functions"));
120
121static cl::opt<bool> PrintImportFailures(
122 "print-import-failures", cl::init(Val: false), cl::Hidden,
123 cl::desc("Print information for functions rejected for importing"));
124
125static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
126 cl::desc("Compute dead symbols"));
127
128static cl::opt<bool> EnableImportMetadata(
129 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
130 cl::desc("Enable import metadata like 'thinlto_src_module' and "
131 "'thinlto_src_file'"));
132
133/// Summary file to use for function importing when using -function-import from
134/// the command line.
135static cl::opt<std::string>
136 SummaryFile("summary-file",
137 cl::desc("The summary file to use for function importing."));
138
139/// Used when testing importing from distributed indexes via opt
140// -function-import.
141static cl::opt<bool>
142 ImportAllIndex("import-all-index",
143 cl::desc("Import all external functions in index."));
144
145/// This is a test-only option.
146/// If this option is enabled, the ThinLTO indexing step will import each
147/// function declaration as a fallback. In a real build this may increase ram
148/// usage of the indexing step unnecessarily.
149/// TODO: Implement selective import (based on combined summary analysis) to
150/// ensure the imported function has a use case in the postlink pipeline.
151static cl::opt<bool> ImportDeclaration(
152 "import-declaration", cl::init(Val: false), cl::Hidden,
153 cl::desc("If true, import function declaration as fallback if the function "
154 "definition is not imported."));
155
156/// Pass a workload description file - an example of workload would be the
157/// functions executed to satisfy a RPC request. A workload is defined by a root
158/// function and the list of functions that are (frequently) needed to satisfy
159/// it. The module that defines the root will have all those functions imported.
160/// The file contains a JSON dictionary. The keys are root functions, the values
161/// are lists of functions to import in the module defining the root. It is
162/// assumed -funique-internal-linkage-names was used, thus ensuring function
163/// names are unique even for local linkage ones.
164static cl::opt<std::string> WorkloadDefinitions(
165 "thinlto-workload-def",
166 cl::desc("Pass a workload definition. This is a file containing a JSON "
167 "dictionary. The keys are root functions, the values are lists of "
168 "functions to import in the module defining the root. It is "
169 "assumed -funique-internal-linkage-names was used, to ensure "
170 "local linkage functions have unique names. For example: \n"
171 "{\n"
172 " \"rootFunction_1\": [\"function_to_import_1\", "
173 "\"function_to_import_2\"], \n"
174 " \"rootFunction_2\": [\"function_to_import_3\", "
175 "\"function_to_import_4\"] \n"
176 "}"),
177 cl::Hidden);
178
179extern cl::opt<std::string> UseCtxProfile;
180
181static cl::opt<bool> CtxprofMoveRootsToOwnModule(
182 "thinlto-move-ctxprof-trees",
183 cl::desc("Move contextual profiling roots and the graphs under them in "
184 "their own module."),
185 cl::Hidden, cl::init(Val: false));
186
187extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
188
189extern cl::opt<bool> EnableMemProfContextDisambiguation;
190} // end namespace llvm
191
192// Load lazily a module from \p FileName in \p Context.
193static std::unique_ptr<Module> loadFile(const std::string &FileName,
194 LLVMContext &Context) {
195 SMDiagnostic Err;
196 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
197 // Metadata isn't loaded until functions are imported, to minimize
198 // the memory overhead.
199 std::unique_ptr<Module> Result =
200 getLazyIRFileModule(Filename: FileName, Err, Context,
201 /* ShouldLazyLoadMetadata = */ true);
202 if (!Result) {
203 Err.print(ProgName: "function-import", S&: errs());
204 report_fatal_error(reason: "Abort");
205 }
206
207 return Result;
208}
209
210static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
211 size_t NumDefs,
212 StringRef ImporterModule) {
213 // We can import a local when there is one definition.
214 if (NumDefs == 1)
215 return false;
216 // In other cases, make sure we import the copy in the caller's module if the
217 // referenced value has local linkage. The only time a local variable can
218 // share an entry in the index is if there is a local with the same name in
219 // another module that had the same source file name (in a different
220 // directory), where each was compiled in their own directory so there was not
221 // distinguishing path.
222 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
223 RefSummary->modulePath() != ImporterModule;
224}
225
226/// Given a list of possible callee implementation for a call site, qualify the
227/// legality of importing each. The return is a range of pairs. Each pair
228/// corresponds to a candidate. The first value is the ImportFailureReason for
229/// that candidate, the second is the candidate.
230static auto qualifyCalleeCandidates(
231 const ModuleSummaryIndex &Index,
232 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
233 StringRef CallerModulePath) {
234 return llvm::map_range(
235 C&: CalleeSummaryList,
236 F: [&Index, CalleeSummaryList,
237 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
238 -> std::pair<FunctionImporter::ImportFailureReason,
239 const GlobalValueSummary *> {
240 auto *GVSummary = SummaryPtr.get();
241 if (!Index.isGlobalValueLive(GVS: GVSummary))
242 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
243
244 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
245 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
246 GVSummary};
247
248 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
249
250 // Ignore any callees that aren't actually functions. This could happen
251 // in the case of GUID hash collisions. It could also happen in theory
252 // for SamplePGO profiles collected on old versions of the code after
253 // renaming, since we synthesize edges to any inlined callees appearing
254 // in the profile.
255 if (!Summary)
256 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
257
258 // If this is a local function, make sure we import the copy in the
259 // caller's module. The only time a local function can share an entry in
260 // the index is if there is a local with the same name in another module
261 // that had the same source file name (in a different directory), where
262 // each was compiled in their own directory so there was not
263 // distinguishing path.
264 // If the local function is from another module, it must be a reference
265 // due to indirect call profile data since a function pointer can point
266 // to a local in another module. Do the import from another module if
267 // there is only one entry in the list or when all files in the program
268 // are compiled with full path - in both cases the local function has
269 // unique PGO name and GUID.
270 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
271 ImporterModule: CallerModulePath))
272 return {
273 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
274 GVSummary};
275
276 // Skip if it isn't legal to import (e.g. may reference unpromotable
277 // locals).
278 if (Summary->notEligibleToImport())
279 return {FunctionImporter::ImportFailureReason::NotEligible,
280 GVSummary};
281
282 return {FunctionImporter::ImportFailureReason::None, GVSummary};
283 });
284}
285
286/// Given a list of possible callee implementation for a call site, select one
287/// that fits the \p Threshold for function definition import. If none are
288/// found, the Reason will give the last reason for the failure (last, in the
289/// order of CalleeSummaryList entries). While looking for a callee definition,
290/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
291/// candidate; other modules may want to know the function summary or
292/// declaration even if a definition is not needed.
293///
294/// FIXME: select "best" instead of first that fits. But what is "best"?
295/// - The smallest: more likely to be inlined.
296/// - The one with the least outgoing edges (already well optimized).
297/// - One from a module already being imported from in order to reduce the
298/// number of source modules parsed/linked.
299/// - One that has PGO data attached.
300/// - [insert you fancy metric here]
301static const GlobalValueSummary *
302selectCallee(const ModuleSummaryIndex &Index,
303 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
304 unsigned Threshold, StringRef CallerModulePath,
305 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
306 FunctionImporter::ImportFailureReason &Reason) {
307 // Records the last summary with reason noinline or too-large.
308 TooLargeOrNoInlineSummary = nullptr;
309 auto QualifiedCandidates =
310 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
311 for (auto QualifiedValue : QualifiedCandidates) {
312 Reason = QualifiedValue.first;
313 // Skip a summary if its import is not (proved to be) legal.
314 if (Reason != FunctionImporter::ImportFailureReason::None)
315 continue;
316 auto *Summary =
317 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
318
319 // Don't bother importing the definition if the chance of inlining it is
320 // not high enough (except under `--force-import-all`).
321 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
322 !ForceImportAll) {
323 TooLargeOrNoInlineSummary = Summary;
324 Reason = FunctionImporter::ImportFailureReason::TooLarge;
325 continue;
326 }
327
328 // Don't bother importing the definition if we can't inline it anyway.
329 if (Summary->fflags().NoInline && !ForceImportAll) {
330 TooLargeOrNoInlineSummary = Summary;
331 Reason = FunctionImporter::ImportFailureReason::NoInline;
332 continue;
333 }
334
335 return Summary;
336 }
337 return nullptr;
338}
339
340namespace {
341
342using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
343
344} // anonymous namespace
345
346FunctionImporter::ImportMapTy::AddDefinitionStatus
347FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
348 GlobalValue::GUID GUID) {
349 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
350 if (!Imports.insert(V: Def).second)
351 // Already there.
352 return AddDefinitionStatus::NoChange;
353
354 // Remove Decl in case it's there. Note that a definition takes precedence
355 // over a declaration for a given GUID.
356 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
357 : AddDefinitionStatus::Inserted;
358}
359
360void FunctionImporter::ImportMapTy::maybeAddDeclaration(
361 StringRef FromModule, GlobalValue::GUID GUID) {
362 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
363 // Insert Decl only if Def is not present. Note that a definition takes
364 // precedence over a declaration for a given GUID.
365 if (!Imports.contains(V: Def))
366 Imports.insert(V: Decl);
367}
368
369SmallVector<StringRef, 0>
370FunctionImporter::ImportMapTy::getSourceModules() const {
371 SetVector<StringRef> ModuleSet;
372 for (const auto &[SrcMod, GUID, ImportType] : *this)
373 ModuleSet.insert(X: SrcMod);
374 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
375 llvm::sort(C&: Modules);
376 return Modules;
377}
378
379std::optional<GlobalValueSummary::ImportKind>
380FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
381 GlobalValue::GUID GUID) const {
382 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
383 auto [Def, Decl] = *IDPair;
384 if (Imports.contains(V: Def))
385 return GlobalValueSummary::Definition;
386 if (Imports.contains(V: Decl))
387 return GlobalValueSummary::Declaration;
388 }
389 return std::nullopt;
390}
391
392/// Import globals referenced by a function or other globals that are being
393/// imported, if importing such global is possible.
394class GlobalsImporter final {
395 const ModuleSummaryIndex &Index;
396 const GVSummaryMapTy &DefinedGVSummaries;
397 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
398 IsPrevailing;
399 FunctionImporter::ImportMapTy &ImportList;
400 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
401
402 bool shouldImportGlobal(const ValueInfo &VI) {
403 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
404 if (GVS == DefinedGVSummaries.end())
405 return true;
406 // We should not skip import if the module contains a non-prevailing
407 // definition with interposable linkage type. This is required for
408 // correctness in the situation where there is a prevailing def available
409 // for import and marked read-only. In this case, the non-prevailing def
410 // will be converted to a declaration, while the prevailing one becomes
411 // internal, thus no definitions will be available for linking. In order to
412 // prevent undefined symbol link error, the prevailing definition must be
413 // imported.
414 // FIXME: Consider adding a check that the suitable prevailing definition
415 // exists and marked read-only.
416 if (VI.getSummaryList().size() > 1 &&
417 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
418 !IsPrevailing(VI.getGUID(), GVS->second))
419 return true;
420
421 return false;
422 }
423
424 void
425 onImportingSummaryImpl(const GlobalValueSummary &Summary,
426 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
427 for (const auto &VI : Summary.refs()) {
428 if (!shouldImportGlobal(VI)) {
429 LLVM_DEBUG(
430 dbgs() << "Ref ignored! Target already in destination module.\n");
431 continue;
432 }
433
434 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
435
436 for (const auto &RefSummary : VI.getSummaryList()) {
437 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
438 // Stop looking if this is not a global variable, e.g. a function.
439 // Functions could be referenced by global vars - e.g. a vtable; but we
440 // don't currently imagine a reason those would be imported here, rather
441 // than as part of the logic deciding which functions to import (i.e.
442 // based on profile information). Should we decide to handle them here,
443 // we can refactor accordingly at that time.
444 // Note that it is safe to stop looking because the one case where we
445 // might have to import (a read/write-only global variable) cannot occur
446 // if this GUID has a non-variable summary. The only case where we even
447 // might find another summary in the list that is a variable is in the
448 // case of same-named locals in different modules not compiled with
449 // enough path, and during attribute propagation we will mark all
450 // summaries for a GUID (ValueInfo) as non read/write-only if any is not
451 // a global variable.
452 if (!GVS)
453 break;
454 bool CanImportDecl = false;
455 if (shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
456 ImporterModule: Summary.modulePath()) ||
457 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
458 CanImportDecl)) {
459 if (ImportDeclaration && CanImportDecl)
460 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
461 GUID: VI.getGUID());
462
463 continue;
464 }
465
466 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
467 // Otherwise, definition should take precedence over declaration.
468 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
469 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
470 break;
471
472 // Only update stat and exports if we haven't already imported this
473 // variable.
474 NumImportedGlobalVarsThinLink++;
475 // Any references made by this variable will be marked exported
476 // later, in ComputeCrossModuleImport, after import decisions are
477 // complete, which is more efficient than adding them here.
478 if (ExportLists)
479 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
480
481 // If variable is not writeonly we attempt to recursively analyze
482 // its references in order to import referenced constants.
483 if (!Index.isWriteOnly(GVS))
484 Worklist.emplace_back(Args&: GVS);
485 break;
486 }
487 }
488 }
489
490public:
491 GlobalsImporter(
492 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
493 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
494 IsPrevailing,
495 FunctionImporter::ImportMapTy &ImportList,
496 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
497 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
498 IsPrevailing(IsPrevailing), ImportList(ImportList),
499 ExportLists(ExportLists) {}
500
501 void onImportingSummary(const GlobalValueSummary &Summary) {
502 SmallVector<const GlobalVarSummary *, 128> Worklist;
503 onImportingSummaryImpl(Summary, Worklist);
504 while (!Worklist.empty())
505 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
506 }
507};
508
509static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
510
511/// Determine the list of imports and exports for each module.
512class ModuleImportsManager {
513 void computeImportForFunction(
514 const FunctionSummary &Summary, unsigned Threshold,
515 const GVSummaryMapTy &DefinedGVSummaries,
516 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
517 FunctionImporter::ImportMapTy &ImportList,
518 FunctionImporter::ImportThresholdsTy &ImportThresholds);
519
520protected:
521 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
522 IsPrevailing;
523 const ModuleSummaryIndex &Index;
524 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
525
526 ModuleImportsManager(
527 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
528 IsPrevailing,
529 const ModuleSummaryIndex &Index,
530 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
531 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
532 virtual bool canImport(ValueInfo VI) { return true; }
533
534public:
535 virtual ~ModuleImportsManager() = default;
536
537 /// Given the list of globals defined in a module, compute the list of imports
538 /// as well as the list of "exports", i.e. the list of symbols referenced from
539 /// another module (that may require promotion).
540 virtual void
541 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
542 StringRef ModName,
543 FunctionImporter::ImportMapTy &ImportList);
544
545 static std::unique_ptr<ModuleImportsManager>
546 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
547 IsPrevailing,
548 const ModuleSummaryIndex &Index,
549 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
550 nullptr);
551};
552
553/// A ModuleImportsManager that operates based on a workload definition (see
554/// -thinlto-workload-def). For modules that do not define workload roots, it
555/// applies the base ModuleImportsManager import policy.
556class WorkloadImportsManager : public ModuleImportsManager {
557 // Keep a module name -> value infos to import association. We use it to
558 // determine if a module's import list should be done by the base
559 // ModuleImportsManager or by us.
560 StringMap<DenseSet<ValueInfo>> Workloads;
561 // Track the roots to avoid importing them due to other callers. We want there
562 // to be only one variant), for which we optimize according to the contextual
563 // profile. "Variants" refers to copies due to importing - we want there to be
564 // just one instance of this function.
565 DenseSet<ValueInfo> Roots;
566
567 void
568 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
569 StringRef ModName,
570 FunctionImporter::ImportMapTy &ImportList) override {
571 StringRef Filename = ModName;
572 if (CtxprofMoveRootsToOwnModule) {
573 Filename = sys::path::filename(path: ModName);
574 // Drop the file extension.
575 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
576 }
577 auto SetIter = Workloads.find(Key: Filename);
578
579 if (SetIter == Workloads.end()) {
580 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
581 << " does not contain the root of any context.\n");
582 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
583 ModName, ImportList);
584 }
585 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
586 << " contains the root(s) of context(s).\n");
587
588 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
589 ExportLists);
590 auto &ValueInfos = SetIter->second;
591 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
592 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
593 if (It != DefinedGVSummaries.end() &&
594 IsPrevailing(VI.getGUID(), It->second)) {
595 LLVM_DEBUG(
596 dbgs() << "[Workload] " << VI.name()
597 << " has the prevailing variant already in the module "
598 << ModName << ". No need to import\n");
599 continue;
600 }
601 auto Candidates =
602 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
603
604 const GlobalValueSummary *GVS = nullptr;
605 auto PotentialCandidates = llvm::map_range(
606 C: llvm::make_filter_range(
607 Range&: Candidates,
608 Pred: [&](const auto &Candidate) {
609 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
610 << " from " << Candidate.second->modulePath()
611 << " ImportFailureReason: "
612 << getFailureName(Candidate.first) << "\n");
613 return Candidate.first ==
614 FunctionImporter::ImportFailureReason::None;
615 }),
616 F: [](const auto &Candidate) { return Candidate.second; });
617 if (PotentialCandidates.empty()) {
618 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
619 << " because can't find eligible Callee. Guid is: "
620 << VI.getGUID() << "\n");
621 continue;
622 }
623 /// We will prefer importing the prevailing candidate, if not, we'll
624 /// still pick the first available candidate. The reason we want to make
625 /// sure we do import the prevailing candidate is because the goal of
626 /// workload-awareness is to enable optimizations specializing the call
627 /// graph of that workload. Suppose a function is already defined in the
628 /// module, but it's not the prevailing variant. Suppose also we do not
629 /// inline it (in fact, if it were interposable, we can't inline it),
630 /// but we could specialize it to the workload in other ways. However,
631 /// the linker would drop it in the favor of the prevailing copy.
632 /// Instead, by importing the prevailing variant (assuming also the use
633 /// of `-avail-extern-to-local`), we keep the specialization. We could
634 /// alteranatively make the non-prevailing variant local, but the
635 /// prevailing one is also the one for which we would have previously
636 /// collected profiles, making it preferrable.
637 auto PrevailingCandidates = llvm::make_filter_range(
638 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
639 return IsPrevailing(VI.getGUID(), Candidate);
640 });
641 if (PrevailingCandidates.empty()) {
642 GVS = *PotentialCandidates.begin();
643 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
644 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
645 LLVM_DEBUG(
646 dbgs()
647 << "[Workload] Found multiple non-prevailing candidates for "
648 << VI.name()
649 << ". This is unexpected. Are module paths passed to the "
650 "compiler unique for the modules passed to the linker?");
651 // We could in theory have multiple (interposable) copies of a symbol
652 // when there is no prevailing candidate, if say the prevailing copy was
653 // in a native object being linked in. However, we should in theory be
654 // marking all of these non-prevailing IR copies dead in that case, in
655 // which case they won't be candidates.
656 assert(GVS->isLive());
657 } else {
658 assert(llvm::hasSingleElement(PrevailingCandidates));
659 GVS = *PrevailingCandidates.begin();
660 }
661
662 auto ExportingModule = GVS->modulePath();
663 // We checked that for the prevailing case, but if we happen to have for
664 // example an internal that's defined in this module, it'd have no
665 // PrevailingCandidates.
666 if (ExportingModule == ModName) {
667 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
668 << " because its defining module is the same as the "
669 "current module\n");
670 continue;
671 }
672 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
673 << ExportingModule << " : " << VI.getGUID() << "\n");
674 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
675 GVI.onImportingSummary(Summary: *GVS);
676 if (ExportLists)
677 (*ExportLists)[ExportingModule].insert(V: VI);
678 }
679 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
680 }
681
682 void loadFromJson() {
683 // Since the workload def uses names, we need a quick lookup
684 // name->ValueInfo.
685 StringMap<ValueInfo> NameToValueInfo;
686 StringSet<> AmbiguousNames;
687 for (auto &I : Index) {
688 ValueInfo VI = Index.getValueInfo(R: I);
689 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
690 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
691 }
692 auto DbgReportIfAmbiguous = [&](StringRef Name) {
693 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
694 dbgs() << "[Workload] Function name " << Name
695 << " present in the workload definition is ambiguous. Consider "
696 "compiling with -funique-internal-linkage-names.";
697 });
698 };
699 std::error_code EC;
700 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
701 if (std::error_code EC = BufferOrErr.getError()) {
702 report_fatal_error(reason: "Failed to open context file");
703 return;
704 }
705 auto Buffer = std::move(BufferOrErr.get());
706 std::map<std::string, std::vector<std::string>> WorkloadDefs;
707 json::Path::Root NullRoot;
708 // The JSON is supposed to contain a dictionary matching the type of
709 // WorkloadDefs. For example:
710 // {
711 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
712 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
713 // }
714 auto Parsed = json::parse(JSON: Buffer->getBuffer());
715 if (!Parsed)
716 report_fatal_error(Err: Parsed.takeError());
717 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
718 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
719 for (const auto &Workload : WorkloadDefs) {
720 const auto &Root = Workload.first;
721 DbgReportIfAmbiguous(Root);
722 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
723 const auto &AllCallees = Workload.second;
724 auto RootIt = NameToValueInfo.find(Key: Root);
725 if (RootIt == NameToValueInfo.end()) {
726 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
727 << " not found in this linkage unit.\n");
728 continue;
729 }
730 auto RootVI = RootIt->second;
731 if (RootVI.getSummaryList().size() != 1) {
732 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
733 << " should have exactly one summary, but has "
734 << RootVI.getSummaryList().size() << ". Skipping.\n");
735 continue;
736 }
737 StringRef RootDefiningModule =
738 RootVI.getSummaryList().front()->modulePath();
739 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
740 << " is : " << RootDefiningModule << "\n");
741 auto &Set = Workloads[RootDefiningModule];
742 for (const auto &Callee : AllCallees) {
743 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
744 DbgReportIfAmbiguous(Callee);
745 auto ElemIt = NameToValueInfo.find(Key: Callee);
746 if (ElemIt == NameToValueInfo.end()) {
747 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
748 continue;
749 }
750 Set.insert(V: ElemIt->second);
751 }
752 }
753 }
754
755 void loadFromCtxProf() {
756 std::error_code EC;
757 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
758 if (std::error_code EC = BufferOrErr.getError()) {
759 report_fatal_error(reason: "Failed to open contextual profile file");
760 return;
761 }
762 auto Buffer = std::move(BufferOrErr.get());
763
764 PGOCtxProfileReader Reader(Buffer->getBuffer());
765 auto Ctx = Reader.loadProfiles();
766 if (!Ctx) {
767 report_fatal_error(reason: "Failed to parse contextual profiles");
768 return;
769 }
770 const auto &CtxMap = Ctx->Contexts;
771 SetVector<GlobalValue::GUID> ContainedGUIDs;
772 for (const auto &[RootGuid, Root] : CtxMap) {
773 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
774 // subsequent roots, but clear its contents.
775 ContainedGUIDs.clear();
776
777 auto RootVI = Index.getValueInfo(GUID: RootGuid);
778 if (!RootVI) {
779 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
780 << " not found in this linkage unit.\n");
781 continue;
782 }
783 if (RootVI.getSummaryList().size() != 1) {
784 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
785 << " should have exactly one summary, but has "
786 << RootVI.getSummaryList().size() << ". Skipping.\n");
787 continue;
788 }
789 std::string RootDefiningModule =
790 RootVI.getSummaryList().front()->modulePath().str();
791 if (CtxprofMoveRootsToOwnModule) {
792 RootDefiningModule = std::to_string(val: RootGuid);
793 LLVM_DEBUG(
794 dbgs() << "[Workload] Moving " << RootGuid
795 << " to a module with the filename without extension : "
796 << RootDefiningModule << "\n");
797 } else {
798 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
799 << " is : " << RootDefiningModule << "\n");
800 }
801 auto &Set = Workloads[RootDefiningModule];
802 Root.getContainedGuids(Guids&: ContainedGUIDs);
803 Roots.insert(V: RootVI);
804 for (auto Guid : ContainedGUIDs)
805 if (auto VI = Index.getValueInfo(GUID: Guid))
806 Set.insert(V: VI);
807 }
808 }
809
810 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
811
812public:
813 WorkloadImportsManager(
814 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
815 IsPrevailing,
816 const ModuleSummaryIndex &Index,
817 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
818 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
819 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
820 report_fatal_error(
821 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
822 return;
823 }
824 if (!UseCtxProfile.empty())
825 loadFromCtxProf();
826 else
827 loadFromJson();
828 LLVM_DEBUG({
829 for (const auto &[Root, Set] : Workloads) {
830 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
831 << " distinct callees.\n";
832 for (const auto &VI : Set) {
833 dbgs() << "[Workload] Root: " << Root
834 << " Would include: " << VI.getGUID() << "\n";
835 }
836 }
837 });
838 }
839};
840
841std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
842 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
843 IsPrevailing,
844 const ModuleSummaryIndex &Index,
845 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
846 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
847 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
848 return std::unique_ptr<ModuleImportsManager>(
849 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
850 }
851 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
852 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
853 args&: ExportLists);
854}
855
856static const char *
857getFailureName(FunctionImporter::ImportFailureReason Reason) {
858 switch (Reason) {
859 case FunctionImporter::ImportFailureReason::None:
860 return "None";
861 case FunctionImporter::ImportFailureReason::GlobalVar:
862 return "GlobalVar";
863 case FunctionImporter::ImportFailureReason::NotLive:
864 return "NotLive";
865 case FunctionImporter::ImportFailureReason::TooLarge:
866 return "TooLarge";
867 case FunctionImporter::ImportFailureReason::InterposableLinkage:
868 return "InterposableLinkage";
869 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
870 return "LocalLinkageNotInModule";
871 case FunctionImporter::ImportFailureReason::NotEligible:
872 return "NotEligible";
873 case FunctionImporter::ImportFailureReason::NoInline:
874 return "NoInline";
875 }
876 llvm_unreachable("invalid reason");
877}
878
879/// Compute the list of functions to import for a given caller. Mark these
880/// imported functions and the symbols they reference in their source module as
881/// exported from their source module.
882void ModuleImportsManager::computeImportForFunction(
883 const FunctionSummary &Summary, const unsigned Threshold,
884 const GVSummaryMapTy &DefinedGVSummaries,
885 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
886 FunctionImporter::ImportMapTy &ImportList,
887 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
888 GVImporter.onImportingSummary(Summary);
889 static int ImportCount = 0;
890 for (const auto &Edge : Summary.calls()) {
891 ValueInfo VI = Edge.first;
892 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
893 << "\n");
894
895 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
896 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
897 << " reached.\n");
898 continue;
899 }
900
901 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
902 // FIXME: Consider not skipping import if the module contains
903 // a non-prevailing def with interposable linkage. The prevailing copy
904 // can safely be imported (see shouldImportGlobal()).
905 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
906 continue;
907 }
908
909 if (!canImport(VI)) {
910 LLVM_DEBUG(
911 dbgs() << "Skipping over " << VI.getGUID()
912 << " because its import is handled in a different module.");
913 assert(VI.getSummaryList().size() == 1 &&
914 "The root was expected to be an external symbol");
915 continue;
916 }
917
918 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
919 if (Hotness == CalleeInfo::HotnessType::Hot)
920 return ImportHotMultiplier;
921 if (Hotness == CalleeInfo::HotnessType::Cold)
922 return ImportColdMultiplier;
923 if (Hotness == CalleeInfo::HotnessType::Critical)
924 return ImportCriticalMultiplier;
925 return 1.0;
926 };
927
928 const auto NewThreshold =
929 Threshold * GetBonusMultiplier(Edge.second.getHotness());
930
931 auto IT = ImportThresholds.insert(KV: std::make_pair(
932 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
933 bool PreviouslyVisited = !IT.second;
934 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
935 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
936 auto &FailureInfo = std::get<2>(t&: IT.first->second);
937
938 bool IsHotCallsite =
939 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
940 bool IsCriticalCallsite =
941 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
942
943 const FunctionSummary *ResolvedCalleeSummary = nullptr;
944 if (CalleeSummary) {
945 assert(PreviouslyVisited);
946 // Since the traversal of the call graph is DFS, we can revisit a function
947 // a second time with a higher threshold. In this case, it is added back
948 // to the worklist with the new threshold (so that its own callee chains
949 // can be considered with the higher threshold).
950 if (NewThreshold <= ProcessedThreshold) {
951 LLVM_DEBUG(
952 dbgs() << "ignored! Target was already imported with Threshold "
953 << ProcessedThreshold << "\n");
954 continue;
955 }
956 // Update with new larger threshold.
957 ProcessedThreshold = NewThreshold;
958 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
959 } else {
960 // If we already rejected importing a callee at the same or higher
961 // threshold, don't waste time calling selectCallee.
962 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
963 LLVM_DEBUG(
964 dbgs() << "ignored! Target was already rejected with Threshold "
965 << ProcessedThreshold << "\n");
966 if (PrintImportFailures) {
967 assert(FailureInfo &&
968 "Expected FailureInfo for previously rejected candidate");
969 FailureInfo->Attempts++;
970 }
971 continue;
972 }
973
974 FunctionImporter::ImportFailureReason Reason{};
975
976 // `SummaryForDeclImport` is an summary eligible for declaration import.
977 const GlobalValueSummary *SummaryForDeclImport = nullptr;
978 CalleeSummary =
979 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
980 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
981 if (!CalleeSummary) {
982 // There isn't a callee for definition import but one for declaration
983 // import.
984 if (ImportDeclaration && SummaryForDeclImport) {
985 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
986
987 // Note `ExportLists` only keeps track of exports due to imported
988 // definitions.
989 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
990 }
991 // Update with new larger threshold if this was a retry (otherwise
992 // we would have already inserted with NewThreshold above). Also
993 // update failure info if requested.
994 if (PreviouslyVisited) {
995 ProcessedThreshold = NewThreshold;
996 if (PrintImportFailures) {
997 assert(FailureInfo &&
998 "Expected FailureInfo for previously rejected candidate");
999 FailureInfo->Reason = Reason;
1000 FailureInfo->Attempts++;
1001 FailureInfo->MaxHotness =
1002 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
1003 }
1004 } else if (PrintImportFailures) {
1005 assert(!FailureInfo &&
1006 "Expected no FailureInfo for newly rejected candidate");
1007 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
1008 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
1009 }
1010 if (ForceImportAll) {
1011 std::string Msg = std::string("Failed to import function ") +
1012 VI.name().str() + " due to " +
1013 getFailureName(Reason);
1014 auto Error = make_error<StringError>(
1015 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1016 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1017 ErrorBanner: "Error importing module: ");
1018 break;
1019 } else {
1020 LLVM_DEBUG(dbgs()
1021 << "ignored! No qualifying callee with summary found.\n");
1022 continue;
1023 }
1024 }
1025
1026 // "Resolve" the summary
1027 CalleeSummary = CalleeSummary->getBaseObject();
1028 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1029
1030 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1031 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1032 "selectCallee() didn't honor the threshold");
1033
1034 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1035
1036 // Try emplace the definition entry, and update stats based on insertion
1037 // status.
1038 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1039 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1040 NumImportedFunctionsThinLink++;
1041 if (IsHotCallsite)
1042 NumImportedHotFunctionsThinLink++;
1043 if (IsCriticalCallsite)
1044 NumImportedCriticalFunctionsThinLink++;
1045 }
1046
1047 // Any calls/references made by this function will be marked exported
1048 // later, in ComputeCrossModuleImport, after import decisions are
1049 // complete, which is more efficient than adding them here.
1050 if (ExportLists)
1051 (*ExportLists)[ExportModulePath].insert(V: VI);
1052 }
1053
1054 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1055 // Adjust the threshold for next level of imported functions.
1056 // The threshold is different for hot callsites because we can then
1057 // inline chains of hot calls.
1058 if (IsHotCallsite)
1059 return Threshold * ImportHotInstrFactor;
1060 return Threshold * ImportInstrFactor;
1061 };
1062
1063 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1064
1065 ImportCount++;
1066
1067 // Insert the newly imported function to the worklist.
1068 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1069 }
1070}
1071
1072void ModuleImportsManager::computeImportForModule(
1073 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1074 FunctionImporter::ImportMapTy &ImportList) {
1075 // Worklist contains the list of function imported in this module, for which
1076 // we will analyse the callees and may import further down the callgraph.
1077 SmallVector<EdgeInfo, 128> Worklist;
1078 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1079 ExportLists);
1080 FunctionImporter::ImportThresholdsTy ImportThresholds;
1081
1082 // Populate the worklist with the import for the functions in the current
1083 // module
1084 for (const auto &GVSummary : DefinedGVSummaries) {
1085#ifndef NDEBUG
1086 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1087 // so this map look up (and possibly others) can be avoided.
1088 auto VI = Index.getValueInfo(GVSummary.first);
1089#endif
1090 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1091 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1092 continue;
1093 }
1094 auto *FuncSummary =
1095 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1096 if (!FuncSummary)
1097 // Skip import for global variables
1098 continue;
1099 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1100 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1101 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1102 }
1103
1104 // Process the newly imported functions and add callees to the worklist.
1105 while (!Worklist.empty()) {
1106 auto GVInfo = Worklist.pop_back_val();
1107 auto *Summary = std::get<0>(t&: GVInfo);
1108 auto Threshold = std::get<1>(t&: GVInfo);
1109
1110 computeImportForFunction(Summary: *Summary, Threshold, DefinedGVSummaries, Worklist,
1111 GVImporter&: GVI, ImportList, ImportThresholds);
1112 }
1113
1114 // Print stats about functions considered but rejected for importing
1115 // when requested.
1116 if (PrintImportFailures) {
1117 dbgs() << "Missed imports into module " << ModName << "\n";
1118 for (auto &I : ImportThresholds) {
1119 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1120 auto &CalleeSummary = std::get<1>(t&: I.second);
1121 auto &FailureInfo = std::get<2>(t&: I.second);
1122 if (CalleeSummary)
1123 continue; // We are going to import.
1124 assert(FailureInfo);
1125 FunctionSummary *FS = nullptr;
1126 if (!FailureInfo->VI.getSummaryList().empty())
1127 FS = dyn_cast<FunctionSummary>(
1128 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1129 dbgs() << FailureInfo->VI
1130 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1131 << ", Threshold = " << ProcessedThreshold
1132 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1133 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1134 << ", Attempts = " << FailureInfo->Attempts << "\n";
1135 }
1136 }
1137}
1138
1139#ifndef NDEBUG
1140static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1141 auto SL = VI.getSummaryList();
1142 return SL.empty()
1143 ? false
1144 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1145}
1146
1147static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1148 GlobalValue::GUID G) {
1149 if (const auto &VI = Index.getValueInfo(G))
1150 return isGlobalVarSummary(Index, VI);
1151 return false;
1152}
1153
1154// Return the number of global variable summaries in ExportSet.
1155static unsigned
1156numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1157 FunctionImporter::ExportSetTy &ExportSet) {
1158 unsigned NumGVS = 0;
1159 for (auto &VI : ExportSet)
1160 if (isGlobalVarSummary(Index, VI.getGUID()))
1161 ++NumGVS;
1162 return NumGVS;
1163}
1164
1165struct ImportStatistics {
1166 unsigned NumGVS = 0;
1167 unsigned DefinedFS = 0;
1168 unsigned Count = 0;
1169};
1170
1171// Compute import statistics for each source module in ImportList.
1172static DenseMap<StringRef, ImportStatistics>
1173collectImportStatistics(const ModuleSummaryIndex &Index,
1174 const FunctionImporter::ImportMapTy &ImportList) {
1175 DenseMap<StringRef, ImportStatistics> Histogram;
1176
1177 for (const auto &[FromModule, GUID, Type] : ImportList) {
1178 ImportStatistics &Entry = Histogram[FromModule];
1179 ++Entry.Count;
1180 if (isGlobalVarSummary(Index, GUID))
1181 ++Entry.NumGVS;
1182 else if (Type == GlobalValueSummary::Definition)
1183 ++Entry.DefinedFS;
1184 }
1185 return Histogram;
1186}
1187#endif
1188
1189#ifndef NDEBUG
1190static bool checkVariableImport(
1191 const ModuleSummaryIndex &Index,
1192 FunctionImporter::ImportListsTy &ImportLists,
1193 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1194 DenseSet<GlobalValue::GUID> FlattenedImports;
1195
1196 for (const auto &ImportPerModule : ImportLists)
1197 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1198 FlattenedImports.insert(GUID);
1199
1200 // Checks that all GUIDs of read/writeonly vars we see in export lists
1201 // are also in the import lists. Otherwise we my face linker undefs,
1202 // because readonly and writeonly vars are internalized in their
1203 // source modules. The exception would be if it has a linkage type indicating
1204 // that there may have been a copy existing in the importing module (e.g.
1205 // linkonce_odr). In that case we cannot accurately do this checking.
1206 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1207 const ValueInfo &VI) {
1208 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1209 Index.findSummaryInModule(VI, ModulePath));
1210 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1211 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1212 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1213 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1214 };
1215
1216 for (auto &ExportPerModule : ExportLists)
1217 for (auto &VI : ExportPerModule.second)
1218 if (!FlattenedImports.count(VI.getGUID()) &&
1219 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1220 return false;
1221
1222 return true;
1223}
1224#endif
1225
1226/// Compute all the import and export for every module using the Index.
1227void llvm::ComputeCrossModuleImport(
1228 const ModuleSummaryIndex &Index,
1229 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1230 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1231 isPrevailing,
1232 FunctionImporter::ImportListsTy &ImportLists,
1233 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1234 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1235 // For each module that has function defined, compute the import/export lists.
1236 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1237 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1238 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1239 << DefinedGVSummaries.first << "'\n");
1240 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1241 ModName: DefinedGVSummaries.first, ImportList);
1242 }
1243
1244 // When computing imports we only added the variables and functions being
1245 // imported to the export list. We also need to mark any references and calls
1246 // they make as exported as well. We do this here, as it is more efficient
1247 // since we may import the same values multiple times into different modules
1248 // during the import computation.
1249 for (auto &ELI : ExportLists) {
1250 // `NewExports` tracks the VI that gets exported because the full definition
1251 // of its user/referencer gets exported.
1252 FunctionImporter::ExportSetTy NewExports;
1253 const auto &DefinedGVSummaries =
1254 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1255 for (auto &EI : ELI.second) {
1256 // Find the copy defined in the exporting module so that we can mark the
1257 // values it references in that specific definition as exported.
1258 // Below we will add all references and called values, without regard to
1259 // whether they are also defined in this module. We subsequently prune the
1260 // list to only include those defined in the exporting module, see comment
1261 // there as to why.
1262 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1263 // Anything marked exported during the import computation must have been
1264 // defined in the exporting module.
1265 assert(DS != DefinedGVSummaries.end());
1266 auto *S = DS->getSecond();
1267 S = S->getBaseObject();
1268 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1269 // Export referenced functions and variables. We don't export/promote
1270 // objects referenced by writeonly variable initializer, because
1271 // we convert such variables initializers to "zeroinitializer".
1272 // See processGlobalForThinLTO.
1273 if (!Index.isWriteOnly(GVS))
1274 NewExports.insert_range(R: GVS->refs());
1275 } else {
1276 auto *FS = cast<FunctionSummary>(Val: S);
1277 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1278 NewExports.insert_range(R: FS->refs());
1279 }
1280 }
1281 // Prune list computed above to only include values defined in the
1282 // exporting module. We do this after the above insertion since we may hit
1283 // the same ref/call target multiple times in above loop, and it is more
1284 // efficient to avoid a set lookup each time.
1285 for (auto EI = NewExports.begin(); EI != NewExports.end();) {
1286 if (!DefinedGVSummaries.count(Val: EI->getGUID()))
1287 NewExports.erase(I: EI++);
1288 else
1289 ++EI;
1290 }
1291 ELI.second.insert_range(R&: NewExports);
1292 }
1293
1294 assert(checkVariableImport(Index, ImportLists, ExportLists));
1295#ifndef NDEBUG
1296 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1297 << " modules:\n");
1298 for (const auto &ModuleImports : ImportLists) {
1299 auto ModName = ModuleImports.first;
1300 auto &Exports = ExportLists[ModName];
1301 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1302 DenseMap<StringRef, ImportStatistics> Histogram =
1303 collectImportStatistics(Index, ModuleImports.second);
1304 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1305 << Exports.size() - NumGVS << " functions and " << NumGVS
1306 << " vars. Imports from " << Histogram.size()
1307 << " modules.\n");
1308 for (const auto &[SrcModName, Stats] : Histogram) {
1309 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1310 << " function definitions and "
1311 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1312 << " function declarations imported from " << SrcModName
1313 << "\n");
1314 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1315 << " global vars imported from " << SrcModName << "\n");
1316 }
1317 }
1318#endif
1319}
1320
1321#ifndef NDEBUG
1322static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1323 StringRef ModulePath,
1324 FunctionImporter::ImportMapTy &ImportList) {
1325 DenseMap<StringRef, ImportStatistics> Histogram =
1326 collectImportStatistics(Index, ImportList);
1327 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1328 << Histogram.size() << " modules.\n");
1329 for (const auto &[SrcModName, Stats] : Histogram) {
1330 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1331 << " function definitions and "
1332 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1333 << " function declarations imported from " << SrcModName
1334 << "\n");
1335 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1336 << SrcModName << "\n");
1337 }
1338}
1339#endif
1340
1341/// Compute all the imports for the given module using the Index.
1342///
1343/// \p isPrevailing is a callback that will be called with a global value's GUID
1344/// and summary and should return whether the module corresponding to the
1345/// summary contains the linker-prevailing copy of that value.
1346///
1347/// \p ImportList will be populated with a map that can be passed to
1348/// FunctionImporter::importFunctions() above (see description there).
1349static void ComputeCrossModuleImportForModuleForTest(
1350 StringRef ModulePath,
1351 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1352 isPrevailing,
1353 const ModuleSummaryIndex &Index,
1354 FunctionImporter::ImportMapTy &ImportList) {
1355 // Collect the list of functions this module defines.
1356 // GUID -> Summary
1357 GVSummaryMapTy FunctionSummaryMap;
1358 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1359
1360 // Compute the import list for this module.
1361 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1362 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1363 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1364
1365#ifndef NDEBUG
1366 dumpImportListForModule(Index, ModulePath, ImportList);
1367#endif
1368}
1369
1370/// Mark all external summaries in \p Index for import into the given module.
1371/// Used for testing the case of distributed builds using a distributed index.
1372///
1373/// \p ImportList will be populated with a map that can be passed to
1374/// FunctionImporter::importFunctions() above (see description there).
1375static void ComputeCrossModuleImportForModuleFromIndexForTest(
1376 StringRef ModulePath, const ModuleSummaryIndex &Index,
1377 FunctionImporter::ImportMapTy &ImportList) {
1378 for (const auto &GlobalList : Index) {
1379 // Ignore entries for undefined references.
1380 if (GlobalList.second.getSummaryList().empty())
1381 continue;
1382
1383 auto GUID = GlobalList.first;
1384 assert(GlobalList.second.getSummaryList().size() == 1 &&
1385 "Expected individual combined index to have one summary per GUID");
1386 auto &Summary = GlobalList.second.getSummaryList()[0];
1387 // Skip the summaries for the importing module. These are included to
1388 // e.g. record required linkage changes.
1389 if (Summary->modulePath() == ModulePath)
1390 continue;
1391 // Add an entry to provoke importing by thinBackend.
1392 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1393 }
1394#ifndef NDEBUG
1395 dumpImportListForModule(Index, ModulePath, ImportList);
1396#endif
1397}
1398
1399// For SamplePGO, the indirect call targets for local functions will
1400// have its original name annotated in profile. We try to find the
1401// corresponding PGOFuncName as the GUID, and fix up the edges
1402// accordingly.
1403void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1404 FunctionSummary *FS) {
1405 for (auto &EI : FS->mutableCalls()) {
1406 if (!EI.first.getSummaryList().empty())
1407 continue;
1408 auto GUID = Index.getGUIDFromOriginalID(OriginalID: EI.first.getGUID());
1409 if (GUID == 0)
1410 continue;
1411 // Update the edge to point directly to the correct GUID.
1412 auto VI = Index.getValueInfo(GUID);
1413 if (llvm::any_of(
1414 Range: VI.getSummaryList(),
1415 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1416 // The mapping from OriginalId to GUID may return a GUID
1417 // that corresponds to a static variable. Filter it out here.
1418 // This can happen when
1419 // 1) There is a call to a library function which is not defined
1420 // in the index.
1421 // 2) There is a static variable with the OriginalGUID identical
1422 // to the GUID of the library function in 1);
1423 // When this happens the static variable in 2) will be found,
1424 // which needs to be filtered out.
1425 return SummaryPtr->getSummaryKind() ==
1426 GlobalValueSummary::GlobalVarKind;
1427 }))
1428 continue;
1429 EI.first = VI;
1430 }
1431}
1432
1433void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1434 for (const auto &Entry : Index) {
1435 for (const auto &S : Entry.second.getSummaryList()) {
1436 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1437 updateValueInfoForIndirectCalls(Index, FS);
1438 }
1439 }
1440}
1441
1442void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1443 ModuleSummaryIndex &Index,
1444 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1445 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1446 assert(!Index.withGlobalValueDeadStripping());
1447 if (!ComputeDead ||
1448 // Don't do anything when nothing is live, this is friendly with tests.
1449 GUIDPreservedSymbols.empty()) {
1450 // Still need to update indirect calls.
1451 updateIndirectCalls(Index);
1452 return;
1453 }
1454 unsigned LiveSymbols = 0;
1455 SmallVector<ValueInfo, 128> Worklist;
1456 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1457 for (auto GUID : GUIDPreservedSymbols) {
1458 ValueInfo VI = Index.getValueInfo(GUID);
1459 if (!VI)
1460 continue;
1461 for (const auto &S : VI.getSummaryList())
1462 S->setLive(true);
1463 }
1464
1465 // Add values flagged in the index as live roots to the worklist.
1466 for (const auto &Entry : Index) {
1467 auto VI = Index.getValueInfo(R: Entry);
1468 for (const auto &S : Entry.second.getSummaryList()) {
1469 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1470 updateValueInfoForIndirectCalls(Index, FS);
1471 if (S->isLive()) {
1472 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1473 Worklist.push_back(Elt: VI);
1474 ++LiveSymbols;
1475 break;
1476 }
1477 }
1478 }
1479
1480 // Make value live and add it to the worklist if it was not live before.
1481 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1482 // FIXME: If we knew which edges were created for indirect call profiles,
1483 // we could skip them here. Any that are live should be reached via
1484 // other edges, e.g. reference edges. Otherwise, using a profile collected
1485 // on a slightly different binary might provoke preserving, importing
1486 // and ultimately promoting calls to functions not linked into this
1487 // binary, which increases the binary size unnecessarily. Note that
1488 // if this code changes, the importer needs to change so that edges
1489 // to functions marked dead are skipped.
1490
1491 if (llvm::any_of(Range: VI.getSummaryList(),
1492 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1493 return S->isLive();
1494 }))
1495 return;
1496
1497 // We only keep live symbols that are known to be non-prevailing if any are
1498 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1499 // later in the EliminateAvailableExternally pass and setting them to
1500 // not-live could break downstreams users of liveness information (PR36483)
1501 // or limit optimization opportunities.
1502 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1503 bool KeepAliveLinkage = false;
1504 bool Interposable = false;
1505 for (const auto &S : VI.getSummaryList()) {
1506 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1507 S->linkage() == GlobalValue::WeakODRLinkage ||
1508 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1509 KeepAliveLinkage = true;
1510 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1511 Interposable = true;
1512 }
1513
1514 if (!IsAliasee) {
1515 if (!KeepAliveLinkage)
1516 return;
1517
1518 if (Interposable)
1519 report_fatal_error(
1520 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1521 "symbol");
1522 }
1523 }
1524
1525 for (const auto &S : VI.getSummaryList())
1526 S->setLive(true);
1527 ++LiveSymbols;
1528 Worklist.push_back(Elt: VI);
1529 };
1530
1531 while (!Worklist.empty()) {
1532 auto VI = Worklist.pop_back_val();
1533 for (const auto &Summary : VI.getSummaryList()) {
1534 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1535 // If this is an alias, visit the aliasee VI to ensure that all copies
1536 // are marked live and it is added to the worklist for further
1537 // processing of its references.
1538 visit(AS->getAliaseeVI(), true);
1539 continue;
1540 }
1541 for (auto Ref : Summary->refs())
1542 visit(Ref, false);
1543 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1544 for (auto Call : FS->calls())
1545 visit(Call.first, false);
1546 }
1547 }
1548 Index.setWithGlobalValueDeadStripping();
1549
1550 unsigned DeadSymbols = Index.size() - LiveSymbols;
1551 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1552 << " symbols Dead \n");
1553 NumDeadSymbols += DeadSymbols;
1554 NumLiveSymbols += LiveSymbols;
1555}
1556
1557// Compute dead symbols and propagate constants in combined index.
1558void llvm::computeDeadSymbolsWithConstProp(
1559 ModuleSummaryIndex &Index,
1560 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1561 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1562 bool ImportEnabled) {
1563 llvm::TimeTraceScope timeScope("Drop dead symbols and propagate attributes");
1564 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1565 isPrevailing);
1566 if (ImportEnabled)
1567 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1568}
1569
1570/// Compute the set of summaries needed for a ThinLTO backend compilation of
1571/// \p ModulePath.
1572void llvm::gatherImportedSummariesForModule(
1573 StringRef ModulePath,
1574 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1575 const FunctionImporter::ImportMapTy &ImportList,
1576 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1577 GVSummaryPtrSet &DecSummaries) {
1578 // Include all summaries from the importing module.
1579 ModuleToSummariesForIndex[std::string(ModulePath)] =
1580 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1581
1582 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1583 // lets us look up the map without allocating an instance of std::string when
1584 // the key-value pair exists in the map.
1585 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1586 // C++26 when it becomes available for our codebase.
1587 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1588 StringRef Key) -> GVSummaryMapTy & {
1589 auto It = Map.find(x: Key);
1590 if (It == Map.end())
1591 std::tie(args&: It, args: std::ignore) =
1592 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1593 return It->second;
1594 };
1595
1596 // Include summaries for imports.
1597 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1598 auto &SummariesForIndex =
1599 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1600
1601 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1602 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1603 assert(DS != DefinedGVSummaries.end() &&
1604 "Expected a defined summary for imported global value");
1605 if (ImportType == GlobalValueSummary::Declaration)
1606 DecSummaries.insert(x: DS->second);
1607
1608 SummariesForIndex[GUID] = DS->second;
1609 }
1610}
1611
1612/// Emit the files \p ModulePath will import from into \p OutputFilename.
1613Error llvm::EmitImportsFiles(
1614 StringRef ModulePath, StringRef OutputFilename,
1615 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1616 std::error_code EC;
1617 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1618 if (EC)
1619 return createFileError(F: "cannot open " + OutputFilename,
1620 E: errorCodeToError(EC));
1621 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1622 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1623 return Error::success();
1624}
1625
1626/// Invoke callback \p F on the file paths from which \p ModulePath
1627/// will import.
1628void llvm::processImportsFiles(
1629 StringRef ModulePath,
1630 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1631 function_ref<void(const std::string &)> F) {
1632 for (const auto &ILI : ModuleToSummariesForIndex)
1633 // The ModuleToSummariesForIndex map includes an entry for the current
1634 // Module (needed for writing out the index files). We don't want to
1635 // include it in the imports file, however, so filter it out.
1636 if (ILI.first != ModulePath)
1637 F(ILI.first);
1638}
1639
1640bool llvm::convertToDeclaration(GlobalValue &GV) {
1641 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1642 << "\n");
1643 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1644 F->deleteBody();
1645 F->clearMetadata();
1646 F->setComdat(nullptr);
1647 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1648 V->setInitializer(nullptr);
1649 V->setLinkage(GlobalValue::ExternalLinkage);
1650 V->clearMetadata();
1651 V->setComdat(nullptr);
1652 } else {
1653 GlobalValue *NewGV;
1654 if (GV.getValueType()->isFunctionTy())
1655 NewGV =
1656 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1657 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1658 N: "", M: GV.getParent());
1659 else
1660 NewGV =
1661 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1662 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1663 /*init*/ nullptr, "",
1664 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1665 GV.getType()->getAddressSpace());
1666 NewGV->takeName(V: &GV);
1667 GV.replaceAllUsesWith(V: NewGV);
1668 return false;
1669 }
1670 if (!GV.isImplicitDSOLocal())
1671 GV.setDSOLocal(false);
1672 return true;
1673}
1674
1675void llvm::thinLTOFinalizeInModule(Module &TheModule,
1676 const GVSummaryMapTy &DefinedGlobals,
1677 bool PropagateAttrs) {
1678 llvm::TimeTraceScope timeScope("ThinLTO finalize in module");
1679 DenseSet<Comdat *> NonPrevailingComdats;
1680 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1681 // See if the global summary analysis computed a new resolved linkage.
1682 const auto &GS = DefinedGlobals.find(Val: GV.getGUID());
1683 if (GS == DefinedGlobals.end())
1684 return;
1685
1686 if (Propagate)
1687 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1688 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1689 // TODO: propagate ReadNone and ReadOnly.
1690 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1691 F->setDoesNotAccessMemory();
1692
1693 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1694 F->setOnlyReadsMemory();
1695
1696 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1697 F->setDoesNotRecurse();
1698
1699 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1700 F->setDoesNotThrow();
1701 }
1702 }
1703
1704 auto NewLinkage = GS->second->linkage();
1705 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1706 // Don't internalize anything here, because the code below
1707 // lacks necessary correctness checks. Leave this job to
1708 // LLVM 'internalize' pass.
1709 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1710 // In case it was dead and already converted to declaration.
1711 GV.isDeclaration())
1712 return;
1713
1714 // Set the potentially more constraining visibility computed from summaries.
1715 // The DefaultVisibility condition is because older GlobalValueSummary does
1716 // not record DefaultVisibility and we don't want to change protected/hidden
1717 // to default.
1718 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1719 GV.setVisibility(GS->second->getVisibility());
1720
1721 if (NewLinkage == GV.getLinkage())
1722 return;
1723
1724 // Check for a non-prevailing def that has interposable linkage
1725 // (e.g. non-odr weak or linkonce). In that case we can't simply
1726 // convert to available_externally, since it would lose the
1727 // interposable property and possibly get inlined. Simply drop
1728 // the definition in that case.
1729 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1730 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1731 if (!convertToDeclaration(GV))
1732 // FIXME: Change this to collect replaced GVs and later erase
1733 // them from the parent module once thinLTOResolvePrevailingGUID is
1734 // changed to enable this for aliases.
1735 llvm_unreachable("Expected GV to be converted");
1736 } else {
1737 // If all copies of the original symbol had global unnamed addr and
1738 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1739 // and are constants, then it should be an auto hide symbol. In that case
1740 // the thin link would have marked it as CanAutoHide. Add hidden
1741 // visibility to the symbol to preserve the property.
1742 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1743 GS->second->canAutoHide()) {
1744 assert(GV.canBeOmittedFromSymbolTable());
1745 GV.setVisibility(GlobalValue::HiddenVisibility);
1746 }
1747
1748 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1749 << "` from " << GV.getLinkage() << " to " << NewLinkage
1750 << "\n");
1751 GV.setLinkage(NewLinkage);
1752 }
1753 // Remove declarations from comdats, including available_externally
1754 // as this is a declaration for the linker, and will be dropped eventually.
1755 // It is illegal for comdats to contain declarations.
1756 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1757 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1758 if (GO->getComdat()->getName() == GO->getName())
1759 NonPrevailingComdats.insert(V: GO->getComdat());
1760 GO->setComdat(nullptr);
1761 }
1762 };
1763
1764 // Process functions and global now
1765 for (auto &GV : TheModule)
1766 FinalizeInModule(GV, PropagateAttrs);
1767 for (auto &GV : TheModule.globals())
1768 FinalizeInModule(GV);
1769 for (auto &GV : TheModule.aliases())
1770 FinalizeInModule(GV);
1771
1772 // For a non-prevailing comdat, all its members must be available_externally.
1773 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1774 // local linkage GlobalValues.
1775 if (NonPrevailingComdats.empty())
1776 return;
1777 for (auto &GO : TheModule.global_objects()) {
1778 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1779 GO.setComdat(nullptr);
1780 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1781 }
1782 }
1783 bool Changed;
1784 do {
1785 Changed = false;
1786 // If an alias references a GlobalValue in a non-prevailing comdat, change
1787 // it to available_externally. For simplicity we only handle GlobalValue and
1788 // ConstantExpr with a base object. ConstantExpr without a base object is
1789 // unlikely used in a COMDAT.
1790 for (auto &GA : TheModule.aliases()) {
1791 if (GA.hasAvailableExternallyLinkage())
1792 continue;
1793 GlobalObject *Obj = GA.getAliaseeObject();
1794 assert(Obj && "aliasee without an base object is unimplemented");
1795 if (Obj->hasAvailableExternallyLinkage()) {
1796 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1797 Changed = true;
1798 }
1799 }
1800 } while (Changed);
1801}
1802
1803/// Run internalization on \p TheModule based on symmary analysis.
1804void llvm::thinLTOInternalizeModule(Module &TheModule,
1805 const GVSummaryMapTy &DefinedGlobals) {
1806 llvm::TimeTraceScope timeScope("ThinLTO internalize module");
1807 // Declare a callback for the internalize pass that will ask for every
1808 // candidate GlobalValue if it can be internalized or not.
1809 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1810 // It may be the case that GV is on a chain of an ifunc, its alias and
1811 // subsequent aliases. In this case, the summary for the value is not
1812 // available.
1813 if (isa<GlobalIFunc>(Val: &GV) ||
1814 (isa<GlobalAlias>(Val: &GV) &&
1815 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1816 return true;
1817
1818 // Lookup the linkage recorded in the summaries during global analysis.
1819 auto GS = DefinedGlobals.find(Val: GV.getGUID());
1820 if (GS == DefinedGlobals.end()) {
1821 // Must have been promoted (possibly conservatively). Find original
1822 // name so that we can access the correct summary and see if it can
1823 // be internalized again.
1824 // FIXME: Eventually we should control promotion instead of promoting
1825 // and internalizing again.
1826 StringRef OrigName =
1827 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1828 std::string OrigId = GlobalValue::getGlobalIdentifier(
1829 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1830 FileName: TheModule.getSourceFileName());
1831 GS = DefinedGlobals.find(
1832 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1833 if (GS == DefinedGlobals.end()) {
1834 // Also check the original non-promoted non-globalized name. In some
1835 // cases a preempted weak value is linked in as a local copy because
1836 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1837 // In that case, since it was originally not a local value, it was
1838 // recorded in the index using the original name.
1839 // FIXME: This may not be needed once PR27866 is fixed.
1840 GS = DefinedGlobals.find(
1841 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1842 assert(GS != DefinedGlobals.end());
1843 }
1844 }
1845 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1846 };
1847
1848 // FIXME: See if we can just internalize directly here via linkage changes
1849 // based on the index, rather than invoking internalizeModule.
1850 internalizeModule(TheModule, MustPreserveGV);
1851}
1852
1853/// Make alias a clone of its aliasee.
1854static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1855 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1856
1857 ValueToValueMapTy VMap;
1858 Function *NewFn = CloneFunction(F: Fn, VMap);
1859 // Clone should use the original alias's linkage, visibility and name, and we
1860 // ensure all uses of alias instead use the new clone (casted if necessary).
1861 NewFn->setLinkage(GA->getLinkage());
1862 NewFn->setVisibility(GA->getVisibility());
1863 GA->replaceAllUsesWith(V: NewFn);
1864 NewFn->takeName(V: GA);
1865 return NewFn;
1866}
1867
1868// Internalize values that we marked with specific attribute
1869// in processGlobalForThinLTO.
1870static void internalizeGVsAfterImport(Module &M) {
1871 for (auto &GV : M.globals())
1872 // Skip GVs which have been converted to declarations
1873 // by dropDeadSymbols.
1874 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1875 GV.setLinkage(GlobalValue::InternalLinkage);
1876 GV.setVisibility(GlobalValue::DefaultVisibility);
1877 }
1878}
1879
1880// Automatically import functions in Module \p DestModule based on the summaries
1881// index.
1882Expected<bool> FunctionImporter::importFunctions(
1883 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1884 LLVM_DEBUG(dbgs() << "Starting import for Module "
1885 << DestModule.getModuleIdentifier() << "\n");
1886 unsigned ImportedCount = 0, ImportedGVCount = 0;
1887 // Before carrying out any imports, see if this module defines functions in
1888 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1889 // The function will be imported elsewhere, as extenal linkage, and the
1890 // destination doesn't yet have its definition.
1891 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1892 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1893 for (auto &F : DestModule)
1894 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUID()))
1895 F.deleteBody();
1896
1897 IRMover Mover(DestModule);
1898
1899 // Do the actual import of functions now, one Module at a time
1900 for (const auto &ModName : ImportList.getSourceModules()) {
1901 llvm::TimeTraceScope timeScope("Import", ModName);
1902 // Get the module for the import
1903 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1904 if (!SrcModuleOrErr)
1905 return SrcModuleOrErr.takeError();
1906 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1907 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1908 "Context mismatch");
1909
1910 // If modules were created with lazy metadata loading, materialize it
1911 // now, before linking it (otherwise this will be a noop).
1912 if (Error Err = SrcModule->materializeMetadata())
1913 return std::move(Err);
1914
1915 // Find the globals to import
1916 SetVector<GlobalValue *> GlobalsToImport;
1917 {
1918 llvm::TimeTraceScope functionsScope("Functions");
1919 for (Function &F : *SrcModule) {
1920 if (!F.hasName())
1921 continue;
1922 auto GUID = F.getGUID();
1923 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1924 bool ImportDefinition =
1925 MaybeImportType == GlobalValueSummary::Definition;
1926
1927 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1928 << " importing function"
1929 << (ImportDefinition
1930 ? " definition "
1931 : (MaybeImportType ? " declaration " : " "))
1932 << GUID << " " << F.getName() << " from "
1933 << SrcModule->getSourceFileName() << "\n");
1934 if (ImportDefinition) {
1935 if (Error Err = F.materialize())
1936 return std::move(Err);
1937 // MemProf should match function's definition and summary,
1938 // 'thinlto_src_module' is needed.
1939 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
1940 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
1941 // statistics and debugging.
1942 F.setMetadata(
1943 Kind: "thinlto_src_module",
1944 Node: MDNode::get(Context&: DestModule.getContext(),
1945 MDs: {MDString::get(Context&: DestModule.getContext(),
1946 Str: SrcModule->getModuleIdentifier())}));
1947 F.setMetadata(
1948 Kind: "thinlto_src_file",
1949 Node: MDNode::get(Context&: DestModule.getContext(),
1950 MDs: {MDString::get(Context&: DestModule.getContext(),
1951 Str: SrcModule->getSourceFileName())}));
1952 }
1953 GlobalsToImport.insert(X: &F);
1954 }
1955 }
1956 }
1957 {
1958 llvm::TimeTraceScope globalsScope("Globals");
1959 for (GlobalVariable &GV : SrcModule->globals()) {
1960 if (!GV.hasName())
1961 continue;
1962 auto GUID = GV.getGUID();
1963 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1964 bool ImportDefinition =
1965 MaybeImportType == GlobalValueSummary::Definition;
1966
1967 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1968 << " importing global"
1969 << (ImportDefinition
1970 ? " definition "
1971 : (MaybeImportType ? " declaration " : " "))
1972 << GUID << " " << GV.getName() << " from "
1973 << SrcModule->getSourceFileName() << "\n");
1974 if (ImportDefinition) {
1975 if (Error Err = GV.materialize())
1976 return std::move(Err);
1977 ImportedGVCount += GlobalsToImport.insert(X: &GV);
1978 }
1979 }
1980 }
1981 {
1982 llvm::TimeTraceScope aliasesScope("Aliases");
1983 for (GlobalAlias &GA : SrcModule->aliases()) {
1984 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
1985 continue;
1986 auto GUID = GA.getGUID();
1987 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1988 bool ImportDefinition =
1989 MaybeImportType == GlobalValueSummary::Definition;
1990
1991 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1992 << " importing alias"
1993 << (ImportDefinition
1994 ? " definition "
1995 : (MaybeImportType ? " declaration " : " "))
1996 << GUID << " " << GA.getName() << " from "
1997 << SrcModule->getSourceFileName() << "\n");
1998 if (ImportDefinition) {
1999 if (Error Err = GA.materialize())
2000 return std::move(Err);
2001 // Import alias as a copy of its aliasee.
2002 GlobalObject *GO = GA.getAliaseeObject();
2003 if (Error Err = GO->materialize())
2004 return std::move(Err);
2005 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
2006 LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID()
2007 << " " << GO->getName() << " from "
2008 << SrcModule->getSourceFileName() << "\n");
2009 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2010 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2011 // statistics and debugging.
2012 Fn->setMetadata(
2013 Kind: "thinlto_src_module",
2014 Node: MDNode::get(Context&: DestModule.getContext(),
2015 MDs: {MDString::get(Context&: DestModule.getContext(),
2016 Str: SrcModule->getModuleIdentifier())}));
2017 Fn->setMetadata(
2018 Kind: "thinlto_src_file",
2019 Node: MDNode::get(Context&: DestModule.getContext(),
2020 MDs: {MDString::get(Context&: DestModule.getContext(),
2021 Str: SrcModule->getSourceFileName())}));
2022 }
2023 GlobalsToImport.insert(X: Fn);
2024 }
2025 }
2026 }
2027
2028 // Upgrade debug info after we're done materializing all the globals and we
2029 // have loaded all the required metadata!
2030 UpgradeDebugInfo(M&: *SrcModule);
2031
2032 // Set the partial sample profile ratio in the profile summary module flag
2033 // of the imported source module, if applicable, so that the profile summary
2034 // module flag will match with that of the destination module when it's
2035 // imported.
2036 SrcModule->setPartialSampleProfileRatio(Index);
2037
2038 // Link in the specified functions.
2039 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2040 GlobalsToImport: &GlobalsToImport);
2041
2042 if (PrintImports) {
2043 for (const auto *GV : GlobalsToImport)
2044 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2045 << " from " << SrcModule->getSourceFileName() << "\n";
2046 }
2047
2048 if (Error Err = Mover.move(Src: std::move(SrcModule),
2049 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2050 /*IsPerformingImport=*/true))
2051 return createStringError(EC: errc::invalid_argument,
2052 S: Twine("Function Import: link error: ") +
2053 toString(E: std::move(Err)));
2054
2055 ImportedCount += GlobalsToImport.size();
2056 NumImportedModules++;
2057 }
2058
2059 internalizeGVsAfterImport(M&: DestModule);
2060
2061 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2062 NumImportedGlobalVars += ImportedGVCount;
2063
2064 // TODO: Print counters for definitions and declarations in the debugging log.
2065 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2066 << " functions for Module "
2067 << DestModule.getModuleIdentifier() << "\n");
2068 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2069 << " global variables for Module "
2070 << DestModule.getModuleIdentifier() << "\n");
2071 return ImportedCount;
2072}
2073
2074static bool doImportingForModuleForTest(
2075 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2076 isPrevailing) {
2077 if (SummaryFile.empty())
2078 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2079 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2080 getModuleSummaryIndexForFile(Path: SummaryFile);
2081 if (!IndexPtrOrErr) {
2082 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2083 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2084 return false;
2085 }
2086 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2087
2088 // First step is collecting the import list.
2089 FunctionImporter::ImportIDTable ImportIDs;
2090 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2091 // If requested, simply import all functions in the index. This is used
2092 // when testing distributed backend handling via the opt tool, when
2093 // we have distributed indexes containing exactly the summaries to import.
2094 if (ImportAllIndex)
2095 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2096 Index: *Index, ImportList);
2097 else
2098 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2099 isPrevailing, Index: *Index, ImportList);
2100
2101 // Conservatively mark all internal values as promoted. This interface is
2102 // only used when doing importing via the function importing pass. The pass
2103 // is only enabled when testing importing via the 'opt' tool, which does
2104 // not do the ThinLink that would normally determine what values to promote.
2105 for (auto &I : *Index) {
2106 for (auto &S : I.second.getSummaryList()) {
2107 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2108 S->setExternalLinkageForTest();
2109 }
2110 }
2111
2112 // Next we need to promote to global scope and rename any local values that
2113 // are potentially exported to other modules.
2114 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2115 /*GlobalsToImport=*/nullptr);
2116
2117 // Perform the import now.
2118 auto ModuleLoader = [&M](StringRef Identifier) {
2119 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2120 };
2121 FunctionImporter Importer(*Index, ModuleLoader,
2122 /*ClearDSOLocalOnDeclarations=*/false);
2123 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2124
2125 // FIXME: Probably need to propagate Errors through the pass manager.
2126 if (!Result) {
2127 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2128 ErrorBanner: "Error importing module: ");
2129 return true;
2130 }
2131
2132 return true;
2133}
2134
2135PreservedAnalyses FunctionImportPass::run(Module &M,
2136 ModuleAnalysisManager &AM) {
2137 // This is only used for testing the function import pass via opt, where we
2138 // don't have prevailing information from the LTO context available, so just
2139 // conservatively assume everything is prevailing (which is fine for the very
2140 // limited use of prevailing checking in this pass).
2141 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2142 return true;
2143 };
2144 if (!doImportingForModuleForTest(M, isPrevailing))
2145 return PreservedAnalyses::all();
2146
2147 return PreservedAnalyses::none();
2148}
2149