1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Bitcode/BitcodeReader.h"
21#include "llvm/IR/AutoUpgrade.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/GlobalObject.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/ModuleSummaryIndex.h"
30#include "llvm/IRReader/IRReader.h"
31#include "llvm/Linker/IRMover.h"
32#include "llvm/ProfileData/PGOCtxProfReader.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Errc.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Support/ErrorHandling.h"
39#include "llvm/Support/FileSystem.h"
40#include "llvm/Support/JSON.h"
41#include "llvm/Support/Path.h"
42#include "llvm/Support/SourceMgr.h"
43#include "llvm/Support/raw_ostream.h"
44#include "llvm/Transforms/IPO/Internalize.h"
45#include "llvm/Transforms/Utils/Cloning.h"
46#include "llvm/Transforms/Utils/FunctionImportUtils.h"
47#include "llvm/Transforms/Utils/ValueMapper.h"
48#include <cassert>
49#include <memory>
50#include <string>
51#include <system_error>
52#include <tuple>
53#include <utility>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "function-import"
58
59STATISTIC(NumImportedFunctionsThinLink,
60 "Number of functions thin link decided to import");
61STATISTIC(NumImportedHotFunctionsThinLink,
62 "Number of hot functions thin link decided to import");
63STATISTIC(NumImportedCriticalFunctionsThinLink,
64 "Number of critical functions thin link decided to import");
65STATISTIC(NumImportedGlobalVarsThinLink,
66 "Number of global variables thin link decided to import");
67STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
68STATISTIC(NumImportedGlobalVars,
69 "Number of global variables imported in backend");
70STATISTIC(NumImportedModules, "Number of modules imported from");
71STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
72STATISTIC(NumLiveSymbols, "Number of live symbols in index");
73
74cl::opt<bool>
75 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
76 cl::desc("Import functions with noinline attribute"));
77
78/// Limit on instruction count of imported functions.
79static cl::opt<unsigned> ImportInstrLimit(
80 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
81 cl::desc("Only import functions with less than N instructions"));
82
83static cl::opt<int> ImportCutoff(
84 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
85 cl::desc("Only import first N functions if N>=0 (default -1)"));
86
87static cl::opt<float>
88 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
89 cl::Hidden, cl::value_desc("x"),
90 cl::desc("As we import functions, multiply the "
91 "`import-instr-limit` threshold by this factor "
92 "before processing newly imported functions"));
93
94static cl::opt<float> ImportHotInstrFactor(
95 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
96 cl::value_desc("x"),
97 cl::desc("As we import functions called from hot callsite, multiply the "
98 "`import-instr-limit` threshold by this factor "
99 "before processing newly imported functions"));
100
101static cl::opt<float> ImportHotMultiplier(
102 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
103 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
104
105static cl::opt<float> ImportCriticalMultiplier(
106 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
107 cl::value_desc("x"),
108 cl::desc(
109 "Multiply the `import-instr-limit` threshold for critical callsites"));
110
111// FIXME: This multiplier was not really tuned up.
112static cl::opt<float> ImportColdMultiplier(
113 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
114 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
115
116static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
117 cl::desc("Print imported functions"));
118
119static cl::opt<bool> PrintImportFailures(
120 "print-import-failures", cl::init(Val: false), cl::Hidden,
121 cl::desc("Print information for functions rejected for importing"));
122
123static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
124 cl::desc("Compute dead symbols"));
125
126static cl::opt<bool> EnableImportMetadata(
127 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
128 cl::desc("Enable import metadata like 'thinlto_src_module' and "
129 "'thinlto_src_file'"));
130
131/// Summary file to use for function importing when using -function-import from
132/// the command line.
133static cl::opt<std::string>
134 SummaryFile("summary-file",
135 cl::desc("The summary file to use for function importing."));
136
137/// Used when testing importing from distributed indexes via opt
138// -function-import.
139static cl::opt<bool>
140 ImportAllIndex("import-all-index",
141 cl::desc("Import all external functions in index."));
142
143/// This is a test-only option.
144/// If this option is enabled, the ThinLTO indexing step will import each
145/// function declaration as a fallback. In a real build this may increase ram
146/// usage of the indexing step unnecessarily.
147/// TODO: Implement selective import (based on combined summary analysis) to
148/// ensure the imported function has a use case in the postlink pipeline.
149static cl::opt<bool> ImportDeclaration(
150 "import-declaration", cl::init(Val: false), cl::Hidden,
151 cl::desc("If true, import function declaration as fallback if the function "
152 "definition is not imported."));
153
154/// Pass a workload description file - an example of workload would be the
155/// functions executed to satisfy a RPC request. A workload is defined by a root
156/// function and the list of functions that are (frequently) needed to satisfy
157/// it. The module that defines the root will have all those functions imported.
158/// The file contains a JSON dictionary. The keys are root functions, the values
159/// are lists of functions to import in the module defining the root. It is
160/// assumed -funique-internal-linkage-names was used, thus ensuring function
161/// names are unique even for local linkage ones.
162static cl::opt<std::string> WorkloadDefinitions(
163 "thinlto-workload-def",
164 cl::desc("Pass a workload definition. This is a file containing a JSON "
165 "dictionary. The keys are root functions, the values are lists of "
166 "functions to import in the module defining the root. It is "
167 "assumed -funique-internal-linkage-names was used, to ensure "
168 "local linkage functions have unique names. For example: \n"
169 "{\n"
170 " \"rootFunction_1\": [\"function_to_import_1\", "
171 "\"function_to_import_2\"], \n"
172 " \"rootFunction_2\": [\"function_to_import_3\", "
173 "\"function_to_import_4\"] \n"
174 "}"),
175 cl::Hidden);
176
177extern cl::opt<std::string> UseCtxProfile;
178
179static cl::opt<bool> CtxprofMoveRootsToOwnModule(
180 "thinlto-move-ctxprof-trees",
181 cl::desc("Move contextual profiling roots and the graphs under them in "
182 "their own module."),
183 cl::Hidden, cl::init(Val: false));
184
185extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
186
187namespace llvm {
188extern cl::opt<bool> EnableMemProfContextDisambiguation;
189}
190
191// Load lazily a module from \p FileName in \p Context.
192static std::unique_ptr<Module> loadFile(const std::string &FileName,
193 LLVMContext &Context) {
194 SMDiagnostic Err;
195 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
196 // Metadata isn't loaded until functions are imported, to minimize
197 // the memory overhead.
198 std::unique_ptr<Module> Result =
199 getLazyIRFileModule(Filename: FileName, Err, Context,
200 /* ShouldLazyLoadMetadata = */ true);
201 if (!Result) {
202 Err.print(ProgName: "function-import", S&: errs());
203 report_fatal_error(reason: "Abort");
204 }
205
206 return Result;
207}
208
209static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
210 size_t NumDefs,
211 StringRef ImporterModule) {
212 // We can import a local when there is one definition.
213 if (NumDefs == 1)
214 return false;
215 // In other cases, make sure we import the copy in the caller's module if the
216 // referenced value has local linkage. The only time a local variable can
217 // share an entry in the index is if there is a local with the same name in
218 // another module that had the same source file name (in a different
219 // directory), where each was compiled in their own directory so there was not
220 // distinguishing path.
221 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
222 RefSummary->modulePath() != ImporterModule;
223}
224
225/// Given a list of possible callee implementation for a call site, qualify the
226/// legality of importing each. The return is a range of pairs. Each pair
227/// corresponds to a candidate. The first value is the ImportFailureReason for
228/// that candidate, the second is the candidate.
229static auto qualifyCalleeCandidates(
230 const ModuleSummaryIndex &Index,
231 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
232 StringRef CallerModulePath) {
233 return llvm::map_range(
234 C&: CalleeSummaryList,
235 F: [&Index, CalleeSummaryList,
236 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
237 -> std::pair<FunctionImporter::ImportFailureReason,
238 const GlobalValueSummary *> {
239 auto *GVSummary = SummaryPtr.get();
240 if (!Index.isGlobalValueLive(GVS: GVSummary))
241 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
242
243 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
244 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
245 GVSummary};
246
247 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
248
249 // Ignore any callees that aren't actually functions. This could happen
250 // in the case of GUID hash collisions. It could also happen in theory
251 // for SamplePGO profiles collected on old versions of the code after
252 // renaming, since we synthesize edges to any inlined callees appearing
253 // in the profile.
254 if (!Summary)
255 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
256
257 // If this is a local function, make sure we import the copy in the
258 // caller's module. The only time a local function can share an entry in
259 // the index is if there is a local with the same name in another module
260 // that had the same source file name (in a different directory), where
261 // each was compiled in their own directory so there was not
262 // distinguishing path.
263 // If the local function is from another module, it must be a reference
264 // due to indirect call profile data since a function pointer can point
265 // to a local in another module. Do the import from another module if
266 // there is only one entry in the list or when all files in the program
267 // are compiled with full path - in both cases the local function has
268 // unique PGO name and GUID.
269 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
270 ImporterModule: CallerModulePath))
271 return {
272 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
273 GVSummary};
274
275 // Skip if it isn't legal to import (e.g. may reference unpromotable
276 // locals).
277 if (Summary->notEligibleToImport())
278 return {FunctionImporter::ImportFailureReason::NotEligible,
279 GVSummary};
280
281 return {FunctionImporter::ImportFailureReason::None, GVSummary};
282 });
283}
284
285/// Given a list of possible callee implementation for a call site, select one
286/// that fits the \p Threshold for function definition import. If none are
287/// found, the Reason will give the last reason for the failure (last, in the
288/// order of CalleeSummaryList entries). While looking for a callee definition,
289/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
290/// candidate; other modules may want to know the function summary or
291/// declaration even if a definition is not needed.
292///
293/// FIXME: select "best" instead of first that fits. But what is "best"?
294/// - The smallest: more likely to be inlined.
295/// - The one with the least outgoing edges (already well optimized).
296/// - One from a module already being imported from in order to reduce the
297/// number of source modules parsed/linked.
298/// - One that has PGO data attached.
299/// - [insert you fancy metric here]
300static const GlobalValueSummary *
301selectCallee(const ModuleSummaryIndex &Index,
302 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
303 unsigned Threshold, StringRef CallerModulePath,
304 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
305 FunctionImporter::ImportFailureReason &Reason) {
306 // Records the last summary with reason noinline or too-large.
307 TooLargeOrNoInlineSummary = nullptr;
308 auto QualifiedCandidates =
309 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
310 for (auto QualifiedValue : QualifiedCandidates) {
311 Reason = QualifiedValue.first;
312 // Skip a summary if its import is not (proved to be) legal.
313 if (Reason != FunctionImporter::ImportFailureReason::None)
314 continue;
315 auto *Summary =
316 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
317
318 // Don't bother importing the definition if the chance of inlining it is
319 // not high enough (except under `--force-import-all`).
320 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
321 !ForceImportAll) {
322 TooLargeOrNoInlineSummary = Summary;
323 Reason = FunctionImporter::ImportFailureReason::TooLarge;
324 continue;
325 }
326
327 // Don't bother importing the definition if we can't inline it anyway.
328 if (Summary->fflags().NoInline && !ForceImportAll) {
329 TooLargeOrNoInlineSummary = Summary;
330 Reason = FunctionImporter::ImportFailureReason::NoInline;
331 continue;
332 }
333
334 return Summary;
335 }
336 return nullptr;
337}
338
339namespace {
340
341using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
342
343} // anonymous namespace
344
345FunctionImporter::ImportMapTy::AddDefinitionStatus
346FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
347 GlobalValue::GUID GUID) {
348 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
349 if (!Imports.insert(V: Def).second)
350 // Already there.
351 return AddDefinitionStatus::NoChange;
352
353 // Remove Decl in case it's there. Note that a definition takes precedence
354 // over a declaration for a given GUID.
355 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
356 : AddDefinitionStatus::Inserted;
357}
358
359void FunctionImporter::ImportMapTy::maybeAddDeclaration(
360 StringRef FromModule, GlobalValue::GUID GUID) {
361 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
362 // Insert Decl only if Def is not present. Note that a definition takes
363 // precedence over a declaration for a given GUID.
364 if (!Imports.contains(V: Def))
365 Imports.insert(V: Decl);
366}
367
368SmallVector<StringRef, 0>
369FunctionImporter::ImportMapTy::getSourceModules() const {
370 SetVector<StringRef> ModuleSet;
371 for (const auto &[SrcMod, GUID, ImportType] : *this)
372 ModuleSet.insert(X: SrcMod);
373 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
374 llvm::sort(C&: Modules);
375 return Modules;
376}
377
378std::optional<GlobalValueSummary::ImportKind>
379FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
380 GlobalValue::GUID GUID) const {
381 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
382 auto [Def, Decl] = *IDPair;
383 if (Imports.contains(V: Def))
384 return GlobalValueSummary::Definition;
385 if (Imports.contains(V: Decl))
386 return GlobalValueSummary::Declaration;
387 }
388 return std::nullopt;
389}
390
391/// Import globals referenced by a function or other globals that are being
392/// imported, if importing such global is possible.
393class GlobalsImporter final {
394 const ModuleSummaryIndex &Index;
395 const GVSummaryMapTy &DefinedGVSummaries;
396 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
397 IsPrevailing;
398 FunctionImporter::ImportMapTy &ImportList;
399 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
400
401 bool shouldImportGlobal(const ValueInfo &VI) {
402 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
403 if (GVS == DefinedGVSummaries.end())
404 return true;
405 // We should not skip import if the module contains a non-prevailing
406 // definition with interposable linkage type. This is required for
407 // correctness in the situation where there is a prevailing def available
408 // for import and marked read-only. In this case, the non-prevailing def
409 // will be converted to a declaration, while the prevailing one becomes
410 // internal, thus no definitions will be available for linking. In order to
411 // prevent undefined symbol link error, the prevailing definition must be
412 // imported.
413 // FIXME: Consider adding a check that the suitable prevailing definition
414 // exists and marked read-only.
415 if (VI.getSummaryList().size() > 1 &&
416 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
417 !IsPrevailing(VI.getGUID(), GVS->second))
418 return true;
419
420 return false;
421 }
422
423 void
424 onImportingSummaryImpl(const GlobalValueSummary &Summary,
425 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
426 for (const auto &VI : Summary.refs()) {
427 if (!shouldImportGlobal(VI)) {
428 LLVM_DEBUG(
429 dbgs() << "Ref ignored! Target already in destination module.\n");
430 continue;
431 }
432
433 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
434
435 for (const auto &RefSummary : VI.getSummaryList()) {
436 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
437 // Functions could be referenced by global vars - e.g. a vtable; but we
438 // don't currently imagine a reason those would be imported here, rather
439 // than as part of the logic deciding which functions to import (i.e.
440 // based on profile information). Should we decide to handle them here,
441 // we can refactor accordingly at that time.
442 bool CanImportDecl = false;
443 if (!GVS ||
444 shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
445 ImporterModule: Summary.modulePath()) ||
446 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
447 CanImportDecl)) {
448 if (ImportDeclaration && CanImportDecl)
449 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
450 GUID: VI.getGUID());
451
452 continue;
453 }
454
455 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
456 // Otherwise, definition should take precedence over declaration.
457 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
458 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
459 break;
460
461 // Only update stat and exports if we haven't already imported this
462 // variable.
463 NumImportedGlobalVarsThinLink++;
464 // Any references made by this variable will be marked exported
465 // later, in ComputeCrossModuleImport, after import decisions are
466 // complete, which is more efficient than adding them here.
467 if (ExportLists)
468 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
469
470 // If variable is not writeonly we attempt to recursively analyze
471 // its references in order to import referenced constants.
472 if (!Index.isWriteOnly(GVS))
473 Worklist.emplace_back(Args&: GVS);
474 break;
475 }
476 }
477 }
478
479public:
480 GlobalsImporter(
481 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
482 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
483 IsPrevailing,
484 FunctionImporter::ImportMapTy &ImportList,
485 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
486 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
487 IsPrevailing(IsPrevailing), ImportList(ImportList),
488 ExportLists(ExportLists) {}
489
490 void onImportingSummary(const GlobalValueSummary &Summary) {
491 SmallVector<const GlobalVarSummary *, 128> Worklist;
492 onImportingSummaryImpl(Summary, Worklist);
493 while (!Worklist.empty())
494 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
495 }
496};
497
498static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
499
500/// Determine the list of imports and exports for each module.
501class ModuleImportsManager {
502 void computeImportForFunction(
503 const FunctionSummary &Summary, unsigned Threshold,
504 const GVSummaryMapTy &DefinedGVSummaries,
505 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
506 FunctionImporter::ImportMapTy &ImportList,
507 FunctionImporter::ImportThresholdsTy &ImportThresholds);
508
509protected:
510 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
511 IsPrevailing;
512 const ModuleSummaryIndex &Index;
513 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
514
515 ModuleImportsManager(
516 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
517 IsPrevailing,
518 const ModuleSummaryIndex &Index,
519 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
520 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
521 virtual bool canImport(ValueInfo VI) { return true; }
522
523public:
524 virtual ~ModuleImportsManager() = default;
525
526 /// Given the list of globals defined in a module, compute the list of imports
527 /// as well as the list of "exports", i.e. the list of symbols referenced from
528 /// another module (that may require promotion).
529 virtual void
530 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
531 StringRef ModName,
532 FunctionImporter::ImportMapTy &ImportList);
533
534 static std::unique_ptr<ModuleImportsManager>
535 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
536 IsPrevailing,
537 const ModuleSummaryIndex &Index,
538 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
539 nullptr);
540};
541
542/// A ModuleImportsManager that operates based on a workload definition (see
543/// -thinlto-workload-def). For modules that do not define workload roots, it
544/// applies the base ModuleImportsManager import policy.
545class WorkloadImportsManager : public ModuleImportsManager {
546 // Keep a module name -> value infos to import association. We use it to
547 // determine if a module's import list should be done by the base
548 // ModuleImportsManager or by us.
549 StringMap<DenseSet<ValueInfo>> Workloads;
550 // Track the roots to avoid importing them due to other callers. We want there
551 // to be only one variant), for which we optimize according to the contextual
552 // profile. "Variants" refers to copies due to importing - we want there to be
553 // just one instance of this function.
554 DenseSet<ValueInfo> Roots;
555
556 void
557 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
558 StringRef ModName,
559 FunctionImporter::ImportMapTy &ImportList) override {
560 StringRef Filename = ModName;
561 if (CtxprofMoveRootsToOwnModule) {
562 Filename = sys::path::filename(path: ModName);
563 // Drop the file extension.
564 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
565 }
566 auto SetIter = Workloads.find(Key: Filename);
567
568 if (SetIter == Workloads.end()) {
569 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
570 << " does not contain the root of any context.\n");
571 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
572 ModName, ImportList);
573 }
574 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
575 << " contains the root(s) of context(s).\n");
576
577 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
578 ExportLists);
579 auto &ValueInfos = SetIter->second;
580 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
581 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
582 if (It != DefinedGVSummaries.end() &&
583 IsPrevailing(VI.getGUID(), It->second)) {
584 LLVM_DEBUG(
585 dbgs() << "[Workload] " << VI.name()
586 << " has the prevailing variant already in the module "
587 << ModName << ". No need to import\n");
588 continue;
589 }
590 auto Candidates =
591 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
592
593 const GlobalValueSummary *GVS = nullptr;
594 auto PotentialCandidates = llvm::map_range(
595 C: llvm::make_filter_range(
596 Range&: Candidates,
597 Pred: [&](const auto &Candidate) {
598 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
599 << " from " << Candidate.second->modulePath()
600 << " ImportFailureReason: "
601 << getFailureName(Candidate.first) << "\n");
602 return Candidate.first ==
603 FunctionImporter::ImportFailureReason::None;
604 }),
605 F: [](const auto &Candidate) { return Candidate.second; });
606 if (PotentialCandidates.empty()) {
607 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
608 << " because can't find eligible Callee. Guid is: "
609 << VI.getGUID() << "\n");
610 continue;
611 }
612 /// We will prefer importing the prevailing candidate, if not, we'll
613 /// still pick the first available candidate. The reason we want to make
614 /// sure we do import the prevailing candidate is because the goal of
615 /// workload-awareness is to enable optimizations specializing the call
616 /// graph of that workload. Suppose a function is already defined in the
617 /// module, but it's not the prevailing variant. Suppose also we do not
618 /// inline it (in fact, if it were interposable, we can't inline it),
619 /// but we could specialize it to the workload in other ways. However,
620 /// the linker would drop it in the favor of the prevailing copy.
621 /// Instead, by importing the prevailing variant (assuming also the use
622 /// of `-avail-extern-to-local`), we keep the specialization. We could
623 /// alteranatively make the non-prevailing variant local, but the
624 /// prevailing one is also the one for which we would have previously
625 /// collected profiles, making it preferrable.
626 auto PrevailingCandidates = llvm::make_filter_range(
627 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
628 return IsPrevailing(VI.getGUID(), Candidate);
629 });
630 if (PrevailingCandidates.empty()) {
631 GVS = *PotentialCandidates.begin();
632 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
633 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
634 LLVM_DEBUG(
635 dbgs()
636 << "[Workload] Found multiple non-prevailing candidates for "
637 << VI.name()
638 << ". This is unexpected. Are module paths passed to the "
639 "compiler unique for the modules passed to the linker?");
640 // We could in theory have multiple (interposable) copies of a symbol
641 // when there is no prevailing candidate, if say the prevailing copy was
642 // in a native object being linked in. However, we should in theory be
643 // marking all of these non-prevailing IR copies dead in that case, in
644 // which case they won't be candidates.
645 assert(GVS->isLive());
646 } else {
647 assert(llvm::hasSingleElement(PrevailingCandidates));
648 GVS = *PrevailingCandidates.begin();
649 }
650
651 auto ExportingModule = GVS->modulePath();
652 // We checked that for the prevailing case, but if we happen to have for
653 // example an internal that's defined in this module, it'd have no
654 // PrevailingCandidates.
655 if (ExportingModule == ModName) {
656 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
657 << " because its defining module is the same as the "
658 "current module\n");
659 continue;
660 }
661 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
662 << ExportingModule << " : " << VI.getGUID() << "\n");
663 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
664 GVI.onImportingSummary(Summary: *GVS);
665 if (ExportLists)
666 (*ExportLists)[ExportingModule].insert(V: VI);
667 }
668 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
669 }
670
671 void loadFromJson() {
672 // Since the workload def uses names, we need a quick lookup
673 // name->ValueInfo.
674 StringMap<ValueInfo> NameToValueInfo;
675 StringSet<> AmbiguousNames;
676 for (auto &I : Index) {
677 ValueInfo VI = Index.getValueInfo(R: I);
678 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
679 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
680 }
681 auto DbgReportIfAmbiguous = [&](StringRef Name) {
682 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
683 dbgs() << "[Workload] Function name " << Name
684 << " present in the workload definition is ambiguous. Consider "
685 "compiling with -funique-internal-linkage-names.";
686 });
687 };
688 std::error_code EC;
689 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
690 if (std::error_code EC = BufferOrErr.getError()) {
691 report_fatal_error(reason: "Failed to open context file");
692 return;
693 }
694 auto Buffer = std::move(BufferOrErr.get());
695 std::map<std::string, std::vector<std::string>> WorkloadDefs;
696 json::Path::Root NullRoot;
697 // The JSON is supposed to contain a dictionary matching the type of
698 // WorkloadDefs. For example:
699 // {
700 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
701 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
702 // }
703 auto Parsed = json::parse(JSON: Buffer->getBuffer());
704 if (!Parsed)
705 report_fatal_error(Err: Parsed.takeError());
706 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
707 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
708 for (const auto &Workload : WorkloadDefs) {
709 const auto &Root = Workload.first;
710 DbgReportIfAmbiguous(Root);
711 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
712 const auto &AllCallees = Workload.second;
713 auto RootIt = NameToValueInfo.find(Key: Root);
714 if (RootIt == NameToValueInfo.end()) {
715 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
716 << " not found in this linkage unit.\n");
717 continue;
718 }
719 auto RootVI = RootIt->second;
720 if (RootVI.getSummaryList().size() != 1) {
721 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
722 << " should have exactly one summary, but has "
723 << RootVI.getSummaryList().size() << ". Skipping.\n");
724 continue;
725 }
726 StringRef RootDefiningModule =
727 RootVI.getSummaryList().front()->modulePath();
728 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
729 << " is : " << RootDefiningModule << "\n");
730 auto &Set = Workloads[RootDefiningModule];
731 for (const auto &Callee : AllCallees) {
732 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
733 DbgReportIfAmbiguous(Callee);
734 auto ElemIt = NameToValueInfo.find(Key: Callee);
735 if (ElemIt == NameToValueInfo.end()) {
736 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
737 continue;
738 }
739 Set.insert(V: ElemIt->second);
740 }
741 }
742 }
743
744 void loadFromCtxProf() {
745 std::error_code EC;
746 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
747 if (std::error_code EC = BufferOrErr.getError()) {
748 report_fatal_error(reason: "Failed to open contextual profile file");
749 return;
750 }
751 auto Buffer = std::move(BufferOrErr.get());
752
753 PGOCtxProfileReader Reader(Buffer->getBuffer());
754 auto Ctx = Reader.loadProfiles();
755 if (!Ctx) {
756 report_fatal_error(reason: "Failed to parse contextual profiles");
757 return;
758 }
759 const auto &CtxMap = Ctx->Contexts;
760 SetVector<GlobalValue::GUID> ContainedGUIDs;
761 for (const auto &[RootGuid, Root] : CtxMap) {
762 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
763 // subsequent roots, but clear its contents.
764 ContainedGUIDs.clear();
765
766 auto RootVI = Index.getValueInfo(GUID: RootGuid);
767 if (!RootVI) {
768 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
769 << " not found in this linkage unit.\n");
770 continue;
771 }
772 if (RootVI.getSummaryList().size() != 1) {
773 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
774 << " should have exactly one summary, but has "
775 << RootVI.getSummaryList().size() << ". Skipping.\n");
776 continue;
777 }
778 std::string RootDefiningModule =
779 RootVI.getSummaryList().front()->modulePath().str();
780 if (CtxprofMoveRootsToOwnModule) {
781 RootDefiningModule = std::to_string(val: RootGuid);
782 LLVM_DEBUG(
783 dbgs() << "[Workload] Moving " << RootGuid
784 << " to a module with the filename without extension : "
785 << RootDefiningModule << "\n");
786 } else {
787 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
788 << " is : " << RootDefiningModule << "\n");
789 }
790 auto &Set = Workloads[RootDefiningModule];
791 Root.getContainedGuids(Guids&: ContainedGUIDs);
792 Roots.insert(V: RootVI);
793 for (auto Guid : ContainedGUIDs)
794 if (auto VI = Index.getValueInfo(GUID: Guid))
795 Set.insert(V: VI);
796 }
797 }
798
799 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
800
801public:
802 WorkloadImportsManager(
803 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
804 IsPrevailing,
805 const ModuleSummaryIndex &Index,
806 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
807 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
808 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
809 report_fatal_error(
810 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
811 return;
812 }
813 if (!UseCtxProfile.empty())
814 loadFromCtxProf();
815 else
816 loadFromJson();
817 LLVM_DEBUG({
818 for (const auto &[Root, Set] : Workloads) {
819 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
820 << " distinct callees.\n";
821 for (const auto &VI : Set) {
822 dbgs() << "[Workload] Root: " << Root
823 << " Would include: " << VI.getGUID() << "\n";
824 }
825 }
826 });
827 }
828};
829
830std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
831 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
832 IsPrevailing,
833 const ModuleSummaryIndex &Index,
834 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
835 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
836 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
837 return std::unique_ptr<ModuleImportsManager>(
838 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
839 }
840 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
841 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
842 args&: ExportLists);
843}
844
845static const char *
846getFailureName(FunctionImporter::ImportFailureReason Reason) {
847 switch (Reason) {
848 case FunctionImporter::ImportFailureReason::None:
849 return "None";
850 case FunctionImporter::ImportFailureReason::GlobalVar:
851 return "GlobalVar";
852 case FunctionImporter::ImportFailureReason::NotLive:
853 return "NotLive";
854 case FunctionImporter::ImportFailureReason::TooLarge:
855 return "TooLarge";
856 case FunctionImporter::ImportFailureReason::InterposableLinkage:
857 return "InterposableLinkage";
858 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
859 return "LocalLinkageNotInModule";
860 case FunctionImporter::ImportFailureReason::NotEligible:
861 return "NotEligible";
862 case FunctionImporter::ImportFailureReason::NoInline:
863 return "NoInline";
864 }
865 llvm_unreachable("invalid reason");
866}
867
868/// Compute the list of functions to import for a given caller. Mark these
869/// imported functions and the symbols they reference in their source module as
870/// exported from their source module.
871void ModuleImportsManager::computeImportForFunction(
872 const FunctionSummary &Summary, const unsigned Threshold,
873 const GVSummaryMapTy &DefinedGVSummaries,
874 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
875 FunctionImporter::ImportMapTy &ImportList,
876 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
877 GVImporter.onImportingSummary(Summary);
878 static int ImportCount = 0;
879 for (const auto &Edge : Summary.calls()) {
880 ValueInfo VI = Edge.first;
881 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
882 << "\n");
883
884 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
885 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
886 << " reached.\n");
887 continue;
888 }
889
890 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
891 // FIXME: Consider not skipping import if the module contains
892 // a non-prevailing def with interposable linkage. The prevailing copy
893 // can safely be imported (see shouldImportGlobal()).
894 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
895 continue;
896 }
897
898 if (!canImport(VI)) {
899 LLVM_DEBUG(
900 dbgs() << "Skipping over " << VI.getGUID()
901 << " because its import is handled in a different module.");
902 assert(VI.getSummaryList().size() == 1 &&
903 "The root was expected to be an external symbol");
904 continue;
905 }
906
907 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
908 if (Hotness == CalleeInfo::HotnessType::Hot)
909 return ImportHotMultiplier;
910 if (Hotness == CalleeInfo::HotnessType::Cold)
911 return ImportColdMultiplier;
912 if (Hotness == CalleeInfo::HotnessType::Critical)
913 return ImportCriticalMultiplier;
914 return 1.0;
915 };
916
917 const auto NewThreshold =
918 Threshold * GetBonusMultiplier(Edge.second.getHotness());
919
920 auto IT = ImportThresholds.insert(KV: std::make_pair(
921 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
922 bool PreviouslyVisited = !IT.second;
923 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
924 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
925 auto &FailureInfo = std::get<2>(t&: IT.first->second);
926
927 bool IsHotCallsite =
928 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
929 bool IsCriticalCallsite =
930 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
931
932 const FunctionSummary *ResolvedCalleeSummary = nullptr;
933 if (CalleeSummary) {
934 assert(PreviouslyVisited);
935 // Since the traversal of the call graph is DFS, we can revisit a function
936 // a second time with a higher threshold. In this case, it is added back
937 // to the worklist with the new threshold (so that its own callee chains
938 // can be considered with the higher threshold).
939 if (NewThreshold <= ProcessedThreshold) {
940 LLVM_DEBUG(
941 dbgs() << "ignored! Target was already imported with Threshold "
942 << ProcessedThreshold << "\n");
943 continue;
944 }
945 // Update with new larger threshold.
946 ProcessedThreshold = NewThreshold;
947 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
948 } else {
949 // If we already rejected importing a callee at the same or higher
950 // threshold, don't waste time calling selectCallee.
951 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
952 LLVM_DEBUG(
953 dbgs() << "ignored! Target was already rejected with Threshold "
954 << ProcessedThreshold << "\n");
955 if (PrintImportFailures) {
956 assert(FailureInfo &&
957 "Expected FailureInfo for previously rejected candidate");
958 FailureInfo->Attempts++;
959 }
960 continue;
961 }
962
963 FunctionImporter::ImportFailureReason Reason{};
964
965 // `SummaryForDeclImport` is an summary eligible for declaration import.
966 const GlobalValueSummary *SummaryForDeclImport = nullptr;
967 CalleeSummary =
968 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
969 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
970 if (!CalleeSummary) {
971 // There isn't a callee for definition import but one for declaration
972 // import.
973 if (ImportDeclaration && SummaryForDeclImport) {
974 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
975
976 // Note `ExportLists` only keeps track of exports due to imported
977 // definitions.
978 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
979 }
980 // Update with new larger threshold if this was a retry (otherwise
981 // we would have already inserted with NewThreshold above). Also
982 // update failure info if requested.
983 if (PreviouslyVisited) {
984 ProcessedThreshold = NewThreshold;
985 if (PrintImportFailures) {
986 assert(FailureInfo &&
987 "Expected FailureInfo for previously rejected candidate");
988 FailureInfo->Reason = Reason;
989 FailureInfo->Attempts++;
990 FailureInfo->MaxHotness =
991 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
992 }
993 } else if (PrintImportFailures) {
994 assert(!FailureInfo &&
995 "Expected no FailureInfo for newly rejected candidate");
996 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
997 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
998 }
999 if (ForceImportAll) {
1000 std::string Msg = std::string("Failed to import function ") +
1001 VI.name().str() + " due to " +
1002 getFailureName(Reason);
1003 auto Error = make_error<StringError>(
1004 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1005 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1006 ErrorBanner: "Error importing module: ");
1007 break;
1008 } else {
1009 LLVM_DEBUG(dbgs()
1010 << "ignored! No qualifying callee with summary found.\n");
1011 continue;
1012 }
1013 }
1014
1015 // "Resolve" the summary
1016 CalleeSummary = CalleeSummary->getBaseObject();
1017 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1018
1019 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1020 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1021 "selectCallee() didn't honor the threshold");
1022
1023 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1024
1025 // Try emplace the definition entry, and update stats based on insertion
1026 // status.
1027 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1028 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1029 NumImportedFunctionsThinLink++;
1030 if (IsHotCallsite)
1031 NumImportedHotFunctionsThinLink++;
1032 if (IsCriticalCallsite)
1033 NumImportedCriticalFunctionsThinLink++;
1034 }
1035
1036 // Any calls/references made by this function will be marked exported
1037 // later, in ComputeCrossModuleImport, after import decisions are
1038 // complete, which is more efficient than adding them here.
1039 if (ExportLists)
1040 (*ExportLists)[ExportModulePath].insert(V: VI);
1041 }
1042
1043 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1044 // Adjust the threshold for next level of imported functions.
1045 // The threshold is different for hot callsites because we can then
1046 // inline chains of hot calls.
1047 if (IsHotCallsite)
1048 return Threshold * ImportHotInstrFactor;
1049 return Threshold * ImportInstrFactor;
1050 };
1051
1052 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1053
1054 ImportCount++;
1055
1056 // Insert the newly imported function to the worklist.
1057 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1058 }
1059}
1060
1061void ModuleImportsManager::computeImportForModule(
1062 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1063 FunctionImporter::ImportMapTy &ImportList) {
1064 // Worklist contains the list of function imported in this module, for which
1065 // we will analyse the callees and may import further down the callgraph.
1066 SmallVector<EdgeInfo, 128> Worklist;
1067 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1068 ExportLists);
1069 FunctionImporter::ImportThresholdsTy ImportThresholds;
1070
1071 // Populate the worklist with the import for the functions in the current
1072 // module
1073 for (const auto &GVSummary : DefinedGVSummaries) {
1074#ifndef NDEBUG
1075 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1076 // so this map look up (and possibly others) can be avoided.
1077 auto VI = Index.getValueInfo(GVSummary.first);
1078#endif
1079 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1080 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1081 continue;
1082 }
1083 auto *FuncSummary =
1084 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1085 if (!FuncSummary)
1086 // Skip import for global variables
1087 continue;
1088 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1089 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1090 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1091 }
1092
1093 // Process the newly imported functions and add callees to the worklist.
1094 while (!Worklist.empty()) {
1095 auto GVInfo = Worklist.pop_back_val();
1096 auto *Summary = std::get<0>(t&: GVInfo);
1097 auto Threshold = std::get<1>(t&: GVInfo);
1098
1099 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary))
1100 computeImportForFunction(Summary: *FS, Threshold, DefinedGVSummaries, Worklist,
1101 GVImporter&: GVI, ImportList, ImportThresholds);
1102 }
1103
1104 // Print stats about functions considered but rejected for importing
1105 // when requested.
1106 if (PrintImportFailures) {
1107 dbgs() << "Missed imports into module " << ModName << "\n";
1108 for (auto &I : ImportThresholds) {
1109 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1110 auto &CalleeSummary = std::get<1>(t&: I.second);
1111 auto &FailureInfo = std::get<2>(t&: I.second);
1112 if (CalleeSummary)
1113 continue; // We are going to import.
1114 assert(FailureInfo);
1115 FunctionSummary *FS = nullptr;
1116 if (!FailureInfo->VI.getSummaryList().empty())
1117 FS = dyn_cast<FunctionSummary>(
1118 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1119 dbgs() << FailureInfo->VI
1120 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1121 << ", Threshold = " << ProcessedThreshold
1122 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1123 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1124 << ", Attempts = " << FailureInfo->Attempts << "\n";
1125 }
1126 }
1127}
1128
1129#ifndef NDEBUG
1130static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1131 auto SL = VI.getSummaryList();
1132 return SL.empty()
1133 ? false
1134 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1135}
1136
1137static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1138 GlobalValue::GUID G) {
1139 if (const auto &VI = Index.getValueInfo(G))
1140 return isGlobalVarSummary(Index, VI);
1141 return false;
1142}
1143
1144// Return the number of global variable summaries in ExportSet.
1145static unsigned
1146numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1147 FunctionImporter::ExportSetTy &ExportSet) {
1148 unsigned NumGVS = 0;
1149 for (auto &VI : ExportSet)
1150 if (isGlobalVarSummary(Index, VI.getGUID()))
1151 ++NumGVS;
1152 return NumGVS;
1153}
1154
1155struct ImportStatistics {
1156 unsigned NumGVS = 0;
1157 unsigned DefinedFS = 0;
1158 unsigned Count = 0;
1159};
1160
1161// Compute import statistics for each source module in ImportList.
1162static DenseMap<StringRef, ImportStatistics>
1163collectImportStatistics(const ModuleSummaryIndex &Index,
1164 const FunctionImporter::ImportMapTy &ImportList) {
1165 DenseMap<StringRef, ImportStatistics> Histogram;
1166
1167 for (const auto &[FromModule, GUID, Type] : ImportList) {
1168 ImportStatistics &Entry = Histogram[FromModule];
1169 ++Entry.Count;
1170 if (isGlobalVarSummary(Index, GUID))
1171 ++Entry.NumGVS;
1172 else if (Type == GlobalValueSummary::Definition)
1173 ++Entry.DefinedFS;
1174 }
1175 return Histogram;
1176}
1177#endif
1178
1179#ifndef NDEBUG
1180static bool checkVariableImport(
1181 const ModuleSummaryIndex &Index,
1182 FunctionImporter::ImportListsTy &ImportLists,
1183 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1184 DenseSet<GlobalValue::GUID> FlattenedImports;
1185
1186 for (const auto &ImportPerModule : ImportLists)
1187 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1188 FlattenedImports.insert(GUID);
1189
1190 // Checks that all GUIDs of read/writeonly vars we see in export lists
1191 // are also in the import lists. Otherwise we my face linker undefs,
1192 // because readonly and writeonly vars are internalized in their
1193 // source modules. The exception would be if it has a linkage type indicating
1194 // that there may have been a copy existing in the importing module (e.g.
1195 // linkonce_odr). In that case we cannot accurately do this checking.
1196 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1197 const ValueInfo &VI) {
1198 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1199 Index.findSummaryInModule(VI, ModulePath));
1200 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1201 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1202 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1203 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1204 };
1205
1206 for (auto &ExportPerModule : ExportLists)
1207 for (auto &VI : ExportPerModule.second)
1208 if (!FlattenedImports.count(VI.getGUID()) &&
1209 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1210 return false;
1211
1212 return true;
1213}
1214#endif
1215
1216/// Compute all the import and export for every module using the Index.
1217void llvm::ComputeCrossModuleImport(
1218 const ModuleSummaryIndex &Index,
1219 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1220 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1221 isPrevailing,
1222 FunctionImporter::ImportListsTy &ImportLists,
1223 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1224 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1225 // For each module that has function defined, compute the import/export lists.
1226 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1227 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1228 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1229 << DefinedGVSummaries.first << "'\n");
1230 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1231 ModName: DefinedGVSummaries.first, ImportList);
1232 }
1233
1234 // When computing imports we only added the variables and functions being
1235 // imported to the export list. We also need to mark any references and calls
1236 // they make as exported as well. We do this here, as it is more efficient
1237 // since we may import the same values multiple times into different modules
1238 // during the import computation.
1239 for (auto &ELI : ExportLists) {
1240 // `NewExports` tracks the VI that gets exported because the full definition
1241 // of its user/referencer gets exported.
1242 FunctionImporter::ExportSetTy NewExports;
1243 const auto &DefinedGVSummaries =
1244 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1245 for (auto &EI : ELI.second) {
1246 // Find the copy defined in the exporting module so that we can mark the
1247 // values it references in that specific definition as exported.
1248 // Below we will add all references and called values, without regard to
1249 // whether they are also defined in this module. We subsequently prune the
1250 // list to only include those defined in the exporting module, see comment
1251 // there as to why.
1252 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1253 // Anything marked exported during the import computation must have been
1254 // defined in the exporting module.
1255 assert(DS != DefinedGVSummaries.end());
1256 auto *S = DS->getSecond();
1257 S = S->getBaseObject();
1258 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1259 // Export referenced functions and variables. We don't export/promote
1260 // objects referenced by writeonly variable initializer, because
1261 // we convert such variables initializers to "zeroinitializer".
1262 // See processGlobalForThinLTO.
1263 if (!Index.isWriteOnly(GVS))
1264 NewExports.insert_range(R: GVS->refs());
1265 } else {
1266 auto *FS = cast<FunctionSummary>(Val: S);
1267 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1268 NewExports.insert_range(R: FS->refs());
1269 }
1270 }
1271 // Prune list computed above to only include values defined in the
1272 // exporting module. We do this after the above insertion since we may hit
1273 // the same ref/call target multiple times in above loop, and it is more
1274 // efficient to avoid a set lookup each time.
1275 for (auto EI = NewExports.begin(); EI != NewExports.end();) {
1276 if (!DefinedGVSummaries.count(Val: EI->getGUID()))
1277 NewExports.erase(I: EI++);
1278 else
1279 ++EI;
1280 }
1281 ELI.second.insert_range(R&: NewExports);
1282 }
1283
1284 assert(checkVariableImport(Index, ImportLists, ExportLists));
1285#ifndef NDEBUG
1286 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1287 << " modules:\n");
1288 for (const auto &ModuleImports : ImportLists) {
1289 auto ModName = ModuleImports.first;
1290 auto &Exports = ExportLists[ModName];
1291 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1292 DenseMap<StringRef, ImportStatistics> Histogram =
1293 collectImportStatistics(Index, ModuleImports.second);
1294 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1295 << Exports.size() - NumGVS << " functions and " << NumGVS
1296 << " vars. Imports from " << Histogram.size()
1297 << " modules.\n");
1298 for (const auto &[SrcModName, Stats] : Histogram) {
1299 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1300 << " function definitions and "
1301 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1302 << " function declarations imported from " << SrcModName
1303 << "\n");
1304 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1305 << " global vars imported from " << SrcModName << "\n");
1306 }
1307 }
1308#endif
1309}
1310
1311#ifndef NDEBUG
1312static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1313 StringRef ModulePath,
1314 FunctionImporter::ImportMapTy &ImportList) {
1315 DenseMap<StringRef, ImportStatistics> Histogram =
1316 collectImportStatistics(Index, ImportList);
1317 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1318 << Histogram.size() << " modules.\n");
1319 for (const auto &[SrcModName, Stats] : Histogram) {
1320 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1321 << " function definitions and "
1322 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1323 << " function declarations imported from " << SrcModName
1324 << "\n");
1325 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1326 << SrcModName << "\n");
1327 }
1328}
1329#endif
1330
1331/// Compute all the imports for the given module using the Index.
1332///
1333/// \p isPrevailing is a callback that will be called with a global value's GUID
1334/// and summary and should return whether the module corresponding to the
1335/// summary contains the linker-prevailing copy of that value.
1336///
1337/// \p ImportList will be populated with a map that can be passed to
1338/// FunctionImporter::importFunctions() above (see description there).
1339static void ComputeCrossModuleImportForModuleForTest(
1340 StringRef ModulePath,
1341 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1342 isPrevailing,
1343 const ModuleSummaryIndex &Index,
1344 FunctionImporter::ImportMapTy &ImportList) {
1345 // Collect the list of functions this module defines.
1346 // GUID -> Summary
1347 GVSummaryMapTy FunctionSummaryMap;
1348 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1349
1350 // Compute the import list for this module.
1351 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1352 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1353 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1354
1355#ifndef NDEBUG
1356 dumpImportListForModule(Index, ModulePath, ImportList);
1357#endif
1358}
1359
1360/// Mark all external summaries in \p Index for import into the given module.
1361/// Used for testing the case of distributed builds using a distributed index.
1362///
1363/// \p ImportList will be populated with a map that can be passed to
1364/// FunctionImporter::importFunctions() above (see description there).
1365static void ComputeCrossModuleImportForModuleFromIndexForTest(
1366 StringRef ModulePath, const ModuleSummaryIndex &Index,
1367 FunctionImporter::ImportMapTy &ImportList) {
1368 for (const auto &GlobalList : Index) {
1369 // Ignore entries for undefined references.
1370 if (GlobalList.second.SummaryList.empty())
1371 continue;
1372
1373 auto GUID = GlobalList.first;
1374 assert(GlobalList.second.SummaryList.size() == 1 &&
1375 "Expected individual combined index to have one summary per GUID");
1376 auto &Summary = GlobalList.second.SummaryList[0];
1377 // Skip the summaries for the importing module. These are included to
1378 // e.g. record required linkage changes.
1379 if (Summary->modulePath() == ModulePath)
1380 continue;
1381 // Add an entry to provoke importing by thinBackend.
1382 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1383 }
1384#ifndef NDEBUG
1385 dumpImportListForModule(Index, ModulePath, ImportList);
1386#endif
1387}
1388
1389// For SamplePGO, the indirect call targets for local functions will
1390// have its original name annotated in profile. We try to find the
1391// corresponding PGOFuncName as the GUID, and fix up the edges
1392// accordingly.
1393void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1394 FunctionSummary *FS) {
1395 for (auto &EI : FS->mutableCalls()) {
1396 if (!EI.first.getSummaryList().empty())
1397 continue;
1398 auto GUID = Index.getGUIDFromOriginalID(OriginalID: EI.first.getGUID());
1399 if (GUID == 0)
1400 continue;
1401 // Update the edge to point directly to the correct GUID.
1402 auto VI = Index.getValueInfo(GUID);
1403 if (llvm::any_of(
1404 Range: VI.getSummaryList(),
1405 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1406 // The mapping from OriginalId to GUID may return a GUID
1407 // that corresponds to a static variable. Filter it out here.
1408 // This can happen when
1409 // 1) There is a call to a library function which is not defined
1410 // in the index.
1411 // 2) There is a static variable with the OriginalGUID identical
1412 // to the GUID of the library function in 1);
1413 // When this happens the static variable in 2) will be found,
1414 // which needs to be filtered out.
1415 return SummaryPtr->getSummaryKind() ==
1416 GlobalValueSummary::GlobalVarKind;
1417 }))
1418 continue;
1419 EI.first = VI;
1420 }
1421}
1422
1423void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1424 for (const auto &Entry : Index) {
1425 for (const auto &S : Entry.second.SummaryList) {
1426 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1427 updateValueInfoForIndirectCalls(Index, FS);
1428 }
1429 }
1430}
1431
1432void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1433 ModuleSummaryIndex &Index,
1434 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1435 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1436 assert(!Index.withGlobalValueDeadStripping());
1437 if (!ComputeDead ||
1438 // Don't do anything when nothing is live, this is friendly with tests.
1439 GUIDPreservedSymbols.empty()) {
1440 // Still need to update indirect calls.
1441 updateIndirectCalls(Index);
1442 return;
1443 }
1444 unsigned LiveSymbols = 0;
1445 SmallVector<ValueInfo, 128> Worklist;
1446 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1447 for (auto GUID : GUIDPreservedSymbols) {
1448 ValueInfo VI = Index.getValueInfo(GUID);
1449 if (!VI)
1450 continue;
1451 for (const auto &S : VI.getSummaryList())
1452 S->setLive(true);
1453 }
1454
1455 // Add values flagged in the index as live roots to the worklist.
1456 for (const auto &Entry : Index) {
1457 auto VI = Index.getValueInfo(R: Entry);
1458 for (const auto &S : Entry.second.SummaryList) {
1459 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1460 updateValueInfoForIndirectCalls(Index, FS);
1461 if (S->isLive()) {
1462 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1463 Worklist.push_back(Elt: VI);
1464 ++LiveSymbols;
1465 break;
1466 }
1467 }
1468 }
1469
1470 // Make value live and add it to the worklist if it was not live before.
1471 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1472 // FIXME: If we knew which edges were created for indirect call profiles,
1473 // we could skip them here. Any that are live should be reached via
1474 // other edges, e.g. reference edges. Otherwise, using a profile collected
1475 // on a slightly different binary might provoke preserving, importing
1476 // and ultimately promoting calls to functions not linked into this
1477 // binary, which increases the binary size unnecessarily. Note that
1478 // if this code changes, the importer needs to change so that edges
1479 // to functions marked dead are skipped.
1480
1481 if (llvm::any_of(Range: VI.getSummaryList(),
1482 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1483 return S->isLive();
1484 }))
1485 return;
1486
1487 // We only keep live symbols that are known to be non-prevailing if any are
1488 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1489 // later in the EliminateAvailableExternally pass and setting them to
1490 // not-live could break downstreams users of liveness information (PR36483)
1491 // or limit optimization opportunities.
1492 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1493 bool KeepAliveLinkage = false;
1494 bool Interposable = false;
1495 for (const auto &S : VI.getSummaryList()) {
1496 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1497 S->linkage() == GlobalValue::WeakODRLinkage ||
1498 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1499 KeepAliveLinkage = true;
1500 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1501 Interposable = true;
1502 }
1503
1504 if (!IsAliasee) {
1505 if (!KeepAliveLinkage)
1506 return;
1507
1508 if (Interposable)
1509 report_fatal_error(
1510 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1511 "symbol");
1512 }
1513 }
1514
1515 for (const auto &S : VI.getSummaryList())
1516 S->setLive(true);
1517 ++LiveSymbols;
1518 Worklist.push_back(Elt: VI);
1519 };
1520
1521 while (!Worklist.empty()) {
1522 auto VI = Worklist.pop_back_val();
1523 for (const auto &Summary : VI.getSummaryList()) {
1524 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1525 // If this is an alias, visit the aliasee VI to ensure that all copies
1526 // are marked live and it is added to the worklist for further
1527 // processing of its references.
1528 visit(AS->getAliaseeVI(), true);
1529 continue;
1530 }
1531 for (auto Ref : Summary->refs())
1532 visit(Ref, false);
1533 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1534 for (auto Call : FS->calls())
1535 visit(Call.first, false);
1536 }
1537 }
1538 Index.setWithGlobalValueDeadStripping();
1539
1540 unsigned DeadSymbols = Index.size() - LiveSymbols;
1541 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1542 << " symbols Dead \n");
1543 NumDeadSymbols += DeadSymbols;
1544 NumLiveSymbols += LiveSymbols;
1545}
1546
1547// Compute dead symbols and propagate constants in combined index.
1548void llvm::computeDeadSymbolsWithConstProp(
1549 ModuleSummaryIndex &Index,
1550 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1551 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1552 bool ImportEnabled) {
1553 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1554 isPrevailing);
1555 if (ImportEnabled)
1556 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1557}
1558
1559/// Compute the set of summaries needed for a ThinLTO backend compilation of
1560/// \p ModulePath.
1561void llvm::gatherImportedSummariesForModule(
1562 StringRef ModulePath,
1563 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1564 const FunctionImporter::ImportMapTy &ImportList,
1565 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1566 GVSummaryPtrSet &DecSummaries) {
1567 // Include all summaries from the importing module.
1568 ModuleToSummariesForIndex[std::string(ModulePath)] =
1569 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1570
1571 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1572 // lets us look up the map without allocating an instance of std::string when
1573 // the key-value pair exists in the map.
1574 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1575 // C++26 when it becomes available for our codebase.
1576 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1577 StringRef Key) -> GVSummaryMapTy & {
1578 auto It = Map.find(x: Key);
1579 if (It == Map.end())
1580 std::tie(args&: It, args: std::ignore) =
1581 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1582 return It->second;
1583 };
1584
1585 // Include summaries for imports.
1586 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1587 auto &SummariesForIndex =
1588 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1589
1590 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1591 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1592 assert(DS != DefinedGVSummaries.end() &&
1593 "Expected a defined summary for imported global value");
1594 if (ImportType == GlobalValueSummary::Declaration)
1595 DecSummaries.insert(x: DS->second);
1596
1597 SummariesForIndex[GUID] = DS->second;
1598 }
1599}
1600
1601/// Emit the files \p ModulePath will import from into \p OutputFilename.
1602Error llvm::EmitImportsFiles(
1603 StringRef ModulePath, StringRef OutputFilename,
1604 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1605 std::error_code EC;
1606 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1607 if (EC)
1608 return createFileError(F: "cannot open " + OutputFilename,
1609 E: errorCodeToError(EC));
1610 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1611 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1612 return Error::success();
1613}
1614
1615/// Invoke callback \p F on the file paths from which \p ModulePath
1616/// will import.
1617void llvm::processImportsFiles(
1618 StringRef ModulePath,
1619 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1620 function_ref<void(const std::string &)> F) {
1621 for (const auto &ILI : ModuleToSummariesForIndex)
1622 // The ModuleToSummariesForIndex map includes an entry for the current
1623 // Module (needed for writing out the index files). We don't want to
1624 // include it in the imports file, however, so filter it out.
1625 if (ILI.first != ModulePath)
1626 F(ILI.first);
1627}
1628
1629bool llvm::convertToDeclaration(GlobalValue &GV) {
1630 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1631 << "\n");
1632 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1633 F->deleteBody();
1634 F->clearMetadata();
1635 F->setComdat(nullptr);
1636 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1637 V->setInitializer(nullptr);
1638 V->setLinkage(GlobalValue::ExternalLinkage);
1639 V->clearMetadata();
1640 V->setComdat(nullptr);
1641 } else {
1642 GlobalValue *NewGV;
1643 if (GV.getValueType()->isFunctionTy())
1644 NewGV =
1645 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1646 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1647 N: "", M: GV.getParent());
1648 else
1649 NewGV =
1650 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1651 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1652 /*init*/ nullptr, "",
1653 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1654 GV.getType()->getAddressSpace());
1655 NewGV->takeName(V: &GV);
1656 GV.replaceAllUsesWith(V: NewGV);
1657 return false;
1658 }
1659 if (!GV.isImplicitDSOLocal())
1660 GV.setDSOLocal(false);
1661 return true;
1662}
1663
1664void llvm::thinLTOFinalizeInModule(Module &TheModule,
1665 const GVSummaryMapTy &DefinedGlobals,
1666 bool PropagateAttrs) {
1667 DenseSet<Comdat *> NonPrevailingComdats;
1668 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1669 // See if the global summary analysis computed a new resolved linkage.
1670 const auto &GS = DefinedGlobals.find(Val: GV.getGUID());
1671 if (GS == DefinedGlobals.end())
1672 return;
1673
1674 if (Propagate)
1675 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1676 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1677 // TODO: propagate ReadNone and ReadOnly.
1678 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1679 F->setDoesNotAccessMemory();
1680
1681 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1682 F->setOnlyReadsMemory();
1683
1684 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1685 F->setDoesNotRecurse();
1686
1687 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1688 F->setDoesNotThrow();
1689 }
1690 }
1691
1692 auto NewLinkage = GS->second->linkage();
1693 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1694 // Don't internalize anything here, because the code below
1695 // lacks necessary correctness checks. Leave this job to
1696 // LLVM 'internalize' pass.
1697 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1698 // In case it was dead and already converted to declaration.
1699 GV.isDeclaration())
1700 return;
1701
1702 // Set the potentially more constraining visibility computed from summaries.
1703 // The DefaultVisibility condition is because older GlobalValueSummary does
1704 // not record DefaultVisibility and we don't want to change protected/hidden
1705 // to default.
1706 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1707 GV.setVisibility(GS->second->getVisibility());
1708
1709 if (NewLinkage == GV.getLinkage())
1710 return;
1711
1712 // Check for a non-prevailing def that has interposable linkage
1713 // (e.g. non-odr weak or linkonce). In that case we can't simply
1714 // convert to available_externally, since it would lose the
1715 // interposable property and possibly get inlined. Simply drop
1716 // the definition in that case.
1717 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1718 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1719 if (!convertToDeclaration(GV))
1720 // FIXME: Change this to collect replaced GVs and later erase
1721 // them from the parent module once thinLTOResolvePrevailingGUID is
1722 // changed to enable this for aliases.
1723 llvm_unreachable("Expected GV to be converted");
1724 } else {
1725 // If all copies of the original symbol had global unnamed addr and
1726 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1727 // and are constants, then it should be an auto hide symbol. In that case
1728 // the thin link would have marked it as CanAutoHide. Add hidden
1729 // visibility to the symbol to preserve the property.
1730 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1731 GS->second->canAutoHide()) {
1732 assert(GV.canBeOmittedFromSymbolTable());
1733 GV.setVisibility(GlobalValue::HiddenVisibility);
1734 }
1735
1736 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1737 << "` from " << GV.getLinkage() << " to " << NewLinkage
1738 << "\n");
1739 GV.setLinkage(NewLinkage);
1740 }
1741 // Remove declarations from comdats, including available_externally
1742 // as this is a declaration for the linker, and will be dropped eventually.
1743 // It is illegal for comdats to contain declarations.
1744 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1745 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1746 if (GO->getComdat()->getName() == GO->getName())
1747 NonPrevailingComdats.insert(V: GO->getComdat());
1748 GO->setComdat(nullptr);
1749 }
1750 };
1751
1752 // Process functions and global now
1753 for (auto &GV : TheModule)
1754 FinalizeInModule(GV, PropagateAttrs);
1755 for (auto &GV : TheModule.globals())
1756 FinalizeInModule(GV);
1757 for (auto &GV : TheModule.aliases())
1758 FinalizeInModule(GV);
1759
1760 // For a non-prevailing comdat, all its members must be available_externally.
1761 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1762 // local linkage GlobalValues.
1763 if (NonPrevailingComdats.empty())
1764 return;
1765 for (auto &GO : TheModule.global_objects()) {
1766 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1767 GO.setComdat(nullptr);
1768 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1769 }
1770 }
1771 bool Changed;
1772 do {
1773 Changed = false;
1774 // If an alias references a GlobalValue in a non-prevailing comdat, change
1775 // it to available_externally. For simplicity we only handle GlobalValue and
1776 // ConstantExpr with a base object. ConstantExpr without a base object is
1777 // unlikely used in a COMDAT.
1778 for (auto &GA : TheModule.aliases()) {
1779 if (GA.hasAvailableExternallyLinkage())
1780 continue;
1781 GlobalObject *Obj = GA.getAliaseeObject();
1782 assert(Obj && "aliasee without an base object is unimplemented");
1783 if (Obj->hasAvailableExternallyLinkage()) {
1784 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1785 Changed = true;
1786 }
1787 }
1788 } while (Changed);
1789}
1790
1791/// Run internalization on \p TheModule based on symmary analysis.
1792void llvm::thinLTOInternalizeModule(Module &TheModule,
1793 const GVSummaryMapTy &DefinedGlobals) {
1794 // Declare a callback for the internalize pass that will ask for every
1795 // candidate GlobalValue if it can be internalized or not.
1796 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1797 // It may be the case that GV is on a chain of an ifunc, its alias and
1798 // subsequent aliases. In this case, the summary for the value is not
1799 // available.
1800 if (isa<GlobalIFunc>(Val: &GV) ||
1801 (isa<GlobalAlias>(Val: &GV) &&
1802 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1803 return true;
1804
1805 // Lookup the linkage recorded in the summaries during global analysis.
1806 auto GS = DefinedGlobals.find(Val: GV.getGUID());
1807 if (GS == DefinedGlobals.end()) {
1808 // Must have been promoted (possibly conservatively). Find original
1809 // name so that we can access the correct summary and see if it can
1810 // be internalized again.
1811 // FIXME: Eventually we should control promotion instead of promoting
1812 // and internalizing again.
1813 StringRef OrigName =
1814 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1815 std::string OrigId = GlobalValue::getGlobalIdentifier(
1816 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1817 FileName: TheModule.getSourceFileName());
1818 GS = DefinedGlobals.find(
1819 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1820 if (GS == DefinedGlobals.end()) {
1821 // Also check the original non-promoted non-globalized name. In some
1822 // cases a preempted weak value is linked in as a local copy because
1823 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1824 // In that case, since it was originally not a local value, it was
1825 // recorded in the index using the original name.
1826 // FIXME: This may not be needed once PR27866 is fixed.
1827 GS = DefinedGlobals.find(
1828 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1829 assert(GS != DefinedGlobals.end());
1830 }
1831 }
1832 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1833 };
1834
1835 // FIXME: See if we can just internalize directly here via linkage changes
1836 // based on the index, rather than invoking internalizeModule.
1837 internalizeModule(TheModule, MustPreserveGV);
1838}
1839
1840/// Make alias a clone of its aliasee.
1841static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1842 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1843
1844 ValueToValueMapTy VMap;
1845 Function *NewFn = CloneFunction(F: Fn, VMap);
1846 // Clone should use the original alias's linkage, visibility and name, and we
1847 // ensure all uses of alias instead use the new clone (casted if necessary).
1848 NewFn->setLinkage(GA->getLinkage());
1849 NewFn->setVisibility(GA->getVisibility());
1850 GA->replaceAllUsesWith(V: NewFn);
1851 NewFn->takeName(V: GA);
1852 return NewFn;
1853}
1854
1855// Internalize values that we marked with specific attribute
1856// in processGlobalForThinLTO.
1857static void internalizeGVsAfterImport(Module &M) {
1858 for (auto &GV : M.globals())
1859 // Skip GVs which have been converted to declarations
1860 // by dropDeadSymbols.
1861 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1862 GV.setLinkage(GlobalValue::InternalLinkage);
1863 GV.setVisibility(GlobalValue::DefaultVisibility);
1864 }
1865}
1866
1867// Automatically import functions in Module \p DestModule based on the summaries
1868// index.
1869Expected<bool> FunctionImporter::importFunctions(
1870 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1871 LLVM_DEBUG(dbgs() << "Starting import for Module "
1872 << DestModule.getModuleIdentifier() << "\n");
1873 unsigned ImportedCount = 0, ImportedGVCount = 0;
1874 // Before carrying out any imports, see if this module defines functions in
1875 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1876 // The function will be imported elsewhere, as extenal linkage, and the
1877 // destination doesn't yet have its definition.
1878 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1879 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1880 for (auto &F : DestModule)
1881 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUID()))
1882 F.deleteBody();
1883
1884 IRMover Mover(DestModule);
1885
1886 // Do the actual import of functions now, one Module at a time
1887 for (const auto &ModName : ImportList.getSourceModules()) {
1888 // Get the module for the import
1889 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1890 if (!SrcModuleOrErr)
1891 return SrcModuleOrErr.takeError();
1892 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1893 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1894 "Context mismatch");
1895
1896 // If modules were created with lazy metadata loading, materialize it
1897 // now, before linking it (otherwise this will be a noop).
1898 if (Error Err = SrcModule->materializeMetadata())
1899 return std::move(Err);
1900
1901 // Find the globals to import
1902 SetVector<GlobalValue *> GlobalsToImport;
1903 for (Function &F : *SrcModule) {
1904 if (!F.hasName())
1905 continue;
1906 auto GUID = F.getGUID();
1907 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1908 bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition;
1909
1910 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1911 << " importing function"
1912 << (ImportDefinition
1913 ? " definition "
1914 : (MaybeImportType ? " declaration " : " "))
1915 << GUID << " " << F.getName() << " from "
1916 << SrcModule->getSourceFileName() << "\n");
1917 if (ImportDefinition) {
1918 if (Error Err = F.materialize())
1919 return std::move(Err);
1920 // MemProf should match function's definition and summary,
1921 // 'thinlto_src_module' is needed.
1922 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
1923 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
1924 // statistics and debugging.
1925 F.setMetadata(
1926 Kind: "thinlto_src_module",
1927 Node: MDNode::get(Context&: DestModule.getContext(),
1928 MDs: {MDString::get(Context&: DestModule.getContext(),
1929 Str: SrcModule->getModuleIdentifier())}));
1930 F.setMetadata(
1931 Kind: "thinlto_src_file",
1932 Node: MDNode::get(Context&: DestModule.getContext(),
1933 MDs: {MDString::get(Context&: DestModule.getContext(),
1934 Str: SrcModule->getSourceFileName())}));
1935 }
1936 GlobalsToImport.insert(X: &F);
1937 }
1938 }
1939 for (GlobalVariable &GV : SrcModule->globals()) {
1940 if (!GV.hasName())
1941 continue;
1942 auto GUID = GV.getGUID();
1943 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1944 bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition;
1945
1946 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1947 << " importing global"
1948 << (ImportDefinition
1949 ? " definition "
1950 : (MaybeImportType ? " declaration " : " "))
1951 << GUID << " " << GV.getName() << " from "
1952 << SrcModule->getSourceFileName() << "\n");
1953 if (ImportDefinition) {
1954 if (Error Err = GV.materialize())
1955 return std::move(Err);
1956 ImportedGVCount += GlobalsToImport.insert(X: &GV);
1957 }
1958 }
1959 for (GlobalAlias &GA : SrcModule->aliases()) {
1960 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
1961 continue;
1962 auto GUID = GA.getGUID();
1963 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1964 bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition;
1965
1966 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1967 << " importing alias"
1968 << (ImportDefinition
1969 ? " definition "
1970 : (MaybeImportType ? " declaration " : " "))
1971 << GUID << " " << GA.getName() << " from "
1972 << SrcModule->getSourceFileName() << "\n");
1973 if (ImportDefinition) {
1974 if (Error Err = GA.materialize())
1975 return std::move(Err);
1976 // Import alias as a copy of its aliasee.
1977 GlobalObject *GO = GA.getAliaseeObject();
1978 if (Error Err = GO->materialize())
1979 return std::move(Err);
1980 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
1981 LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID() << " "
1982 << GO->getName() << " from "
1983 << SrcModule->getSourceFileName() << "\n");
1984 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
1985 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
1986 // statistics and debugging.
1987 Fn->setMetadata(
1988 Kind: "thinlto_src_module",
1989 Node: MDNode::get(Context&: DestModule.getContext(),
1990 MDs: {MDString::get(Context&: DestModule.getContext(),
1991 Str: SrcModule->getModuleIdentifier())}));
1992 Fn->setMetadata(
1993 Kind: "thinlto_src_file",
1994 Node: MDNode::get(Context&: DestModule.getContext(),
1995 MDs: {MDString::get(Context&: DestModule.getContext(),
1996 Str: SrcModule->getSourceFileName())}));
1997 }
1998 GlobalsToImport.insert(X: Fn);
1999 }
2000 }
2001
2002 // Upgrade debug info after we're done materializing all the globals and we
2003 // have loaded all the required metadata!
2004 UpgradeDebugInfo(M&: *SrcModule);
2005
2006 // Set the partial sample profile ratio in the profile summary module flag
2007 // of the imported source module, if applicable, so that the profile summary
2008 // module flag will match with that of the destination module when it's
2009 // imported.
2010 SrcModule->setPartialSampleProfileRatio(Index);
2011
2012 // Link in the specified functions.
2013 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2014 GlobalsToImport: &GlobalsToImport);
2015
2016 if (PrintImports) {
2017 for (const auto *GV : GlobalsToImport)
2018 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2019 << " from " << SrcModule->getSourceFileName() << "\n";
2020 }
2021
2022 if (Error Err = Mover.move(Src: std::move(SrcModule),
2023 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2024 /*IsPerformingImport=*/true))
2025 return createStringError(EC: errc::invalid_argument,
2026 S: Twine("Function Import: link error: ") +
2027 toString(E: std::move(Err)));
2028
2029 ImportedCount += GlobalsToImport.size();
2030 NumImportedModules++;
2031 }
2032
2033 internalizeGVsAfterImport(M&: DestModule);
2034
2035 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2036 NumImportedGlobalVars += ImportedGVCount;
2037
2038 // TODO: Print counters for definitions and declarations in the debugging log.
2039 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2040 << " functions for Module "
2041 << DestModule.getModuleIdentifier() << "\n");
2042 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2043 << " global variables for Module "
2044 << DestModule.getModuleIdentifier() << "\n");
2045 return ImportedCount;
2046}
2047
2048static bool doImportingForModuleForTest(
2049 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2050 isPrevailing) {
2051 if (SummaryFile.empty())
2052 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2053 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2054 getModuleSummaryIndexForFile(Path: SummaryFile);
2055 if (!IndexPtrOrErr) {
2056 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2057 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2058 return false;
2059 }
2060 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2061
2062 // First step is collecting the import list.
2063 FunctionImporter::ImportIDTable ImportIDs;
2064 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2065 // If requested, simply import all functions in the index. This is used
2066 // when testing distributed backend handling via the opt tool, when
2067 // we have distributed indexes containing exactly the summaries to import.
2068 if (ImportAllIndex)
2069 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2070 Index: *Index, ImportList);
2071 else
2072 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2073 isPrevailing, Index: *Index, ImportList);
2074
2075 // Conservatively mark all internal values as promoted. This interface is
2076 // only used when doing importing via the function importing pass. The pass
2077 // is only enabled when testing importing via the 'opt' tool, which does
2078 // not do the ThinLink that would normally determine what values to promote.
2079 for (auto &I : *Index) {
2080 for (auto &S : I.second.SummaryList) {
2081 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2082 S->setLinkage(GlobalValue::ExternalLinkage);
2083 }
2084 }
2085
2086 // Next we need to promote to global scope and rename any local values that
2087 // are potentially exported to other modules.
2088 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2089 /*GlobalsToImport=*/nullptr);
2090
2091 // Perform the import now.
2092 auto ModuleLoader = [&M](StringRef Identifier) {
2093 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2094 };
2095 FunctionImporter Importer(*Index, ModuleLoader,
2096 /*ClearDSOLocalOnDeclarations=*/false);
2097 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2098
2099 // FIXME: Probably need to propagate Errors through the pass manager.
2100 if (!Result) {
2101 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2102 ErrorBanner: "Error importing module: ");
2103 return true;
2104 }
2105
2106 return true;
2107}
2108
2109PreservedAnalyses FunctionImportPass::run(Module &M,
2110 ModuleAnalysisManager &AM) {
2111 // This is only used for testing the function import pass via opt, where we
2112 // don't have prevailing information from the LTO context available, so just
2113 // conservatively assume everything is prevailing (which is fine for the very
2114 // limited use of prevailing checking in this pass).
2115 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2116 return true;
2117 };
2118 if (!doImportingForModuleForTest(M, isPrevailing))
2119 return PreservedAnalyses::all();
2120
2121 return PreservedAnalyses::none();
2122}
2123