1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Bitcode/BitcodeReader.h"
21#include "llvm/IR/AutoUpgrade.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/GlobalObject.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/Metadata.h"
29#include "llvm/IR/Module.h"
30#include "llvm/IR/ModuleSummaryIndex.h"
31#include "llvm/IRReader/IRReader.h"
32#include "llvm/Linker/IRMover.h"
33#include "llvm/ProfileData/PGOCtxProfReader.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/Errc.h"
38#include "llvm/Support/Error.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/FileSystem.h"
41#include "llvm/Support/JSON.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/SourceMgr.h"
44#include "llvm/Support/TimeProfiler.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Transforms/IPO/Internalize.h"
47#include "llvm/Transforms/Utils/Cloning.h"
48#include "llvm/Transforms/Utils/FunctionImportUtils.h"
49#include "llvm/Transforms/Utils/ModuleUtils.h"
50#include "llvm/Transforms/Utils/ValueMapper.h"
51#include <cassert>
52#include <memory>
53#include <string>
54#include <system_error>
55#include <tuple>
56#include <utility>
57
58using namespace llvm;
59
60#define DEBUG_TYPE "function-import"
61
62STATISTIC(NumImportedFunctionsThinLink,
63 "Number of functions thin link decided to import");
64STATISTIC(NumImportedHotFunctionsThinLink,
65 "Number of hot functions thin link decided to import");
66STATISTIC(NumImportedCriticalFunctionsThinLink,
67 "Number of critical functions thin link decided to import");
68STATISTIC(NumImportedGlobalVarsThinLink,
69 "Number of global variables thin link decided to import");
70STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
71STATISTIC(NumImportedGlobalVars,
72 "Number of global variables imported in backend");
73STATISTIC(NumImportedModules, "Number of modules imported from");
74STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
75STATISTIC(NumLiveSymbols, "Number of live symbols in index");
76
77namespace llvm {
78extern cl::opt<bool> AlwaysRenamePromotedLocals;
79
80cl::opt<bool>
81 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
82 cl::desc("Import functions with noinline attribute"));
83
84/// Limit on instruction count of imported functions.
85static cl::opt<unsigned> ImportInstrLimit(
86 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
87 cl::desc("Only import functions with less than N instructions"));
88
89static cl::opt<int> ImportCutoff(
90 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
91 cl::desc("Only import first N functions if N>=0 (default -1)"));
92
93static cl::opt<float>
94 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
95 cl::Hidden, cl::value_desc("x"),
96 cl::desc("As we import functions, multiply the "
97 "`import-instr-limit` threshold by this factor "
98 "before processing newly imported functions"));
99
100static cl::opt<float> ImportHotInstrFactor(
101 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
102 cl::value_desc("x"),
103 cl::desc("As we import functions called from hot callsite, multiply the "
104 "`import-instr-limit` threshold by this factor "
105 "before processing newly imported functions"));
106
107static cl::opt<float> ImportHotMultiplier(
108 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
109 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
110
111static cl::opt<float> ImportCriticalMultiplier(
112 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
113 cl::value_desc("x"),
114 cl::desc(
115 "Multiply the `import-instr-limit` threshold for critical callsites"));
116
117// FIXME: This multiplier was not really tuned up.
118static cl::opt<float> ImportColdMultiplier(
119 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
120 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
121
122static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
123 cl::desc("Print imported functions"));
124
125static cl::opt<bool> PrintImportFailures(
126 "print-import-failures", cl::init(Val: false), cl::Hidden,
127 cl::desc("Print information for functions rejected for importing"));
128
129static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
130 cl::desc("Compute dead symbols"));
131
132static cl::opt<bool> EnableImportMetadata(
133 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
134 cl::desc("Enable import metadata like 'thinlto_src_module' and "
135 "'thinlto_src_file'"));
136
137/// Summary file to use for function importing when using -function-import from
138/// the command line.
139static cl::opt<std::string>
140 SummaryFile("summary-file",
141 cl::desc("The summary file to use for function importing."));
142
143/// Used when testing importing from distributed indexes via opt
144// -function-import.
145static cl::opt<bool>
146 ImportAllIndex("import-all-index",
147 cl::desc("Import all external functions in index."));
148
149/// This is a test-only option.
150/// If this option is enabled, the ThinLTO indexing step will import each
151/// function declaration as a fallback. In a real build this may increase ram
152/// usage of the indexing step unnecessarily.
153/// TODO: Implement selective import (based on combined summary analysis) to
154/// ensure the imported function has a use case in the postlink pipeline.
155static cl::opt<bool> ImportDeclaration(
156 "import-declaration", cl::init(Val: false), cl::Hidden,
157 cl::desc("If true, import function declaration as fallback if the function "
158 "definition is not imported."));
159
160/// Pass a workload description file - an example of workload would be the
161/// functions executed to satisfy a RPC request. A workload is defined by a root
162/// function and the list of functions that are (frequently) needed to satisfy
163/// it. The module that defines the root will have all those functions imported.
164/// The file contains a JSON dictionary. The keys are root functions, the values
165/// are lists of functions to import in the module defining the root. It is
166/// assumed -funique-internal-linkage-names was used, thus ensuring function
167/// names are unique even for local linkage ones.
168static cl::opt<std::string> WorkloadDefinitions(
169 "thinlto-workload-def",
170 cl::desc("Pass a workload definition. This is a file containing a JSON "
171 "dictionary. The keys are root functions, the values are lists of "
172 "functions to import in the module defining the root. It is "
173 "assumed -funique-internal-linkage-names was used, to ensure "
174 "local linkage functions have unique names. For example: \n"
175 "{\n"
176 " \"rootFunction_1\": [\"function_to_import_1\", "
177 "\"function_to_import_2\"], \n"
178 " \"rootFunction_2\": [\"function_to_import_3\", "
179 "\"function_to_import_4\"] \n"
180 "}"),
181 cl::Hidden);
182
183extern cl::opt<std::string> UseCtxProfile;
184
185static cl::opt<bool> CtxprofMoveRootsToOwnModule(
186 "thinlto-move-ctxprof-trees",
187 cl::desc("Move contextual profiling roots and the graphs under them in "
188 "their own module."),
189 cl::Hidden, cl::init(Val: false));
190
191extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
192
193extern cl::opt<bool> EnableMemProfContextDisambiguation;
194} // end namespace llvm
195
196// Load lazily a module from \p FileName in \p Context.
197static std::unique_ptr<Module> loadFile(const std::string &FileName,
198 LLVMContext &Context) {
199 SMDiagnostic Err;
200 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
201 // Metadata isn't loaded until functions are imported, to minimize
202 // the memory overhead.
203 std::unique_ptr<Module> Result =
204 getLazyIRFileModule(Filename: FileName, Err, Context,
205 /* ShouldLazyLoadMetadata = */ true);
206 if (!Result) {
207 Err.print(ProgName: "function-import", S&: errs());
208 report_fatal_error(reason: "Abort");
209 }
210
211 return Result;
212}
213
214static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
215 size_t NumDefs,
216 StringRef ImporterModule) {
217 // We can import a local when there is one definition.
218 if (NumDefs == 1)
219 return false;
220 // In other cases, make sure we import the copy in the caller's module if the
221 // referenced value has local linkage. The only time a local variable can
222 // share an entry in the index is if there is a local with the same name in
223 // another module that had the same source file name (in a different
224 // directory), where each was compiled in their own directory so there was not
225 // distinguishing path.
226 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
227 RefSummary->modulePath() != ImporterModule;
228}
229
230/// Given a list of possible callee implementation for a call site, qualify the
231/// legality of importing each. The return is a range of pairs. Each pair
232/// corresponds to a candidate. The first value is the ImportFailureReason for
233/// that candidate, the second is the candidate.
234static auto qualifyCalleeCandidates(
235 const ModuleSummaryIndex &Index,
236 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
237 StringRef CallerModulePath) {
238 return llvm::map_range(
239 C&: CalleeSummaryList,
240 F: [&Index, CalleeSummaryList,
241 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
242 -> std::pair<FunctionImporter::ImportFailureReason,
243 const GlobalValueSummary *> {
244 auto *GVSummary = SummaryPtr.get();
245 if (!Index.isGlobalValueLive(GVS: GVSummary))
246 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
247
248 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
249 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
250 GVSummary};
251
252 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
253
254 // Ignore any callees that aren't actually functions. This could happen
255 // in the case of GUID hash collisions. It could also happen in theory
256 // for SamplePGO profiles collected on old versions of the code after
257 // renaming, since we synthesize edges to any inlined callees appearing
258 // in the profile.
259 if (!Summary)
260 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
261
262 // If this is a local function, make sure we import the copy in the
263 // caller's module. The only time a local function can share an entry in
264 // the index is if there is a local with the same name in another module
265 // that had the same source file name (in a different directory), where
266 // each was compiled in their own directory so there was not
267 // distinguishing path.
268 // If the local function is from another module, it must be a reference
269 // due to indirect call profile data since a function pointer can point
270 // to a local in another module. Do the import from another module if
271 // there is only one entry in the list or when all files in the program
272 // are compiled with full path - in both cases the local function has
273 // unique PGO name and GUID.
274 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
275 ImporterModule: CallerModulePath))
276 return {
277 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
278 GVSummary};
279
280 // Skip if it isn't legal to import (e.g. may reference unpromotable
281 // locals).
282 if (Summary->notEligibleToImport())
283 return {FunctionImporter::ImportFailureReason::NotEligible,
284 GVSummary};
285
286 return {FunctionImporter::ImportFailureReason::None, GVSummary};
287 });
288}
289
290/// Given a list of possible callee implementation for a call site, select one
291/// that fits the \p Threshold for function definition import. If none are
292/// found, the Reason will give the last reason for the failure (last, in the
293/// order of CalleeSummaryList entries). While looking for a callee definition,
294/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
295/// candidate; other modules may want to know the function summary or
296/// declaration even if a definition is not needed.
297///
298/// FIXME: select "best" instead of first that fits. But what is "best"?
299/// - The smallest: more likely to be inlined.
300/// - The one with the least outgoing edges (already well optimized).
301/// - One from a module already being imported from in order to reduce the
302/// number of source modules parsed/linked.
303/// - One that has PGO data attached.
304/// - [insert you fancy metric here]
305static const GlobalValueSummary *
306selectCallee(const ModuleSummaryIndex &Index,
307 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
308 unsigned Threshold, StringRef CallerModulePath,
309 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
310 FunctionImporter::ImportFailureReason &Reason) {
311 // Records the last summary with reason noinline or too-large.
312 TooLargeOrNoInlineSummary = nullptr;
313 auto QualifiedCandidates =
314 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
315 for (auto QualifiedValue : QualifiedCandidates) {
316 Reason = QualifiedValue.first;
317 // Skip a summary if its import is not (proved to be) legal.
318 if (Reason != FunctionImporter::ImportFailureReason::None)
319 continue;
320 auto *Summary =
321 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
322
323 // Don't bother importing the definition if the chance of inlining it is
324 // not high enough (except under `--force-import-all`).
325 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
326 !ForceImportAll) {
327 TooLargeOrNoInlineSummary = Summary;
328 Reason = FunctionImporter::ImportFailureReason::TooLarge;
329 continue;
330 }
331
332 // Don't bother importing the definition if we can't inline it anyway.
333 if (Summary->fflags().NoInline && !ForceImportAll) {
334 TooLargeOrNoInlineSummary = Summary;
335 Reason = FunctionImporter::ImportFailureReason::NoInline;
336 continue;
337 }
338
339 return Summary;
340 }
341 return nullptr;
342}
343
344namespace {
345
346using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
347
348} // anonymous namespace
349
350FunctionImporter::ImportMapTy::AddDefinitionStatus
351FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
352 GlobalValue::GUID GUID) {
353 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
354 if (!Imports.insert(V: Def).second)
355 // Already there.
356 return AddDefinitionStatus::NoChange;
357
358 // Remove Decl in case it's there. Note that a definition takes precedence
359 // over a declaration for a given GUID.
360 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
361 : AddDefinitionStatus::Inserted;
362}
363
364void FunctionImporter::ImportMapTy::maybeAddDeclaration(
365 StringRef FromModule, GlobalValue::GUID GUID) {
366 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
367 // Insert Decl only if Def is not present. Note that a definition takes
368 // precedence over a declaration for a given GUID.
369 if (!Imports.contains(V: Def))
370 Imports.insert(V: Decl);
371}
372
373SmallVector<StringRef, 0>
374FunctionImporter::ImportMapTy::getSourceModules() const {
375 SetVector<StringRef> ModuleSet;
376 for (const auto &[SrcMod, GUID, ImportType] : *this)
377 ModuleSet.insert(X: SrcMod);
378 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
379 llvm::sort(C&: Modules);
380 return Modules;
381}
382
383std::optional<GlobalValueSummary::ImportKind>
384FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
385 GlobalValue::GUID GUID) const {
386 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
387 auto [Def, Decl] = *IDPair;
388 if (Imports.contains(V: Def))
389 return GlobalValueSummary::Definition;
390 if (Imports.contains(V: Decl))
391 return GlobalValueSummary::Declaration;
392 }
393 return std::nullopt;
394}
395
396/// Import globals referenced by a function or other globals that are being
397/// imported, if importing such global is possible.
398class GlobalsImporter final {
399 const ModuleSummaryIndex &Index;
400 const GVSummaryMapTy &DefinedGVSummaries;
401 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
402 IsPrevailing;
403 FunctionImporter::ImportMapTy &ImportList;
404 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
405
406 bool shouldImportGlobal(const ValueInfo &VI) {
407 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
408 if (GVS == DefinedGVSummaries.end())
409 return true;
410 // We should not skip import if the module contains a non-prevailing
411 // definition with interposable linkage type. This is required for
412 // correctness in the situation where there is a prevailing def available
413 // for import and marked read-only. In this case, the non-prevailing def
414 // will be converted to a declaration, while the prevailing one becomes
415 // internal, thus no definitions will be available for linking. In order to
416 // prevent undefined symbol link error, the prevailing definition must be
417 // imported.
418 // FIXME: Consider adding a check that the suitable prevailing definition
419 // exists and marked read-only.
420 if (VI.getSummaryList().size() > 1 &&
421 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
422 !IsPrevailing(VI.getGUID(), GVS->second))
423 return true;
424
425 return false;
426 }
427
428 void
429 onImportingSummaryImpl(const GlobalValueSummary &Summary,
430 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
431 for (const auto &VI : Summary.refs()) {
432 if (!shouldImportGlobal(VI)) {
433 LLVM_DEBUG(
434 dbgs() << "Ref ignored! Target already in destination module.\n");
435 continue;
436 }
437
438 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
439
440 for (const auto &RefSummary : VI.getSummaryList()) {
441 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
442 // Stop looking if this is not a global variable, e.g. a function.
443 // Functions could be referenced by global vars - e.g. a vtable; but we
444 // don't currently imagine a reason those would be imported here, rather
445 // than as part of the logic deciding which functions to import (i.e.
446 // based on profile information). Should we decide to handle them here,
447 // we can refactor accordingly at that time.
448 // Note that it is safe to stop looking because the one case where we
449 // might have to import (a read/write-only global variable) cannot occur
450 // if this GUID has a non-variable summary. The only case where we even
451 // might find another summary in the list that is a variable is in the
452 // case of same-named locals in different modules not compiled with
453 // enough path, and during attribute propagation we will mark all
454 // summaries for a GUID (ValueInfo) as non read/write-only if any is not
455 // a global variable.
456 if (!GVS)
457 break;
458 bool CanImportDecl = false;
459 if (shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
460 ImporterModule: Summary.modulePath()) ||
461 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
462 CanImportDecl)) {
463 if (ImportDeclaration && CanImportDecl)
464 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
465 GUID: VI.getGUID());
466
467 continue;
468 }
469
470 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
471 // Otherwise, definition should take precedence over declaration.
472 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
473 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
474 break;
475
476 // Only update stat and exports if we haven't already imported this
477 // variable.
478 NumImportedGlobalVarsThinLink++;
479 // Any references made by this variable will be marked exported
480 // later, in ComputeCrossModuleImport, after import decisions are
481 // complete, which is more efficient than adding them here.
482 if (ExportLists)
483 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
484
485 // If variable is not writeonly we attempt to recursively analyze
486 // its references in order to import referenced constants.
487 if (!Index.isWriteOnly(GVS))
488 Worklist.emplace_back(Args&: GVS);
489 break;
490 }
491 }
492 }
493
494public:
495 GlobalsImporter(
496 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
497 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
498 IsPrevailing,
499 FunctionImporter::ImportMapTy &ImportList,
500 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
501 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
502 IsPrevailing(IsPrevailing), ImportList(ImportList),
503 ExportLists(ExportLists) {}
504
505 void onImportingSummary(const GlobalValueSummary &Summary) {
506 SmallVector<const GlobalVarSummary *, 128> Worklist;
507 onImportingSummaryImpl(Summary, Worklist);
508 while (!Worklist.empty())
509 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
510 }
511};
512
513static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
514
515/// Determine the list of imports and exports for each module.
516class ModuleImportsManager {
517 void computeImportForFunction(
518 const FunctionSummary &Summary, unsigned Threshold,
519 const GVSummaryMapTy &DefinedGVSummaries,
520 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
521 FunctionImporter::ImportMapTy &ImportList,
522 FunctionImporter::ImportThresholdsTy &ImportThresholds);
523
524protected:
525 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
526 IsPrevailing;
527 const ModuleSummaryIndex &Index;
528 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
529
530 ModuleImportsManager(
531 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
532 IsPrevailing,
533 const ModuleSummaryIndex &Index,
534 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
535 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
536 virtual bool canImport(ValueInfo VI) { return true; }
537
538public:
539 virtual ~ModuleImportsManager() = default;
540
541 /// Given the list of globals defined in a module, compute the list of imports
542 /// as well as the list of "exports", i.e. the list of symbols referenced from
543 /// another module (that may require promotion).
544 virtual void
545 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
546 StringRef ModName,
547 FunctionImporter::ImportMapTy &ImportList);
548
549 static std::unique_ptr<ModuleImportsManager>
550 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
551 IsPrevailing,
552 const ModuleSummaryIndex &Index,
553 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
554 nullptr);
555};
556
557/// A ModuleImportsManager that operates based on a workload definition (see
558/// -thinlto-workload-def). For modules that do not define workload roots, it
559/// applies the base ModuleImportsManager import policy.
560class WorkloadImportsManager : public ModuleImportsManager {
561 // Keep a module name -> value infos to import association. We use it to
562 // determine if a module's import list should be done by the base
563 // ModuleImportsManager or by us.
564 StringMap<DenseSet<ValueInfo>> Workloads;
565 // Track the roots to avoid importing them due to other callers. We want there
566 // to be only one variant), for which we optimize according to the contextual
567 // profile. "Variants" refers to copies due to importing - we want there to be
568 // just one instance of this function.
569 DenseSet<ValueInfo> Roots;
570
571 void
572 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
573 StringRef ModName,
574 FunctionImporter::ImportMapTy &ImportList) override {
575 StringRef Filename = ModName;
576 if (CtxprofMoveRootsToOwnModule) {
577 Filename = sys::path::filename(path: ModName);
578 // Drop the file extension.
579 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
580 }
581 auto SetIter = Workloads.find(Key: Filename);
582
583 if (SetIter == Workloads.end()) {
584 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
585 << " does not contain the root of any context.\n");
586 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
587 ModName, ImportList);
588 }
589 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
590 << " contains the root(s) of context(s).\n");
591
592 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
593 ExportLists);
594 auto &ValueInfos = SetIter->second;
595 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
596 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
597 if (It != DefinedGVSummaries.end() &&
598 IsPrevailing(VI.getGUID(), It->second)) {
599 LLVM_DEBUG(
600 dbgs() << "[Workload] " << VI.name()
601 << " has the prevailing variant already in the module "
602 << ModName << ". No need to import\n");
603 continue;
604 }
605 auto Candidates =
606 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
607
608 const GlobalValueSummary *GVS = nullptr;
609 auto PotentialCandidates = llvm::map_range(
610 C: llvm::make_filter_range(
611 Range&: Candidates,
612 Pred: [&](const auto &Candidate) {
613 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
614 << " from " << Candidate.second->modulePath()
615 << " ImportFailureReason: "
616 << getFailureName(Candidate.first) << "\n");
617 return Candidate.first ==
618 FunctionImporter::ImportFailureReason::None;
619 }),
620 F: [](const auto &Candidate) { return Candidate.second; });
621 if (PotentialCandidates.empty()) {
622 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
623 << " because can't find eligible Callee. Guid is: "
624 << VI.getGUID() << "\n");
625 continue;
626 }
627 /// We will prefer importing the prevailing candidate, if not, we'll
628 /// still pick the first available candidate. The reason we want to make
629 /// sure we do import the prevailing candidate is because the goal of
630 /// workload-awareness is to enable optimizations specializing the call
631 /// graph of that workload. Suppose a function is already defined in the
632 /// module, but it's not the prevailing variant. Suppose also we do not
633 /// inline it (in fact, if it were interposable, we can't inline it),
634 /// but we could specialize it to the workload in other ways. However,
635 /// the linker would drop it in the favor of the prevailing copy.
636 /// Instead, by importing the prevailing variant (assuming also the use
637 /// of `-avail-extern-to-local`), we keep the specialization. We could
638 /// alteranatively make the non-prevailing variant local, but the
639 /// prevailing one is also the one for which we would have previously
640 /// collected profiles, making it preferrable.
641 auto PrevailingCandidates = llvm::make_filter_range(
642 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
643 return IsPrevailing(VI.getGUID(), Candidate);
644 });
645 if (PrevailingCandidates.empty()) {
646 GVS = *PotentialCandidates.begin();
647 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
648 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
649 LLVM_DEBUG(
650 dbgs()
651 << "[Workload] Found multiple non-prevailing candidates for "
652 << VI.name()
653 << ". This is unexpected. Are module paths passed to the "
654 "compiler unique for the modules passed to the linker?");
655 // We could in theory have multiple (interposable) copies of a symbol
656 // when there is no prevailing candidate, if say the prevailing copy was
657 // in a native object being linked in. However, we should in theory be
658 // marking all of these non-prevailing IR copies dead in that case, in
659 // which case they won't be candidates.
660 assert(GVS->isLive());
661 } else {
662 assert(llvm::hasSingleElement(PrevailingCandidates));
663 GVS = *PrevailingCandidates.begin();
664 }
665
666 auto ExportingModule = GVS->modulePath();
667 // We checked that for the prevailing case, but if we happen to have for
668 // example an internal that's defined in this module, it'd have no
669 // PrevailingCandidates.
670 if (ExportingModule == ModName) {
671 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
672 << " because its defining module is the same as the "
673 "current module\n");
674 continue;
675 }
676 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
677 << ExportingModule << " : " << VI.getGUID() << "\n");
678 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
679 GVI.onImportingSummary(Summary: *GVS);
680 if (ExportLists)
681 (*ExportLists)[ExportingModule].insert(V: VI);
682 }
683 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
684 }
685
686 void loadFromJson() {
687 // Since the workload def uses names, we need a quick lookup
688 // name->ValueInfo.
689 StringMap<ValueInfo> NameToValueInfo;
690 StringSet<> AmbiguousNames;
691 for (auto &I : Index) {
692 ValueInfo VI = Index.getValueInfo(R: I);
693 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
694 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
695 }
696 auto DbgReportIfAmbiguous = [&](StringRef Name) {
697 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
698 dbgs() << "[Workload] Function name " << Name
699 << " present in the workload definition is ambiguous. Consider "
700 "compiling with -funique-internal-linkage-names.";
701 });
702 };
703 std::error_code EC;
704 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
705 if (std::error_code EC = BufferOrErr.getError()) {
706 report_fatal_error(reason: "Failed to open context file");
707 return;
708 }
709 auto Buffer = std::move(BufferOrErr.get());
710 std::map<std::string, std::vector<std::string>> WorkloadDefs;
711 json::Path::Root NullRoot;
712 // The JSON is supposed to contain a dictionary matching the type of
713 // WorkloadDefs. For example:
714 // {
715 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
716 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
717 // }
718 auto Parsed = json::parse(JSON: Buffer->getBuffer());
719 if (!Parsed)
720 report_fatal_error(Err: Parsed.takeError());
721 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
722 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
723 for (const auto &Workload : WorkloadDefs) {
724 const auto &Root = Workload.first;
725 DbgReportIfAmbiguous(Root);
726 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
727 const auto &AllCallees = Workload.second;
728 auto RootIt = NameToValueInfo.find(Key: Root);
729 if (RootIt == NameToValueInfo.end()) {
730 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
731 << " not found in this linkage unit.\n");
732 continue;
733 }
734 auto RootVI = RootIt->second;
735 if (RootVI.getSummaryList().size() != 1) {
736 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
737 << " should have exactly one summary, but has "
738 << RootVI.getSummaryList().size() << ". Skipping.\n");
739 continue;
740 }
741 StringRef RootDefiningModule =
742 RootVI.getSummaryList().front()->modulePath();
743 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
744 << " is : " << RootDefiningModule << "\n");
745 auto &Set = Workloads[RootDefiningModule];
746 for (const auto &Callee : AllCallees) {
747 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
748 DbgReportIfAmbiguous(Callee);
749 auto ElemIt = NameToValueInfo.find(Key: Callee);
750 if (ElemIt == NameToValueInfo.end()) {
751 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
752 continue;
753 }
754 Set.insert(V: ElemIt->second);
755 }
756 }
757 }
758
759 void loadFromCtxProf() {
760 std::error_code EC;
761 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
762 if (std::error_code EC = BufferOrErr.getError()) {
763 report_fatal_error(reason: "Failed to open contextual profile file");
764 return;
765 }
766 auto Buffer = std::move(BufferOrErr.get());
767
768 PGOCtxProfileReader Reader(Buffer->getBuffer());
769 auto Ctx = Reader.loadProfiles();
770 if (!Ctx) {
771 report_fatal_error(reason: "Failed to parse contextual profiles");
772 return;
773 }
774 const auto &CtxMap = Ctx->Contexts;
775 SetVector<GlobalValue::GUID> ContainedGUIDs;
776 for (const auto &[RootGuid, Root] : CtxMap) {
777 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
778 // subsequent roots, but clear its contents.
779 ContainedGUIDs.clear();
780
781 auto RootVI = Index.getValueInfo(GUID: RootGuid);
782 if (!RootVI) {
783 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
784 << " not found in this linkage unit.\n");
785 continue;
786 }
787 if (RootVI.getSummaryList().size() != 1) {
788 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
789 << " should have exactly one summary, but has "
790 << RootVI.getSummaryList().size() << ". Skipping.\n");
791 continue;
792 }
793 std::string RootDefiningModule =
794 RootVI.getSummaryList().front()->modulePath().str();
795 if (CtxprofMoveRootsToOwnModule) {
796 RootDefiningModule = std::to_string(val: RootGuid);
797 LLVM_DEBUG(
798 dbgs() << "[Workload] Moving " << RootGuid
799 << " to a module with the filename without extension : "
800 << RootDefiningModule << "\n");
801 } else {
802 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
803 << " is : " << RootDefiningModule << "\n");
804 }
805 auto &Set = Workloads[RootDefiningModule];
806 Root.getContainedGuids(Guids&: ContainedGUIDs);
807 Roots.insert(V: RootVI);
808 for (auto Guid : ContainedGUIDs)
809 if (auto VI = Index.getValueInfo(GUID: Guid))
810 Set.insert(V: VI);
811 }
812 }
813
814 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
815
816public:
817 WorkloadImportsManager(
818 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
819 IsPrevailing,
820 const ModuleSummaryIndex &Index,
821 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
822 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
823 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
824 report_fatal_error(
825 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
826 return;
827 }
828 if (!UseCtxProfile.empty())
829 loadFromCtxProf();
830 else
831 loadFromJson();
832 LLVM_DEBUG({
833 for (const auto &[Root, Set] : Workloads) {
834 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
835 << " distinct callees.\n";
836 for (const auto &VI : Set) {
837 dbgs() << "[Workload] Root: " << Root
838 << " Would include: " << VI.getGUID() << "\n";
839 }
840 }
841 });
842 }
843};
844
845std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
846 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
847 IsPrevailing,
848 const ModuleSummaryIndex &Index,
849 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
850 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
851 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
852 return std::unique_ptr<ModuleImportsManager>(
853 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
854 }
855 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
856 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
857 args&: ExportLists);
858}
859
860static const char *
861getFailureName(FunctionImporter::ImportFailureReason Reason) {
862 switch (Reason) {
863 case FunctionImporter::ImportFailureReason::None:
864 return "None";
865 case FunctionImporter::ImportFailureReason::GlobalVar:
866 return "GlobalVar";
867 case FunctionImporter::ImportFailureReason::NotLive:
868 return "NotLive";
869 case FunctionImporter::ImportFailureReason::TooLarge:
870 return "TooLarge";
871 case FunctionImporter::ImportFailureReason::InterposableLinkage:
872 return "InterposableLinkage";
873 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
874 return "LocalLinkageNotInModule";
875 case FunctionImporter::ImportFailureReason::NotEligible:
876 return "NotEligible";
877 case FunctionImporter::ImportFailureReason::NoInline:
878 return "NoInline";
879 }
880 llvm_unreachable("invalid reason");
881}
882
883/// Compute the list of functions to import for a given caller. Mark these
884/// imported functions and the symbols they reference in their source module as
885/// exported from their source module.
886void ModuleImportsManager::computeImportForFunction(
887 const FunctionSummary &Summary, const unsigned Threshold,
888 const GVSummaryMapTy &DefinedGVSummaries,
889 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
890 FunctionImporter::ImportMapTy &ImportList,
891 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
892 GVImporter.onImportingSummary(Summary);
893 static int ImportCount = 0;
894 for (const auto &Edge : Summary.calls()) {
895 ValueInfo VI = Edge.first;
896 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
897 << "\n");
898
899 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
900 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
901 << " reached.\n");
902 continue;
903 }
904
905 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
906 // FIXME: Consider not skipping import if the module contains
907 // a non-prevailing def with interposable linkage. The prevailing copy
908 // can safely be imported (see shouldImportGlobal()).
909 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
910 continue;
911 }
912
913 if (!canImport(VI)) {
914 LLVM_DEBUG(
915 dbgs() << "Skipping over " << VI.getGUID()
916 << " because its import is handled in a different module.");
917 assert(VI.getSummaryList().size() == 1 &&
918 "The root was expected to be an external symbol");
919 continue;
920 }
921
922 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
923 if (Hotness == CalleeInfo::HotnessType::Hot)
924 return ImportHotMultiplier;
925 if (Hotness == CalleeInfo::HotnessType::Cold)
926 return ImportColdMultiplier;
927 if (Hotness == CalleeInfo::HotnessType::Critical)
928 return ImportCriticalMultiplier;
929 return 1.0;
930 };
931
932 const auto NewThreshold =
933 Threshold * GetBonusMultiplier(Edge.second.getHotness());
934
935 auto IT = ImportThresholds.insert(KV: std::make_pair(
936 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
937 bool PreviouslyVisited = !IT.second;
938 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
939 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
940 auto &FailureInfo = std::get<2>(t&: IT.first->second);
941
942 bool IsHotCallsite =
943 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
944 bool IsCriticalCallsite =
945 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
946
947 const FunctionSummary *ResolvedCalleeSummary = nullptr;
948 if (CalleeSummary) {
949 assert(PreviouslyVisited);
950 // Since the traversal of the call graph is DFS, we can revisit a function
951 // a second time with a higher threshold. In this case, it is added back
952 // to the worklist with the new threshold (so that its own callee chains
953 // can be considered with the higher threshold).
954 if (NewThreshold <= ProcessedThreshold) {
955 LLVM_DEBUG(
956 dbgs() << "ignored! Target was already imported with Threshold "
957 << ProcessedThreshold << "\n");
958 continue;
959 }
960 // Update with new larger threshold.
961 ProcessedThreshold = NewThreshold;
962 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
963 } else {
964 // If we already rejected importing a callee at the same or higher
965 // threshold, don't waste time calling selectCallee.
966 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
967 LLVM_DEBUG(
968 dbgs() << "ignored! Target was already rejected with Threshold "
969 << ProcessedThreshold << "\n");
970 if (PrintImportFailures) {
971 assert(FailureInfo &&
972 "Expected FailureInfo for previously rejected candidate");
973 FailureInfo->Attempts++;
974 }
975 continue;
976 }
977
978 FunctionImporter::ImportFailureReason Reason{};
979
980 // `SummaryForDeclImport` is an summary eligible for declaration import.
981 const GlobalValueSummary *SummaryForDeclImport = nullptr;
982 CalleeSummary =
983 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
984 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
985 if (!CalleeSummary) {
986 // There isn't a callee for definition import but one for declaration
987 // import.
988 if (ImportDeclaration && SummaryForDeclImport) {
989 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
990
991 // Note `ExportLists` only keeps track of exports due to imported
992 // definitions.
993 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
994 }
995 // Update with new larger threshold if this was a retry (otherwise
996 // we would have already inserted with NewThreshold above). Also
997 // update failure info if requested.
998 if (PreviouslyVisited) {
999 ProcessedThreshold = NewThreshold;
1000 if (PrintImportFailures) {
1001 assert(FailureInfo &&
1002 "Expected FailureInfo for previously rejected candidate");
1003 FailureInfo->Reason = Reason;
1004 FailureInfo->Attempts++;
1005 FailureInfo->MaxHotness =
1006 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
1007 }
1008 } else if (PrintImportFailures) {
1009 assert(!FailureInfo &&
1010 "Expected no FailureInfo for newly rejected candidate");
1011 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
1012 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
1013 }
1014 if (ForceImportAll) {
1015 std::string Msg = std::string("Failed to import function ") +
1016 VI.name().str() + " due to " +
1017 getFailureName(Reason);
1018 auto Error = make_error<StringError>(
1019 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1020 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1021 ErrorBanner: "Error importing module: ");
1022 break;
1023 } else {
1024 LLVM_DEBUG(dbgs()
1025 << "ignored! No qualifying callee with summary found.\n");
1026 continue;
1027 }
1028 }
1029
1030 // "Resolve" the summary
1031 CalleeSummary = CalleeSummary->getBaseObject();
1032 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1033
1034 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1035 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1036 "selectCallee() didn't honor the threshold");
1037
1038 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1039
1040 // Try emplace the definition entry, and update stats based on insertion
1041 // status.
1042 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1043 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1044 NumImportedFunctionsThinLink++;
1045 if (IsHotCallsite)
1046 NumImportedHotFunctionsThinLink++;
1047 if (IsCriticalCallsite)
1048 NumImportedCriticalFunctionsThinLink++;
1049 }
1050
1051 // Any calls/references made by this function will be marked exported
1052 // later, in ComputeCrossModuleImport, after import decisions are
1053 // complete, which is more efficient than adding them here.
1054 if (ExportLists)
1055 (*ExportLists)[ExportModulePath].insert(V: VI);
1056 }
1057
1058 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1059 // Adjust the threshold for next level of imported functions.
1060 // The threshold is different for hot callsites because we can then
1061 // inline chains of hot calls.
1062 if (IsHotCallsite)
1063 return Threshold * ImportHotInstrFactor;
1064 return Threshold * ImportInstrFactor;
1065 };
1066
1067 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1068
1069 ImportCount++;
1070
1071 // Insert the newly imported function to the worklist.
1072 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1073 }
1074}
1075
1076void ModuleImportsManager::computeImportForModule(
1077 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1078 FunctionImporter::ImportMapTy &ImportList) {
1079 // Worklist contains the list of function imported in this module, for which
1080 // we will analyse the callees and may import further down the callgraph.
1081 SmallVector<EdgeInfo, 128> Worklist;
1082 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1083 ExportLists);
1084 FunctionImporter::ImportThresholdsTy ImportThresholds;
1085
1086 // Populate the worklist with the import for the functions in the current
1087 // module
1088 for (const auto &GVSummary : DefinedGVSummaries) {
1089#ifndef NDEBUG
1090 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1091 // so this map look up (and possibly others) can be avoided.
1092 auto VI = Index.getValueInfo(GVSummary.first);
1093#endif
1094 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1095 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1096 continue;
1097 }
1098 auto *FuncSummary =
1099 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1100 if (!FuncSummary)
1101 // Skip import for global variables
1102 continue;
1103 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1104 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1105 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1106 }
1107
1108 // Process the newly imported functions and add callees to the worklist.
1109 while (!Worklist.empty()) {
1110 auto GVInfo = Worklist.pop_back_val();
1111 auto *Summary = std::get<0>(t&: GVInfo);
1112 auto Threshold = std::get<1>(t&: GVInfo);
1113
1114 computeImportForFunction(Summary: *Summary, Threshold, DefinedGVSummaries, Worklist,
1115 GVImporter&: GVI, ImportList, ImportThresholds);
1116 }
1117
1118 // Print stats about functions considered but rejected for importing
1119 // when requested.
1120 if (PrintImportFailures) {
1121 dbgs() << "Missed imports into module " << ModName << "\n";
1122 for (auto &I : ImportThresholds) {
1123 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1124 auto &CalleeSummary = std::get<1>(t&: I.second);
1125 auto &FailureInfo = std::get<2>(t&: I.second);
1126 if (CalleeSummary)
1127 continue; // We are going to import.
1128 assert(FailureInfo);
1129 FunctionSummary *FS = nullptr;
1130 if (!FailureInfo->VI.getSummaryList().empty())
1131 FS = dyn_cast<FunctionSummary>(
1132 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1133 dbgs() << FailureInfo->VI
1134 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1135 << ", Threshold = " << ProcessedThreshold
1136 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1137 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1138 << ", Attempts = " << FailureInfo->Attempts << "\n";
1139 }
1140 }
1141}
1142
1143#ifndef NDEBUG
1144static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1145 auto SL = VI.getSummaryList();
1146 return SL.empty()
1147 ? false
1148 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1149}
1150
1151static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1152 GlobalValue::GUID G) {
1153 if (const auto &VI = Index.getValueInfo(G))
1154 return isGlobalVarSummary(Index, VI);
1155 return false;
1156}
1157
1158// Return the number of global variable summaries in ExportSet.
1159static unsigned
1160numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1161 FunctionImporter::ExportSetTy &ExportSet) {
1162 unsigned NumGVS = 0;
1163 for (auto &VI : ExportSet)
1164 if (isGlobalVarSummary(Index, VI.getGUID()))
1165 ++NumGVS;
1166 return NumGVS;
1167}
1168
1169struct ImportStatistics {
1170 unsigned NumGVS = 0;
1171 unsigned DefinedFS = 0;
1172 unsigned Count = 0;
1173};
1174
1175// Compute import statistics for each source module in ImportList.
1176static DenseMap<StringRef, ImportStatistics>
1177collectImportStatistics(const ModuleSummaryIndex &Index,
1178 const FunctionImporter::ImportMapTy &ImportList) {
1179 DenseMap<StringRef, ImportStatistics> Histogram;
1180
1181 for (const auto &[FromModule, GUID, Type] : ImportList) {
1182 ImportStatistics &Entry = Histogram[FromModule];
1183 ++Entry.Count;
1184 if (isGlobalVarSummary(Index, GUID))
1185 ++Entry.NumGVS;
1186 else if (Type == GlobalValueSummary::Definition)
1187 ++Entry.DefinedFS;
1188 }
1189 return Histogram;
1190}
1191#endif
1192
1193#ifndef NDEBUG
1194static bool checkVariableImport(
1195 const ModuleSummaryIndex &Index,
1196 FunctionImporter::ImportListsTy &ImportLists,
1197 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1198 DenseSet<GlobalValue::GUID> FlattenedImports;
1199
1200 for (const auto &ImportPerModule : ImportLists)
1201 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1202 FlattenedImports.insert(GUID);
1203
1204 // Checks that all GUIDs of read/writeonly vars we see in export lists
1205 // are also in the import lists. Otherwise we my face linker undefs,
1206 // because readonly and writeonly vars are internalized in their
1207 // source modules. The exception would be if it has a linkage type indicating
1208 // that there may have been a copy existing in the importing module (e.g.
1209 // linkonce_odr). In that case we cannot accurately do this checking.
1210 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1211 const ValueInfo &VI) {
1212 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1213 Index.findSummaryInModule(VI, ModulePath));
1214 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1215 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1216 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1217 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1218 };
1219
1220 for (auto &ExportPerModule : ExportLists)
1221 for (auto &VI : ExportPerModule.second)
1222 if (!FlattenedImports.count(VI.getGUID()) &&
1223 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1224 return false;
1225
1226 return true;
1227}
1228#endif
1229
1230/// Compute all the import and export for every module using the Index.
1231void llvm::ComputeCrossModuleImport(
1232 const ModuleSummaryIndex &Index,
1233 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1234 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1235 isPrevailing,
1236 FunctionImporter::ImportListsTy &ImportLists,
1237 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1238 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1239 // For each module that has function defined, compute the import/export lists.
1240 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1241 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1242 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1243 << DefinedGVSummaries.first << "'\n");
1244 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1245 ModName: DefinedGVSummaries.first, ImportList);
1246 }
1247
1248 // When computing imports we only added the variables and functions being
1249 // imported to the export list. We also need to mark any references and calls
1250 // they make as exported as well. We do this here, as it is more efficient
1251 // since we may import the same values multiple times into different modules
1252 // during the import computation.
1253 for (auto &ELI : ExportLists) {
1254 // `NewExports` tracks the VI that gets exported because the full definition
1255 // of its user/referencer gets exported.
1256 FunctionImporter::ExportSetTy NewExports;
1257 const auto &DefinedGVSummaries =
1258 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1259 for (auto &EI : ELI.second) {
1260 // Find the copy defined in the exporting module so that we can mark the
1261 // values it references in that specific definition as exported.
1262 // Below we will add all references and called values, without regard to
1263 // whether they are also defined in this module. We subsequently prune the
1264 // list to only include those defined in the exporting module, see comment
1265 // there as to why.
1266 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1267 // Anything marked exported during the import computation must have been
1268 // defined in the exporting module.
1269 assert(DS != DefinedGVSummaries.end());
1270 auto *S = DS->getSecond();
1271 S = S->getBaseObject();
1272 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1273 // Export referenced functions and variables. We don't export/promote
1274 // objects referenced by writeonly variable initializer, because
1275 // we convert such variables initializers to "zeroinitializer".
1276 // See processGlobalForThinLTO.
1277 if (!Index.isWriteOnly(GVS))
1278 NewExports.insert_range(R: GVS->refs());
1279 } else {
1280 auto *FS = cast<FunctionSummary>(Val: S);
1281 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1282 NewExports.insert_range(R: FS->refs());
1283 }
1284 }
1285 // Prune list computed above to only include values defined in the
1286 // exporting module. We do this after the above insertion since we may hit
1287 // the same ref/call target multiple times in above loop, and it is more
1288 // efficient to avoid a set lookup each time.
1289 NewExports.remove_if(
1290 Pred: [&](ValueInfo VI) { return !DefinedGVSummaries.count(Val: VI.getGUID()); });
1291 ELI.second.insert_range(R&: NewExports);
1292 }
1293
1294 assert(checkVariableImport(Index, ImportLists, ExportLists));
1295#ifndef NDEBUG
1296 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1297 << " modules:\n");
1298 for (const auto &ModuleImports : ImportLists) {
1299 auto ModName = ModuleImports.first;
1300 auto &Exports = ExportLists[ModName];
1301 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1302 DenseMap<StringRef, ImportStatistics> Histogram =
1303 collectImportStatistics(Index, ModuleImports.second);
1304 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1305 << Exports.size() - NumGVS << " functions and " << NumGVS
1306 << " vars. Imports from " << Histogram.size()
1307 << " modules.\n");
1308 for (const auto &[SrcModName, Stats] : Histogram) {
1309 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1310 << " function definitions and "
1311 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1312 << " function declarations imported from " << SrcModName
1313 << "\n");
1314 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1315 << " global vars imported from " << SrcModName << "\n");
1316 }
1317 }
1318#endif
1319}
1320
1321#ifndef NDEBUG
1322static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1323 StringRef ModulePath,
1324 FunctionImporter::ImportMapTy &ImportList) {
1325 DenseMap<StringRef, ImportStatistics> Histogram =
1326 collectImportStatistics(Index, ImportList);
1327 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1328 << Histogram.size() << " modules.\n");
1329 for (const auto &[SrcModName, Stats] : Histogram) {
1330 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1331 << " function definitions and "
1332 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1333 << " function declarations imported from " << SrcModName
1334 << "\n");
1335 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1336 << SrcModName << "\n");
1337 }
1338}
1339#endif
1340
1341/// Compute all the imports for the given module using the Index.
1342///
1343/// \p isPrevailing is a callback that will be called with a global value's GUID
1344/// and summary and should return whether the module corresponding to the
1345/// summary contains the linker-prevailing copy of that value.
1346///
1347/// \p ImportList will be populated with a map that can be passed to
1348/// FunctionImporter::importFunctions() above (see description there).
1349static void ComputeCrossModuleImportForModuleForTest(
1350 StringRef ModulePath,
1351 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1352 isPrevailing,
1353 const ModuleSummaryIndex &Index,
1354 FunctionImporter::ImportMapTy &ImportList) {
1355 // Collect the list of functions this module defines.
1356 // GUID -> Summary
1357 GVSummaryMapTy FunctionSummaryMap;
1358 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1359
1360 // Compute the import list for this module.
1361 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1362 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1363 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1364
1365#ifndef NDEBUG
1366 dumpImportListForModule(Index, ModulePath, ImportList);
1367#endif
1368}
1369
1370/// Mark all external summaries in \p Index for import into the given module.
1371/// Used for testing the case of distributed builds using a distributed index.
1372///
1373/// \p ImportList will be populated with a map that can be passed to
1374/// FunctionImporter::importFunctions() above (see description there).
1375static void ComputeCrossModuleImportForModuleFromIndexForTest(
1376 StringRef ModulePath, const ModuleSummaryIndex &Index,
1377 FunctionImporter::ImportMapTy &ImportList) {
1378 for (const auto &GlobalList : Index) {
1379 // Ignore entries for undefined references.
1380 if (GlobalList.second.getSummaryList().empty())
1381 continue;
1382
1383 auto GUID = GlobalList.first;
1384 assert(GlobalList.second.getSummaryList().size() == 1 &&
1385 "Expected individual combined index to have one summary per GUID");
1386 auto &Summary = GlobalList.second.getSummaryList()[0];
1387 // Skip the summaries for the importing module. These are included to
1388 // e.g. record required linkage changes.
1389 if (Summary->modulePath() == ModulePath)
1390 continue;
1391 // Add an entry to provoke importing by thinBackend.
1392 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1393 }
1394#ifndef NDEBUG
1395 dumpImportListForModule(Index, ModulePath, ImportList);
1396#endif
1397}
1398
1399// For SamplePGO, the indirect call targets for local functions will
1400// have its original name annotated in profile. We try to find the
1401// corresponding PGOFuncName as the GUID, and fix up the edges
1402// accordingly.
1403void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1404 FunctionSummary *FS) {
1405 for (auto &EI : FS->mutableCalls()) {
1406 if (!EI.first.getSummaryList().empty())
1407 continue;
1408 auto GUID = Index.getGUIDFromOriginalID(OriginalID: EI.first.getGUID());
1409 if (GUID == 0)
1410 continue;
1411 // Update the edge to point directly to the correct GUID.
1412 auto VI = Index.getValueInfo(GUID);
1413 if (llvm::any_of(
1414 Range: VI.getSummaryList(),
1415 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1416 // The mapping from OriginalId to GUID may return a GUID
1417 // that corresponds to a static variable. Filter it out here.
1418 // This can happen when
1419 // 1) There is a call to a library function which is not defined
1420 // in the index.
1421 // 2) There is a static variable with the OriginalGUID identical
1422 // to the GUID of the library function in 1);
1423 // When this happens the static variable in 2) will be found,
1424 // which needs to be filtered out.
1425 return SummaryPtr->getSummaryKind() ==
1426 GlobalValueSummary::GlobalVarKind;
1427 }))
1428 continue;
1429 EI.first = VI;
1430 }
1431}
1432
1433void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1434 for (const auto &Entry : Index) {
1435 for (const auto &S : Entry.second.getSummaryList()) {
1436 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1437 updateValueInfoForIndirectCalls(Index, FS);
1438 }
1439 }
1440}
1441
1442void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1443 ModuleSummaryIndex &Index,
1444 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1445 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1446 assert(!Index.withGlobalValueDeadStripping());
1447 if (!ComputeDead ||
1448 // Don't do anything when nothing is live, this is friendly with tests.
1449 GUIDPreservedSymbols.empty()) {
1450 // Still need to update indirect calls.
1451 updateIndirectCalls(Index);
1452 return;
1453 }
1454 unsigned LiveSymbols = 0;
1455 SmallVector<ValueInfo, 128> Worklist;
1456 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1457 for (auto GUID : GUIDPreservedSymbols) {
1458 ValueInfo VI = Index.getValueInfo(GUID);
1459 if (!VI)
1460 continue;
1461 for (const auto &S : VI.getSummaryList())
1462 S->setLive(true);
1463 }
1464
1465 // Add values flagged in the index as live roots to the worklist.
1466 for (const auto &Entry : Index) {
1467 auto VI = Index.getValueInfo(R: Entry);
1468 for (const auto &S : Entry.second.getSummaryList()) {
1469 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1470 updateValueInfoForIndirectCalls(Index, FS);
1471 if (S->isLive()) {
1472 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1473 Worklist.push_back(Elt: VI);
1474 ++LiveSymbols;
1475 break;
1476 }
1477 }
1478 }
1479
1480 // Make value live and add it to the worklist if it was not live before.
1481 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1482 // FIXME: If we knew which edges were created for indirect call profiles,
1483 // we could skip them here. Any that are live should be reached via
1484 // other edges, e.g. reference edges. Otherwise, using a profile collected
1485 // on a slightly different binary might provoke preserving, importing
1486 // and ultimately promoting calls to functions not linked into this
1487 // binary, which increases the binary size unnecessarily. Note that
1488 // if this code changes, the importer needs to change so that edges
1489 // to functions marked dead are skipped.
1490
1491 if (llvm::any_of(Range: VI.getSummaryList(),
1492 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1493 return S->isLive();
1494 }))
1495 return;
1496
1497 // We only keep live symbols that are known to be non-prevailing if any are
1498 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1499 // later in the EliminateAvailableExternally pass and setting them to
1500 // not-live could break downstreams users of liveness information (PR36483)
1501 // or limit optimization opportunities.
1502 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1503 bool KeepAliveLinkage = false;
1504 bool Interposable = false;
1505 for (const auto &S : VI.getSummaryList()) {
1506 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1507 S->linkage() == GlobalValue::WeakODRLinkage ||
1508 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1509 KeepAliveLinkage = true;
1510 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1511 Interposable = true;
1512 }
1513
1514 if (!IsAliasee) {
1515 if (!KeepAliveLinkage)
1516 return;
1517
1518 if (Interposable)
1519 report_fatal_error(
1520 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1521 "symbol");
1522 }
1523 }
1524
1525 for (const auto &S : VI.getSummaryList())
1526 S->setLive(true);
1527 ++LiveSymbols;
1528 Worklist.push_back(Elt: VI);
1529 };
1530
1531 while (!Worklist.empty()) {
1532 auto VI = Worklist.pop_back_val();
1533 for (const auto &Summary : VI.getSummaryList()) {
1534 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1535 // If this is an alias, visit the aliasee VI to ensure that all copies
1536 // are marked live and it is added to the worklist for further
1537 // processing of its references.
1538 visit(AS->getAliaseeVI(), true);
1539 continue;
1540 }
1541 for (auto Ref : Summary->refs())
1542 visit(Ref, false);
1543 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1544 for (auto Call : FS->calls())
1545 visit(Call.first, false);
1546 }
1547 }
1548 Index.setWithGlobalValueDeadStripping();
1549
1550 unsigned DeadSymbols = Index.size() - LiveSymbols;
1551 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1552 << " symbols Dead \n");
1553 NumDeadSymbols += DeadSymbols;
1554 NumLiveSymbols += LiveSymbols;
1555}
1556
1557// Compute dead symbols and propagate constants in combined index.
1558void llvm::computeDeadSymbolsWithConstProp(
1559 ModuleSummaryIndex &Index,
1560 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1561 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1562 bool ImportEnabled) {
1563 llvm::TimeTraceScope timeScope("Drop dead symbols and propagate attributes");
1564 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1565 isPrevailing);
1566 if (ImportEnabled)
1567 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1568}
1569
1570/// Compute the set of summaries needed for a ThinLTO backend compilation of
1571/// \p ModulePath.
1572void llvm::gatherImportedSummariesForModule(
1573 StringRef ModulePath,
1574 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1575 const FunctionImporter::ImportMapTy &ImportList,
1576 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1577 GVSummaryPtrSet &DecSummaries) {
1578 // Include all summaries from the importing module.
1579 ModuleToSummariesForIndex[std::string(ModulePath)] =
1580 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1581
1582 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1583 // lets us look up the map without allocating an instance of std::string when
1584 // the key-value pair exists in the map.
1585 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1586 // C++26 when it becomes available for our codebase.
1587 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1588 StringRef Key) -> GVSummaryMapTy & {
1589 auto It = Map.find(x: Key);
1590 if (It == Map.end())
1591 std::tie(args&: It, args: std::ignore) =
1592 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1593 return It->second;
1594 };
1595
1596 // Include summaries for imports.
1597 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1598 auto &SummariesForIndex =
1599 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1600
1601 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1602 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1603 assert(DS != DefinedGVSummaries.end() &&
1604 "Expected a defined summary for imported global value");
1605 if (ImportType == GlobalValueSummary::Declaration)
1606 DecSummaries.insert(Ptr: DS->second);
1607
1608 SummariesForIndex[GUID] = DS->second;
1609 }
1610
1611 // When AlwaysRenamePromotedLocals is false, for each source module we import
1612 // from, also include summaries for local functions that have
1613 // NoRenameOnPromotion set. This is needed for distributed ThinLTO. Otherwise,
1614 // the local function of the source module will keep its origin name, e.g.,
1615 // foo() while the function in destination module will have name
1616 // foo.llvm.<...>() and this will cause a link failure.
1617 //
1618 // Note: this imports a superset of the necessary declarations — all locals
1619 // with NoRenameOnPromotion in each source module, not just those referenced
1620 // by the importing module. Computing the precise set would require walking
1621 // the summary reference graph from each imported function, which is more
1622 // expensive than the simple scan here.
1623 if (!AlwaysRenamePromotedLocals) {
1624 for (auto &[ModPath, SummariesForIndex] : ModuleToSummariesForIndex) {
1625 if (ModPath == ModulePath)
1626 continue;
1627 auto It = ModuleToDefinedGVSummaries.find(Val: ModPath);
1628 if (It == ModuleToDefinedGVSummaries.end())
1629 continue;
1630 for (const auto &[GUID, Summary] : It->second) {
1631 if (Summary->noRenameOnPromotion()) {
1632 DecSummaries.insert(Ptr: Summary);
1633 SummariesForIndex.try_emplace(Key: GUID, Args: Summary);
1634 }
1635 }
1636 }
1637 }
1638}
1639
1640/// Emit the files \p ModulePath will import from into \p OutputFilename.
1641Error llvm::EmitImportsFiles(
1642 StringRef ModulePath, StringRef OutputFilename,
1643 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1644 std::error_code EC;
1645 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1646 if (EC)
1647 return createFileError(F: "cannot open " + OutputFilename,
1648 E: errorCodeToError(EC));
1649 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1650 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1651 return Error::success();
1652}
1653
1654/// Invoke callback \p F on the file paths from which \p ModulePath
1655/// will import.
1656void llvm::processImportsFiles(
1657 StringRef ModulePath,
1658 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1659 function_ref<void(const std::string &)> F) {
1660 for (const auto &ILI : ModuleToSummariesForIndex)
1661 // The ModuleToSummariesForIndex map includes an entry for the current
1662 // Module (needed for writing out the index files). We don't want to
1663 // include it in the imports file, however, so filter it out.
1664 if (ILI.first != ModulePath)
1665 F(ILI.first);
1666}
1667
1668bool llvm::convertToDeclaration(GlobalValue &GV) {
1669 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1670 << "\n");
1671 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1672 F->deleteBody();
1673 F->clearMetadata();
1674 F->setComdat(nullptr);
1675 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1676 V->setInitializer(nullptr);
1677 V->setLinkage(GlobalValue::ExternalLinkage);
1678 V->clearMetadata();
1679 V->setComdat(nullptr);
1680 } else {
1681 GlobalValue *NewGV;
1682 if (GV.getValueType()->isFunctionTy())
1683 NewGV =
1684 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1685 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1686 N: "", M: GV.getParent());
1687 else
1688 NewGV =
1689 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1690 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1691 /*init*/ nullptr, "",
1692 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1693 GV.getType()->getAddressSpace());
1694 NewGV->takeName(V: &GV);
1695 GV.replaceAllUsesWith(V: NewGV);
1696 return false;
1697 }
1698 if (!GV.isImplicitDSOLocal())
1699 GV.setDSOLocal(false);
1700 return true;
1701}
1702
1703void llvm::thinLTOFinalizeInModule(Module &TheModule,
1704 const GVSummaryMapTy &DefinedGlobals,
1705 bool PropagateAttrs) {
1706 llvm::TimeTraceScope timeScope("ThinLTO finalize in module");
1707 DenseSet<Comdat *> NonPrevailingComdats;
1708 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1709 // See if the global summary analysis computed a new resolved linkage.
1710 const auto &GS = DefinedGlobals.find(Val: GV.getGUID());
1711 if (GS == DefinedGlobals.end())
1712 return;
1713
1714 if (Propagate)
1715 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1716 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1717 // TODO: propagate ReadNone and ReadOnly.
1718 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1719 F->setDoesNotAccessMemory();
1720
1721 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1722 F->setOnlyReadsMemory();
1723
1724 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1725 F->setDoesNotRecurse();
1726
1727 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1728 F->setDoesNotThrow();
1729 }
1730 }
1731
1732 auto NewLinkage = GS->second->linkage();
1733 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1734 // Don't internalize anything here, because the code below
1735 // lacks necessary correctness checks. Leave this job to
1736 // LLVM 'internalize' pass.
1737 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1738 // In case it was dead and already converted to declaration.
1739 GV.isDeclaration())
1740 return;
1741
1742 // Set the potentially more constraining visibility computed from summaries.
1743 // The DefaultVisibility condition is because older GlobalValueSummary does
1744 // not record DefaultVisibility and we don't want to change protected/hidden
1745 // to default.
1746 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1747 GV.setVisibility(GS->second->getVisibility());
1748
1749 if (NewLinkage == GV.getLinkage())
1750 return;
1751
1752 // Check for a non-prevailing def that has interposable linkage
1753 // (e.g. non-odr weak or linkonce). In that case we can't simply
1754 // convert to available_externally, since it would lose the
1755 // interposable property and possibly get inlined. Simply drop
1756 // the definition in that case.
1757 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1758 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1759 if (!convertToDeclaration(GV))
1760 // FIXME: Change this to collect replaced GVs and later erase
1761 // them from the parent module once thinLTOResolvePrevailingGUID is
1762 // changed to enable this for aliases.
1763 llvm_unreachable("Expected GV to be converted");
1764 } else {
1765 // If all copies of the original symbol had global unnamed addr and
1766 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1767 // and are constants, then it should be an auto hide symbol. In that case
1768 // the thin link would have marked it as CanAutoHide. Add hidden
1769 // visibility to the symbol to preserve the property.
1770 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1771 GS->second->canAutoHide()) {
1772 assert(GV.canBeOmittedFromSymbolTable());
1773 GV.setVisibility(GlobalValue::HiddenVisibility);
1774 }
1775
1776 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1777 << "` from " << GV.getLinkage() << " to " << NewLinkage
1778 << "\n");
1779 GV.setLinkage(NewLinkage);
1780 }
1781 // Remove declarations from comdats, including available_externally
1782 // as this is a declaration for the linker, and will be dropped eventually.
1783 // It is illegal for comdats to contain declarations.
1784 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1785 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1786 if (GO->getComdat()->getName() == GO->getName())
1787 NonPrevailingComdats.insert(V: GO->getComdat());
1788 GO->setComdat(nullptr);
1789 }
1790 };
1791
1792 // Process functions and global now
1793 for (auto &GV : TheModule)
1794 FinalizeInModule(GV, PropagateAttrs);
1795 for (auto &GV : TheModule.globals())
1796 FinalizeInModule(GV);
1797 for (auto &GV : TheModule.aliases())
1798 FinalizeInModule(GV);
1799
1800 // For a non-prevailing comdat, all its members must be available_externally.
1801 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1802 // local linkage GlobalValues.
1803 if (NonPrevailingComdats.empty())
1804 return;
1805 for (auto &GO : TheModule.global_objects()) {
1806 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1807 GO.setComdat(nullptr);
1808 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1809 }
1810 }
1811 bool Changed;
1812 do {
1813 Changed = false;
1814 // If an alias references a GlobalValue in a non-prevailing comdat, change
1815 // it to available_externally. For simplicity we only handle GlobalValue and
1816 // ConstantExpr with a base object. ConstantExpr without a base object is
1817 // unlikely used in a COMDAT.
1818 for (auto &GA : TheModule.aliases()) {
1819 if (GA.hasAvailableExternallyLinkage())
1820 continue;
1821 GlobalObject *Obj = GA.getAliaseeObject();
1822 assert(Obj && "aliasee without an base object is unimplemented");
1823 if (Obj->hasAvailableExternallyLinkage()) {
1824 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1825 Changed = true;
1826 }
1827 }
1828 } while (Changed);
1829}
1830
1831/// Run internalization on \p TheModule based on symmary analysis.
1832void llvm::thinLTOInternalizeModule(Module &TheModule,
1833 const GVSummaryMapTy &DefinedGlobals) {
1834 llvm::TimeTraceScope timeScope("ThinLTO internalize module");
1835 // Declare a callback for the internalize pass that will ask for every
1836 // candidate GlobalValue if it can be internalized or not.
1837 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1838 // It may be the case that GV is on a chain of an ifunc, its alias and
1839 // subsequent aliases. In this case, the summary for the value is not
1840 // available.
1841 if (isa<GlobalIFunc>(Val: &GV) ||
1842 (isa<GlobalAlias>(Val: &GV) &&
1843 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1844 return true;
1845
1846 // Lookup the linkage recorded in the summaries during global analysis.
1847 auto GS = DefinedGlobals.find(Val: GV.getGUID());
1848 if (GS == DefinedGlobals.end()) {
1849 // Must have been promoted (possibly conservatively). Find original
1850 // name so that we can access the correct summary and see if it can
1851 // be internalized again.
1852 // FIXME: Eventually we should control promotion instead of promoting
1853 // and internalizing again.
1854 StringRef OrigName =
1855 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1856 std::string OrigId = GlobalValue::getGlobalIdentifier(
1857 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1858 FileName: TheModule.getSourceFileName());
1859 GS = DefinedGlobals.find(
1860 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1861 if (GS == DefinedGlobals.end()) {
1862 // Also check the original non-promoted non-globalized name. In some
1863 // cases a preempted weak value is linked in as a local copy because
1864 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1865 // In that case, since it was originally not a local value, it was
1866 // recorded in the index using the original name.
1867 // FIXME: This may not be needed once PR27866 is fixed.
1868 GS = DefinedGlobals.find(
1869 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1870 assert(GS != DefinedGlobals.end());
1871 }
1872 }
1873 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1874 };
1875
1876 // FIXME: See if we can just internalize directly here via linkage changes
1877 // based on the index, rather than invoking internalizeModule.
1878 internalizeModule(TheModule, MustPreserveGV);
1879}
1880
1881/// Make alias a clone of its aliasee.
1882static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1883 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1884
1885 ValueToValueMapTy VMap;
1886 Function *NewFn = CloneFunction(F: Fn, VMap);
1887 // Clone should use the original alias's linkage, visibility and name, and we
1888 // ensure all uses of alias instead use the new clone (casted if necessary).
1889 NewFn->setLinkage(GA->getLinkage());
1890 NewFn->setVisibility(GA->getVisibility());
1891 GA->replaceAllUsesWith(V: NewFn);
1892 NewFn->takeName(V: GA);
1893 return NewFn;
1894}
1895
1896// Internalize values that we marked with specific attribute
1897// in processGlobalForThinLTO.
1898static void internalizeGVsAfterImport(Module &M) {
1899 for (auto &GV : M.globals())
1900 // Skip GVs which have been converted to declarations
1901 // by dropDeadSymbols.
1902 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1903 GV.setLinkage(GlobalValue::InternalLinkage);
1904 GV.setVisibility(GlobalValue::DefaultVisibility);
1905 }
1906}
1907
1908/// When a function carrying !implicit.ref is imported via ThinLTO, the
1909/// referenced global arrives as available_externally. This string can be dead
1910/// code eliminated since it has no IR uses -- only metadata references. Adding
1911/// it to llvm.compiler.used prevents elimination.
1912static void protectImplicitRefGlobals(Module &M) {
1913 SmallPtrSet<GlobalValue *, 4> Seen;
1914 SmallVector<GlobalValue *, 4> ToProtect;
1915 for (Function &F : M) {
1916 if (!F.hasAvailableExternallyLinkage() ||
1917 !F.hasMetadata(KindID: LLVMContext::MD_implicit_ref))
1918 continue;
1919 SmallVector<MDNode *> MDs;
1920 F.getMetadata(KindID: LLVMContext::MD_implicit_ref, MDs);
1921 for (MDNode *MD : MDs) {
1922 auto *Op = MD->getOperand(I: 0).get();
1923 if (!Op)
1924 continue;
1925 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: Op))
1926 if (auto *GV = dyn_cast<GlobalVariable>(Val: VAM->getValue()))
1927 if (GV->hasAvailableExternallyLinkage() && Seen.insert(Ptr: GV).second)
1928 ToProtect.push_back(Elt: GV);
1929 }
1930 }
1931 if (!ToProtect.empty())
1932 appendToCompilerUsed(M, Values: ToProtect);
1933}
1934
1935// Automatically import functions in Module \p DestModule based on the summaries
1936// index.
1937Expected<bool> FunctionImporter::importFunctions(
1938 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1939 LLVM_DEBUG(dbgs() << "Starting import for Module "
1940 << DestModule.getModuleIdentifier() << "\n");
1941 unsigned ImportedCount = 0, ImportedGVCount = 0;
1942 // Before carrying out any imports, see if this module defines functions in
1943 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1944 // The function will be imported elsewhere, as extenal linkage, and the
1945 // destination doesn't yet have its definition.
1946 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1947 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1948 for (auto &F : DestModule)
1949 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUID()))
1950 F.deleteBody();
1951
1952 IRMover Mover(DestModule);
1953
1954 // Do the actual import of functions now, one Module at a time
1955 for (const auto &ModName : ImportList.getSourceModules()) {
1956 llvm::TimeTraceScope timeScope("Import", ModName);
1957 // Get the module for the import
1958 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1959 if (!SrcModuleOrErr)
1960 return SrcModuleOrErr.takeError();
1961 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1962 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1963 "Context mismatch");
1964
1965 // If modules were created with lazy metadata loading, materialize it
1966 // now, before linking it (otherwise this will be a noop).
1967 if (Error Err = SrcModule->materializeMetadata())
1968 return std::move(Err);
1969
1970 // Find the globals to import
1971 SetVector<GlobalValue *> GlobalsToImport;
1972 {
1973 llvm::TimeTraceScope functionsScope("Functions");
1974 for (Function &F : *SrcModule) {
1975 if (!F.hasName())
1976 continue;
1977 auto GUID = F.getGUID();
1978 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1979 bool ImportDefinition =
1980 MaybeImportType == GlobalValueSummary::Definition;
1981
1982 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1983 << " importing function"
1984 << (ImportDefinition
1985 ? " definition "
1986 : (MaybeImportType ? " declaration " : " "))
1987 << GUID << " " << F.getName() << " from "
1988 << SrcModule->getSourceFileName() << "\n");
1989 if (ImportDefinition) {
1990 if (Error Err = F.materialize())
1991 return std::move(Err);
1992 // MemProf should match function's definition and summary,
1993 // 'thinlto_src_module' is needed.
1994 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
1995 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
1996 // statistics and debugging.
1997 F.setMetadata(
1998 Kind: "thinlto_src_module",
1999 Node: MDNode::get(Context&: DestModule.getContext(),
2000 MDs: {MDString::get(Context&: DestModule.getContext(),
2001 Str: SrcModule->getModuleIdentifier())}));
2002 F.setMetadata(
2003 Kind: "thinlto_src_file",
2004 Node: MDNode::get(Context&: DestModule.getContext(),
2005 MDs: {MDString::get(Context&: DestModule.getContext(),
2006 Str: SrcModule->getSourceFileName())}));
2007 }
2008 GlobalsToImport.insert(X: &F);
2009 }
2010 }
2011 }
2012 {
2013 llvm::TimeTraceScope globalsScope("Globals");
2014 for (GlobalVariable &GV : SrcModule->globals()) {
2015 if (!GV.hasName())
2016 continue;
2017 auto GUID = GV.getGUID();
2018 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2019 bool ImportDefinition =
2020 MaybeImportType == GlobalValueSummary::Definition;
2021
2022 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2023 << " importing global"
2024 << (ImportDefinition
2025 ? " definition "
2026 : (MaybeImportType ? " declaration " : " "))
2027 << GUID << " " << GV.getName() << " from "
2028 << SrcModule->getSourceFileName() << "\n");
2029 if (ImportDefinition) {
2030 if (Error Err = GV.materialize())
2031 return std::move(Err);
2032 ImportedGVCount += GlobalsToImport.insert(X: &GV);
2033 }
2034 }
2035 }
2036 {
2037 llvm::TimeTraceScope aliasesScope("Aliases");
2038 for (GlobalAlias &GA : SrcModule->aliases()) {
2039 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
2040 continue;
2041 auto GUID = GA.getGUID();
2042 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2043 bool ImportDefinition =
2044 MaybeImportType == GlobalValueSummary::Definition;
2045
2046 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2047 << " importing alias"
2048 << (ImportDefinition
2049 ? " definition "
2050 : (MaybeImportType ? " declaration " : " "))
2051 << GUID << " " << GA.getName() << " from "
2052 << SrcModule->getSourceFileName() << "\n");
2053 if (ImportDefinition) {
2054 if (Error Err = GA.materialize())
2055 return std::move(Err);
2056 // Import alias as a copy of its aliasee.
2057 GlobalObject *GO = GA.getAliaseeObject();
2058 if (Error Err = GO->materialize())
2059 return std::move(Err);
2060 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
2061 LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID()
2062 << " " << GO->getName() << " from "
2063 << SrcModule->getSourceFileName() << "\n");
2064 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2065 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2066 // statistics and debugging.
2067 Fn->setMetadata(
2068 Kind: "thinlto_src_module",
2069 Node: MDNode::get(Context&: DestModule.getContext(),
2070 MDs: {MDString::get(Context&: DestModule.getContext(),
2071 Str: SrcModule->getModuleIdentifier())}));
2072 Fn->setMetadata(
2073 Kind: "thinlto_src_file",
2074 Node: MDNode::get(Context&: DestModule.getContext(),
2075 MDs: {MDString::get(Context&: DestModule.getContext(),
2076 Str: SrcModule->getSourceFileName())}));
2077 }
2078 GlobalsToImport.insert(X: Fn);
2079 }
2080 }
2081 }
2082
2083 // Upgrade debug info after we're done materializing all the globals and we
2084 // have loaded all the required metadata!
2085 UpgradeDebugInfo(M&: *SrcModule);
2086
2087 // Set the partial sample profile ratio in the profile summary module flag
2088 // of the imported source module, if applicable, so that the profile summary
2089 // module flag will match with that of the destination module when it's
2090 // imported.
2091 SrcModule->setPartialSampleProfileRatio(Index);
2092
2093 // Link in the specified functions.
2094 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2095 GlobalsToImport: &GlobalsToImport);
2096
2097 if (PrintImports) {
2098 for (const auto *GV : GlobalsToImport)
2099 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2100 << " from " << SrcModule->getSourceFileName() << "\n";
2101 }
2102
2103 if (Error Err = Mover.move(Src: std::move(SrcModule),
2104 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2105 /*IsPerformingImport=*/true))
2106 return createStringError(EC: errc::invalid_argument,
2107 S: Twine("Function Import: link error: ") +
2108 toString(E: std::move(Err)));
2109
2110 ImportedCount += GlobalsToImport.size();
2111 NumImportedModules++;
2112 }
2113
2114 internalizeGVsAfterImport(M&: DestModule);
2115
2116 // Protect !implicit.ref-referenced globals imported as available_externally
2117 // from DCE'd. Only needed when globals were actually imported.
2118 if (ImportedGVCount > 0)
2119 protectImplicitRefGlobals(M&: DestModule);
2120
2121 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2122 NumImportedGlobalVars += ImportedGVCount;
2123
2124 // TODO: Print counters for definitions and declarations in the debugging log.
2125 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2126 << " functions for Module "
2127 << DestModule.getModuleIdentifier() << "\n");
2128 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2129 << " global variables for Module "
2130 << DestModule.getModuleIdentifier() << "\n");
2131 return ImportedCount;
2132}
2133
2134static bool doImportingForModuleForTest(
2135 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2136 isPrevailing) {
2137 if (SummaryFile.empty())
2138 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2139 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2140 getModuleSummaryIndexForFile(Path: SummaryFile);
2141 if (!IndexPtrOrErr) {
2142 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2143 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2144 return false;
2145 }
2146 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2147
2148 // First step is collecting the import list.
2149 FunctionImporter::ImportIDTable ImportIDs;
2150 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2151 // If requested, simply import all functions in the index. This is used
2152 // when testing distributed backend handling via the opt tool, when
2153 // we have distributed indexes containing exactly the summaries to import.
2154 if (ImportAllIndex)
2155 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2156 Index: *Index, ImportList);
2157 else
2158 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2159 isPrevailing, Index: *Index, ImportList);
2160
2161 // Conservatively mark all internal values as promoted. This interface is
2162 // only used when doing importing via the function importing pass. The pass
2163 // is only enabled when testing importing via the 'opt' tool, which does
2164 // not do the ThinLink that would normally determine what values to promote.
2165 for (auto &I : *Index) {
2166 for (auto &S : I.second.getSummaryList()) {
2167 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2168 S->setExternalLinkageForTest();
2169 }
2170 }
2171
2172 // Next we need to promote to global scope and rename any local values that
2173 // are potentially exported to other modules.
2174 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2175 /*GlobalsToImport=*/nullptr);
2176
2177 // Perform the import now.
2178 auto ModuleLoader = [&M](StringRef Identifier) {
2179 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2180 };
2181 FunctionImporter Importer(*Index, ModuleLoader,
2182 /*ClearDSOLocalOnDeclarations=*/false);
2183 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2184
2185 // FIXME: Probably need to propagate Errors through the pass manager.
2186 if (!Result) {
2187 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2188 ErrorBanner: "Error importing module: ");
2189 return true;
2190 }
2191
2192 return true;
2193}
2194
2195PreservedAnalyses FunctionImportPass::run(Module &M,
2196 ModuleAnalysisManager &AM) {
2197 // This is only used for testing the function import pass via opt, where we
2198 // don't have prevailing information from the LTO context available, so just
2199 // conservatively assume everything is prevailing (which is fine for the very
2200 // limited use of prevailing checking in this pass).
2201 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2202 return true;
2203 };
2204 if (!doImportingForModuleForTest(M, isPrevailing))
2205 return PreservedAnalyses::all();
2206
2207 return PreservedAnalyses::none();
2208}
2209