1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Bitcode/BitcodeReader.h"
22#include "llvm/IR/AutoUpgrade.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalAlias.h"
25#include "llvm/IR/GlobalObject.h"
26#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/GlobalVariable.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/ModuleSummaryIndex.h"
32#include "llvm/IRReader/IRReader.h"
33#include "llvm/Linker/IRMover.h"
34#include "llvm/ProfileData/PGOCtxProfReader.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Support/Errc.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/FileSystem.h"
42#include "llvm/Support/JSON.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SourceMgr.h"
45#include "llvm/Support/TimeProfiler.h"
46#include "llvm/Support/raw_ostream.h"
47#include "llvm/Transforms/IPO/Internalize.h"
48#include "llvm/Transforms/Utils/Cloning.h"
49#include "llvm/Transforms/Utils/FunctionImportUtils.h"
50#include "llvm/Transforms/Utils/ModuleUtils.h"
51#include "llvm/Transforms/Utils/ValueMapper.h"
52#include <cassert>
53#include <memory>
54#include <string>
55#include <system_error>
56#include <tuple>
57#include <utility>
58
59using namespace llvm;
60
61#define DEBUG_TYPE "function-import"
62
63STATISTIC(NumImportedFunctionsThinLink,
64 "Number of functions thin link decided to import");
65STATISTIC(NumImportedHotFunctionsThinLink,
66 "Number of hot functions thin link decided to import");
67STATISTIC(NumImportedCriticalFunctionsThinLink,
68 "Number of critical functions thin link decided to import");
69STATISTIC(NumImportedGlobalVarsThinLink,
70 "Number of global variables thin link decided to import");
71STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
72STATISTIC(NumImportedGlobalVars,
73 "Number of global variables imported in backend");
74STATISTIC(NumImportedModules, "Number of modules imported from");
75STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
76STATISTIC(NumLiveSymbols, "Number of live symbols in index");
77
78namespace llvm {
79extern cl::opt<bool> AlwaysRenamePromotedLocals;
80
81cl::opt<bool>
82 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
83 cl::desc("Import functions with noinline attribute"));
84
85/// Limit on instruction count of imported functions.
86static cl::opt<unsigned> ImportInstrLimit(
87 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
88 cl::desc("Only import functions with less than N instructions"));
89
90static cl::opt<int> ImportCutoff(
91 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
92 cl::desc("Only import first N functions if N>=0 (default -1)"));
93
94static cl::opt<float>
95 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
96 cl::Hidden, cl::value_desc("x"),
97 cl::desc("As we import functions, multiply the "
98 "`import-instr-limit` threshold by this factor "
99 "before processing newly imported functions"));
100
101static cl::opt<float> ImportHotInstrFactor(
102 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
103 cl::value_desc("x"),
104 cl::desc("As we import functions called from hot callsite, multiply the "
105 "`import-instr-limit` threshold by this factor "
106 "before processing newly imported functions"));
107
108static cl::opt<float> ImportHotMultiplier(
109 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
110 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
111
112static cl::opt<float> ImportCriticalMultiplier(
113 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
114 cl::value_desc("x"),
115 cl::desc(
116 "Multiply the `import-instr-limit` threshold for critical callsites"));
117
118// FIXME: This multiplier was not really tuned up.
119static cl::opt<float> ImportColdMultiplier(
120 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
121 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
122
123static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
124 cl::desc("Print imported functions"));
125
126static cl::opt<bool> PrintImportFailures(
127 "print-import-failures", cl::init(Val: false), cl::Hidden,
128 cl::desc("Print information for functions rejected for importing"));
129
130static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
131 cl::desc("Compute dead symbols"));
132
133static cl::opt<bool> EnableImportMetadata(
134 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
135 cl::desc("Enable import metadata like 'thinlto_src_module' and "
136 "'thinlto_src_file'"));
137
138/// Summary file to use for function importing when using -function-import from
139/// the command line.
140static cl::opt<std::string>
141 SummaryFile("summary-file",
142 cl::desc("The summary file to use for function importing."));
143
144/// Used when testing importing from distributed indexes via opt
145// -function-import.
146static cl::opt<bool>
147 ImportAllIndex("import-all-index",
148 cl::desc("Import all external functions in index."));
149
150/// This is a test-only option.
151/// If this option is enabled, the ThinLTO indexing step will import each
152/// function declaration as a fallback. In a real build this may increase ram
153/// usage of the indexing step unnecessarily.
154/// TODO: Implement selective import (based on combined summary analysis) to
155/// ensure the imported function has a use case in the postlink pipeline.
156static cl::opt<bool> ImportDeclaration(
157 "import-declaration", cl::init(Val: false), cl::Hidden,
158 cl::desc("If true, import function declaration as fallback if the function "
159 "definition is not imported."));
160
161/// Pass a workload description file - an example of workload would be the
162/// functions executed to satisfy a RPC request. A workload is defined by a root
163/// function and the list of functions that are (frequently) needed to satisfy
164/// it. The module that defines the root will have all those functions imported.
165/// The file contains a JSON dictionary. The keys are root functions, the values
166/// are lists of functions to import in the module defining the root. It is
167/// assumed -funique-internal-linkage-names was used, thus ensuring function
168/// names are unique even for local linkage ones.
169static cl::opt<std::string> WorkloadDefinitions(
170 "thinlto-workload-def",
171 cl::desc("Pass a workload definition. This is a file containing a JSON "
172 "dictionary. The keys are root functions, the values are lists of "
173 "functions to import in the module defining the root. It is "
174 "assumed -funique-internal-linkage-names was used, to ensure "
175 "local linkage functions have unique names. For example: \n"
176 "{\n"
177 " \"rootFunction_1\": [\"function_to_import_1\", "
178 "\"function_to_import_2\"], \n"
179 " \"rootFunction_2\": [\"function_to_import_3\", "
180 "\"function_to_import_4\"] \n"
181 "}"),
182 cl::Hidden);
183
184extern cl::opt<std::string> UseCtxProfile;
185
186static cl::opt<bool> CtxprofMoveRootsToOwnModule(
187 "thinlto-move-ctxprof-trees",
188 cl::desc("Move contextual profiling roots and the graphs under them in "
189 "their own module."),
190 cl::Hidden, cl::init(Val: false));
191
192extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
193
194extern cl::opt<bool> EnableMemProfContextDisambiguation;
195} // end namespace llvm
196
197// Load lazily a module from \p FileName in \p Context.
198static std::unique_ptr<Module> loadFile(const std::string &FileName,
199 LLVMContext &Context) {
200 SMDiagnostic Err;
201 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
202 // Metadata isn't loaded until functions are imported, to minimize
203 // the memory overhead.
204 std::unique_ptr<Module> Result =
205 getLazyIRFileModule(Filename: FileName, Err, Context,
206 /* ShouldLazyLoadMetadata = */ true);
207 if (!Result) {
208 Err.print(ProgName: "function-import", S&: errs());
209 report_fatal_error(reason: "Abort");
210 }
211
212 return Result;
213}
214
215static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
216 size_t NumDefs,
217 StringRef ImporterModule) {
218 // We can import a local when there is one definition.
219 if (NumDefs == 1)
220 return false;
221 // In other cases, make sure we import the copy in the caller's module if the
222 // referenced value has local linkage. The only time a local variable can
223 // share an entry in the index is if there is a local with the same name in
224 // another module that had the same source file name (in a different
225 // directory), where each was compiled in their own directory so there was not
226 // distinguishing path.
227 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
228 RefSummary->modulePath() != ImporterModule;
229}
230
231/// Given a list of possible callee implementation for a call site, qualify the
232/// legality of importing each. The return is a range of pairs. Each pair
233/// corresponds to a candidate. The first value is the ImportFailureReason for
234/// that candidate, the second is the candidate.
235static auto qualifyCalleeCandidates(
236 const ModuleSummaryIndex &Index,
237 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
238 StringRef CallerModulePath) {
239 return llvm::map_range(
240 C&: CalleeSummaryList,
241 F: [&Index, CalleeSummaryList,
242 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
243 -> std::pair<FunctionImporter::ImportFailureReason,
244 const GlobalValueSummary *> {
245 auto *GVSummary = SummaryPtr.get();
246 if (!Index.isGlobalValueLive(GVS: GVSummary))
247 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
248
249 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
250 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
251 GVSummary};
252
253 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
254
255 // Ignore any callees that aren't actually functions. This could happen
256 // in the case of GUID hash collisions. It could also happen in theory
257 // for SamplePGO profiles collected on old versions of the code after
258 // renaming, since we synthesize edges to any inlined callees appearing
259 // in the profile.
260 if (!Summary)
261 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
262
263 // If this is a local function, make sure we import the copy in the
264 // caller's module. The only time a local function can share an entry in
265 // the index is if there is a local with the same name in another module
266 // that had the same source file name (in a different directory), where
267 // each was compiled in their own directory so there was not
268 // distinguishing path.
269 // If the local function is from another module, it must be a reference
270 // due to indirect call profile data since a function pointer can point
271 // to a local in another module. Do the import from another module if
272 // there is only one entry in the list or when all files in the program
273 // are compiled with full path - in both cases the local function has
274 // unique PGO name and GUID.
275 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
276 ImporterModule: CallerModulePath))
277 return {
278 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
279 GVSummary};
280
281 // Skip if it isn't legal to import (e.g. may reference unpromotable
282 // locals).
283 if (Summary->notEligibleToImport())
284 return {FunctionImporter::ImportFailureReason::NotEligible,
285 GVSummary};
286
287 return {FunctionImporter::ImportFailureReason::None, GVSummary};
288 });
289}
290
291/// Given a list of possible callee implementation for a call site, select one
292/// that fits the \p Threshold for function definition import. If none are
293/// found, the Reason will give the last reason for the failure (last, in the
294/// order of CalleeSummaryList entries). While looking for a callee definition,
295/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
296/// candidate; other modules may want to know the function summary or
297/// declaration even if a definition is not needed.
298///
299/// FIXME: select "best" instead of first that fits. But what is "best"?
300/// - The smallest: more likely to be inlined.
301/// - The one with the least outgoing edges (already well optimized).
302/// - One from a module already being imported from in order to reduce the
303/// number of source modules parsed/linked.
304/// - One that has PGO data attached.
305/// - [insert you fancy metric here]
306static const GlobalValueSummary *
307selectCallee(const ModuleSummaryIndex &Index,
308 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
309 unsigned Threshold, StringRef CallerModulePath,
310 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
311 FunctionImporter::ImportFailureReason &Reason) {
312 // Records the last summary with reason noinline or too-large.
313 TooLargeOrNoInlineSummary = nullptr;
314 auto QualifiedCandidates =
315 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
316 for (auto QualifiedValue : QualifiedCandidates) {
317 Reason = QualifiedValue.first;
318 // Skip a summary if its import is not (proved to be) legal.
319 if (Reason != FunctionImporter::ImportFailureReason::None)
320 continue;
321 auto *Summary =
322 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
323
324 // Don't bother importing the definition if the chance of inlining it is
325 // not high enough (except under `--force-import-all`).
326 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
327 !ForceImportAll) {
328 TooLargeOrNoInlineSummary = Summary;
329 Reason = FunctionImporter::ImportFailureReason::TooLarge;
330 continue;
331 }
332
333 // Don't bother importing the definition if we can't inline it anyway.
334 if (Summary->fflags().NoInline && !ForceImportAll) {
335 TooLargeOrNoInlineSummary = Summary;
336 Reason = FunctionImporter::ImportFailureReason::NoInline;
337 continue;
338 }
339
340 return Summary;
341 }
342 return nullptr;
343}
344
345namespace {
346
347using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
348
349} // anonymous namespace
350
351FunctionImporter::ImportMapTy::AddDefinitionStatus
352FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
353 GlobalValue::GUID GUID) {
354 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
355 if (!Imports.insert(V: Def).second)
356 // Already there.
357 return AddDefinitionStatus::NoChange;
358
359 // Remove Decl in case it's there. Note that a definition takes precedence
360 // over a declaration for a given GUID.
361 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
362 : AddDefinitionStatus::Inserted;
363}
364
365void FunctionImporter::ImportMapTy::maybeAddDeclaration(
366 StringRef FromModule, GlobalValue::GUID GUID) {
367 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
368 // Insert Decl only if Def is not present. Note that a definition takes
369 // precedence over a declaration for a given GUID.
370 if (!Imports.contains(V: Def))
371 Imports.insert(V: Decl);
372}
373
374SmallVector<StringRef, 0>
375FunctionImporter::ImportMapTy::getSourceModules() const {
376 SetVector<StringRef> ModuleSet;
377 for (const auto &[SrcMod, GUID, ImportType] : *this)
378 ModuleSet.insert(X: SrcMod);
379 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
380 llvm::sort(C&: Modules);
381 return Modules;
382}
383
384std::optional<GlobalValueSummary::ImportKind>
385FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
386 GlobalValue::GUID GUID) const {
387 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
388 auto [Def, Decl] = *IDPair;
389 if (Imports.contains(V: Def))
390 return GlobalValueSummary::Definition;
391 if (Imports.contains(V: Decl))
392 return GlobalValueSummary::Declaration;
393 }
394 return std::nullopt;
395}
396
397/// Import globals referenced by a function or other globals that are being
398/// imported, if importing such global is possible.
399class GlobalsImporter final {
400 const ModuleSummaryIndex &Index;
401 const GVSummaryMapTy &DefinedGVSummaries;
402 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
403 IsPrevailing;
404 FunctionImporter::ImportMapTy &ImportList;
405 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
406
407 bool shouldImportGlobal(const ValueInfo &VI) {
408 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
409 if (GVS == DefinedGVSummaries.end())
410 return true;
411 // We should not skip import if the module contains a non-prevailing
412 // definition with interposable linkage type. This is required for
413 // correctness in the situation where there is a prevailing def available
414 // for import and marked read-only. In this case, the non-prevailing def
415 // will be converted to a declaration, while the prevailing one becomes
416 // internal, thus no definitions will be available for linking. In order to
417 // prevent undefined symbol link error, the prevailing definition must be
418 // imported.
419 // FIXME: Consider adding a check that the suitable prevailing definition
420 // exists and marked read-only.
421 if (VI.getSummaryList().size() > 1 &&
422 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
423 !IsPrevailing(VI.getGUID(), GVS->second))
424 return true;
425
426 return false;
427 }
428
429 void
430 onImportingSummaryImpl(const GlobalValueSummary &Summary,
431 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
432 for (const auto &VI : Summary.refs()) {
433 if (!shouldImportGlobal(VI)) {
434 LLVM_DEBUG(
435 dbgs() << "Ref ignored! Target already in destination module.\n");
436 continue;
437 }
438
439 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
440
441 for (const auto &RefSummary : VI.getSummaryList()) {
442 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
443 // Stop looking if this is not a global variable, e.g. a function.
444 // Functions could be referenced by global vars - e.g. a vtable; but we
445 // don't currently imagine a reason those would be imported here, rather
446 // than as part of the logic deciding which functions to import (i.e.
447 // based on profile information). Should we decide to handle them here,
448 // we can refactor accordingly at that time.
449 // Note that it is safe to stop looking because the one case where we
450 // might have to import (a read/write-only global variable) cannot occur
451 // if this GUID has a non-variable summary. The only case where we even
452 // might find another summary in the list that is a variable is in the
453 // case of same-named locals in different modules not compiled with
454 // enough path, and during attribute propagation we will mark all
455 // summaries for a GUID (ValueInfo) as non read/write-only if any is not
456 // a global variable.
457 if (!GVS)
458 break;
459 bool CanImportDecl = false;
460 if (shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
461 ImporterModule: Summary.modulePath()) ||
462 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
463 CanImportDecl)) {
464 if (ImportDeclaration && CanImportDecl)
465 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
466 GUID: VI.getGUID());
467
468 continue;
469 }
470
471 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
472 // Otherwise, definition should take precedence over declaration.
473 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
474 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
475 break;
476
477 // Only update stat and exports if we haven't already imported this
478 // variable.
479 NumImportedGlobalVarsThinLink++;
480 // Any references made by this variable will be marked exported
481 // later, in ComputeCrossModuleImport, after import decisions are
482 // complete, which is more efficient than adding them here.
483 if (ExportLists)
484 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
485
486 // If variable is not writeonly we attempt to recursively analyze
487 // its references in order to import referenced constants.
488 if (!Index.isWriteOnly(GVS))
489 Worklist.emplace_back(Args&: GVS);
490 break;
491 }
492 }
493 }
494
495public:
496 GlobalsImporter(
497 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
498 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
499 IsPrevailing,
500 FunctionImporter::ImportMapTy &ImportList,
501 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
502 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
503 IsPrevailing(IsPrevailing), ImportList(ImportList),
504 ExportLists(ExportLists) {}
505
506 void onImportingSummary(const GlobalValueSummary &Summary) {
507 SmallVector<const GlobalVarSummary *, 128> Worklist;
508 onImportingSummaryImpl(Summary, Worklist);
509 while (!Worklist.empty())
510 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
511 }
512};
513
514static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
515
516/// Determine the list of imports and exports for each module.
517class ModuleImportsManager {
518 void computeImportForFunction(
519 const FunctionSummary &Summary, unsigned Threshold,
520 const GVSummaryMapTy &DefinedGVSummaries,
521 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
522 FunctionImporter::ImportMapTy &ImportList,
523 FunctionImporter::ImportThresholdsTy &ImportThresholds);
524
525protected:
526 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
527 IsPrevailing;
528 const ModuleSummaryIndex &Index;
529 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
530
531 ModuleImportsManager(
532 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
533 IsPrevailing,
534 const ModuleSummaryIndex &Index,
535 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
536 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
537 virtual bool canImport(ValueInfo VI) { return true; }
538
539public:
540 virtual ~ModuleImportsManager() = default;
541
542 /// Given the list of globals defined in a module, compute the list of imports
543 /// as well as the list of "exports", i.e. the list of symbols referenced from
544 /// another module (that may require promotion).
545 virtual void
546 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
547 StringRef ModName,
548 FunctionImporter::ImportMapTy &ImportList);
549
550 static std::unique_ptr<ModuleImportsManager>
551 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
552 IsPrevailing,
553 const ModuleSummaryIndex &Index,
554 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
555 nullptr);
556};
557
558/// A ModuleImportsManager that operates based on a workload definition (see
559/// -thinlto-workload-def). For modules that do not define workload roots, it
560/// applies the base ModuleImportsManager import policy.
561class WorkloadImportsManager : public ModuleImportsManager {
562 // Keep a module name -> value infos to import association. We use it to
563 // determine if a module's import list should be done by the base
564 // ModuleImportsManager or by us.
565 StringMap<DenseSet<ValueInfo>> Workloads;
566 // Track the roots to avoid importing them due to other callers. We want there
567 // to be only one variant), for which we optimize according to the contextual
568 // profile. "Variants" refers to copies due to importing - we want there to be
569 // just one instance of this function.
570 DenseSet<ValueInfo> Roots;
571
572 void
573 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
574 StringRef ModName,
575 FunctionImporter::ImportMapTy &ImportList) override {
576 StringRef Filename = ModName;
577 if (CtxprofMoveRootsToOwnModule) {
578 Filename = sys::path::filename(path: ModName);
579 // Drop the file extension.
580 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
581 }
582 auto SetIter = Workloads.find(Key: Filename);
583
584 if (SetIter == Workloads.end()) {
585 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
586 << " does not contain the root of any context.\n");
587 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
588 ModName, ImportList);
589 }
590 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
591 << " contains the root(s) of context(s).\n");
592
593 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
594 ExportLists);
595 auto &ValueInfos = SetIter->second;
596 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
597 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
598 if (It != DefinedGVSummaries.end() &&
599 IsPrevailing(VI.getGUID(), It->second)) {
600 LLVM_DEBUG(
601 dbgs() << "[Workload] " << VI.name()
602 << " has the prevailing variant already in the module "
603 << ModName << ". No need to import\n");
604 continue;
605 }
606 auto Candidates =
607 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
608
609 const GlobalValueSummary *GVS = nullptr;
610 auto PotentialCandidates = llvm::map_range(
611 C: llvm::make_filter_range(
612 Range&: Candidates,
613 Pred: [&](const auto &Candidate) {
614 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
615 << " from " << Candidate.second->modulePath()
616 << " ImportFailureReason: "
617 << getFailureName(Candidate.first) << "\n");
618 return Candidate.first ==
619 FunctionImporter::ImportFailureReason::None;
620 }),
621 F: [](const auto &Candidate) { return Candidate.second; });
622 if (PotentialCandidates.empty()) {
623 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
624 << " because can't find eligible Callee. Guid is: "
625 << VI.getGUID() << "\n");
626 continue;
627 }
628 /// We will prefer importing the prevailing candidate, if not, we'll
629 /// still pick the first available candidate. The reason we want to make
630 /// sure we do import the prevailing candidate is because the goal of
631 /// workload-awareness is to enable optimizations specializing the call
632 /// graph of that workload. Suppose a function is already defined in the
633 /// module, but it's not the prevailing variant. Suppose also we do not
634 /// inline it (in fact, if it were interposable, we can't inline it),
635 /// but we could specialize it to the workload in other ways. However,
636 /// the linker would drop it in the favor of the prevailing copy.
637 /// Instead, by importing the prevailing variant (assuming also the use
638 /// of `-avail-extern-to-local`), we keep the specialization. We could
639 /// alteranatively make the non-prevailing variant local, but the
640 /// prevailing one is also the one for which we would have previously
641 /// collected profiles, making it preferrable.
642 auto PrevailingCandidates = llvm::make_filter_range(
643 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
644 return IsPrevailing(VI.getGUID(), Candidate);
645 });
646 if (PrevailingCandidates.empty()) {
647 GVS = *PotentialCandidates.begin();
648 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
649 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
650 LLVM_DEBUG(
651 dbgs()
652 << "[Workload] Found multiple non-prevailing candidates for "
653 << VI.name()
654 << ". This is unexpected. Are module paths passed to the "
655 "compiler unique for the modules passed to the linker?");
656 // We could in theory have multiple (interposable) copies of a symbol
657 // when there is no prevailing candidate, if say the prevailing copy was
658 // in a native object being linked in. However, we should in theory be
659 // marking all of these non-prevailing IR copies dead in that case, in
660 // which case they won't be candidates.
661 assert(GVS->isLive());
662 } else {
663 assert(llvm::hasSingleElement(PrevailingCandidates));
664 GVS = *PrevailingCandidates.begin();
665 }
666
667 auto ExportingModule = GVS->modulePath();
668 // We checked that for the prevailing case, but if we happen to have for
669 // example an internal that's defined in this module, it'd have no
670 // PrevailingCandidates.
671 if (ExportingModule == ModName) {
672 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
673 << " because its defining module is the same as the "
674 "current module\n");
675 continue;
676 }
677 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
678 << ExportingModule << " : " << VI.getGUID() << "\n");
679 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
680 GVI.onImportingSummary(Summary: *GVS);
681 if (ExportLists)
682 (*ExportLists)[ExportingModule].insert(V: VI);
683 }
684 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
685 }
686
687 void loadFromJson() {
688 // Since the workload def uses names, we need a quick lookup
689 // name->ValueInfo.
690 StringMap<ValueInfo> NameToValueInfo;
691 StringSet<> AmbiguousNames;
692 for (auto &I : Index) {
693 ValueInfo VI = Index.getValueInfo(R: I);
694 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
695 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
696 }
697 auto DbgReportIfAmbiguous = [&](StringRef Name) {
698 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
699 dbgs() << "[Workload] Function name " << Name
700 << " present in the workload definition is ambiguous. Consider "
701 "compiling with -funique-internal-linkage-names.";
702 });
703 };
704 std::error_code EC;
705 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
706 if (std::error_code EC = BufferOrErr.getError()) {
707 report_fatal_error(reason: "Failed to open context file");
708 return;
709 }
710 auto Buffer = std::move(BufferOrErr.get());
711 std::map<std::string, std::vector<std::string>> WorkloadDefs;
712 json::Path::Root NullRoot;
713 // The JSON is supposed to contain a dictionary matching the type of
714 // WorkloadDefs. For example:
715 // {
716 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
717 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
718 // }
719 auto Parsed = json::parse(JSON: Buffer->getBuffer());
720 if (!Parsed)
721 report_fatal_error(Err: Parsed.takeError());
722 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
723 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
724 for (const auto &Workload : WorkloadDefs) {
725 const auto &Root = Workload.first;
726 DbgReportIfAmbiguous(Root);
727 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
728 const auto &AllCallees = Workload.second;
729 auto RootIt = NameToValueInfo.find(Key: Root);
730 if (RootIt == NameToValueInfo.end()) {
731 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
732 << " not found in this linkage unit.\n");
733 continue;
734 }
735 auto RootVI = RootIt->second;
736 if (RootVI.getSummaryList().size() != 1) {
737 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
738 << " should have exactly one summary, but has "
739 << RootVI.getSummaryList().size() << ". Skipping.\n");
740 continue;
741 }
742 StringRef RootDefiningModule =
743 RootVI.getSummaryList().front()->modulePath();
744 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
745 << " is : " << RootDefiningModule << "\n");
746 auto &Set = Workloads[RootDefiningModule];
747 for (const auto &Callee : AllCallees) {
748 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
749 DbgReportIfAmbiguous(Callee);
750 auto ElemIt = NameToValueInfo.find(Key: Callee);
751 if (ElemIt == NameToValueInfo.end()) {
752 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
753 continue;
754 }
755 Set.insert(V: ElemIt->second);
756 }
757 }
758 }
759
760 void loadFromCtxProf() {
761 std::error_code EC;
762 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
763 if (std::error_code EC = BufferOrErr.getError()) {
764 report_fatal_error(reason: "Failed to open contextual profile file");
765 return;
766 }
767 auto Buffer = std::move(BufferOrErr.get());
768
769 PGOCtxProfileReader Reader(Buffer->getBuffer());
770 auto Ctx = Reader.loadProfiles();
771 if (!Ctx) {
772 report_fatal_error(reason: "Failed to parse contextual profiles");
773 return;
774 }
775 const auto &CtxMap = Ctx->Contexts;
776 SetVector<GlobalValue::GUID> ContainedGUIDs;
777 for (const auto &[RootGuid, Root] : CtxMap) {
778 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
779 // subsequent roots, but clear its contents.
780 ContainedGUIDs.clear();
781
782 auto RootVI = Index.getValueInfo(GUID: RootGuid);
783 if (!RootVI) {
784 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
785 << " not found in this linkage unit.\n");
786 continue;
787 }
788 if (RootVI.getSummaryList().size() != 1) {
789 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
790 << " should have exactly one summary, but has "
791 << RootVI.getSummaryList().size() << ". Skipping.\n");
792 continue;
793 }
794 std::string RootDefiningModule =
795 RootVI.getSummaryList().front()->modulePath().str();
796 if (CtxprofMoveRootsToOwnModule) {
797 RootDefiningModule = std::to_string(val: RootGuid);
798 LLVM_DEBUG(
799 dbgs() << "[Workload] Moving " << RootGuid
800 << " to a module with the filename without extension : "
801 << RootDefiningModule << "\n");
802 } else {
803 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
804 << " is : " << RootDefiningModule << "\n");
805 }
806 auto &Set = Workloads[RootDefiningModule];
807 Root.getContainedGuids(Guids&: ContainedGUIDs);
808 Roots.insert(V: RootVI);
809 for (auto Guid : ContainedGUIDs)
810 if (auto VI = Index.getValueInfo(GUID: Guid))
811 Set.insert(V: VI);
812 }
813 }
814
815 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
816
817public:
818 WorkloadImportsManager(
819 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
820 IsPrevailing,
821 const ModuleSummaryIndex &Index,
822 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
823 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
824 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
825 report_fatal_error(
826 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
827 return;
828 }
829 if (!UseCtxProfile.empty())
830 loadFromCtxProf();
831 else
832 loadFromJson();
833 LLVM_DEBUG({
834 for (const auto &[Root, Set] : Workloads) {
835 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
836 << " distinct callees.\n";
837 for (const auto &VI : Set) {
838 dbgs() << "[Workload] Root: " << Root
839 << " Would include: " << VI.getGUID() << "\n";
840 }
841 }
842 });
843 }
844};
845
846std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
847 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
848 IsPrevailing,
849 const ModuleSummaryIndex &Index,
850 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
851 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
852 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
853 return std::unique_ptr<ModuleImportsManager>(
854 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
855 }
856 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
857 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
858 args&: ExportLists);
859}
860
861static const char *
862getFailureName(FunctionImporter::ImportFailureReason Reason) {
863 switch (Reason) {
864 case FunctionImporter::ImportFailureReason::None:
865 return "None";
866 case FunctionImporter::ImportFailureReason::GlobalVar:
867 return "GlobalVar";
868 case FunctionImporter::ImportFailureReason::NotLive:
869 return "NotLive";
870 case FunctionImporter::ImportFailureReason::TooLarge:
871 return "TooLarge";
872 case FunctionImporter::ImportFailureReason::InterposableLinkage:
873 return "InterposableLinkage";
874 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
875 return "LocalLinkageNotInModule";
876 case FunctionImporter::ImportFailureReason::NotEligible:
877 return "NotEligible";
878 case FunctionImporter::ImportFailureReason::NoInline:
879 return "NoInline";
880 }
881 llvm_unreachable("invalid reason");
882}
883
884/// Compute the list of functions to import for a given caller. Mark these
885/// imported functions and the symbols they reference in their source module as
886/// exported from their source module.
887void ModuleImportsManager::computeImportForFunction(
888 const FunctionSummary &Summary, const unsigned Threshold,
889 const GVSummaryMapTy &DefinedGVSummaries,
890 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
891 FunctionImporter::ImportMapTy &ImportList,
892 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
893 GVImporter.onImportingSummary(Summary);
894 static int ImportCount = 0;
895 for (const auto &Edge : Summary.calls()) {
896 ValueInfo VI = Edge.first;
897 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
898 << "\n");
899
900 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
901 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
902 << " reached.\n");
903 continue;
904 }
905
906 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
907 // FIXME: Consider not skipping import if the module contains
908 // a non-prevailing def with interposable linkage. The prevailing copy
909 // can safely be imported (see shouldImportGlobal()).
910 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
911 continue;
912 }
913
914 if (!canImport(VI)) {
915 LLVM_DEBUG(
916 dbgs() << "Skipping over " << VI.getGUID()
917 << " because its import is handled in a different module.");
918 assert(VI.getSummaryList().size() == 1 &&
919 "The root was expected to be an external symbol");
920 continue;
921 }
922
923 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
924 if (Hotness == CalleeInfo::HotnessType::Hot)
925 return ImportHotMultiplier;
926 if (Hotness == CalleeInfo::HotnessType::Cold)
927 return ImportColdMultiplier;
928 if (Hotness == CalleeInfo::HotnessType::Critical)
929 return ImportCriticalMultiplier;
930 return 1.0;
931 };
932
933 const auto NewThreshold =
934 Threshold * GetBonusMultiplier(Edge.second.getHotness());
935
936 auto IT = ImportThresholds.insert(KV: std::make_pair(
937 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
938 bool PreviouslyVisited = !IT.second;
939 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
940 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
941 auto &FailureInfo = std::get<2>(t&: IT.first->second);
942
943 bool IsHotCallsite =
944 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
945 bool IsCriticalCallsite =
946 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
947
948 const FunctionSummary *ResolvedCalleeSummary = nullptr;
949 if (CalleeSummary) {
950 assert(PreviouslyVisited);
951 // Since the traversal of the call graph is DFS, we can revisit a function
952 // a second time with a higher threshold. In this case, it is added back
953 // to the worklist with the new threshold (so that its own callee chains
954 // can be considered with the higher threshold).
955 if (NewThreshold <= ProcessedThreshold) {
956 LLVM_DEBUG(
957 dbgs() << "ignored! Target was already imported with Threshold "
958 << ProcessedThreshold << "\n");
959 continue;
960 }
961 // Update with new larger threshold.
962 ProcessedThreshold = NewThreshold;
963 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
964 } else {
965 // If we already rejected importing a callee at the same or higher
966 // threshold, don't waste time calling selectCallee.
967 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
968 LLVM_DEBUG(
969 dbgs() << "ignored! Target was already rejected with Threshold "
970 << ProcessedThreshold << "\n");
971 if (PrintImportFailures) {
972 assert(FailureInfo &&
973 "Expected FailureInfo for previously rejected candidate");
974 FailureInfo->Attempts++;
975 }
976 continue;
977 }
978
979 FunctionImporter::ImportFailureReason Reason{};
980
981 // `SummaryForDeclImport` is an summary eligible for declaration import.
982 const GlobalValueSummary *SummaryForDeclImport = nullptr;
983 CalleeSummary =
984 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
985 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
986 if (!CalleeSummary) {
987 // There isn't a callee for definition import but one for declaration
988 // import.
989 if (ImportDeclaration && SummaryForDeclImport) {
990 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
991
992 // Note `ExportLists` only keeps track of exports due to imported
993 // definitions.
994 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
995 }
996 // Update with new larger threshold if this was a retry (otherwise
997 // we would have already inserted with NewThreshold above). Also
998 // update failure info if requested.
999 if (PreviouslyVisited) {
1000 ProcessedThreshold = NewThreshold;
1001 if (PrintImportFailures) {
1002 assert(FailureInfo &&
1003 "Expected FailureInfo for previously rejected candidate");
1004 FailureInfo->Reason = Reason;
1005 FailureInfo->Attempts++;
1006 FailureInfo->MaxHotness =
1007 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
1008 }
1009 } else if (PrintImportFailures) {
1010 assert(!FailureInfo &&
1011 "Expected no FailureInfo for newly rejected candidate");
1012 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
1013 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
1014 }
1015 if (ForceImportAll) {
1016 std::string Msg = std::string("Failed to import function ") +
1017 VI.name().str() + " due to " +
1018 getFailureName(Reason);
1019 auto Error = make_error<StringError>(
1020 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1021 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1022 ErrorBanner: "Error importing module: ");
1023 break;
1024 } else {
1025 LLVM_DEBUG(dbgs()
1026 << "ignored! No qualifying callee with summary found.\n");
1027 continue;
1028 }
1029 }
1030
1031 // "Resolve" the summary
1032 CalleeSummary = CalleeSummary->getBaseObject();
1033 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1034
1035 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1036 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1037 "selectCallee() didn't honor the threshold");
1038
1039 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1040
1041 // Try emplace the definition entry, and update stats based on insertion
1042 // status.
1043 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1044 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1045 NumImportedFunctionsThinLink++;
1046 if (IsHotCallsite)
1047 NumImportedHotFunctionsThinLink++;
1048 if (IsCriticalCallsite)
1049 NumImportedCriticalFunctionsThinLink++;
1050 }
1051
1052 // Any calls/references made by this function will be marked exported
1053 // later, in ComputeCrossModuleImport, after import decisions are
1054 // complete, which is more efficient than adding them here.
1055 if (ExportLists)
1056 (*ExportLists)[ExportModulePath].insert(V: VI);
1057 }
1058
1059 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1060 // Adjust the threshold for next level of imported functions.
1061 // The threshold is different for hot callsites because we can then
1062 // inline chains of hot calls.
1063 if (IsHotCallsite)
1064 return Threshold * ImportHotInstrFactor;
1065 return Threshold * ImportInstrFactor;
1066 };
1067
1068 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1069
1070 ImportCount++;
1071
1072 // Insert the newly imported function to the worklist.
1073 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1074 }
1075}
1076
1077void ModuleImportsManager::computeImportForModule(
1078 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1079 FunctionImporter::ImportMapTy &ImportList) {
1080 // Worklist contains the list of function imported in this module, for which
1081 // we will analyse the callees and may import further down the callgraph.
1082 SmallVector<EdgeInfo, 128> Worklist;
1083 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1084 ExportLists);
1085 FunctionImporter::ImportThresholdsTy ImportThresholds;
1086
1087 // Populate the worklist with the import for the functions in the current
1088 // module
1089 for (const auto &GVSummary : DefinedGVSummaries) {
1090#ifndef NDEBUG
1091 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1092 // so this map look up (and possibly others) can be avoided.
1093 auto VI = Index.getValueInfo(GVSummary.first);
1094#endif
1095 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1096 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1097 continue;
1098 }
1099 auto *FuncSummary =
1100 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1101 if (!FuncSummary)
1102 // Skip import for global variables
1103 continue;
1104 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1105 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1106 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1107 }
1108
1109 // Process the newly imported functions and add callees to the worklist.
1110 while (!Worklist.empty()) {
1111 auto GVInfo = Worklist.pop_back_val();
1112 auto *Summary = std::get<0>(t&: GVInfo);
1113 auto Threshold = std::get<1>(t&: GVInfo);
1114
1115 computeImportForFunction(Summary: *Summary, Threshold, DefinedGVSummaries, Worklist,
1116 GVImporter&: GVI, ImportList, ImportThresholds);
1117 }
1118
1119 // Print stats about functions considered but rejected for importing
1120 // when requested.
1121 if (PrintImportFailures) {
1122 dbgs() << "Missed imports into module " << ModName << "\n";
1123 for (auto &I : ImportThresholds) {
1124 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1125 auto &CalleeSummary = std::get<1>(t&: I.second);
1126 auto &FailureInfo = std::get<2>(t&: I.second);
1127 if (CalleeSummary)
1128 continue; // We are going to import.
1129 assert(FailureInfo);
1130 FunctionSummary *FS = nullptr;
1131 if (!FailureInfo->VI.getSummaryList().empty())
1132 FS = dyn_cast<FunctionSummary>(
1133 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1134 dbgs() << FailureInfo->VI
1135 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1136 << ", Threshold = " << ProcessedThreshold
1137 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1138 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1139 << ", Attempts = " << FailureInfo->Attempts << "\n";
1140 }
1141 }
1142}
1143
1144#ifndef NDEBUG
1145static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1146 auto SL = VI.getSummaryList();
1147 return SL.empty()
1148 ? false
1149 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1150}
1151
1152static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1153 GlobalValue::GUID G) {
1154 if (const auto &VI = Index.getValueInfo(G))
1155 return isGlobalVarSummary(Index, VI);
1156 return false;
1157}
1158
1159// Return the number of global variable summaries in ExportSet.
1160static unsigned
1161numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1162 FunctionImporter::ExportSetTy &ExportSet) {
1163 unsigned NumGVS = 0;
1164 for (auto &VI : ExportSet)
1165 if (isGlobalVarSummary(Index, VI.getGUID()))
1166 ++NumGVS;
1167 return NumGVS;
1168}
1169
1170struct ImportStatistics {
1171 unsigned NumGVS = 0;
1172 unsigned DefinedFS = 0;
1173 unsigned Count = 0;
1174};
1175
1176// Compute import statistics for each source module in ImportList.
1177static DenseMap<StringRef, ImportStatistics>
1178collectImportStatistics(const ModuleSummaryIndex &Index,
1179 const FunctionImporter::ImportMapTy &ImportList) {
1180 DenseMap<StringRef, ImportStatistics> Histogram;
1181
1182 for (const auto &[FromModule, GUID, Type] : ImportList) {
1183 ImportStatistics &Entry = Histogram[FromModule];
1184 ++Entry.Count;
1185 if (isGlobalVarSummary(Index, GUID))
1186 ++Entry.NumGVS;
1187 else if (Type == GlobalValueSummary::Definition)
1188 ++Entry.DefinedFS;
1189 }
1190 return Histogram;
1191}
1192#endif
1193
1194#ifndef NDEBUG
1195static bool checkVariableImport(
1196 const ModuleSummaryIndex &Index,
1197 FunctionImporter::ImportListsTy &ImportLists,
1198 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1199 DenseSet<GlobalValue::GUID> FlattenedImports;
1200
1201 for (const auto &ImportPerModule : ImportLists)
1202 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1203 FlattenedImports.insert(GUID);
1204
1205 // Checks that all GUIDs of read/writeonly vars we see in export lists
1206 // are also in the import lists. Otherwise we my face linker undefs,
1207 // because readonly and writeonly vars are internalized in their
1208 // source modules. The exception would be if it has a linkage type indicating
1209 // that there may have been a copy existing in the importing module (e.g.
1210 // linkonce_odr). In that case we cannot accurately do this checking.
1211 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1212 const ValueInfo &VI) {
1213 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1214 Index.findSummaryInModule(VI, ModulePath));
1215 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1216 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1217 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1218 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1219 };
1220
1221 for (auto &ExportPerModule : ExportLists)
1222 for (auto &VI : ExportPerModule.second)
1223 if (!FlattenedImports.count(VI.getGUID()) &&
1224 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1225 return false;
1226
1227 return true;
1228}
1229#endif
1230
1231/// Compute all the import and export for every module using the Index.
1232void llvm::ComputeCrossModuleImport(
1233 const ModuleSummaryIndex &Index,
1234 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1235 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1236 isPrevailing,
1237 FunctionImporter::ImportListsTy &ImportLists,
1238 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1239 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1240 // For each module that has function defined, compute the import/export lists.
1241 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1242 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1243 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1244 << DefinedGVSummaries.first << "'\n");
1245 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1246 ModName: DefinedGVSummaries.first, ImportList);
1247 }
1248
1249 // When computing imports we only added the variables and functions being
1250 // imported to the export list. We also need to mark any references and calls
1251 // they make as exported as well. We do this here, as it is more efficient
1252 // since we may import the same values multiple times into different modules
1253 // during the import computation.
1254 for (auto &ELI : ExportLists) {
1255 // `NewExports` tracks the VI that gets exported because the full definition
1256 // of its user/referencer gets exported.
1257 FunctionImporter::ExportSetTy NewExports;
1258 const auto &DefinedGVSummaries =
1259 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1260 for (auto &EI : ELI.second) {
1261 // Find the copy defined in the exporting module so that we can mark the
1262 // values it references in that specific definition as exported.
1263 // Below we will add all references and called values, without regard to
1264 // whether they are also defined in this module. We subsequently prune the
1265 // list to only include those defined in the exporting module, see comment
1266 // there as to why.
1267 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1268 // Anything marked exported during the import computation must have been
1269 // defined in the exporting module.
1270 assert(DS != DefinedGVSummaries.end());
1271 auto *S = DS->getSecond();
1272 S = S->getBaseObject();
1273 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1274 // Export referenced functions and variables. We don't export/promote
1275 // objects referenced by writeonly variable initializer, because
1276 // we convert such variables initializers to "zeroinitializer".
1277 // See processGlobalForThinLTO.
1278 if (!Index.isWriteOnly(GVS))
1279 NewExports.insert_range(R: GVS->refs());
1280 } else {
1281 auto *FS = cast<FunctionSummary>(Val: S);
1282 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1283 NewExports.insert_range(R: FS->refs());
1284 }
1285 }
1286 // Prune list computed above to only include values defined in the
1287 // exporting module. We do this after the above insertion since we may hit
1288 // the same ref/call target multiple times in above loop, and it is more
1289 // efficient to avoid a set lookup each time.
1290 NewExports.remove_if(
1291 Pred: [&](ValueInfo VI) { return !DefinedGVSummaries.count(Val: VI.getGUID()); });
1292 ELI.second.insert_range(R&: NewExports);
1293 }
1294
1295 assert(checkVariableImport(Index, ImportLists, ExportLists));
1296#ifndef NDEBUG
1297 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1298 << " modules:\n");
1299 for (const auto &ModuleImports : ImportLists) {
1300 auto ModName = ModuleImports.first;
1301 auto &Exports = ExportLists[ModName];
1302 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1303 DenseMap<StringRef, ImportStatistics> Histogram =
1304 collectImportStatistics(Index, ModuleImports.second);
1305 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1306 << Exports.size() - NumGVS << " functions and " << NumGVS
1307 << " vars. Imports from " << Histogram.size()
1308 << " modules.\n");
1309 for (const auto &[SrcModName, Stats] : Histogram) {
1310 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1311 << " function definitions and "
1312 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1313 << " function declarations imported from " << SrcModName
1314 << "\n");
1315 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1316 << " global vars imported from " << SrcModName << "\n");
1317 }
1318 }
1319#endif
1320}
1321
1322#ifndef NDEBUG
1323static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1324 StringRef ModulePath,
1325 FunctionImporter::ImportMapTy &ImportList) {
1326 DenseMap<StringRef, ImportStatistics> Histogram =
1327 collectImportStatistics(Index, ImportList);
1328 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1329 << Histogram.size() << " modules.\n");
1330 for (const auto &[SrcModName, Stats] : Histogram) {
1331 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1332 << " function definitions and "
1333 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1334 << " function declarations imported from " << SrcModName
1335 << "\n");
1336 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1337 << SrcModName << "\n");
1338 }
1339}
1340#endif
1341
1342/// Compute all the imports for the given module using the Index.
1343///
1344/// \p isPrevailing is a callback that will be called with a global value's GUID
1345/// and summary and should return whether the module corresponding to the
1346/// summary contains the linker-prevailing copy of that value.
1347///
1348/// \p ImportList will be populated with a map that can be passed to
1349/// FunctionImporter::importFunctions() above (see description there).
1350static void ComputeCrossModuleImportForModuleForTest(
1351 StringRef ModulePath,
1352 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1353 isPrevailing,
1354 const ModuleSummaryIndex &Index,
1355 FunctionImporter::ImportMapTy &ImportList) {
1356 // Collect the list of functions this module defines.
1357 // GUID -> Summary
1358 GVSummaryMapTy FunctionSummaryMap;
1359 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1360
1361 // Compute the import list for this module.
1362 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1363 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1364 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1365
1366#ifndef NDEBUG
1367 dumpImportListForModule(Index, ModulePath, ImportList);
1368#endif
1369}
1370
1371/// Mark all external summaries in \p Index for import into the given module.
1372/// Used for testing the case of distributed builds using a distributed index.
1373///
1374/// \p ImportList will be populated with a map that can be passed to
1375/// FunctionImporter::importFunctions() above (see description there).
1376static void ComputeCrossModuleImportForModuleFromIndexForTest(
1377 StringRef ModulePath, const ModuleSummaryIndex &Index,
1378 FunctionImporter::ImportMapTy &ImportList) {
1379 for (const auto &GlobalList : Index) {
1380 // Ignore entries for undefined references.
1381 if (GlobalList.second.getSummaryList().empty())
1382 continue;
1383
1384 auto GUID = GlobalList.first;
1385 assert(GlobalList.second.getSummaryList().size() == 1 &&
1386 "Expected individual combined index to have one summary per GUID");
1387 auto &Summary = GlobalList.second.getSummaryList()[0];
1388 // Skip the summaries for the importing module. These are included to
1389 // e.g. record required linkage changes.
1390 if (Summary->modulePath() == ModulePath)
1391 continue;
1392 // Add an entry to provoke importing by thinBackend.
1393 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1394 }
1395#ifndef NDEBUG
1396 dumpImportListForModule(Index, ModulePath, ImportList);
1397#endif
1398}
1399
1400// For SamplePGO, the indirect call targets for local functions will
1401// have its original name annotated in profile. We try to find the
1402// corresponding PGOFuncName as the GUID, and fix up the edges
1403// accordingly.
1404void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1405 FunctionSummary *FS) {
1406 for (auto &EI : FS->mutableCalls()) {
1407 if (!EI.first.getSummaryList().empty())
1408 continue;
1409 auto GUID = Index.getGUIDFromOriginalID(OriginalID: EI.first.getGUID());
1410 if (GUID == 0)
1411 continue;
1412 // Update the edge to point directly to the correct GUID.
1413 auto VI = Index.getValueInfo(GUID);
1414 if (llvm::any_of(
1415 Range: VI.getSummaryList(),
1416 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1417 // The mapping from OriginalId to GUID may return a GUID
1418 // that corresponds to a static variable. Filter it out here.
1419 // This can happen when
1420 // 1) There is a call to a library function which is not defined
1421 // in the index.
1422 // 2) There is a static variable with the OriginalGUID identical
1423 // to the GUID of the library function in 1);
1424 // When this happens the static variable in 2) will be found,
1425 // which needs to be filtered out.
1426 return SummaryPtr->getSummaryKind() ==
1427 GlobalValueSummary::GlobalVarKind;
1428 }))
1429 continue;
1430 EI.first = VI;
1431 }
1432}
1433
1434void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1435 for (const auto &Entry : Index) {
1436 for (const auto &S : Entry.second.getSummaryList()) {
1437 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1438 updateValueInfoForIndirectCalls(Index, FS);
1439 }
1440 }
1441}
1442
1443void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1444 ModuleSummaryIndex &Index,
1445 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1446 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1447 assert(!Index.withGlobalValueDeadStripping());
1448 if (!ComputeDead ||
1449 // Don't do anything when nothing is live, this is friendly with tests.
1450 GUIDPreservedSymbols.empty()) {
1451 // Still need to update indirect calls.
1452 updateIndirectCalls(Index);
1453 return;
1454 }
1455 unsigned LiveSymbols = 0;
1456 SmallVector<ValueInfo, 128> Worklist;
1457 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1458 for (auto GUID : GUIDPreservedSymbols) {
1459 ValueInfo VI = Index.getValueInfo(GUID);
1460 if (!VI)
1461 continue;
1462 for (const auto &S : VI.getSummaryList())
1463 S->setLive(true);
1464 }
1465
1466 // Add values flagged in the index as live roots to the worklist.
1467 for (const auto &Entry : Index) {
1468 auto VI = Index.getValueInfo(R: Entry);
1469 for (const auto &S : Entry.second.getSummaryList()) {
1470 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1471 updateValueInfoForIndirectCalls(Index, FS);
1472 if (S->isLive()) {
1473 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1474 Worklist.push_back(Elt: VI);
1475 ++LiveSymbols;
1476 break;
1477 }
1478 }
1479 }
1480
1481 // Make value live and add it to the worklist if it was not live before.
1482 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1483 // FIXME: If we knew which edges were created for indirect call profiles,
1484 // we could skip them here. Any that are live should be reached via
1485 // other edges, e.g. reference edges. Otherwise, using a profile collected
1486 // on a slightly different binary might provoke preserving, importing
1487 // and ultimately promoting calls to functions not linked into this
1488 // binary, which increases the binary size unnecessarily. Note that
1489 // if this code changes, the importer needs to change so that edges
1490 // to functions marked dead are skipped.
1491
1492 if (llvm::any_of(Range: VI.getSummaryList(),
1493 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1494 return S->isLive();
1495 }))
1496 return;
1497
1498 // We only keep live symbols that are known to be non-prevailing if any are
1499 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1500 // later in the EliminateAvailableExternally pass and setting them to
1501 // not-live could break downstreams users of liveness information (PR36483)
1502 // or limit optimization opportunities.
1503 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1504 bool KeepAliveLinkage = false;
1505 bool Interposable = false;
1506 for (const auto &S : VI.getSummaryList()) {
1507 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1508 S->linkage() == GlobalValue::WeakODRLinkage ||
1509 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1510 KeepAliveLinkage = true;
1511 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1512 Interposable = true;
1513 }
1514
1515 if (!IsAliasee) {
1516 if (!KeepAliveLinkage)
1517 return;
1518
1519 if (Interposable)
1520 report_fatal_error(
1521 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1522 "symbol");
1523 }
1524 }
1525
1526 for (const auto &S : VI.getSummaryList())
1527 S->setLive(true);
1528 ++LiveSymbols;
1529 Worklist.push_back(Elt: VI);
1530 };
1531
1532 while (!Worklist.empty()) {
1533 auto VI = Worklist.pop_back_val();
1534 for (const auto &Summary : VI.getSummaryList()) {
1535 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1536 // If this is an alias, visit the aliasee VI to ensure that all copies
1537 // are marked live and it is added to the worklist for further
1538 // processing of its references.
1539 visit(AS->getAliaseeVI(), true);
1540 continue;
1541 }
1542 for (auto Ref : Summary->refs())
1543 visit(Ref, false);
1544 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1545 for (auto Call : FS->calls())
1546 visit(Call.first, false);
1547 }
1548 }
1549 Index.setWithGlobalValueDeadStripping();
1550
1551 unsigned DeadSymbols = Index.size() - LiveSymbols;
1552 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1553 << " symbols Dead \n");
1554 NumDeadSymbols += DeadSymbols;
1555 NumLiveSymbols += LiveSymbols;
1556}
1557
1558// Compute dead symbols and propagate constants in combined index.
1559void llvm::computeDeadSymbolsWithConstProp(
1560 ModuleSummaryIndex &Index,
1561 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1562 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1563 bool ImportEnabled) {
1564 llvm::TimeTraceScope timeScope("Drop dead symbols and propagate attributes");
1565 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1566 isPrevailing);
1567 if (ImportEnabled)
1568 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1569}
1570
1571/// Compute the set of summaries needed for a ThinLTO backend compilation of
1572/// \p ModulePath.
1573void llvm::gatherImportedSummariesForModule(
1574 StringRef ModulePath,
1575 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1576 const FunctionImporter::ImportMapTy &ImportList,
1577 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1578 GVSummaryPtrSet &DecSummaries) {
1579 // Include all summaries from the importing module.
1580 ModuleToSummariesForIndex[std::string(ModulePath)] =
1581 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1582
1583 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1584 // lets us look up the map without allocating an instance of std::string when
1585 // the key-value pair exists in the map.
1586 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1587 // C++26 when it becomes available for our codebase.
1588 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1589 StringRef Key) -> GVSummaryMapTy & {
1590 auto It = Map.find(x: Key);
1591 if (It == Map.end())
1592 std::tie(args&: It, args: std::ignore) =
1593 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1594 return It->second;
1595 };
1596
1597 // Include summaries for imports.
1598 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1599 auto &SummariesForIndex =
1600 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1601
1602 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1603 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1604 assert(DS != DefinedGVSummaries.end() &&
1605 "Expected a defined summary for imported global value");
1606 if (ImportType == GlobalValueSummary::Declaration)
1607 DecSummaries.insert(Ptr: DS->second);
1608
1609 SummariesForIndex[GUID] = DS->second;
1610 }
1611
1612 // When AlwaysRenamePromotedLocals is false, for each source module we import
1613 // from, also include summaries for local functions that have
1614 // NoRenameOnPromotion set. This is needed for distributed ThinLTO. Otherwise,
1615 // the local function of the source module will keep its origin name, e.g.,
1616 // foo() while the function in destination module will have name
1617 // foo.llvm.<...>() and this will cause a link failure.
1618 //
1619 // Note: this imports a superset of the necessary declarations — all locals
1620 // with NoRenameOnPromotion in each source module, not just those referenced
1621 // by the importing module. Computing the precise set would require walking
1622 // the summary reference graph from each imported function, which is more
1623 // expensive than the simple scan here.
1624 if (!AlwaysRenamePromotedLocals) {
1625 for (auto &[ModPath, SummariesForIndex] : ModuleToSummariesForIndex) {
1626 if (ModPath == ModulePath)
1627 continue;
1628 auto It = ModuleToDefinedGVSummaries.find(Val: ModPath);
1629 if (It == ModuleToDefinedGVSummaries.end())
1630 continue;
1631 for (const auto &[GUID, Summary] : It->second) {
1632 if (Summary->noRenameOnPromotion()) {
1633 DecSummaries.insert(Ptr: Summary);
1634 SummariesForIndex.try_emplace(Key: GUID, Args: Summary);
1635 }
1636 }
1637 }
1638 }
1639}
1640
1641/// Emit the files \p ModulePath will import from into \p OutputFilename.
1642Error llvm::EmitImportsFiles(
1643 StringRef ModulePath, StringRef OutputFilename,
1644 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1645 std::error_code EC;
1646 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1647 if (EC)
1648 return createFileError(F: "cannot open " + OutputFilename,
1649 E: errorCodeToError(EC));
1650 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1651 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1652 return Error::success();
1653}
1654
1655/// Invoke callback \p F on the file paths from which \p ModulePath
1656/// will import.
1657void llvm::processImportsFiles(
1658 StringRef ModulePath,
1659 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1660 function_ref<void(const std::string &)> F) {
1661 for (const auto &ILI : ModuleToSummariesForIndex)
1662 // The ModuleToSummariesForIndex map includes an entry for the current
1663 // Module (needed for writing out the index files). We don't want to
1664 // include it in the imports file, however, so filter it out.
1665 if (ILI.first != ModulePath)
1666 F(ILI.first);
1667}
1668
1669bool llvm::convertToDeclaration(GlobalValue &GV) {
1670 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1671 << "\n");
1672 MDNode *UniqueID = nullptr;
1673 if (auto *GO = dyn_cast<GlobalObject>(Val: &GV))
1674 UniqueID = GO->getMetadata(KindID: LLVMContext::MD_guid);
1675
1676 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1677 F->deleteBody();
1678 F->clearMetadata();
1679 if (UniqueID)
1680 F->setMetadata(KindID: LLVMContext::MD_guid, Node: UniqueID);
1681 F->setComdat(nullptr);
1682 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1683 V->setInitializer(nullptr);
1684 V->setLinkage(GlobalValue::ExternalLinkage);
1685 V->clearMetadata();
1686 if (UniqueID)
1687 V->setMetadata(KindID: LLVMContext::MD_guid, Node: UniqueID);
1688 V->setComdat(nullptr);
1689 } else {
1690 GlobalValue *NewGV;
1691 if (GV.getValueType()->isFunctionTy())
1692 NewGV =
1693 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1694 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1695 N: "", M: GV.getParent());
1696 else
1697 NewGV =
1698 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1699 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1700 /*init*/ nullptr, "",
1701 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1702 GV.getType()->getAddressSpace());
1703 NewGV->takeName(V: &GV);
1704 GV.replaceAllUsesWith(V: NewGV);
1705 return false;
1706 }
1707 if (!GV.isImplicitDSOLocal())
1708 GV.setDSOLocal(false);
1709 return true;
1710}
1711
1712void llvm::thinLTOFinalizeInModule(Module &TheModule,
1713 const GVSummaryMapTy &DefinedGlobals,
1714 bool PropagateAttrs) {
1715 llvm::TimeTraceScope timeScope("ThinLTO finalize in module");
1716 DenseSet<Comdat *> NonPrevailingComdats;
1717 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1718 // See if the global summary analysis computed a new resolved linkage.
1719 const auto &GS = DefinedGlobals.find(Val: GV.getGUIDOrFallback());
1720 if (GS == DefinedGlobals.end())
1721 return;
1722
1723 if (Propagate)
1724 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1725 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1726 // TODO: propagate ReadNone and ReadOnly.
1727 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1728 F->setDoesNotAccessMemory();
1729
1730 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1731 F->setOnlyReadsMemory();
1732
1733 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1734 F->setDoesNotRecurse();
1735
1736 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1737 F->setDoesNotThrow();
1738 }
1739 }
1740
1741 auto NewLinkage = GS->second->linkage();
1742 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1743 // Don't internalize anything here, because the code below
1744 // lacks necessary correctness checks. Leave this job to
1745 // LLVM 'internalize' pass.
1746 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1747 // In case it was dead and already converted to declaration.
1748 GV.isDeclaration())
1749 return;
1750
1751 // Set the potentially more constraining visibility computed from summaries.
1752 // The DefaultVisibility condition is because older GlobalValueSummary does
1753 // not record DefaultVisibility and we don't want to change protected/hidden
1754 // to default.
1755 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1756 GV.setVisibility(GS->second->getVisibility());
1757
1758 if (NewLinkage == GV.getLinkage())
1759 return;
1760
1761 // Check for a non-prevailing def that has interposable linkage
1762 // (e.g. non-odr weak or linkonce). In that case we can't simply
1763 // convert to available_externally, since it would lose the
1764 // interposable property and possibly get inlined. Simply drop
1765 // the definition in that case.
1766 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1767 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1768 if (!convertToDeclaration(GV))
1769 // FIXME: Change this to collect replaced GVs and later erase
1770 // them from the parent module once thinLTOResolvePrevailingGUID is
1771 // changed to enable this for aliases.
1772 llvm_unreachable("Expected GV to be converted");
1773 } else {
1774 // If all copies of the original symbol had global unnamed addr and
1775 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1776 // and are constants, then it should be an auto hide symbol. In that case
1777 // the thin link would have marked it as CanAutoHide. Add hidden
1778 // visibility to the symbol to preserve the property.
1779 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1780 GS->second->canAutoHide()) {
1781 assert(GV.canBeOmittedFromSymbolTable());
1782 GV.setVisibility(GlobalValue::HiddenVisibility);
1783 }
1784
1785 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1786 << "` from " << GV.getLinkage() << " to " << NewLinkage
1787 << "\n");
1788 GV.setLinkage(NewLinkage);
1789 }
1790 // Remove declarations from comdats, including available_externally
1791 // as this is a declaration for the linker, and will be dropped eventually.
1792 // It is illegal for comdats to contain declarations.
1793 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1794 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1795 if (GO->getComdat()->getName() == GO->getName())
1796 NonPrevailingComdats.insert(V: GO->getComdat());
1797 GO->setComdat(nullptr);
1798 }
1799 };
1800
1801 // Process functions and global now
1802 for (auto &GV : TheModule)
1803 FinalizeInModule(GV, PropagateAttrs);
1804 for (auto &GV : TheModule.globals())
1805 FinalizeInModule(GV);
1806 for (auto &GV : TheModule.aliases())
1807 FinalizeInModule(GV);
1808
1809 // For a non-prevailing comdat, all its members must be available_externally.
1810 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1811 // local linkage GlobalValues.
1812 if (NonPrevailingComdats.empty())
1813 return;
1814 for (auto &GO : TheModule.global_objects()) {
1815 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1816 GO.setComdat(nullptr);
1817 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1818 }
1819 }
1820 bool Changed;
1821 do {
1822 Changed = false;
1823 // If an alias references a GlobalValue in a non-prevailing comdat, change
1824 // it to available_externally. For simplicity we only handle GlobalValue and
1825 // ConstantExpr with a base object. ConstantExpr without a base object is
1826 // unlikely used in a COMDAT.
1827 for (auto &GA : TheModule.aliases()) {
1828 if (GA.hasAvailableExternallyLinkage())
1829 continue;
1830 GlobalObject *Obj = GA.getAliaseeObject();
1831 assert(Obj && "aliasee without an base object is unimplemented");
1832 if (Obj->hasAvailableExternallyLinkage()) {
1833 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1834 Changed = true;
1835 }
1836 }
1837 } while (Changed);
1838}
1839
1840/// Run internalization on \p TheModule based on symmary analysis.
1841void llvm::thinLTOInternalizeModule(Module &TheModule,
1842 const GVSummaryMapTy &DefinedGlobals) {
1843 llvm::TimeTraceScope timeScope("ThinLTO internalize module");
1844 // Declare a callback for the internalize pass that will ask for every
1845 // candidate GlobalValue if it can be internalized or not.
1846 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1847 // It may be the case that GV is on a chain of an ifunc, its alias and
1848 // subsequent aliases. In this case, the summary for the value is not
1849 // available.
1850 if (isa<GlobalIFunc>(Val: &GV) ||
1851 (isa<GlobalAlias>(Val: &GV) &&
1852 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1853 return true;
1854
1855 // Lookup the linkage recorded in the summaries during global analysis.
1856 auto GS = DefinedGlobals.find(Val: GV.getGUIDOrFallback());
1857 if (GS == DefinedGlobals.end()) {
1858 // Must have been promoted (possibly conservatively). Find original
1859 // name so that we can access the correct summary and see if it can
1860 // be internalized again.
1861 // FIXME: Eventually we should control promotion instead of promoting
1862 // and internalizing again.
1863 StringRef OrigName =
1864 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1865 std::string OrigId = GlobalValue::getGlobalIdentifier(
1866 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1867 FileName: TheModule.getSourceFileName());
1868 GS = DefinedGlobals.find(
1869 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1870 if (GS == DefinedGlobals.end()) {
1871 // Also check the original non-promoted non-globalized name. In some
1872 // cases a preempted weak value is linked in as a local copy because
1873 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1874 // In that case, since it was originally not a local value, it was
1875 // recorded in the index using the original name.
1876 // FIXME: This may not be needed once PR27866 is fixed.
1877 GS = DefinedGlobals.find(
1878 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1879 assert(GS != DefinedGlobals.end());
1880 }
1881 }
1882 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1883 };
1884
1885 // FIXME: See if we can just internalize directly here via linkage changes
1886 // based on the index, rather than invoking internalizeModule.
1887 internalizeModule(TheModule, MustPreserveGV);
1888}
1889
1890/// Make alias a clone of its aliasee.
1891static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1892 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1893
1894 ValueToValueMapTy VMap;
1895 Function *NewFn = CloneFunction(F: Fn, VMap);
1896 // Clone should use the original alias's linkage, visibility and name, and we
1897 // ensure all uses of alias instead use the new clone (casted if necessary).
1898 NewFn->setLinkage(GA->getLinkage());
1899 NewFn->setVisibility(GA->getVisibility());
1900 GA->replaceAllUsesWith(V: NewFn);
1901 NewFn->takeName(V: GA);
1902 return NewFn;
1903}
1904
1905// Internalize values that we marked with specific attribute
1906// in processGlobalForThinLTO.
1907static void internalizeGVsAfterImport(Module &M) {
1908 for (auto &GV : M.globals())
1909 // Skip GVs which have been converted to declarations
1910 // by dropDeadSymbols.
1911 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1912 GV.setLinkage(GlobalValue::InternalLinkage);
1913 GV.setVisibility(GlobalValue::DefaultVisibility);
1914 }
1915}
1916
1917/// When a function carrying !implicit.ref is imported via ThinLTO, the
1918/// referenced global arrives as available_externally. This string can be dead
1919/// code eliminated since it has no IR uses -- only metadata references. Adding
1920/// it to llvm.compiler.used prevents elimination.
1921static void protectImplicitRefGlobals(Module &M) {
1922 SmallPtrSet<GlobalValue *, 4> Seen;
1923 SmallVector<GlobalValue *, 4> ToProtect;
1924 for (Function &F : M) {
1925 if (!F.hasAvailableExternallyLinkage() ||
1926 !F.hasMetadata(KindID: LLVMContext::MD_implicit_ref))
1927 continue;
1928 SmallVector<MDNode *> MDs;
1929 F.getMetadata(KindID: LLVMContext::MD_implicit_ref, MDs);
1930 for (MDNode *MD : MDs) {
1931 auto *Op = MD->getOperand(I: 0).get();
1932 if (!Op)
1933 continue;
1934 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: Op))
1935 if (auto *GV = dyn_cast<GlobalVariable>(Val: VAM->getValue()))
1936 if (GV->hasAvailableExternallyLinkage() && Seen.insert(Ptr: GV).second)
1937 ToProtect.push_back(Elt: GV);
1938 }
1939 }
1940 if (!ToProtect.empty())
1941 appendToCompilerUsed(M, Values: ToProtect);
1942}
1943
1944// Automatically import functions in Module \p DestModule based on the summaries
1945// index.
1946Expected<bool> FunctionImporter::importFunctions(
1947 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1948 LLVM_DEBUG(dbgs() << "Starting import for Module "
1949 << DestModule.getModuleIdentifier() << "\n");
1950 unsigned ImportedCount = 0, ImportedGVCount = 0;
1951 // Before carrying out any imports, see if this module defines functions in
1952 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1953 // The function will be imported elsewhere, as extenal linkage, and the
1954 // destination doesn't yet have its definition.
1955 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1956 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1957 for (auto &F : DestModule)
1958 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUIDOrFallback()))
1959 F.deleteBody();
1960
1961 IRMover Mover(DestModule);
1962
1963 // Do the actual import of functions now, one Module at a time
1964 for (const auto &ModName : ImportList.getSourceModules()) {
1965 llvm::TimeTraceScope timeScope("Import", ModName);
1966 // Get the module for the import
1967 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1968 if (!SrcModuleOrErr)
1969 return SrcModuleOrErr.takeError();
1970 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1971 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1972 "Context mismatch");
1973
1974 // If modules were created with lazy metadata loading, materialize it
1975 // now, before linking it (otherwise this will be a noop).
1976 if (Error Err = SrcModule->materializeMetadata())
1977 return std::move(Err);
1978
1979 // Find the globals to import
1980 SetVector<GlobalValue *> GlobalsToImport;
1981 {
1982 llvm::TimeTraceScope functionsScope("Functions");
1983 for (Function &F : *SrcModule) {
1984 if (!F.hasName())
1985 continue;
1986 auto GUID = F.getGUIDOrFallback();
1987 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
1988 bool ImportDefinition =
1989 MaybeImportType == GlobalValueSummary::Definition;
1990
1991 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
1992 << " importing function"
1993 << (ImportDefinition
1994 ? " definition "
1995 : (MaybeImportType ? " declaration " : " "))
1996 << GUID << " " << F.getName() << " from "
1997 << SrcModule->getSourceFileName() << "\n");
1998 if (ImportDefinition) {
1999 if (Error Err = F.materialize())
2000 return std::move(Err);
2001 // MemProf should match function's definition and summary,
2002 // 'thinlto_src_module' is needed.
2003 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2004 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2005 // statistics and debugging.
2006 F.setMetadata(
2007 Kind: "thinlto_src_module",
2008 Node: MDNode::get(Context&: DestModule.getContext(),
2009 MDs: {MDString::get(Context&: DestModule.getContext(),
2010 Str: SrcModule->getModuleIdentifier())}));
2011 F.setMetadata(
2012 Kind: "thinlto_src_file",
2013 Node: MDNode::get(Context&: DestModule.getContext(),
2014 MDs: {MDString::get(Context&: DestModule.getContext(),
2015 Str: SrcModule->getSourceFileName())}));
2016 }
2017 GlobalsToImport.insert(X: &F);
2018 }
2019 }
2020 }
2021 {
2022 llvm::TimeTraceScope globalsScope("Globals");
2023 for (GlobalVariable &GV : SrcModule->globals()) {
2024 if (!GV.hasName())
2025 continue;
2026 auto GUID = GV.getGUIDOrFallback();
2027 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2028 bool ImportDefinition =
2029 MaybeImportType == GlobalValueSummary::Definition;
2030
2031 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2032 << " importing global"
2033 << (ImportDefinition
2034 ? " definition "
2035 : (MaybeImportType ? " declaration " : " "))
2036 << GUID << " " << GV.getName() << " from "
2037 << SrcModule->getSourceFileName() << "\n");
2038 if (ImportDefinition) {
2039 if (Error Err = GV.materialize())
2040 return std::move(Err);
2041 ImportedGVCount += GlobalsToImport.insert(X: &GV);
2042 }
2043 }
2044 }
2045 {
2046 llvm::TimeTraceScope aliasesScope("Aliases");
2047 for (GlobalAlias &GA : SrcModule->aliases()) {
2048 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
2049 continue;
2050 auto GUID = GA.getGUIDOrFallback();
2051 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2052 bool ImportDefinition =
2053 MaybeImportType == GlobalValueSummary::Definition;
2054
2055 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2056 << " importing alias"
2057 << (ImportDefinition
2058 ? " definition "
2059 : (MaybeImportType ? " declaration " : " "))
2060 << GUID << " " << GA.getName() << " from "
2061 << SrcModule->getSourceFileName() << "\n");
2062 if (ImportDefinition) {
2063 if (Error Err = GA.materialize())
2064 return std::move(Err);
2065 // Import alias as a copy of its aliasee.
2066 GlobalObject *GO = GA.getAliaseeObject();
2067 if (Error Err = GO->materialize())
2068 return std::move(Err);
2069 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
2070 assert(Fn);
2071 (void)Fn;
2072 LLVM_DEBUG(dbgs()
2073 << "Is importing aliasee fn " << GO->getGUIDOrFallback()
2074 << " " << GO->getName() << " from "
2075 << SrcModule->getSourceFileName() << "\n");
2076 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2077 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2078 // statistics and debugging.
2079 Fn->setMetadata(
2080 Kind: "thinlto_src_module",
2081 Node: MDNode::get(Context&: DestModule.getContext(),
2082 MDs: {MDString::get(Context&: DestModule.getContext(),
2083 Str: SrcModule->getModuleIdentifier())}));
2084 Fn->setMetadata(
2085 Kind: "thinlto_src_file",
2086 Node: MDNode::get(Context&: DestModule.getContext(),
2087 MDs: {MDString::get(Context&: DestModule.getContext(),
2088 Str: SrcModule->getSourceFileName())}));
2089 }
2090 GlobalsToImport.insert(X: Fn);
2091 }
2092 }
2093 }
2094
2095 // Upgrade debug info after we're done materializing all the globals and we
2096 // have loaded all the required metadata!
2097 UpgradeDebugInfo(M&: *SrcModule);
2098
2099 // Set the partial sample profile ratio in the profile summary module flag
2100 // of the imported source module, if applicable, so that the profile summary
2101 // module flag will match with that of the destination module when it's
2102 // imported.
2103 SrcModule->setPartialSampleProfileRatio(Index);
2104
2105 // Link in the specified functions.
2106 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2107 GlobalsToImport: &GlobalsToImport);
2108
2109 if (PrintImports) {
2110 for (const auto *GV : GlobalsToImport)
2111 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2112 << " from " << SrcModule->getSourceFileName() << "\n";
2113 }
2114
2115 if (Error Err = Mover.move(Src: std::move(SrcModule),
2116 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2117 /*IsPerformingImport=*/true))
2118 return createStringError(EC: errc::invalid_argument,
2119 S: Twine("Function Import: link error: ") +
2120 toString(E: std::move(Err)));
2121
2122 ImportedCount += GlobalsToImport.size();
2123 NumImportedModules++;
2124 }
2125
2126 internalizeGVsAfterImport(M&: DestModule);
2127
2128 // Protect !implicit.ref-referenced globals imported as available_externally
2129 // from DCE'd. Only needed when globals were actually imported.
2130 if (ImportedGVCount > 0)
2131 protectImplicitRefGlobals(M&: DestModule);
2132
2133 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2134 NumImportedGlobalVars += ImportedGVCount;
2135
2136 // TODO: Print counters for definitions and declarations in the debugging log.
2137 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2138 << " functions for Module "
2139 << DestModule.getModuleIdentifier() << "\n");
2140 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2141 << " global variables for Module "
2142 << DestModule.getModuleIdentifier() << "\n");
2143 return ImportedCount;
2144}
2145
2146static bool doImportingForModuleForTest(
2147 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2148 isPrevailing) {
2149 if (SummaryFile.empty())
2150 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2151 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2152 getModuleSummaryIndexForFile(Path: SummaryFile);
2153 if (!IndexPtrOrErr) {
2154 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2155 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2156 return false;
2157 }
2158 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2159
2160 // First step is collecting the import list.
2161 FunctionImporter::ImportIDTable ImportIDs;
2162 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2163 // If requested, simply import all functions in the index. This is used
2164 // when testing distributed backend handling via the opt tool, when
2165 // we have distributed indexes containing exactly the summaries to import.
2166 if (ImportAllIndex)
2167 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2168 Index: *Index, ImportList);
2169 else
2170 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2171 isPrevailing, Index: *Index, ImportList);
2172
2173 // Conservatively mark all internal values as promoted. This interface is
2174 // only used when doing importing via the function importing pass. The pass
2175 // is only enabled when testing importing via the 'opt' tool, which does
2176 // not do the ThinLink that would normally determine what values to promote.
2177 for (auto &I : *Index) {
2178 for (auto &S : I.second.getSummaryList()) {
2179 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2180 S->setExternalLinkageForTest();
2181 }
2182 }
2183
2184 // Next we need to promote to global scope and rename any local values that
2185 // are potentially exported to other modules.
2186 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2187 /*GlobalsToImport=*/nullptr);
2188
2189 // Perform the import now.
2190 auto ModuleLoader = [&M](StringRef Identifier) {
2191 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2192 };
2193 FunctionImporter Importer(*Index, ModuleLoader,
2194 /*ClearDSOLocalOnDeclarations=*/false);
2195 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2196
2197 // FIXME: Probably need to propagate Errors through the pass manager.
2198 if (!Result) {
2199 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2200 ErrorBanner: "Error importing module: ");
2201 return true;
2202 }
2203
2204 return true;
2205}
2206
2207PreservedAnalyses FunctionImportPass::run(Module &M,
2208 ModuleAnalysisManager &AM) {
2209 // This is only used for testing the function import pass via opt, where we
2210 // don't have prevailing information from the LTO context available, so just
2211 // conservatively assume everything is prevailing (which is fine for the very
2212 // limited use of prevailing checking in this pass).
2213 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2214 return true;
2215 };
2216 if (!doImportingForModuleForTest(M, isPrevailing))
2217 return PreservedAnalyses::all();
2218
2219 return PreservedAnalyses::none();
2220}
2221