1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements Function import based on summaries.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/FunctionImport.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Bitcode/BitcodeReader.h"
22#include "llvm/IR/AutoUpgrade.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalAlias.h"
25#include "llvm/IR/GlobalObject.h"
26#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/GlobalVariable.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/ModuleSummaryIndex.h"
32#include "llvm/IRReader/IRReader.h"
33#include "llvm/Linker/IRMover.h"
34#include "llvm/ProfileData/PGOCtxProfReader.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Support/Errc.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/FileSystem.h"
42#include "llvm/Support/JSON.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SourceMgr.h"
45#include "llvm/Support/TimeProfiler.h"
46#include "llvm/Support/raw_ostream.h"
47#include "llvm/Transforms/IPO/Internalize.h"
48#include "llvm/Transforms/Utils/Cloning.h"
49#include "llvm/Transforms/Utils/FunctionImportUtils.h"
50#include "llvm/Transforms/Utils/ModuleUtils.h"
51#include "llvm/Transforms/Utils/ValueMapper.h"
52#include <cassert>
53#include <memory>
54#include <string>
55#include <system_error>
56#include <tuple>
57#include <utility>
58
59using namespace llvm;
60
61#define DEBUG_TYPE "function-import"
62
63STATISTIC(NumImportedFunctionsThinLink,
64 "Number of functions thin link decided to import");
65STATISTIC(NumImportedHotFunctionsThinLink,
66 "Number of hot functions thin link decided to import");
67STATISTIC(NumImportedCriticalFunctionsThinLink,
68 "Number of critical functions thin link decided to import");
69STATISTIC(NumImportedGlobalVarsThinLink,
70 "Number of global variables thin link decided to import");
71STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
72STATISTIC(NumImportedGlobalVars,
73 "Number of global variables imported in backend");
74STATISTIC(NumImportedModules, "Number of modules imported from");
75STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
76STATISTIC(NumLiveSymbols, "Number of live symbols in index");
77
78namespace llvm {
79extern cl::opt<bool> AlwaysRenamePromotedLocals;
80
81cl::opt<bool>
82 ForceImportAll("force-import-all", cl::init(Val: false), cl::Hidden,
83 cl::desc("Import functions with noinline attribute"));
84
85/// Limit on instruction count of imported functions.
86static cl::opt<unsigned> ImportInstrLimit(
87 "import-instr-limit", cl::init(Val: 100), cl::Hidden, cl::value_desc("N"),
88 cl::desc("Only import functions with less than N instructions"));
89
90static cl::opt<int> ImportCutoff(
91 "import-cutoff", cl::init(Val: -1), cl::Hidden, cl::value_desc("N"),
92 cl::desc("Only import first N functions if N>=0 (default -1)"));
93
94static cl::opt<float>
95 ImportInstrFactor("import-instr-evolution-factor", cl::init(Val: 0.7),
96 cl::Hidden, cl::value_desc("x"),
97 cl::desc("As we import functions, multiply the "
98 "`import-instr-limit` threshold by this factor "
99 "before processing newly imported functions"));
100
101static cl::opt<float> ImportHotInstrFactor(
102 "import-hot-evolution-factor", cl::init(Val: 1.0), cl::Hidden,
103 cl::value_desc("x"),
104 cl::desc("As we import functions called from hot callsite, multiply the "
105 "`import-instr-limit` threshold by this factor "
106 "before processing newly imported functions"));
107
108static cl::opt<float> ImportHotMultiplier(
109 "import-hot-multiplier", cl::init(Val: 10.0), cl::Hidden, cl::value_desc("x"),
110 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
111
112static cl::opt<float> ImportCriticalMultiplier(
113 "import-critical-multiplier", cl::init(Val: 100.0), cl::Hidden,
114 cl::value_desc("x"),
115 cl::desc(
116 "Multiply the `import-instr-limit` threshold for critical callsites"));
117
118// FIXME: This multiplier was not really tuned up.
119static cl::opt<float> ImportColdMultiplier(
120 "import-cold-multiplier", cl::init(Val: 0), cl::Hidden, cl::value_desc("N"),
121 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
122
123static cl::opt<bool> PrintImports("print-imports", cl::init(Val: false), cl::Hidden,
124 cl::desc("Print imported functions"));
125
126static cl::opt<bool> PrintImportFailures(
127 "print-import-failures", cl::init(Val: false), cl::Hidden,
128 cl::desc("Print information for functions rejected for importing"));
129
130static cl::opt<bool> ComputeDead("compute-dead", cl::init(Val: true), cl::Hidden,
131 cl::desc("Compute dead symbols"));
132
133static cl::opt<bool> EnableImportMetadata(
134 "enable-import-metadata", cl::init(Val: false), cl::Hidden,
135 cl::desc("Enable import metadata like 'thinlto_src_module' and "
136 "'thinlto_src_file'"));
137
138/// Summary file to use for function importing when using -function-import from
139/// the command line.
140static cl::opt<std::string>
141 SummaryFile("summary-file",
142 cl::desc("The summary file to use for function importing."));
143
144/// Used when testing importing from distributed indexes via opt
145// -function-import.
146static cl::opt<bool>
147 ImportAllIndex("import-all-index",
148 cl::desc("Import all external functions in index."));
149
150/// This is a test-only option.
151/// If this option is enabled, the ThinLTO indexing step will import each
152/// function declaration as a fallback. In a real build this may increase ram
153/// usage of the indexing step unnecessarily.
154/// TODO: Implement selective import (based on combined summary analysis) to
155/// ensure the imported function has a use case in the postlink pipeline.
156static cl::opt<bool> ImportDeclaration(
157 "import-declaration", cl::init(Val: false), cl::Hidden,
158 cl::desc("If true, import function declaration as fallback if the function "
159 "definition is not imported."));
160
161/// Pass a workload description file - an example of workload would be the
162/// functions executed to satisfy a RPC request. A workload is defined by a root
163/// function and the list of functions that are (frequently) needed to satisfy
164/// it. The module that defines the root will have all those functions imported.
165/// The file contains a JSON dictionary. The keys are root functions, the values
166/// are lists of functions to import in the module defining the root. It is
167/// assumed -funique-internal-linkage-names was used, thus ensuring function
168/// names are unique even for local linkage ones.
169static cl::opt<std::string> WorkloadDefinitions(
170 "thinlto-workload-def",
171 cl::desc("Pass a workload definition. This is a file containing a JSON "
172 "dictionary. The keys are root functions, the values are lists of "
173 "functions to import in the module defining the root. It is "
174 "assumed -funique-internal-linkage-names was used, to ensure "
175 "local linkage functions have unique names. For example: \n"
176 "{\n"
177 " \"rootFunction_1\": [\"function_to_import_1\", "
178 "\"function_to_import_2\"], \n"
179 " \"rootFunction_2\": [\"function_to_import_3\", "
180 "\"function_to_import_4\"] \n"
181 "}"),
182 cl::Hidden);
183
184extern cl::opt<std::string> UseCtxProfile;
185
186static cl::opt<bool> CtxprofMoveRootsToOwnModule(
187 "thinlto-move-ctxprof-trees",
188 cl::desc("Move contextual profiling roots and the graphs under them in "
189 "their own module."),
190 cl::Hidden, cl::init(Val: false));
191
192extern cl::list<GlobalValue::GUID> MoveSymbolGUID;
193
194extern cl::opt<bool> EnableMemProfContextDisambiguation;
195} // end namespace llvm
196
197// Load lazily a module from \p FileName in \p Context.
198static std::unique_ptr<Module> loadFile(const std::string &FileName,
199 LLVMContext &Context) {
200 SMDiagnostic Err;
201 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
202 // Metadata isn't loaded until functions are imported, to minimize
203 // the memory overhead.
204 std::unique_ptr<Module> Result =
205 getLazyIRFileModule(Filename: FileName, Err, Context,
206 /* ShouldLazyLoadMetadata = */ true);
207 if (!Result) {
208 Err.print(ProgName: "function-import", S&: errs());
209 report_fatal_error(reason: "Abort");
210 }
211
212 return Result;
213}
214
215static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary,
216 size_t NumDefs,
217 StringRef ImporterModule) {
218 // We can import a local when there is one definition.
219 if (NumDefs == 1)
220 return false;
221 // In other cases, make sure we import the copy in the caller's module if the
222 // referenced value has local linkage. The only time a local variable can
223 // share an entry in the index is if there is a local with the same name in
224 // another module that had the same source file name (in a different
225 // directory), where each was compiled in their own directory so there was not
226 // distinguishing path.
227 return GlobalValue::isLocalLinkage(Linkage: RefSummary->linkage()) &&
228 RefSummary->modulePath() != ImporterModule;
229}
230
231/// Given a list of possible callee implementation for a call site, qualify the
232/// legality of importing each. The return is a range of pairs. Each pair
233/// corresponds to a candidate. The first value is the ImportFailureReason for
234/// that candidate, the second is the candidate.
235static auto qualifyCalleeCandidates(
236 const ModuleSummaryIndex &Index,
237 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
238 StringRef CallerModulePath) {
239 return llvm::map_range(
240 C&: CalleeSummaryList,
241 F: [&Index, CalleeSummaryList,
242 CallerModulePath](const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
243 -> std::pair<FunctionImporter::ImportFailureReason,
244 const GlobalValueSummary *> {
245 auto *GVSummary = SummaryPtr.get();
246 if (!Index.isGlobalValueLive(GVS: GVSummary))
247 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
248
249 if (GlobalValue::isInterposableLinkage(Linkage: GVSummary->linkage()))
250 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
251 GVSummary};
252
253 auto *Summary = dyn_cast<FunctionSummary>(Val: GVSummary->getBaseObject());
254
255 // Ignore any callees that aren't actually functions. This could happen
256 // in the case of GUID hash collisions. It could also happen in theory
257 // for SamplePGO profiles collected on old versions of the code after
258 // renaming, since we synthesize edges to any inlined callees appearing
259 // in the profile.
260 if (!Summary)
261 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
262
263 // If this is a local function, make sure we import the copy in the
264 // caller's module. The only time a local function can share an entry in
265 // the index is if there is a local with the same name in another module
266 // that had the same source file name (in a different directory), where
267 // each was compiled in their own directory so there was not
268 // distinguishing path.
269 // If the local function is from another module, it must be a reference
270 // due to indirect call profile data since a function pointer can point
271 // to a local in another module. Do the import from another module if
272 // there is only one entry in the list or when all files in the program
273 // are compiled with full path - in both cases the local function has
274 // unique PGO name and GUID.
275 if (shouldSkipLocalInAnotherModule(RefSummary: Summary, NumDefs: CalleeSummaryList.size(),
276 ImporterModule: CallerModulePath))
277 return {
278 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
279 GVSummary};
280
281 // Skip if it isn't legal to import (e.g. may reference unpromotable
282 // locals).
283 if (Summary->notEligibleToImport())
284 return {FunctionImporter::ImportFailureReason::NotEligible,
285 GVSummary};
286
287 return {FunctionImporter::ImportFailureReason::None, GVSummary};
288 });
289}
290
291/// Given a list of possible callee implementation for a call site, select one
292/// that fits the \p Threshold for function definition import. If none are
293/// found, the Reason will give the last reason for the failure (last, in the
294/// order of CalleeSummaryList entries). While looking for a callee definition,
295/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline
296/// candidate; other modules may want to know the function summary or
297/// declaration even if a definition is not needed.
298///
299/// FIXME: select "best" instead of first that fits. But what is "best"?
300/// - The smallest: more likely to be inlined.
301/// - The one with the least outgoing edges (already well optimized).
302/// - One from a module already being imported from in order to reduce the
303/// number of source modules parsed/linked.
304/// - One that has PGO data attached.
305/// - [insert you fancy metric here]
306static const GlobalValueSummary *
307selectCallee(const ModuleSummaryIndex &Index,
308 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
309 unsigned Threshold, StringRef CallerModulePath,
310 const GlobalValueSummary *&TooLargeOrNoInlineSummary,
311 FunctionImporter::ImportFailureReason &Reason) {
312 // Records the last summary with reason noinline or too-large.
313 TooLargeOrNoInlineSummary = nullptr;
314 auto QualifiedCandidates =
315 qualifyCalleeCandidates(Index, CalleeSummaryList, CallerModulePath);
316 for (auto QualifiedValue : QualifiedCandidates) {
317 Reason = QualifiedValue.first;
318 // Skip a summary if its import is not (proved to be) legal.
319 if (Reason != FunctionImporter::ImportFailureReason::None)
320 continue;
321 auto *Summary =
322 cast<FunctionSummary>(Val: QualifiedValue.second->getBaseObject());
323
324 // Don't bother importing the definition if the chance of inlining it is
325 // not high enough (except under `--force-import-all`).
326 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
327 !ForceImportAll) {
328 TooLargeOrNoInlineSummary = Summary;
329 Reason = FunctionImporter::ImportFailureReason::TooLarge;
330 continue;
331 }
332
333 // Don't bother importing the definition if we can't inline it anyway.
334 if (Summary->fflags().NoInline && !ForceImportAll) {
335 TooLargeOrNoInlineSummary = Summary;
336 Reason = FunctionImporter::ImportFailureReason::NoInline;
337 continue;
338 }
339
340 return Summary;
341 }
342 return nullptr;
343}
344
345namespace {
346
347using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
348
349} // anonymous namespace
350
351FunctionImporter::ImportMapTy::AddDefinitionStatus
352FunctionImporter::ImportMapTy::addDefinition(StringRef FromModule,
353 GlobalValue::GUID GUID) {
354 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
355 if (!Imports.insert(V: Def).second)
356 // Already there.
357 return AddDefinitionStatus::NoChange;
358
359 // Remove Decl in case it's there. Note that a definition takes precedence
360 // over a declaration for a given GUID.
361 return Imports.erase(V: Decl) ? AddDefinitionStatus::ChangedToDefinition
362 : AddDefinitionStatus::Inserted;
363}
364
365void FunctionImporter::ImportMapTy::maybeAddDeclaration(
366 StringRef FromModule, GlobalValue::GUID GUID) {
367 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
368 // Insert Decl only if Def is not present. Note that a definition takes
369 // precedence over a declaration for a given GUID.
370 if (!Imports.contains(V: Def))
371 Imports.insert(V: Decl);
372}
373
374SmallVector<StringRef, 0>
375FunctionImporter::ImportMapTy::getSourceModules() const {
376 SetVector<StringRef> ModuleSet;
377 for (const auto &[SrcMod, GUID, ImportType] : *this)
378 ModuleSet.insert(X: SrcMod);
379 SmallVector<StringRef, 0> Modules = ModuleSet.takeVector();
380 llvm::sort(C&: Modules);
381 return Modules;
382}
383
384std::optional<GlobalValueSummary::ImportKind>
385FunctionImporter::ImportMapTy::getImportType(StringRef FromModule,
386 GlobalValue::GUID GUID) const {
387 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
388 auto [Def, Decl] = *IDPair;
389 if (Imports.contains(V: Def))
390 return GlobalValueSummary::Definition;
391 if (Imports.contains(V: Decl))
392 return GlobalValueSummary::Declaration;
393 }
394 return std::nullopt;
395}
396
397/// Import globals referenced by a function or other globals that are being
398/// imported, if importing such global is possible.
399class GlobalsImporter final {
400 const ModuleSummaryIndex &Index;
401 const GVSummaryMapTy &DefinedGVSummaries;
402 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
403 IsPrevailing;
404 FunctionImporter::ImportMapTy &ImportList;
405 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
406
407 bool shouldImportGlobal(const ValueInfo &VI) {
408 const auto &GVS = DefinedGVSummaries.find(Val: VI.getGUID());
409 if (GVS == DefinedGVSummaries.end())
410 return true;
411 // We should not skip import if the module contains a non-prevailing
412 // definition with interposable linkage type. This is required for
413 // correctness in the situation where there is a prevailing def available
414 // for import and marked read-only. In this case, the non-prevailing def
415 // will be converted to a declaration, while the prevailing one becomes
416 // internal, thus no definitions will be available for linking. In order to
417 // prevent undefined symbol link error, the prevailing definition must be
418 // imported.
419 // FIXME: Consider adding a check that the suitable prevailing definition
420 // exists and marked read-only.
421 if (VI.getSummaryList().size() > 1 &&
422 GlobalValue::isInterposableLinkage(Linkage: GVS->second->linkage()) &&
423 !IsPrevailing(VI.getGUID(), GVS->second))
424 return true;
425
426 return false;
427 }
428
429 void
430 onImportingSummaryImpl(const GlobalValueSummary &Summary,
431 SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
432 for (const auto &VI : Summary.refs()) {
433 if (!shouldImportGlobal(VI)) {
434 LLVM_DEBUG(
435 dbgs() << "Ref ignored! Target already in destination module.\n");
436 continue;
437 }
438
439 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
440
441 for (const auto &RefSummary : VI.getSummaryList()) {
442 const auto *GVS = dyn_cast<GlobalVarSummary>(Val: RefSummary.get());
443 // Stop looking if this is not a global variable, e.g. a function.
444 // Functions could be referenced by global vars - e.g. a vtable; but we
445 // don't currently imagine a reason those would be imported here, rather
446 // than as part of the logic deciding which functions to import (i.e.
447 // based on profile information). Should we decide to handle them here,
448 // we can refactor accordingly at that time.
449 // Note that it is safe to stop looking because the one case where we
450 // might have to import (a read/write-only global variable) cannot occur
451 // if this GUID has a non-variable summary. The only case where we even
452 // might find another summary in the list that is a variable is in the
453 // case of same-named locals in different modules not compiled with
454 // enough path, and during attribute propagation we will mark all
455 // summaries for a GUID (ValueInfo) as non read/write-only if any is not
456 // a global variable.
457 if (!GVS)
458 break;
459 bool CanImportDecl = false;
460 if (shouldSkipLocalInAnotherModule(RefSummary: GVS, NumDefs: VI.getSummaryList().size(),
461 ImporterModule: Summary.modulePath()) ||
462 !Index.canImportGlobalVar(S: GVS, /* AnalyzeRefs */ true,
463 CanImportDecl)) {
464 if (ImportDeclaration && CanImportDecl)
465 ImportList.maybeAddDeclaration(FromModule: RefSummary->modulePath(),
466 GUID: VI.getGUID());
467
468 continue;
469 }
470
471 // If there isn't an entry for GUID, insert <GUID, Definition> pair.
472 // Otherwise, definition should take precedence over declaration.
473 if (ImportList.addDefinition(FromModule: RefSummary->modulePath(), GUID: VI.getGUID()) !=
474 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
475 break;
476
477 // Only update stat and exports if we haven't already imported this
478 // variable.
479 NumImportedGlobalVarsThinLink++;
480 // Any references made by this variable will be marked exported
481 // later, in ComputeCrossModuleImport, after import decisions are
482 // complete, which is more efficient than adding them here.
483 if (ExportLists)
484 (*ExportLists)[RefSummary->modulePath()].insert(V: VI);
485
486 // If variable is not writeonly we attempt to recursively analyze
487 // its references in order to import referenced constants.
488 if (!Index.isWriteOnly(GVS))
489 Worklist.emplace_back(Args&: GVS);
490 break;
491 }
492 }
493 }
494
495public:
496 GlobalsImporter(
497 const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
498 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
499 IsPrevailing,
500 FunctionImporter::ImportMapTy &ImportList,
501 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
502 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
503 IsPrevailing(IsPrevailing), ImportList(ImportList),
504 ExportLists(ExportLists) {}
505
506 void onImportingSummary(const GlobalValueSummary &Summary) {
507 SmallVector<const GlobalVarSummary *, 128> Worklist;
508 onImportingSummaryImpl(Summary, Worklist);
509 while (!Worklist.empty())
510 onImportingSummaryImpl(Summary: *Worklist.pop_back_val(), Worklist);
511 }
512};
513
514static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
515
516/// Determine the list of imports and exports for each module.
517class ModuleImportsManager {
518 void computeImportForFunction(
519 const FunctionSummary &Summary, unsigned Threshold,
520 const GVSummaryMapTy &DefinedGVSummaries,
521 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
522 FunctionImporter::ImportMapTy &ImportList,
523 FunctionImporter::ImportThresholdsTy &ImportThresholds);
524
525protected:
526 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
527 IsPrevailing;
528 const ModuleSummaryIndex &Index;
529 DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
530
531 ModuleImportsManager(
532 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
533 IsPrevailing,
534 const ModuleSummaryIndex &Index,
535 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
536 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
537 virtual bool canImport(ValueInfo VI) { return true; }
538
539public:
540 virtual ~ModuleImportsManager() = default;
541
542 /// Given the list of globals defined in a module, compute the list of imports
543 /// as well as the list of "exports", i.e. the list of symbols referenced from
544 /// another module (that may require promotion).
545 virtual void
546 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
547 StringRef ModName,
548 FunctionImporter::ImportMapTy &ImportList);
549
550 static std::unique_ptr<ModuleImportsManager>
551 create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
552 IsPrevailing,
553 const ModuleSummaryIndex &Index,
554 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
555 nullptr);
556};
557
558/// A ModuleImportsManager that operates based on a workload definition (see
559/// -thinlto-workload-def). For modules that do not define workload roots, it
560/// applies the base ModuleImportsManager import policy.
561class WorkloadImportsManager : public ModuleImportsManager {
562 // Keep a module name -> value infos to import association. We use it to
563 // determine if a module's import list should be done by the base
564 // ModuleImportsManager or by us.
565 StringMap<DenseSet<ValueInfo>> Workloads;
566 // Track the roots to avoid importing them due to other callers. We want there
567 // to be only one variant), for which we optimize according to the contextual
568 // profile. "Variants" refers to copies due to importing - we want there to be
569 // just one instance of this function.
570 DenseSet<ValueInfo> Roots;
571
572 void
573 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
574 StringRef ModName,
575 FunctionImporter::ImportMapTy &ImportList) override {
576 StringRef Filename = ModName;
577 if (CtxprofMoveRootsToOwnModule) {
578 Filename = sys::path::filename(path: ModName);
579 // Drop the file extension.
580 Filename = Filename.substr(Start: 0, N: Filename.find_last_of(C: '.'));
581 }
582 auto SetIter = Workloads.find(Key: Filename);
583
584 if (SetIter == Workloads.end()) {
585 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
586 << " does not contain the root of any context.\n");
587 return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
588 ModName, ImportList);
589 }
590 LLVM_DEBUG(dbgs() << "[Workload] " << ModName
591 << " contains the root(s) of context(s).\n");
592
593 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
594 ExportLists);
595 auto &ValueInfos = SetIter->second;
596 for (auto &VI : llvm::make_early_inc_range(Range&: ValueInfos)) {
597 auto It = DefinedGVSummaries.find(Val: VI.getGUID());
598 if (It != DefinedGVSummaries.end() &&
599 IsPrevailing(VI.getGUID(), It->second)) {
600 LLVM_DEBUG(
601 dbgs() << "[Workload] " << VI.name()
602 << " has the prevailing variant already in the module "
603 << ModName << ". No need to import\n");
604 continue;
605 }
606 auto Candidates =
607 qualifyCalleeCandidates(Index, CalleeSummaryList: VI.getSummaryList(), CallerModulePath: ModName);
608
609 const GlobalValueSummary *GVS = nullptr;
610 auto PotentialCandidates = llvm::map_range(
611 C: llvm::make_filter_range(
612 Range&: Candidates,
613 Pred: [&](const auto &Candidate) {
614 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()
615 << " from " << Candidate.second->modulePath()
616 << " ImportFailureReason: "
617 << getFailureName(Candidate.first) << "\n");
618 return Candidate.first ==
619 FunctionImporter::ImportFailureReason::None;
620 }),
621 F: [](const auto &Candidate) { return Candidate.second; });
622 if (PotentialCandidates.empty()) {
623 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
624 << " because can't find eligible Callee. Guid is: "
625 << VI.getGUID() << "\n");
626 continue;
627 }
628 /// We will prefer importing the prevailing candidate, if not, we'll
629 /// still pick the first available candidate. The reason we want to make
630 /// sure we do import the prevailing candidate is because the goal of
631 /// workload-awareness is to enable optimizations specializing the call
632 /// graph of that workload. Suppose a function is already defined in the
633 /// module, but it's not the prevailing variant. Suppose also we do not
634 /// inline it (in fact, if it were interposable, we can't inline it),
635 /// but we could specialize it to the workload in other ways. However,
636 /// the linker would drop it in the favor of the prevailing copy.
637 /// Instead, by importing the prevailing variant (assuming also the use
638 /// of `-avail-extern-to-local`), we keep the specialization. We could
639 /// alteranatively make the non-prevailing variant local, but the
640 /// prevailing one is also the one for which we would have previously
641 /// collected profiles, making it preferrable.
642 auto PrevailingCandidates = llvm::make_filter_range(
643 Range&: PotentialCandidates, Pred: [&](const auto *Candidate) {
644 return IsPrevailing(VI.getGUID(), Candidate);
645 });
646 if (PrevailingCandidates.empty()) {
647 GVS = *PotentialCandidates.begin();
648 if (!llvm::hasSingleElement(C&: PotentialCandidates) &&
649 GlobalValue::isLocalLinkage(Linkage: GVS->linkage()))
650 LLVM_DEBUG(
651 dbgs()
652 << "[Workload] Found multiple non-prevailing candidates for "
653 << VI.name()
654 << ". This is unexpected. Are module paths passed to the "
655 "compiler unique for the modules passed to the linker?");
656 // We could in theory have multiple (interposable) copies of a symbol
657 // when there is no prevailing candidate, if say the prevailing copy was
658 // in a native object being linked in. However, we should in theory be
659 // marking all of these non-prevailing IR copies dead in that case, in
660 // which case they won't be candidates.
661 assert(GVS->isLive());
662 } else {
663 assert(llvm::hasSingleElement(PrevailingCandidates));
664 GVS = *PrevailingCandidates.begin();
665 }
666
667 auto ExportingModule = GVS->modulePath();
668 // We checked that for the prevailing case, but if we happen to have for
669 // example an internal that's defined in this module, it'd have no
670 // PrevailingCandidates.
671 if (ExportingModule == ModName) {
672 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
673 << " because its defining module is the same as the "
674 "current module\n");
675 continue;
676 }
677 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
678 << ExportingModule << " : " << VI.getGUID() << "\n");
679 ImportList.addDefinition(FromModule: ExportingModule, GUID: VI.getGUID());
680 GVI.onImportingSummary(Summary: *GVS);
681 if (ExportLists)
682 (*ExportLists)[ExportingModule].insert(V: VI);
683 }
684 LLVM_DEBUG(dbgs() << "[Workload] Done\n");
685 }
686
687 void loadFromJson() {
688 // Since the workload def uses names, we need a quick lookup
689 // name->ValueInfo.
690 StringMap<ValueInfo> NameToValueInfo;
691 StringSet<> AmbiguousNames;
692 for (auto &I : Index) {
693 ValueInfo VI = Index.getValueInfo(R: I);
694 if (!NameToValueInfo.insert(KV: std::make_pair(x: VI.name(), y&: VI)).second)
695 LLVM_DEBUG(AmbiguousNames.insert(VI.name()));
696 }
697 auto DbgReportIfAmbiguous = [&](StringRef Name) {
698 LLVM_DEBUG(if (AmbiguousNames.count(Name) > 0) {
699 dbgs() << "[Workload] Function name " << Name
700 << " present in the workload definition is ambiguous. Consider "
701 "compiling with -funique-internal-linkage-names.";
702 });
703 };
704 std::error_code EC;
705 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: WorkloadDefinitions);
706 if (std::error_code EC = BufferOrErr.getError()) {
707 report_fatal_error(reason: "Failed to open context file");
708 return;
709 }
710 auto Buffer = std::move(BufferOrErr.get());
711 std::map<std::string, std::vector<std::string>> WorkloadDefs;
712 json::Path::Root NullRoot;
713 // The JSON is supposed to contain a dictionary matching the type of
714 // WorkloadDefs. For example:
715 // {
716 // "rootFunction_1": ["function_to_import_1", "function_to_import_2"],
717 // "rootFunction_2": ["function_to_import_3", "function_to_import_4"]
718 // }
719 auto Parsed = json::parse(JSON: Buffer->getBuffer());
720 if (!Parsed)
721 report_fatal_error(Err: Parsed.takeError());
722 if (!json::fromJSON(E: *Parsed, Out&: WorkloadDefs, P: NullRoot))
723 report_fatal_error(reason: "Invalid thinlto contextual profile format.");
724 for (const auto &Workload : WorkloadDefs) {
725 const auto &Root = Workload.first;
726 DbgReportIfAmbiguous(Root);
727 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
728 const auto &AllCallees = Workload.second;
729 auto RootIt = NameToValueInfo.find(Key: Root);
730 if (RootIt == NameToValueInfo.end()) {
731 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
732 << " not found in this linkage unit.\n");
733 continue;
734 }
735 auto RootVI = RootIt->second;
736 if (RootVI.getSummaryList().size() != 1) {
737 LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
738 << " should have exactly one summary, but has "
739 << RootVI.getSummaryList().size() << ". Skipping.\n");
740 continue;
741 }
742 StringRef RootDefiningModule =
743 RootVI.getSummaryList().front()->modulePath();
744 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
745 << " is : " << RootDefiningModule << "\n");
746 auto &Set = Workloads[RootDefiningModule];
747 for (const auto &Callee : AllCallees) {
748 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
749 DbgReportIfAmbiguous(Callee);
750 auto ElemIt = NameToValueInfo.find(Key: Callee);
751 if (ElemIt == NameToValueInfo.end()) {
752 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
753 continue;
754 }
755 Set.insert(V: ElemIt->second);
756 }
757 }
758 }
759
760 void loadFromCtxProf() {
761 std::error_code EC;
762 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename: UseCtxProfile);
763 if (std::error_code EC = BufferOrErr.getError()) {
764 report_fatal_error(reason: "Failed to open contextual profile file");
765 return;
766 }
767 auto Buffer = std::move(BufferOrErr.get());
768
769 PGOCtxProfileReader Reader(Buffer->getBuffer());
770 auto Ctx = Reader.loadProfiles();
771 if (!Ctx) {
772 report_fatal_error(reason: "Failed to parse contextual profiles");
773 return;
774 }
775 const auto &CtxMap = Ctx->Contexts;
776 SetVector<GlobalValue::GUID> ContainedGUIDs;
777 for (const auto &[RootGuid, Root] : CtxMap) {
778 // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for
779 // subsequent roots, but clear its contents.
780 ContainedGUIDs.clear();
781
782 auto RootVI = Index.getValueInfo(GUID: RootGuid);
783 if (!RootVI) {
784 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
785 << " not found in this linkage unit.\n");
786 continue;
787 }
788 if (RootVI.getSummaryList().size() != 1) {
789 LLVM_DEBUG(dbgs() << "[Workload] Root " << RootGuid
790 << " should have exactly one summary, but has "
791 << RootVI.getSummaryList().size() << ". Skipping.\n");
792 continue;
793 }
794 std::string RootDefiningModule =
795 RootVI.getSummaryList().front()->modulePath().str();
796 if (CtxprofMoveRootsToOwnModule) {
797 RootDefiningModule = std::to_string(val: RootGuid);
798 LLVM_DEBUG(
799 dbgs() << "[Workload] Moving " << RootGuid
800 << " to a module with the filename without extension : "
801 << RootDefiningModule << "\n");
802 } else {
803 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid
804 << " is : " << RootDefiningModule << "\n");
805 }
806 auto &Set = Workloads[RootDefiningModule];
807 Root.getContainedGuids(Guids&: ContainedGUIDs);
808 Roots.insert(V: RootVI);
809 for (auto Guid : ContainedGUIDs)
810 if (auto VI = Index.getValueInfo(GUID: Guid))
811 Set.insert(V: VI);
812 }
813 }
814
815 bool canImport(ValueInfo VI) override { return !Roots.contains(V: VI); }
816
817public:
818 WorkloadImportsManager(
819 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
820 IsPrevailing,
821 const ModuleSummaryIndex &Index,
822 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
823 : ModuleImportsManager(IsPrevailing, Index, ExportLists) {
824 if (UseCtxProfile.empty() == WorkloadDefinitions.empty()) {
825 report_fatal_error(
826 reason: "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
827 return;
828 }
829 if (!UseCtxProfile.empty())
830 loadFromCtxProf();
831 else
832 loadFromJson();
833 LLVM_DEBUG({
834 for (const auto &[Root, Set] : Workloads) {
835 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()
836 << " distinct callees.\n";
837 for (const auto &VI : Set) {
838 dbgs() << "[Workload] Root: " << Root
839 << " Would include: " << VI.getGUID() << "\n";
840 }
841 }
842 });
843 }
844};
845
846std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
847 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
848 IsPrevailing,
849 const ModuleSummaryIndex &Index,
850 DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
851 if (WorkloadDefinitions.empty() && UseCtxProfile.empty()) {
852 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
853 return std::unique_ptr<ModuleImportsManager>(
854 new ModuleImportsManager(IsPrevailing, Index, ExportLists));
855 }
856 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
857 return std::make_unique<WorkloadImportsManager>(args&: IsPrevailing, args: Index,
858 args&: ExportLists);
859}
860
861static const char *
862getFailureName(FunctionImporter::ImportFailureReason Reason) {
863 switch (Reason) {
864 case FunctionImporter::ImportFailureReason::None:
865 return "None";
866 case FunctionImporter::ImportFailureReason::GlobalVar:
867 return "GlobalVar";
868 case FunctionImporter::ImportFailureReason::NotLive:
869 return "NotLive";
870 case FunctionImporter::ImportFailureReason::TooLarge:
871 return "TooLarge";
872 case FunctionImporter::ImportFailureReason::InterposableLinkage:
873 return "InterposableLinkage";
874 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
875 return "LocalLinkageNotInModule";
876 case FunctionImporter::ImportFailureReason::NotEligible:
877 return "NotEligible";
878 case FunctionImporter::ImportFailureReason::NoInline:
879 return "NoInline";
880 }
881 llvm_unreachable("invalid reason");
882}
883
884/// Compute the list of functions to import for a given caller. Mark these
885/// imported functions and the symbols they reference in their source module as
886/// exported from their source module.
887void ModuleImportsManager::computeImportForFunction(
888 const FunctionSummary &Summary, const unsigned Threshold,
889 const GVSummaryMapTy &DefinedGVSummaries,
890 SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
891 FunctionImporter::ImportMapTy &ImportList,
892 FunctionImporter::ImportThresholdsTy &ImportThresholds) {
893 GVImporter.onImportingSummary(Summary);
894 static int ImportCount = 0;
895 for (const auto &Edge : Summary.calls()) {
896 ValueInfo VI = Edge.first;
897 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
898 << "\n");
899
900 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
901 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
902 << " reached.\n");
903 continue;
904 }
905
906 if (DefinedGVSummaries.count(Val: VI.getGUID())) {
907 // FIXME: Consider not skipping import if the module contains
908 // a non-prevailing def with interposable linkage. The prevailing copy
909 // can safely be imported (see shouldImportGlobal()).
910 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
911 continue;
912 }
913
914 if (!canImport(VI)) {
915 LLVM_DEBUG(
916 dbgs() << "Skipping over " << VI.getGUID()
917 << " because its import is handled in a different module.");
918 assert(VI.getSummaryList().size() == 1 &&
919 "The root was expected to be an external symbol");
920 continue;
921 }
922
923 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
924 if (Hotness == CalleeInfo::HotnessType::Hot)
925 return ImportHotMultiplier;
926 if (Hotness == CalleeInfo::HotnessType::Cold)
927 return ImportColdMultiplier;
928 if (Hotness == CalleeInfo::HotnessType::Critical)
929 return ImportCriticalMultiplier;
930 return 1.0;
931 };
932
933 const auto NewThreshold =
934 Threshold * GetBonusMultiplier(Edge.second.getHotness());
935
936 auto IT = ImportThresholds.insert(KV: std::make_pair(
937 x: VI.getGUID(), y: std::make_tuple(args: NewThreshold, args: nullptr, args: nullptr)));
938 bool PreviouslyVisited = !IT.second;
939 auto &ProcessedThreshold = std::get<0>(t&: IT.first->second);
940 auto &CalleeSummary = std::get<1>(t&: IT.first->second);
941 auto &FailureInfo = std::get<2>(t&: IT.first->second);
942
943 bool IsHotCallsite =
944 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
945 bool IsCriticalCallsite =
946 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
947
948 const FunctionSummary *ResolvedCalleeSummary = nullptr;
949 if (CalleeSummary) {
950 assert(PreviouslyVisited);
951 // Since the traversal of the call graph is DFS, we can revisit a function
952 // a second time with a higher threshold. In this case, it is added back
953 // to the worklist with the new threshold (so that its own callee chains
954 // can be considered with the higher threshold).
955 if (NewThreshold <= ProcessedThreshold) {
956 LLVM_DEBUG(
957 dbgs() << "ignored! Target was already imported with Threshold "
958 << ProcessedThreshold << "\n");
959 continue;
960 }
961 // Update with new larger threshold.
962 ProcessedThreshold = NewThreshold;
963 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
964 } else {
965 // If we already rejected importing a callee at the same or higher
966 // threshold, don't waste time calling selectCallee.
967 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
968 LLVM_DEBUG(
969 dbgs() << "ignored! Target was already rejected with Threshold "
970 << ProcessedThreshold << "\n");
971 if (PrintImportFailures) {
972 assert(FailureInfo &&
973 "Expected FailureInfo for previously rejected candidate");
974 FailureInfo->Attempts++;
975 }
976 continue;
977 }
978
979 FunctionImporter::ImportFailureReason Reason{};
980
981 // `SummaryForDeclImport` is an summary eligible for declaration import.
982 const GlobalValueSummary *SummaryForDeclImport = nullptr;
983 CalleeSummary =
984 selectCallee(Index, CalleeSummaryList: VI.getSummaryList(), Threshold: NewThreshold,
985 CallerModulePath: Summary.modulePath(), TooLargeOrNoInlineSummary&: SummaryForDeclImport, Reason);
986 if (!CalleeSummary) {
987 // There isn't a callee for definition import but one for declaration
988 // import.
989 if (ImportDeclaration && SummaryForDeclImport) {
990 StringRef DeclSourceModule = SummaryForDeclImport->modulePath();
991
992 // Note `ExportLists` only keeps track of exports due to imported
993 // definitions.
994 ImportList.maybeAddDeclaration(FromModule: DeclSourceModule, GUID: VI.getGUID());
995 }
996 // Update with new larger threshold if this was a retry (otherwise
997 // we would have already inserted with NewThreshold above). Also
998 // update failure info if requested.
999 if (PreviouslyVisited) {
1000 ProcessedThreshold = NewThreshold;
1001 if (PrintImportFailures) {
1002 assert(FailureInfo &&
1003 "Expected FailureInfo for previously rejected candidate");
1004 FailureInfo->Reason = Reason;
1005 FailureInfo->Attempts++;
1006 FailureInfo->MaxHotness =
1007 std::max(a: FailureInfo->MaxHotness, b: Edge.second.getHotness());
1008 }
1009 } else if (PrintImportFailures) {
1010 assert(!FailureInfo &&
1011 "Expected no FailureInfo for newly rejected candidate");
1012 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
1013 args&: VI, args: Edge.second.getHotness(), args&: Reason, args: 1);
1014 }
1015 if (ForceImportAll) {
1016 std::string Msg = std::string("Failed to import function ") +
1017 VI.name().str() + " due to " +
1018 getFailureName(Reason);
1019 auto Error = make_error<StringError>(
1020 Args&: Msg, Args: make_error_code(E: errc::not_supported));
1021 logAllUnhandledErrors(E: std::move(Error), OS&: errs(),
1022 ErrorBanner: "Error importing module: ");
1023 break;
1024 } else {
1025 LLVM_DEBUG(dbgs()
1026 << "ignored! No qualifying callee with summary found.\n");
1027 continue;
1028 }
1029 }
1030
1031 // "Resolve" the summary
1032 CalleeSummary = CalleeSummary->getBaseObject();
1033 ResolvedCalleeSummary = cast<FunctionSummary>(Val: CalleeSummary);
1034
1035 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll ||
1036 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
1037 "selectCallee() didn't honor the threshold");
1038
1039 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
1040
1041 // Try emplace the definition entry, and update stats based on insertion
1042 // status.
1043 if (ImportList.addDefinition(FromModule: ExportModulePath, GUID: VI.getGUID()) !=
1044 FunctionImporter::ImportMapTy::AddDefinitionStatus::NoChange) {
1045 NumImportedFunctionsThinLink++;
1046 if (IsHotCallsite)
1047 NumImportedHotFunctionsThinLink++;
1048 if (IsCriticalCallsite)
1049 NumImportedCriticalFunctionsThinLink++;
1050 }
1051
1052 // Any calls/references made by this function will be marked exported
1053 // later, in ComputeCrossModuleImport, after import decisions are
1054 // complete, which is more efficient than adding them here.
1055 if (ExportLists)
1056 (*ExportLists)[ExportModulePath].insert(V: VI);
1057 }
1058
1059 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
1060 // Adjust the threshold for next level of imported functions.
1061 // The threshold is different for hot callsites because we can then
1062 // inline chains of hot calls.
1063 if (IsHotCallsite)
1064 return Threshold * ImportHotInstrFactor;
1065 return Threshold * ImportInstrFactor;
1066 };
1067
1068 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1069
1070 ImportCount++;
1071
1072 // Insert the newly imported function to the worklist.
1073 Worklist.emplace_back(Args&: ResolvedCalleeSummary, Args: AdjThreshold);
1074 }
1075}
1076
1077void ModuleImportsManager::computeImportForModule(
1078 const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName,
1079 FunctionImporter::ImportMapTy &ImportList) {
1080 // Worklist contains the list of function imported in this module, for which
1081 // we will analyse the callees and may import further down the callgraph.
1082 SmallVector<EdgeInfo, 128> Worklist;
1083 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1084 ExportLists);
1085 FunctionImporter::ImportThresholdsTy ImportThresholds;
1086
1087 // Populate the worklist with the import for the functions in the current
1088 // module
1089 for (const auto &GVSummary : DefinedGVSummaries) {
1090#ifndef NDEBUG
1091 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
1092 // so this map look up (and possibly others) can be avoided.
1093 auto VI = Index.getValueInfo(GVSummary.first);
1094#endif
1095 if (!Index.isGlobalValueLive(GVS: GVSummary.second)) {
1096 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
1097 continue;
1098 }
1099 auto *FuncSummary =
1100 dyn_cast<FunctionSummary>(Val: GVSummary.second->getBaseObject());
1101 if (!FuncSummary)
1102 // Skip import for global variables
1103 continue;
1104 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
1105 computeImportForFunction(Summary: *FuncSummary, Threshold: ImportInstrLimit, DefinedGVSummaries,
1106 Worklist, GVImporter&: GVI, ImportList, ImportThresholds);
1107 }
1108
1109 // Process the newly imported functions and add callees to the worklist.
1110 while (!Worklist.empty()) {
1111 auto GVInfo = Worklist.pop_back_val();
1112 auto *Summary = std::get<0>(t&: GVInfo);
1113 auto Threshold = std::get<1>(t&: GVInfo);
1114
1115 computeImportForFunction(Summary: *Summary, Threshold, DefinedGVSummaries, Worklist,
1116 GVImporter&: GVI, ImportList, ImportThresholds);
1117 }
1118
1119 // Print stats about functions considered but rejected for importing
1120 // when requested.
1121 if (PrintImportFailures) {
1122 dbgs() << "Missed imports into module " << ModName << "\n";
1123 for (auto &I : ImportThresholds) {
1124 auto &ProcessedThreshold = std::get<0>(t&: I.second);
1125 auto &CalleeSummary = std::get<1>(t&: I.second);
1126 auto &FailureInfo = std::get<2>(t&: I.second);
1127 if (CalleeSummary)
1128 continue; // We are going to import.
1129 assert(FailureInfo);
1130 FunctionSummary *FS = nullptr;
1131 if (!FailureInfo->VI.getSummaryList().empty())
1132 FS = dyn_cast<FunctionSummary>(
1133 Val: FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1134 dbgs() << FailureInfo->VI
1135 << ": Reason = " << getFailureName(Reason: FailureInfo->Reason)
1136 << ", Threshold = " << ProcessedThreshold
1137 << ", Size = " << (FS ? (int)FS->instCount() : -1)
1138 << ", MaxHotness = " << getHotnessName(HT: FailureInfo->MaxHotness)
1139 << ", Attempts = " << FailureInfo->Attempts << "\n";
1140 }
1141 }
1142}
1143
1144#ifndef NDEBUG
1145static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) {
1146 auto SL = VI.getSummaryList();
1147 return SL.empty()
1148 ? false
1149 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
1150}
1151
1152static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
1153 GlobalValue::GUID G) {
1154 if (const auto &VI = Index.getValueInfo(G))
1155 return isGlobalVarSummary(Index, VI);
1156 return false;
1157}
1158
1159// Return the number of global variable summaries in ExportSet.
1160static unsigned
1161numGlobalVarSummaries(const ModuleSummaryIndex &Index,
1162 FunctionImporter::ExportSetTy &ExportSet) {
1163 unsigned NumGVS = 0;
1164 for (auto &VI : ExportSet)
1165 if (isGlobalVarSummary(Index, VI.getGUID()))
1166 ++NumGVS;
1167 return NumGVS;
1168}
1169
1170struct ImportStatistics {
1171 unsigned NumGVS = 0;
1172 unsigned DefinedFS = 0;
1173 unsigned Count = 0;
1174};
1175
1176// Compute import statistics for each source module in ImportList.
1177static DenseMap<StringRef, ImportStatistics>
1178collectImportStatistics(const ModuleSummaryIndex &Index,
1179 const FunctionImporter::ImportMapTy &ImportList) {
1180 DenseMap<StringRef, ImportStatistics> Histogram;
1181
1182 for (const auto &[FromModule, GUID, Type] : ImportList) {
1183 ImportStatistics &Entry = Histogram[FromModule];
1184 ++Entry.Count;
1185 if (isGlobalVarSummary(Index, GUID))
1186 ++Entry.NumGVS;
1187 else if (Type == GlobalValueSummary::Definition)
1188 ++Entry.DefinedFS;
1189 }
1190 return Histogram;
1191}
1192#endif
1193
1194#ifndef NDEBUG
1195static bool checkVariableImport(
1196 const ModuleSummaryIndex &Index,
1197 FunctionImporter::ImportListsTy &ImportLists,
1198 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1199 DenseSet<GlobalValue::GUID> FlattenedImports;
1200
1201 for (const auto &ImportPerModule : ImportLists)
1202 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1203 FlattenedImports.insert(GUID);
1204
1205 // Checks that all GUIDs of read/writeonly vars we see in export lists
1206 // are also in the import lists. Otherwise we my face linker undefs,
1207 // because readonly and writeonly vars are internalized in their
1208 // source modules. The exception would be if it has a linkage type indicating
1209 // that there may have been a copy existing in the importing module (e.g.
1210 // linkonce_odr). In that case we cannot accurately do this checking.
1211 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
1212 const ValueInfo &VI) {
1213 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1214 Index.findSummaryInModule(VI, ModulePath));
1215 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1216 !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
1217 GVS->linkage() == GlobalValue::WeakODRLinkage ||
1218 GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
1219 };
1220
1221 for (auto &ExportPerModule : ExportLists)
1222 for (auto &VI : ExportPerModule.second)
1223 if (!FlattenedImports.count(VI.getGUID()) &&
1224 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1225 return false;
1226
1227 return true;
1228}
1229#endif
1230
1231/// Compute all the import and export for every module using the Index.
1232void llvm::ComputeCrossModuleImport(
1233 const ModuleSummaryIndex &Index,
1234 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1235 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1236 isPrevailing,
1237 FunctionImporter::ImportListsTy &ImportLists,
1238 DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
1239 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index, ExportLists: &ExportLists);
1240 // For each module that has function defined, compute the import/export lists.
1241 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1242 auto &ImportList = ImportLists[DefinedGVSummaries.first];
1243 LLVM_DEBUG(dbgs() << "Computing import for Module '"
1244 << DefinedGVSummaries.first << "'\n");
1245 MIS->computeImportForModule(DefinedGVSummaries: DefinedGVSummaries.second,
1246 ModName: DefinedGVSummaries.first, ImportList);
1247 }
1248
1249 // When computing imports we only added the variables and functions being
1250 // imported to the export list. We also need to mark any references and calls
1251 // they make as exported as well. We do this here, as it is more efficient
1252 // since we may import the same values multiple times into different modules
1253 // during the import computation.
1254 for (auto &ELI : ExportLists) {
1255 // `NewExports` tracks the VI that gets exported because the full definition
1256 // of its user/referencer gets exported.
1257 FunctionImporter::ExportSetTy NewExports;
1258 const auto &DefinedGVSummaries =
1259 ModuleToDefinedGVSummaries.lookup(Val: ELI.first);
1260 for (auto &EI : ELI.second) {
1261 // Find the copy defined in the exporting module so that we can mark the
1262 // values it references in that specific definition as exported.
1263 // Below we will add all references and called values, without regard to
1264 // whether they are also defined in this module. We subsequently prune the
1265 // list to only include those defined in the exporting module, see comment
1266 // there as to why.
1267 auto DS = DefinedGVSummaries.find(Val: EI.getGUID());
1268 // Anything marked exported during the import computation must have been
1269 // defined in the exporting module.
1270 assert(DS != DefinedGVSummaries.end());
1271 auto *S = DS->getSecond();
1272 S = S->getBaseObject();
1273 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: S)) {
1274 // Export referenced functions and variables. We don't export/promote
1275 // objects referenced by writeonly variable initializer, because
1276 // we convert such variables initializers to "zeroinitializer".
1277 // See processGlobalForThinLTO.
1278 if (!Index.isWriteOnly(GVS))
1279 NewExports.insert_range(R: GVS->refs());
1280 } else {
1281 auto *FS = cast<FunctionSummary>(Val: S);
1282 NewExports.insert_range(R: llvm::make_first_range(c: FS->calls()));
1283 NewExports.insert_range(R: FS->refs());
1284 }
1285 }
1286 // Prune list computed above to only include values defined in the
1287 // exporting module. We do this after the above insertion since we may hit
1288 // the same ref/call target multiple times in above loop, and it is more
1289 // efficient to avoid a set lookup each time.
1290 NewExports.remove_if(
1291 Pred: [&](ValueInfo VI) { return !DefinedGVSummaries.count(Val: VI.getGUID()); });
1292 ELI.second.insert_range(R&: NewExports);
1293 }
1294
1295 assert(checkVariableImport(Index, ImportLists, ExportLists));
1296#ifndef NDEBUG
1297 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
1298 << " modules:\n");
1299 for (const auto &ModuleImports : ImportLists) {
1300 auto ModName = ModuleImports.first;
1301 auto &Exports = ExportLists[ModName];
1302 unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
1303 DenseMap<StringRef, ImportStatistics> Histogram =
1304 collectImportStatistics(Index, ModuleImports.second);
1305 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
1306 << Exports.size() - NumGVS << " functions and " << NumGVS
1307 << " vars. Imports from " << Histogram.size()
1308 << " modules.\n");
1309 for (const auto &[SrcModName, Stats] : Histogram) {
1310 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1311 << " function definitions and "
1312 << Stats.Count - Stats.NumGVS - Stats.DefinedFS
1313 << " function declarations imported from " << SrcModName
1314 << "\n");
1315 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS
1316 << " global vars imported from " << SrcModName << "\n");
1317 }
1318 }
1319#endif
1320}
1321
1322#ifndef NDEBUG
1323static void dumpImportListForModule(const ModuleSummaryIndex &Index,
1324 StringRef ModulePath,
1325 FunctionImporter::ImportMapTy &ImportList) {
1326 DenseMap<StringRef, ImportStatistics> Histogram =
1327 collectImportStatistics(Index, ImportList);
1328 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
1329 << Histogram.size() << " modules.\n");
1330 for (const auto &[SrcModName, Stats] : Histogram) {
1331 LLVM_DEBUG(dbgs() << " - " << Stats.DefinedFS
1332 << " function definitions and "
1333 << Stats.Count - Stats.DefinedFS - Stats.NumGVS
1334 << " function declarations imported from " << SrcModName
1335 << "\n");
1336 LLVM_DEBUG(dbgs() << " - " << Stats.NumGVS << " vars imported from "
1337 << SrcModName << "\n");
1338 }
1339}
1340#endif
1341
1342/// Compute all the imports for the given module using the Index.
1343///
1344/// \p isPrevailing is a callback that will be called with a global value's GUID
1345/// and summary and should return whether the module corresponding to the
1346/// summary contains the linker-prevailing copy of that value.
1347///
1348/// \p ImportList will be populated with a map that can be passed to
1349/// FunctionImporter::importFunctions() above (see description there).
1350static void ComputeCrossModuleImportForModuleForTest(
1351 StringRef ModulePath,
1352 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1353 isPrevailing,
1354 const ModuleSummaryIndex &Index,
1355 FunctionImporter::ImportMapTy &ImportList) {
1356 // Collect the list of functions this module defines.
1357 // GUID -> Summary
1358 GVSummaryMapTy FunctionSummaryMap;
1359 Index.collectDefinedFunctionsForModule(ModulePath, GVSummaryMap&: FunctionSummaryMap);
1360
1361 // Compute the import list for this module.
1362 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
1363 auto MIS = ModuleImportsManager::create(IsPrevailing: isPrevailing, Index);
1364 MIS->computeImportForModule(DefinedGVSummaries: FunctionSummaryMap, ModName: ModulePath, ImportList);
1365
1366#ifndef NDEBUG
1367 dumpImportListForModule(Index, ModulePath, ImportList);
1368#endif
1369}
1370
1371/// Mark all external summaries in \p Index for import into the given module.
1372/// Used for testing the case of distributed builds using a distributed index.
1373///
1374/// \p ImportList will be populated with a map that can be passed to
1375/// FunctionImporter::importFunctions() above (see description there).
1376static void ComputeCrossModuleImportForModuleFromIndexForTest(
1377 StringRef ModulePath, const ModuleSummaryIndex &Index,
1378 FunctionImporter::ImportMapTy &ImportList) {
1379 for (const auto &GlobalList : Index) {
1380 // Ignore entries for undefined references.
1381 if (GlobalList.second.getSummaryList().empty())
1382 continue;
1383
1384 auto GUID = GlobalList.first;
1385 assert(GlobalList.second.getSummaryList().size() == 1 &&
1386 "Expected individual combined index to have one summary per GUID");
1387 auto &Summary = GlobalList.second.getSummaryList()[0];
1388 // Skip the summaries for the importing module. These are included to
1389 // e.g. record required linkage changes.
1390 if (Summary->modulePath() == ModulePath)
1391 continue;
1392 // Add an entry to provoke importing by thinBackend.
1393 ImportList.addGUID(FromModule: Summary->modulePath(), GUID, ImportKind: Summary->importType());
1394 }
1395#ifndef NDEBUG
1396 dumpImportListForModule(Index, ModulePath, ImportList);
1397#endif
1398}
1399
1400// For SamplePGO, the indirect call targets for local functions will
1401// have its original name annotated in profile. We try to find the
1402// corresponding PGOFuncName as the GUID, and fix up the edges
1403// accordingly.
1404void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
1405 FunctionSummary *FS) {
1406 // Returns the ValueInfo that should be used in place of \p CalleeVI, or an
1407 // empty ValueInfo if no update is needed.
1408 auto GetUpdatedCallee = [&](ValueInfo CalleeVI) -> ValueInfo {
1409 if (!CalleeVI || !CalleeVI.getSummaryList().empty())
1410 return ValueInfo();
1411 auto GUID = Index.getGUIDFromOriginalID(OriginalID: CalleeVI.getGUID());
1412 if (GUID == 0)
1413 return ValueInfo();
1414 auto VI = Index.getValueInfo(GUID);
1415 if (llvm::any_of(
1416 Range: VI.getSummaryList(),
1417 P: [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1418 // The mapping from OriginalId to GUID may return a GUID
1419 // that corresponds to a static variable. Filter it out here.
1420 // This can happen when
1421 // 1) There is a call to a library function which is not defined
1422 // in the index.
1423 // 2) There is a static variable with the OriginalGUID identical
1424 // to the GUID of the library function in 1);
1425 // When this happens the static variable in 2) will be found,
1426 // which needs to be filtered out.
1427 return SummaryPtr->getSummaryKind() ==
1428 GlobalValueSummary::GlobalVarKind;
1429 }))
1430 return ValueInfo();
1431 return VI;
1432 };
1433
1434 for (auto &EI : FS->mutableCalls()) {
1435 // Update the edge to point directly to the correct GUID.
1436 if (auto VI = GetUpdatedCallee(EI.first))
1437 EI.first = VI;
1438 }
1439
1440 // MemProf summarizes indirect callsites by synthesizing a CallsiteInfo
1441 // record for each profiled target, using the same GUIDs found in the VP
1442 // metadata (which, for local targets, may be the original name GUID when
1443 // e.g. synthesized from the MemProf profile or from a SamplePGO profile).
1444 // Update those callees as well so that they are consistent with the call
1445 // edges above, and so that the thin link's MemProf context disambiguation
1446 // can correlate the callsite with the summary for the target function.
1447 if (FS->callsites().empty())
1448 return;
1449 for (auto &CI : FS->mutableCallsites()) {
1450 if (auto VI = GetUpdatedCallee(CI.Callee))
1451 CI.Callee = VI;
1452 }
1453}
1454
1455void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
1456 for (const auto &Entry : Index) {
1457 for (const auto &S : Entry.second.getSummaryList()) {
1458 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1459 updateValueInfoForIndirectCalls(Index, FS);
1460 }
1461 }
1462}
1463
1464void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
1465 ModuleSummaryIndex &Index,
1466 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1467 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
1468 assert(!Index.withGlobalValueDeadStripping());
1469 if (!ComputeDead ||
1470 // Don't do anything when nothing is live, this is friendly with tests.
1471 GUIDPreservedSymbols.empty()) {
1472 // Still need to update indirect calls.
1473 updateIndirectCalls(Index);
1474 return;
1475 }
1476 unsigned LiveSymbols = 0;
1477 SmallVector<ValueInfo, 128> Worklist;
1478 Worklist.reserve(N: GUIDPreservedSymbols.size() * 2);
1479 for (auto GUID : GUIDPreservedSymbols) {
1480 ValueInfo VI = Index.getValueInfo(GUID);
1481 if (!VI)
1482 continue;
1483 for (const auto &S : VI.getSummaryList())
1484 S->setLive(true);
1485 }
1486
1487 // Add values flagged in the index as live roots to the worklist.
1488 for (const auto &Entry : Index) {
1489 auto VI = Index.getValueInfo(R: Entry);
1490 for (const auto &S : Entry.second.getSummaryList()) {
1491 if (auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
1492 updateValueInfoForIndirectCalls(Index, FS);
1493 if (S->isLive()) {
1494 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
1495 Worklist.push_back(Elt: VI);
1496 ++LiveSymbols;
1497 break;
1498 }
1499 }
1500 }
1501
1502 // Make value live and add it to the worklist if it was not live before.
1503 auto visit = [&](ValueInfo VI, bool IsAliasee) {
1504 // FIXME: If we knew which edges were created for indirect call profiles,
1505 // we could skip them here. Any that are live should be reached via
1506 // other edges, e.g. reference edges. Otherwise, using a profile collected
1507 // on a slightly different binary might provoke preserving, importing
1508 // and ultimately promoting calls to functions not linked into this
1509 // binary, which increases the binary size unnecessarily. Note that
1510 // if this code changes, the importer needs to change so that edges
1511 // to functions marked dead are skipped.
1512
1513 if (llvm::any_of(Range: VI.getSummaryList(),
1514 P: [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1515 return S->isLive();
1516 }))
1517 return;
1518
1519 // We only keep live symbols that are known to be non-prevailing if any are
1520 // available_externally, linkonceodr, weakodr. Those symbols are discarded
1521 // later in the EliminateAvailableExternally pass and setting them to
1522 // not-live could break downstreams users of liveness information (PR36483)
1523 // or limit optimization opportunities.
1524 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
1525 bool KeepAliveLinkage = false;
1526 bool Interposable = false;
1527 for (const auto &S : VI.getSummaryList()) {
1528 if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
1529 S->linkage() == GlobalValue::WeakODRLinkage ||
1530 S->linkage() == GlobalValue::LinkOnceODRLinkage)
1531 KeepAliveLinkage = true;
1532 else if (GlobalValue::isInterposableLinkage(Linkage: S->linkage()))
1533 Interposable = true;
1534 }
1535
1536 if (!IsAliasee) {
1537 if (!KeepAliveLinkage)
1538 return;
1539
1540 if (Interposable)
1541 report_fatal_error(
1542 reason: "Interposable and available_externally/linkonce_odr/weak_odr "
1543 "symbol");
1544 }
1545 }
1546
1547 for (const auto &S : VI.getSummaryList())
1548 S->setLive(true);
1549 ++LiveSymbols;
1550 Worklist.push_back(Elt: VI);
1551 };
1552
1553 while (!Worklist.empty()) {
1554 auto VI = Worklist.pop_back_val();
1555 for (const auto &Summary : VI.getSummaryList()) {
1556 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.get())) {
1557 // If this is an alias, visit the aliasee VI to ensure that all copies
1558 // are marked live and it is added to the worklist for further
1559 // processing of its references.
1560 visit(AS->getAliaseeVI(), true);
1561 continue;
1562 }
1563 for (auto Ref : Summary->refs())
1564 visit(Ref, false);
1565 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get()))
1566 for (auto Call : FS->calls())
1567 visit(Call.first, false);
1568 }
1569 }
1570 Index.setWithGlobalValueDeadStripping();
1571
1572 unsigned DeadSymbols = Index.size() - LiveSymbols;
1573 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
1574 << " symbols Dead \n");
1575 NumDeadSymbols += DeadSymbols;
1576 NumLiveSymbols += LiveSymbols;
1577}
1578
1579// Compute dead symbols and propagate constants in combined index.
1580void llvm::computeDeadSymbolsWithConstProp(
1581 ModuleSummaryIndex &Index,
1582 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
1583 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
1584 bool ImportEnabled) {
1585 llvm::TimeTraceScope timeScope("Drop dead symbols and propagate attributes");
1586 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
1587 isPrevailing);
1588 if (ImportEnabled)
1589 Index.propagateAttributes(PreservedSymbols: GUIDPreservedSymbols);
1590}
1591
1592/// Compute the set of summaries needed for a ThinLTO backend compilation of
1593/// \p ModulePath.
1594void llvm::gatherImportedSummariesForModule(
1595 StringRef ModulePath,
1596 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1597 const FunctionImporter::ImportMapTy &ImportList,
1598 ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1599 GVSummaryPtrSet &DecSummaries) {
1600 // Include all summaries from the importing module.
1601 ModuleToSummariesForIndex[std::string(ModulePath)] =
1602 ModuleToDefinedGVSummaries.lookup(Val: ModulePath);
1603
1604 // Forward port the heterogeneous std::map::operator[]() from C++26, which
1605 // lets us look up the map without allocating an instance of std::string when
1606 // the key-value pair exists in the map.
1607 // TODO: Remove this in favor of the heterogenous std::map::operator[]() from
1608 // C++26 when it becomes available for our codebase.
1609 auto LookupOrCreate = [](ModuleToSummariesForIndexTy &Map,
1610 StringRef Key) -> GVSummaryMapTy & {
1611 auto It = Map.find(x: Key);
1612 if (It == Map.end())
1613 std::tie(args&: It, args: std::ignore) =
1614 Map.try_emplace(k: std::string(Key), args: GVSummaryMapTy());
1615 return It->second;
1616 };
1617
1618 // Include summaries for imports.
1619 for (const auto &[FromModule, GUID, ImportType] : ImportList) {
1620 auto &SummariesForIndex =
1621 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1622
1623 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(Val: FromModule);
1624 const auto &DS = DefinedGVSummaries.find(Val: GUID);
1625 assert(DS != DefinedGVSummaries.end() &&
1626 "Expected a defined summary for imported global value");
1627 if (ImportType == GlobalValueSummary::Declaration)
1628 DecSummaries.insert(Ptr: DS->second);
1629
1630 SummariesForIndex[GUID] = DS->second;
1631 }
1632
1633 // When AlwaysRenamePromotedLocals is false, for each source module we import
1634 // from, also include summaries for local functions that have
1635 // NoRenameOnPromotion set. This is needed for distributed ThinLTO. Otherwise,
1636 // the local function of the source module will keep its origin name, e.g.,
1637 // foo() while the function in destination module will have name
1638 // foo.llvm.<...>() and this will cause a link failure.
1639 //
1640 // Note: this imports a superset of the necessary declarations — all locals
1641 // with NoRenameOnPromotion in each source module, not just those referenced
1642 // by the importing module. Computing the precise set would require walking
1643 // the summary reference graph from each imported function, which is more
1644 // expensive than the simple scan here.
1645 if (!AlwaysRenamePromotedLocals) {
1646 for (auto &[ModPath, SummariesForIndex] : ModuleToSummariesForIndex) {
1647 if (ModPath == ModulePath)
1648 continue;
1649 auto It = ModuleToDefinedGVSummaries.find(Val: ModPath);
1650 if (It == ModuleToDefinedGVSummaries.end())
1651 continue;
1652 for (const auto &[GUID, Summary] : It->second) {
1653 if (Summary->noRenameOnPromotion()) {
1654 DecSummaries.insert(Ptr: Summary);
1655 SummariesForIndex.try_emplace(Key: GUID, Args: Summary);
1656 }
1657 }
1658 }
1659 }
1660}
1661
1662/// Emit the files \p ModulePath will import from into \p OutputFilename.
1663Error llvm::EmitImportsFiles(
1664 StringRef ModulePath, StringRef OutputFilename,
1665 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex) {
1666 std::error_code EC;
1667 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_Text);
1668 if (EC)
1669 return createFileError(F: "cannot open " + OutputFilename,
1670 E: errorCodeToError(EC));
1671 processImportsFiles(ModulePath, ModuleToSummariesForIndex,
1672 F: [&](StringRef M) { ImportsOS << M << "\n"; });
1673 return Error::success();
1674}
1675
1676/// Invoke callback \p F on the file paths from which \p ModulePath
1677/// will import.
1678void llvm::processImportsFiles(
1679 StringRef ModulePath,
1680 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
1681 function_ref<void(const std::string &)> F) {
1682 for (const auto &ILI : ModuleToSummariesForIndex)
1683 // The ModuleToSummariesForIndex map includes an entry for the current
1684 // Module (needed for writing out the index files). We don't want to
1685 // include it in the imports file, however, so filter it out.
1686 if (ILI.first != ModulePath)
1687 F(ILI.first);
1688}
1689
1690bool llvm::convertToDeclaration(GlobalValue &GV) {
1691 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
1692 << "\n");
1693 MDNode *UniqueID = nullptr;
1694 if (auto *GO = dyn_cast<GlobalObject>(Val: &GV))
1695 UniqueID = GO->getMetadata(KindID: LLVMContext::MD_guid);
1696
1697 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1698 F->deleteBody();
1699 F->clearMetadata();
1700 if (UniqueID)
1701 F->setMetadata(KindID: LLVMContext::MD_guid, Node: UniqueID);
1702 F->setComdat(nullptr);
1703 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) {
1704 V->setInitializer(nullptr);
1705 V->setLinkage(GlobalValue::ExternalLinkage);
1706 V->clearMetadata();
1707 if (UniqueID)
1708 V->setMetadata(KindID: LLVMContext::MD_guid, Node: UniqueID);
1709 V->setComdat(nullptr);
1710 } else {
1711 GlobalValue *NewGV;
1712 if (GV.getValueType()->isFunctionTy())
1713 NewGV =
1714 Function::Create(Ty: cast<FunctionType>(Val: GV.getValueType()),
1715 Linkage: GlobalValue::ExternalLinkage, AddrSpace: GV.getAddressSpace(),
1716 N: "", M: GV.getParent());
1717 else
1718 NewGV =
1719 new GlobalVariable(*GV.getParent(), GV.getValueType(),
1720 /*isConstant*/ false, GlobalValue::ExternalLinkage,
1721 /*init*/ nullptr, "",
1722 /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
1723 GV.getType()->getAddressSpace());
1724 NewGV->takeName(V: &GV);
1725 GV.replaceAllUsesWith(V: NewGV);
1726 return false;
1727 }
1728 if (!GV.isImplicitDSOLocal())
1729 GV.setDSOLocal(false);
1730 return true;
1731}
1732
1733void llvm::thinLTOFinalizeInModule(Module &TheModule,
1734 const GVSummaryMapTy &DefinedGlobals,
1735 bool PropagateAttrs) {
1736 llvm::TimeTraceScope timeScope("ThinLTO finalize in module");
1737 DenseSet<Comdat *> NonPrevailingComdats;
1738 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {
1739 // See if the global summary analysis computed a new resolved linkage.
1740 const auto &GS = DefinedGlobals.find(Val: GV.getGUIDOrFallback());
1741 if (GS == DefinedGlobals.end())
1742 return;
1743
1744 if (Propagate)
1745 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(Val: GS->second)) {
1746 if (Function *F = dyn_cast<Function>(Val: &GV)) {
1747 // TODO: propagate ReadNone and ReadOnly.
1748 if (FS->fflags().ReadNone && !F->doesNotAccessMemory())
1749 F->setDoesNotAccessMemory();
1750
1751 if (FS->fflags().ReadOnly && !F->onlyReadsMemory())
1752 F->setOnlyReadsMemory();
1753
1754 if (FS->fflags().NoRecurse && !F->doesNotRecurse())
1755 F->setDoesNotRecurse();
1756
1757 if (FS->fflags().NoUnwind && !F->doesNotThrow())
1758 F->setDoesNotThrow();
1759 }
1760 }
1761
1762 auto NewLinkage = GS->second->linkage();
1763 if (GlobalValue::isLocalLinkage(Linkage: GV.getLinkage()) ||
1764 // Don't internalize anything here, because the code below
1765 // lacks necessary correctness checks. Leave this job to
1766 // LLVM 'internalize' pass.
1767 GlobalValue::isLocalLinkage(Linkage: NewLinkage) ||
1768 // In case it was dead and already converted to declaration.
1769 GV.isDeclaration())
1770 return;
1771
1772 // Set the potentially more constraining visibility computed from summaries.
1773 // The DefaultVisibility condition is because older GlobalValueSummary does
1774 // not record DefaultVisibility and we don't want to change protected/hidden
1775 // to default.
1776 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility)
1777 GV.setVisibility(GS->second->getVisibility());
1778
1779 if (NewLinkage == GV.getLinkage())
1780 return;
1781
1782 // Check for a non-prevailing def that has interposable linkage
1783 // (e.g. non-odr weak or linkonce). In that case we can't simply
1784 // convert to available_externally, since it would lose the
1785 // interposable property and possibly get inlined. Simply drop
1786 // the definition in that case.
1787 if (GlobalValue::isAvailableExternallyLinkage(Linkage: NewLinkage) &&
1788 GlobalValue::isInterposableLinkage(Linkage: GV.getLinkage())) {
1789 if (!convertToDeclaration(GV))
1790 // FIXME: Change this to collect replaced GVs and later erase
1791 // them from the parent module once thinLTOResolvePrevailingGUID is
1792 // changed to enable this for aliases.
1793 llvm_unreachable("Expected GV to be converted");
1794 } else {
1795 // If all copies of the original symbol had global unnamed addr and
1796 // linkonce_odr linkage, or if all of them had local unnamed addr linkage
1797 // and are constants, then it should be an auto hide symbol. In that case
1798 // the thin link would have marked it as CanAutoHide. Add hidden
1799 // visibility to the symbol to preserve the property.
1800 if (NewLinkage == GlobalValue::WeakODRLinkage &&
1801 GS->second->canAutoHide()) {
1802 assert(GV.canBeOmittedFromSymbolTable());
1803 GV.setVisibility(GlobalValue::HiddenVisibility);
1804 }
1805
1806 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
1807 << "` from " << GV.getLinkage() << " to " << NewLinkage
1808 << "\n");
1809 GV.setLinkage(NewLinkage);
1810 }
1811 // Remove declarations from comdats, including available_externally
1812 // as this is a declaration for the linker, and will be dropped eventually.
1813 // It is illegal for comdats to contain declarations.
1814 auto *GO = dyn_cast_or_null<GlobalObject>(Val: &GV);
1815 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1816 if (GO->getComdat()->getName() == GO->getName())
1817 NonPrevailingComdats.insert(V: GO->getComdat());
1818 GO->setComdat(nullptr);
1819 }
1820 };
1821
1822 // Process functions and global now
1823 for (auto &GV : TheModule)
1824 FinalizeInModule(GV, PropagateAttrs);
1825 for (auto &GV : TheModule.globals())
1826 FinalizeInModule(GV);
1827 for (auto &GV : TheModule.aliases())
1828 FinalizeInModule(GV);
1829
1830 // For a non-prevailing comdat, all its members must be available_externally.
1831 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle
1832 // local linkage GlobalValues.
1833 if (NonPrevailingComdats.empty())
1834 return;
1835 for (auto &GO : TheModule.global_objects()) {
1836 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(V: C)) {
1837 GO.setComdat(nullptr);
1838 GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
1839 }
1840 }
1841 bool Changed;
1842 do {
1843 Changed = false;
1844 // If an alias references a GlobalValue in a non-prevailing comdat, change
1845 // it to available_externally. For simplicity we only handle GlobalValue and
1846 // ConstantExpr with a base object. ConstantExpr without a base object is
1847 // unlikely used in a COMDAT.
1848 for (auto &GA : TheModule.aliases()) {
1849 if (GA.hasAvailableExternallyLinkage())
1850 continue;
1851 GlobalObject *Obj = GA.getAliaseeObject();
1852 assert(Obj && "aliasee without an base object is unimplemented");
1853 if (Obj->hasAvailableExternallyLinkage()) {
1854 GA.setLinkage(GlobalValue::AvailableExternallyLinkage);
1855 Changed = true;
1856 }
1857 }
1858 } while (Changed);
1859}
1860
1861/// Run internalization on \p TheModule based on symmary analysis.
1862void llvm::thinLTOInternalizeModule(Module &TheModule,
1863 const GVSummaryMapTy &DefinedGlobals) {
1864 llvm::TimeTraceScope timeScope("ThinLTO internalize module");
1865 // Declare a callback for the internalize pass that will ask for every
1866 // candidate GlobalValue if it can be internalized or not.
1867 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1868 // It may be the case that GV is on a chain of an ifunc, its alias and
1869 // subsequent aliases. In this case, the summary for the value is not
1870 // available.
1871 if (isa<GlobalIFunc>(Val: &GV) ||
1872 (isa<GlobalAlias>(Val: &GV) &&
1873 isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: &GV)->getAliaseeObject())))
1874 return true;
1875
1876 // Lookup the linkage recorded in the summaries during global analysis.
1877 auto GS = DefinedGlobals.find(Val: GV.getGUIDOrFallback());
1878 if (GS == DefinedGlobals.end()) {
1879 // Must have been promoted (possibly conservatively). Find original
1880 // name so that we can access the correct summary and see if it can
1881 // be internalized again.
1882 // FIXME: Eventually we should control promotion instead of promoting
1883 // and internalizing again.
1884 StringRef OrigName =
1885 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
1886 std::string OrigId = GlobalValue::getGlobalIdentifier(
1887 Name: OrigName, Linkage: GlobalValue::InternalLinkage,
1888 FileName: TheModule.getSourceFileName());
1889 GS = DefinedGlobals.find(
1890 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
1891 if (GS == DefinedGlobals.end()) {
1892 // Also check the original non-promoted non-globalized name. In some
1893 // cases a preempted weak value is linked in as a local copy because
1894 // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1895 // In that case, since it was originally not a local value, it was
1896 // recorded in the index using the original name.
1897 // FIXME: This may not be needed once PR27866 is fixed.
1898 GS = DefinedGlobals.find(
1899 Val: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName));
1900 assert(GS != DefinedGlobals.end());
1901 }
1902 }
1903 return !GlobalValue::isLocalLinkage(Linkage: GS->second->linkage());
1904 };
1905
1906 // FIXME: See if we can just internalize directly here via linkage changes
1907 // based on the index, rather than invoking internalizeModule.
1908 internalizeModule(TheModule, MustPreserveGV);
1909}
1910
1911/// Make alias a clone of its aliasee.
1912static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1913 Function *Fn = cast<Function>(Val: GA->getAliaseeObject());
1914
1915 ValueToValueMapTy VMap;
1916 Function *NewFn = CloneFunction(F: Fn, VMap);
1917 // Clone should use the original alias's linkage, visibility and name, and we
1918 // ensure all uses of alias instead use the new clone (casted if necessary).
1919 NewFn->setLinkage(GA->getLinkage());
1920 NewFn->setVisibility(GA->getVisibility());
1921 GA->replaceAllUsesWith(V: NewFn);
1922 NewFn->takeName(V: GA);
1923 return NewFn;
1924}
1925
1926// Internalize values that we marked with specific attribute
1927// in processGlobalForThinLTO.
1928static void internalizeGVsAfterImport(Module &M) {
1929 for (auto &GV : M.globals())
1930 // Skip GVs which have been converted to declarations
1931 // by dropDeadSymbols.
1932 if (!GV.isDeclaration() && GV.hasAttribute(Kind: "thinlto-internalize")) {
1933 GV.setLinkage(GlobalValue::InternalLinkage);
1934 GV.setVisibility(GlobalValue::DefaultVisibility);
1935 }
1936}
1937
1938/// When a function carrying !implicit.ref is imported via ThinLTO, the
1939/// referenced global arrives as available_externally. This string can be dead
1940/// code eliminated since it has no IR uses -- only metadata references. Adding
1941/// it to llvm.compiler.used prevents elimination.
1942static void protectImplicitRefGlobals(Module &M) {
1943 SmallPtrSet<GlobalValue *, 4> Seen;
1944 SmallVector<GlobalValue *, 4> ToProtect;
1945 for (Function &F : M) {
1946 if (!F.hasAvailableExternallyLinkage() ||
1947 !F.hasMetadata(KindID: LLVMContext::MD_implicit_ref))
1948 continue;
1949 SmallVector<MDNode *> MDs;
1950 F.getMetadata(KindID: LLVMContext::MD_implicit_ref, MDs);
1951 for (MDNode *MD : MDs) {
1952 auto *Op = MD->getOperand(I: 0).get();
1953 if (!Op)
1954 continue;
1955 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: Op))
1956 if (auto *GV = dyn_cast<GlobalVariable>(Val: VAM->getValue()))
1957 if (GV->hasAvailableExternallyLinkage() && Seen.insert(Ptr: GV).second)
1958 ToProtect.push_back(Elt: GV);
1959 }
1960 }
1961 if (!ToProtect.empty())
1962 appendToCompilerUsed(M, Values: ToProtect);
1963}
1964
1965// Automatically import functions in Module \p DestModule based on the summaries
1966// index.
1967Expected<bool> FunctionImporter::importFunctions(
1968 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1969 LLVM_DEBUG(dbgs() << "Starting import for Module "
1970 << DestModule.getModuleIdentifier() << "\n");
1971 unsigned ImportedCount = 0, ImportedGVCount = 0;
1972 // Before carrying out any imports, see if this module defines functions in
1973 // MoveSymbolGUID. If it does, delete them here (but leave the declaration).
1974 // The function will be imported elsewhere, as extenal linkage, and the
1975 // destination doesn't yet have its definition.
1976 DenseSet<GlobalValue::GUID> MoveSymbolGUIDSet;
1977 MoveSymbolGUIDSet.insert_range(R&: MoveSymbolGUID);
1978 for (auto &F : DestModule)
1979 if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(V: F.getGUIDOrFallback()))
1980 F.deleteBody();
1981
1982 IRMover Mover(DestModule);
1983
1984 // Do the actual import of functions now, one Module at a time
1985 for (const auto &ModName : ImportList.getSourceModules()) {
1986 llvm::TimeTraceScope timeScope("Import", ModName);
1987 // Get the module for the import
1988 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(ModName);
1989 if (!SrcModuleOrErr)
1990 return SrcModuleOrErr.takeError();
1991 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1992 assert(&DestModule.getContext() == &SrcModule->getContext() &&
1993 "Context mismatch");
1994
1995 // If modules were created with lazy metadata loading, materialize it
1996 // now, before linking it (otherwise this will be a noop).
1997 if (Error Err = SrcModule->materializeMetadata())
1998 return std::move(Err);
1999
2000 // Find the globals to import
2001 SetVector<GlobalValue *> GlobalsToImport;
2002 {
2003 llvm::TimeTraceScope functionsScope("Functions");
2004 for (Function &F : *SrcModule) {
2005 if (!F.hasName())
2006 continue;
2007 auto GUID = F.getGUIDOrFallback();
2008 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2009 bool ImportDefinition =
2010 MaybeImportType == GlobalValueSummary::Definition;
2011
2012 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2013 << " importing function"
2014 << (ImportDefinition
2015 ? " definition "
2016 : (MaybeImportType ? " declaration " : " "))
2017 << GUID << " " << F.getName() << " from "
2018 << SrcModule->getSourceFileName() << "\n");
2019 if (ImportDefinition) {
2020 if (Error Err = F.materialize())
2021 return std::move(Err);
2022 // MemProf should match function's definition and summary,
2023 // 'thinlto_src_module' is needed.
2024 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2025 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2026 // statistics and debugging.
2027 F.setMetadata(
2028 Kind: "thinlto_src_module",
2029 Node: MDNode::get(Context&: DestModule.getContext(),
2030 MDs: {MDString::get(Context&: DestModule.getContext(),
2031 Str: SrcModule->getModuleIdentifier())}));
2032 F.setMetadata(
2033 Kind: "thinlto_src_file",
2034 Node: MDNode::get(Context&: DestModule.getContext(),
2035 MDs: {MDString::get(Context&: DestModule.getContext(),
2036 Str: SrcModule->getSourceFileName())}));
2037 }
2038 GlobalsToImport.insert(X: &F);
2039 }
2040 }
2041 }
2042 {
2043 llvm::TimeTraceScope globalsScope("Globals");
2044 for (GlobalVariable &GV : SrcModule->globals()) {
2045 if (!GV.hasName())
2046 continue;
2047 auto GUID = GV.getGUIDOrFallback();
2048 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2049 bool ImportDefinition =
2050 MaybeImportType == GlobalValueSummary::Definition;
2051
2052 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2053 << " importing global"
2054 << (ImportDefinition
2055 ? " definition "
2056 : (MaybeImportType ? " declaration " : " "))
2057 << GUID << " " << GV.getName() << " from "
2058 << SrcModule->getSourceFileName() << "\n");
2059 if (ImportDefinition) {
2060 if (Error Err = GV.materialize())
2061 return std::move(Err);
2062 ImportedGVCount += GlobalsToImport.insert(X: &GV);
2063 }
2064 }
2065 }
2066 {
2067 llvm::TimeTraceScope aliasesScope("Aliases");
2068 for (GlobalAlias &GA : SrcModule->aliases()) {
2069 if (!GA.hasName() || isa<GlobalIFunc>(Val: GA.getAliaseeObject()))
2070 continue;
2071 auto GUID = GA.getGUIDOrFallback();
2072 auto MaybeImportType = ImportList.getImportType(FromModule: ModName, GUID);
2073 bool ImportDefinition =
2074 MaybeImportType == GlobalValueSummary::Definition;
2075
2076 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")
2077 << " importing alias"
2078 << (ImportDefinition
2079 ? " definition "
2080 : (MaybeImportType ? " declaration " : " "))
2081 << GUID << " " << GA.getName() << " from "
2082 << SrcModule->getSourceFileName() << "\n");
2083 if (ImportDefinition) {
2084 if (Error Err = GA.materialize())
2085 return std::move(Err);
2086 // Import alias as a copy of its aliasee.
2087 GlobalObject *GO = GA.getAliaseeObject();
2088 if (Error Err = GO->materialize())
2089 return std::move(Err);
2090 auto *Fn = replaceAliasWithAliasee(SrcModule: SrcModule.get(), GA: &GA);
2091 assert(Fn);
2092 (void)Fn;
2093 LLVM_DEBUG(dbgs()
2094 << "Is importing aliasee fn " << GO->getGUIDOrFallback()
2095 << " " << GO->getName() << " from "
2096 << SrcModule->getSourceFileName() << "\n");
2097 if (EnableImportMetadata || EnableMemProfContextDisambiguation) {
2098 // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for
2099 // statistics and debugging.
2100 Fn->setMetadata(
2101 Kind: "thinlto_src_module",
2102 Node: MDNode::get(Context&: DestModule.getContext(),
2103 MDs: {MDString::get(Context&: DestModule.getContext(),
2104 Str: SrcModule->getModuleIdentifier())}));
2105 Fn->setMetadata(
2106 Kind: "thinlto_src_file",
2107 Node: MDNode::get(Context&: DestModule.getContext(),
2108 MDs: {MDString::get(Context&: DestModule.getContext(),
2109 Str: SrcModule->getSourceFileName())}));
2110 }
2111 GlobalsToImport.insert(X: Fn);
2112 }
2113 }
2114 }
2115
2116 // Upgrade debug info after we're done materializing all the globals and we
2117 // have loaded all the required metadata!
2118 UpgradeDebugInfo(M&: *SrcModule);
2119
2120 // Set the partial sample profile ratio in the profile summary module flag
2121 // of the imported source module, if applicable, so that the profile summary
2122 // module flag will match with that of the destination module when it's
2123 // imported.
2124 SrcModule->setPartialSampleProfileRatio(Index);
2125
2126 // Link in the specified functions.
2127 renameModuleForThinLTO(M&: *SrcModule, Index, ClearDSOLocalOnDeclarations,
2128 GlobalsToImport: &GlobalsToImport);
2129
2130 if (PrintImports) {
2131 for (const auto *GV : GlobalsToImport)
2132 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
2133 << " from " << SrcModule->getSourceFileName() << "\n";
2134 }
2135
2136 if (Error Err = Mover.move(Src: std::move(SrcModule),
2137 ValuesToLink: GlobalsToImport.getArrayRef(), AddLazyFor: nullptr,
2138 /*IsPerformingImport=*/true))
2139 return createStringError(EC: errc::invalid_argument,
2140 S: Twine("Function Import: link error: ") +
2141 toString(E: std::move(Err)));
2142
2143 ImportedCount += GlobalsToImport.size();
2144 NumImportedModules++;
2145 }
2146
2147 internalizeGVsAfterImport(M&: DestModule);
2148
2149 // Protect !implicit.ref-referenced globals imported as available_externally
2150 // from DCE'd. Only needed when globals were actually imported.
2151 if (ImportedGVCount > 0)
2152 protectImplicitRefGlobals(M&: DestModule);
2153
2154 NumImportedFunctions += (ImportedCount - ImportedGVCount);
2155 NumImportedGlobalVars += ImportedGVCount;
2156
2157 // TODO: Print counters for definitions and declarations in the debugging log.
2158 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
2159 << " functions for Module "
2160 << DestModule.getModuleIdentifier() << "\n");
2161 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
2162 << " global variables for Module "
2163 << DestModule.getModuleIdentifier() << "\n");
2164 return ImportedCount;
2165}
2166
2167static bool doImportingForModuleForTest(
2168 Module &M, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2169 isPrevailing) {
2170 if (SummaryFile.empty())
2171 report_fatal_error(reason: "error: -function-import requires -summary-file\n");
2172 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
2173 getModuleSummaryIndexForFile(Path: SummaryFile);
2174 if (!IndexPtrOrErr) {
2175 logAllUnhandledErrors(E: IndexPtrOrErr.takeError(), OS&: errs(),
2176 ErrorBanner: "Error loading file '" + SummaryFile + "': ");
2177 return false;
2178 }
2179 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2180
2181 // First step is collecting the import list.
2182 FunctionImporter::ImportIDTable ImportIDs;
2183 FunctionImporter::ImportMapTy ImportList(ImportIDs);
2184 // If requested, simply import all functions in the index. This is used
2185 // when testing distributed backend handling via the opt tool, when
2186 // we have distributed indexes containing exactly the summaries to import.
2187 if (ImportAllIndex)
2188 ComputeCrossModuleImportForModuleFromIndexForTest(ModulePath: M.getModuleIdentifier(),
2189 Index: *Index, ImportList);
2190 else
2191 ComputeCrossModuleImportForModuleForTest(ModulePath: M.getModuleIdentifier(),
2192 isPrevailing, Index: *Index, ImportList);
2193
2194 // Conservatively mark all internal values as promoted. This interface is
2195 // only used when doing importing via the function importing pass. The pass
2196 // is only enabled when testing importing via the 'opt' tool, which does
2197 // not do the ThinLink that would normally determine what values to promote.
2198 for (auto &I : *Index) {
2199 for (auto &S : I.second.getSummaryList()) {
2200 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
2201 S->setExternalLinkageForTest();
2202 }
2203 }
2204
2205 // Next we need to promote to global scope and rename any local values that
2206 // are potentially exported to other modules.
2207 renameModuleForThinLTO(M, Index: *Index, /*ClearDSOLocalOnDeclarations=*/false,
2208 /*GlobalsToImport=*/nullptr);
2209
2210 // Perform the import now.
2211 auto ModuleLoader = [&M](StringRef Identifier) {
2212 return loadFile(FileName: std::string(Identifier), Context&: M.getContext());
2213 };
2214 FunctionImporter Importer(*Index, ModuleLoader,
2215 /*ClearDSOLocalOnDeclarations=*/false);
2216 Expected<bool> Result = Importer.importFunctions(DestModule&: M, ImportList);
2217
2218 // FIXME: Probably need to propagate Errors through the pass manager.
2219 if (!Result) {
2220 logAllUnhandledErrors(E: Result.takeError(), OS&: errs(),
2221 ErrorBanner: "Error importing module: ");
2222 return true;
2223 }
2224
2225 return true;
2226}
2227
2228PreservedAnalyses FunctionImportPass::run(Module &M,
2229 ModuleAnalysisManager &AM) {
2230 // This is only used for testing the function import pass via opt, where we
2231 // don't have prevailing information from the LTO context available, so just
2232 // conservatively assume everything is prevailing (which is fine for the very
2233 // limited use of prevailing checking in this pass).
2234 auto isPrevailing = [](GlobalValue::GUID, const GlobalValueSummary *) {
2235 return true;
2236 };
2237 if (!doImportingForModuleForTest(M, isPrevailing))
2238 return PreservedAnalyses::all();
2239
2240 return PreservedAnalyses::none();
2241}
2242