1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ModuleSummaryAnalysis.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/Analysis/BlockFrequencyInfo.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
27#include "llvm/Analysis/LoopInfo.h"
28#include "llvm/Analysis/MemoryProfileInfo.h"
29#include "llvm/Analysis/ProfileSummaryInfo.h"
30#include "llvm/Analysis/StackSafetyAnalysis.h"
31#include "llvm/Analysis/TypeMetadataUtils.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Dominators.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalAlias.h"
39#include "llvm/IR/GlobalValue.h"
40#include "llvm/IR/GlobalVariable.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/ModuleSummaryIndex.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
48#include "llvm/InitializePasses.h"
49#include "llvm/Object/ModuleSymbolTable.h"
50#include "llvm/Object/SymbolicFile.h"
51#include "llvm/Pass.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Compiler.h"
55#include "llvm/Support/FileSystem.h"
56#include <cassert>
57#include <cstdint>
58#include <vector>
59
60using namespace llvm;
61using namespace llvm::memprof;
62
63#define DEBUG_TYPE "module-summary-analysis"
64
65// Option to force edges cold which will block importing when the
66// -import-cold-multiplier is set to 0. Useful for debugging.
67namespace llvm {
68FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold =
69 FunctionSummary::FSHT_None;
70
71static cl::opt<FunctionSummary::ForceSummaryHotnessType, true> FSEC(
72 "force-summary-edges-cold", cl::Hidden, cl::location(L&: ForceSummaryEdgesCold),
73 cl::desc("Force all edges in the function summary to cold"),
74 cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."),
75 clEnumValN(FunctionSummary::FSHT_AllNonCritical,
76 "all-non-critical", "All non-critical edges."),
77 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
78
79static cl::opt<std::string> ModuleSummaryDotFile(
80 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
81 cl::desc("File to emit dot graph of new summary into"));
82
83static cl::opt<bool> EnableMemProfIndirectCallSupport(
84 "enable-memprof-indirect-call-support", cl::init(Val: true), cl::Hidden,
85 cl::desc(
86 "Enable MemProf support for summarizing and cloning indirect calls"));
87
88// This can be used to override the number of callees created from VP metadata
89// normally taken from the -icp-max-prom option with a larger amount, if useful
90// for analysis. Use a separate option so that we can control the number of
91// indirect callees for ThinLTO summary based analysis (e.g. for MemProf which
92// needs this information for a correct and not overly-conservative callsite
93// graph analysis, especially because allocation contexts may not be very
94// frequent), without affecting normal ICP.
95cl::opt<unsigned>
96 MaxSummaryIndirectEdges("module-summary-max-indirect-edges", cl::init(Val: 0),
97 cl::Hidden,
98 cl::desc("Max number of summary edges added from "
99 "indirect call profile metadata"));
100
101LLVM_ABI extern cl::opt<bool> ScalePartialSampleProfileWorkingSetSize;
102
103extern cl::opt<unsigned> MaxNumVTableAnnotations;
104
105extern cl::opt<bool> MemProfReportHintedSizes;
106} // namespace llvm
107
108// Walk through the operands of a given User via worklist iteration and populate
109// the set of GlobalValue references encountered. Invoked either on an
110// Instruction or a GlobalVariable (which walks its initializer).
111// Return true if any of the operands contains blockaddress. This is important
112// to know when computing summary for global var, because if global variable
113// references basic block address we can't import it separately from function
114// containing that basic block. For simplicity we currently don't import such
115// global vars at all. When importing function we aren't interested if any
116// instruction in it takes an address of any basic block, because instruction
117// can only take an address of basic block located in the same function.
118// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
119// local-linkage ifunc.
120static bool
121findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
122 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &RefEdges,
123 SmallPtrSet<const User *, 8> &Visited,
124 bool &RefLocalLinkageIFunc) {
125 bool HasBlockAddress = false;
126 SmallVector<const User *, 32> Worklist;
127 if (Visited.insert(Ptr: CurUser).second)
128 Worklist.push_back(Elt: CurUser);
129
130 while (!Worklist.empty()) {
131 const User *U = Worklist.pop_back_val();
132 const auto *CB = dyn_cast<CallBase>(Val: U);
133
134 for (const auto &OI : U->operands()) {
135 const User *Operand = dyn_cast<User>(Val: OI);
136 if (!Operand)
137 continue;
138 if (isa<BlockAddress>(Val: Operand)) {
139 HasBlockAddress = true;
140 continue;
141 }
142 if (auto *GV = dyn_cast<GlobalValue>(Val: Operand)) {
143 // We have a reference to a global value. This should be added to
144 // the reference set unless it is a callee. Callees are handled
145 // specially by WriteFunction and are added to a separate list.
146 if (!(CB && CB->isCallee(U: &OI))) {
147 // If an ifunc has local linkage, do not add it into ref edges, and
148 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
149 // for import. An ifunc doesn't have summary and ThinLTO cannot
150 // promote it; importing the referencer may cause linkage errors.
151 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: GV);
152 GI && GI->hasLocalLinkage()) {
153 RefLocalLinkageIFunc = true;
154 continue;
155 }
156 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
157 }
158 continue;
159 }
160 if (Visited.insert(Ptr: Operand).second)
161 Worklist.push_back(Elt: Operand);
162 }
163 }
164
165 const Instruction *I = dyn_cast<Instruction>(Val: CurUser);
166 if (I) {
167 uint64_t TotalCount = 0;
168 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
169 // the instruction.
170 auto ValueDataArray = getValueProfDataFromInst(
171 Inst: *I, ValueKind: IPVK_VTableTarget, MaxNumValueData: MaxNumVTableAnnotations, TotalC&: TotalCount);
172
173 for (const auto &V : ValueDataArray)
174 RefEdges.insert(X: Index.getOrInsertValueInfo(/* VTableGUID = */
175 GUID: V.Value));
176 }
177 return HasBlockAddress;
178}
179
180static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
181 ProfileSummaryInfo *PSI) {
182 if (!PSI)
183 return CalleeInfo::HotnessType::Unknown;
184 if (PSI->isHotCount(C: ProfileCount))
185 return CalleeInfo::HotnessType::Hot;
186 if (PSI->isColdCount(C: ProfileCount))
187 return CalleeInfo::HotnessType::Cold;
188 return CalleeInfo::HotnessType::None;
189}
190
191static bool isNonRenamableLocal(const GlobalValue &GV) {
192 return GV.hasSection() && GV.hasLocalLinkage();
193}
194
195/// Determine whether this call has all constant integer arguments (excluding
196/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
197static void addVCallToSet(
198 DevirtCallSite Call, GlobalValue::GUID Guid,
199 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
200 &VCalls,
201 SetVector<FunctionSummary::ConstVCall,
202 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
203 std::vector<uint64_t> Args;
204 // Start from the second argument to skip the "this" pointer.
205 for (auto &Arg : drop_begin(RangeOrContainer: Call.CB.args())) {
206 auto *CI = dyn_cast<ConstantInt>(Val&: Arg);
207 if (!CI || CI->getBitWidth() > 64) {
208 VCalls.insert(X: {.GUID: Guid, .Offset: Call.Offset});
209 return;
210 }
211 Args.push_back(x: CI->getZExtValue());
212 }
213 ConstVCalls.insert(X: {.VFunc: {.GUID: Guid, .Offset: Call.Offset}, .Args: std::move(Args)});
214}
215
216/// If this intrinsic call requires that we add information to the function
217/// summary, do so via the non-constant reference arguments.
218static void addIntrinsicToSummary(
219 const CallInst *CI,
220 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
221 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
222 &TypeTestAssumeVCalls,
223 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
224 &TypeCheckedLoadVCalls,
225 SetVector<FunctionSummary::ConstVCall,
226 std::vector<FunctionSummary::ConstVCall>>
227 &TypeTestAssumeConstVCalls,
228 SetVector<FunctionSummary::ConstVCall,
229 std::vector<FunctionSummary::ConstVCall>>
230 &TypeCheckedLoadConstVCalls,
231 DominatorTree &DT) {
232 switch (CI->getCalledFunction()->getIntrinsicID()) {
233 case Intrinsic::type_test:
234 case Intrinsic::public_type_test: {
235 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
236 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
237 if (!TypeId)
238 break;
239 GlobalValue::GUID Guid =
240 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
241
242 // Produce a summary from type.test intrinsics. We only summarize type.test
243 // intrinsics that are used other than by an llvm.assume intrinsic.
244 // Intrinsics that are assumed are relevant only to the devirtualization
245 // pass, not the type test lowering pass.
246 bool HasNonAssumeUses = llvm::any_of(Range: CI->uses(), P: [](const Use &CIU) {
247 return !isa<AssumeInst>(Val: CIU.getUser());
248 });
249 if (HasNonAssumeUses)
250 TypeTests.insert(X: Guid);
251
252 SmallVector<DevirtCallSite, 4> DevirtCalls;
253 SmallVector<CallInst *, 4> Assumes;
254 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
255 for (auto &Call : DevirtCalls)
256 addVCallToSet(Call, Guid, VCalls&: TypeTestAssumeVCalls,
257 ConstVCalls&: TypeTestAssumeConstVCalls);
258
259 break;
260 }
261
262 case Intrinsic::type_checked_load_relative:
263 case Intrinsic::type_checked_load: {
264 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2));
265 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
266 if (!TypeId)
267 break;
268 GlobalValue::GUID Guid =
269 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
270
271 SmallVector<DevirtCallSite, 4> DevirtCalls;
272 SmallVector<Instruction *, 4> LoadedPtrs;
273 SmallVector<Instruction *, 4> Preds;
274 bool HasNonCallUses = false;
275 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
276 HasNonCallUses, CI, DT);
277 // Any non-call uses of the result of llvm.type.checked.load will
278 // prevent us from optimizing away the llvm.type.test.
279 if (HasNonCallUses)
280 TypeTests.insert(X: Guid);
281 for (auto &Call : DevirtCalls)
282 addVCallToSet(Call, Guid, VCalls&: TypeCheckedLoadVCalls,
283 ConstVCalls&: TypeCheckedLoadConstVCalls);
284
285 break;
286 }
287 default:
288 break;
289 }
290}
291
292static bool isNonVolatileLoad(const Instruction *I) {
293 if (const auto *LI = dyn_cast<LoadInst>(Val: I))
294 return !LI->isVolatile();
295
296 return false;
297}
298
299static bool isNonVolatileStore(const Instruction *I) {
300 if (const auto *SI = dyn_cast<StoreInst>(Val: I))
301 return !SI->isVolatile();
302
303 return false;
304}
305
306// Returns true if the function definition must be unreachable.
307//
308// Note if this helper function returns true, `F` is guaranteed
309// to be unreachable; if it returns false, `F` might still
310// be unreachable but not covered by this helper function.
311static bool mustBeUnreachableFunction(const Function &F) {
312 // A function must be unreachable if its entry block ends with an
313 // 'unreachable'.
314 assert(!F.isDeclaration());
315 return isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator());
316}
317
318static void computeFunctionSummary(
319 ModuleSummaryIndex &Index, const Module &M, const Function &F,
320 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT,
321 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
322 bool IsThinLTO,
323 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
324 // Summary not currently supported for anonymous functions, they should
325 // have been named.
326 assert(F.hasName());
327
328 unsigned NumInsts = 0;
329 // Map from callee ValueId to profile count. Used to accumulate profile
330 // counts for all static calls to a given callee.
331 MapVector<ValueInfo, CalleeInfo, DenseMap<ValueInfo, unsigned>,
332 SmallVector<FunctionSummary::EdgeTy, 0>>
333 CallGraphEdges;
334 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges, LoadRefEdges,
335 StoreRefEdges;
336 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> TypeTests;
337 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
338 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
339 SetVector<FunctionSummary::ConstVCall,
340 std::vector<FunctionSummary::ConstVCall>>
341 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
342 ICallPromotionAnalysis ICallAnalysis;
343 SmallPtrSet<const User *, 8> Visited;
344
345 // Add personality function, prefix data and prologue data to function's ref
346 // list.
347 bool HasLocalIFuncCallOrRef = false;
348 findRefEdges(Index, CurUser: &F, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
349 std::vector<const Instruction *> NonVolatileLoads;
350 std::vector<const Instruction *> NonVolatileStores;
351
352 std::vector<CallsiteInfo> Callsites;
353 std::vector<AllocInfo> Allocs;
354
355#ifndef NDEBUG
356 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
357#endif
358
359 bool HasInlineAsmMaybeReferencingInternal = false;
360 bool HasIndirBranchToBlockAddress = false;
361 bool HasUnknownCall = false;
362 bool MayThrow = false;
363 for (const BasicBlock &BB : F) {
364 // We don't allow inlining of function with indirect branch to blockaddress.
365 // If the blockaddress escapes the function, e.g., via a global variable,
366 // inlining may lead to an invalid cross-function reference. So we shouldn't
367 // import such function either.
368 if (BB.hasAddressTaken()) {
369 for (User *U : BlockAddress::get(BB: const_cast<BasicBlock *>(&BB))->users())
370 if (!isa<CallBrInst>(Val: *U)) {
371 HasIndirBranchToBlockAddress = true;
372 break;
373 }
374 }
375
376 for (const Instruction &I : BB) {
377 if (I.isDebugOrPseudoInst())
378 continue;
379 ++NumInsts;
380
381 // Regular LTO module doesn't participate in ThinLTO import,
382 // so no reference from it can be read/writeonly, since this
383 // would require importing variable as local copy
384 if (IsThinLTO) {
385 if (isNonVolatileLoad(I: &I)) {
386 // Postpone processing of non-volatile load instructions
387 // See comments below
388 Visited.insert(Ptr: &I);
389 NonVolatileLoads.push_back(x: &I);
390 continue;
391 } else if (isNonVolatileStore(I: &I)) {
392 Visited.insert(Ptr: &I);
393 NonVolatileStores.push_back(x: &I);
394 // All references from second operand of store (destination address)
395 // can be considered write-only if they're not referenced by any
396 // non-store instruction. References from first operand of store
397 // (stored value) can't be treated either as read- or as write-only
398 // so we add them to RefEdges as we do with all other instructions
399 // except non-volatile load.
400 Value *Stored = I.getOperand(i: 0);
401 if (auto *GV = dyn_cast<GlobalValue>(Val: Stored))
402 // findRefEdges will try to examine GV operands, so instead
403 // of calling it we should add GV to RefEdges directly.
404 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
405 else if (auto *U = dyn_cast<User>(Val: Stored))
406 findRefEdges(Index, CurUser: U, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
407 continue;
408 }
409 }
410 findRefEdges(Index, CurUser: &I, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
411 const auto *CB = dyn_cast<CallBase>(Val: &I);
412 if (!CB) {
413 if (I.mayThrow())
414 MayThrow = true;
415 continue;
416 }
417
418 const auto *CI = dyn_cast<CallInst>(Val: &I);
419 // Since we don't know exactly which local values are referenced in inline
420 // assembly, conservatively mark the function as possibly referencing
421 // a local value from inline assembly to ensure we don't export a
422 // reference (which would require renaming and promotion of the
423 // referenced value).
424 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
425 HasInlineAsmMaybeReferencingInternal = true;
426
427 // Compute this once per indirect call.
428 uint32_t NumCandidates = 0;
429 uint64_t TotalCount = 0;
430 MutableArrayRef<InstrProfValueData> CandidateProfileData;
431
432 auto *CalledValue = CB->getCalledOperand();
433 auto *CalledFunction = CB->getCalledFunction();
434 if (CalledValue && !CalledFunction) {
435 CalledValue = CalledValue->stripPointerCasts();
436 // Stripping pointer casts can reveal a called function.
437 CalledFunction = dyn_cast<Function>(Val: CalledValue);
438 }
439 // Check if this is an alias to a function. If so, get the
440 // called aliasee for the checks below.
441 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
442 assert(!CalledFunction && "Expected null called function in callsite for alias");
443 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
444 }
445 // Check if this is a direct call to a known function or a known
446 // intrinsic, or an indirect call with profile data.
447 if (CalledFunction) {
448 if (CI && CalledFunction->isIntrinsic()) {
449 addIntrinsicToSummary(
450 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
451 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
452 continue;
453 }
454 // We should have named any anonymous globals
455 assert(CalledFunction->hasName());
456 auto ScaledCount = PSI->getProfileCount(CallInst: *CB, BFI);
457 auto Hotness = ScaledCount ? getHotness(ProfileCount: *ScaledCount, PSI)
458 : CalleeInfo::HotnessType::Unknown;
459 if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
460 Hotness = CalleeInfo::HotnessType::Cold;
461
462 // Use the original CalledValue, in case it was an alias. We want
463 // to record the call edge to the alias in that case. Eventually
464 // an alias summary will be created to associate the alias and
465 // aliasee.
466 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
467 GV: cast<GlobalValue>(Val: CalledValue))];
468 ValueInfo.updateHotness(OtherHotness: Hotness);
469 if (CB->isTailCall())
470 ValueInfo.setHasTailCall(true);
471 } else {
472 HasUnknownCall = true;
473 // If F is imported, a local linkage ifunc (e.g. target_clones on a
474 // static function) called by F will be cloned. Since summaries don't
475 // track ifunc, we do not know implementation functions referenced by
476 // the ifunc resolver need to be promoted in the exporter, and we will
477 // get linker errors due to cloned declarations for implementation
478 // functions. As a simple fix, just mark F as not eligible for import.
479 // Non-local ifunc is not cloned and does not have the issue.
480 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: CalledValue))
481 if (GI->hasLocalLinkage())
482 HasLocalIFuncCallOrRef = true;
483 // Skip inline assembly calls.
484 if (CI && CI->isInlineAsm())
485 continue;
486 // Skip direct calls.
487 if (!CalledValue || isa<Constant>(Val: CalledValue))
488 continue;
489
490 // Check if the instruction has a callees metadata. If so, add callees
491 // to CallGraphEdges to reflect the references from the metadata, and
492 // to enable importing for subsequent indirect call promotion and
493 // inlining.
494 if (auto *MD = I.getMetadata(KindID: LLVMContext::MD_callees)) {
495 for (const auto &Op : MD->operands()) {
496 Function *Callee = mdconst::extract_or_null<Function>(MD: Op);
497 if (Callee)
498 CallGraphEdges[Index.getOrInsertValueInfo(GV: Callee)];
499 }
500 }
501
502 CandidateProfileData =
503 ICallAnalysis.getPromotionCandidatesForInstruction(
504 I: &I, TotalCount, NumCandidates, MaxNumValueData: MaxSummaryIndirectEdges);
505 for (const auto &Candidate : CandidateProfileData)
506 CallGraphEdges[Index.getOrInsertValueInfo(GUID: Candidate.Value)]
507 .updateHotness(OtherHotness: getHotness(ProfileCount: Candidate.Count, PSI));
508 }
509
510 // Summarize memprof related metadata. This is only needed for ThinLTO.
511 if (!IsThinLTO)
512 continue;
513
514 // Skip indirect calls if we haven't enabled memprof ICP.
515 if (!CalledFunction && !EnableMemProfIndirectCallSupport)
516 continue;
517
518 // Ensure we keep this analysis in sync with the handling in the ThinLTO
519 // backend (see MemProfContextDisambiguation::applyImport). Save this call
520 // so that we can skip it in checking the reverse case later.
521 assert(mayHaveMemprofSummary(CB));
522#ifndef NDEBUG
523 CallsThatMayHaveMemprofSummary.insert(CB);
524#endif
525
526 // Compute the list of stack ids first (so we can trim them from the stack
527 // ids on any MIBs).
528 CallStack<MDNode, MDNode::op_iterator> InstCallsite(
529 I.getMetadata(KindID: LLVMContext::MD_callsite));
530 auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
531 if (MemProfMD) {
532 std::vector<MIBInfo> MIBs;
533 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
534 bool HasNonZeroContextSizeInfos = false;
535 for (auto &MDOp : MemProfMD->operands()) {
536 auto *MIBMD = cast<const MDNode>(Val: MDOp);
537 MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
538 assert(StackNode);
539 SmallVector<unsigned> StackIdIndices;
540 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
541 // Collapse out any on the allocation call (inlining).
542 for (auto ContextIter =
543 StackContext.beginAfterSharedPrefix(Other: InstCallsite);
544 ContextIter != StackContext.end(); ++ContextIter) {
545 unsigned StackIdIdx = Index.addOrGetStackIdIndex(StackId: *ContextIter);
546 // If this is a direct recursion, simply skip the duplicate
547 // entries. If this is mutual recursion, handling is left to
548 // the LTO link analysis client.
549 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
550 StackIdIndices.push_back(Elt: StackIdIdx);
551 }
552 // If we have context size information, collect it for inclusion in
553 // the summary.
554 assert(MIBMD->getNumOperands() > 2 ||
555 !metadataIncludesAllContextSizeInfo());
556 if (MIBMD->getNumOperands() > 2) {
557 std::vector<ContextTotalSize> ContextSizes;
558 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
559 MDNode *ContextSizePair = dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
560 assert(ContextSizePair->getNumOperands() == 2);
561 uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
562 MD: ContextSizePair->getOperand(I: 0))
563 ->getZExtValue();
564 uint64_t TS = mdconst::dyn_extract<ConstantInt>(
565 MD: ContextSizePair->getOperand(I: 1))
566 ->getZExtValue();
567 ContextSizes.push_back(x: {.FullStackId: FullStackId, .TotalSize: TS});
568 }
569 // Flag that we need to keep the ContextSizeInfos array for this
570 // alloc as it now contains non-zero context info sizes.
571 HasNonZeroContextSizeInfos = true;
572 ContextSizeInfos.push_back(x: std::move(ContextSizes));
573 } else {
574 // The ContextSizeInfos must be in the same relative position as the
575 // associated MIB. In some cases we only include a ContextSizeInfo
576 // for a subset of MIBs in an allocation. To handle that, eagerly
577 // fill any MIB entries that don't have context size info metadata
578 // with a pair of 0s. Later on we will only use this array if it
579 // ends up containing any non-zero entries (see where we set
580 // HasNonZeroContextSizeInfos above).
581 ContextSizeInfos.push_back(x: {{.FullStackId: 0, .TotalSize: 0}});
582 }
583 MIBs.push_back(
584 x: MIBInfo(getMIBAllocType(MIB: MIBMD), std::move(StackIdIndices)));
585 }
586 Allocs.push_back(x: AllocInfo(std::move(MIBs)));
587 assert(HasNonZeroContextSizeInfos ||
588 !metadataIncludesAllContextSizeInfo());
589 // We eagerly build the ContextSizeInfos array, but it will be filled
590 // with sub arrays of pairs of 0s if no MIBs on this alloc actually
591 // contained context size info metadata. Only save it if any MIBs had
592 // any such metadata.
593 if (HasNonZeroContextSizeInfos) {
594 assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
595 Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
596 }
597 } else if (!InstCallsite.empty()) {
598 SmallVector<unsigned> StackIdIndices;
599 for (auto StackId : InstCallsite)
600 StackIdIndices.push_back(Elt: Index.addOrGetStackIdIndex(StackId));
601 if (CalledFunction) {
602 // Use the original CalledValue, in case it was an alias. We want
603 // to record the call edge to the alias in that case. Eventually
604 // an alias summary will be created to associate the alias and
605 // aliasee.
606 auto CalleeValueInfo =
607 Index.getOrInsertValueInfo(GV: cast<GlobalValue>(Val: CalledValue));
608 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
609 } else {
610 assert(EnableMemProfIndirectCallSupport);
611 // For indirect callsites, create multiple Callsites, one per target.
612 // This enables having a different set of clone versions per target,
613 // and we will apply the cloning decisions while speculatively
614 // devirtualizing in the ThinLTO backends.
615 for (const auto &Candidate : CandidateProfileData) {
616 auto CalleeValueInfo = Index.getOrInsertValueInfo(GUID: Candidate.Value);
617 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
618 }
619 }
620 }
621 }
622 }
623
624 if (PSI->hasPartialSampleProfile() && ScalePartialSampleProfileWorkingSetSize)
625 Index.addBlockCount(C: F.size());
626
627 SmallVector<ValueInfo, 0> Refs;
628 if (IsThinLTO) {
629 auto AddRefEdges =
630 [&](const std::vector<const Instruction *> &Instrs,
631 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &Edges,
632 SmallPtrSet<const User *, 8> &Cache) {
633 for (const auto *I : Instrs) {
634 Cache.erase(Ptr: I);
635 findRefEdges(Index, CurUser: I, RefEdges&: Edges, Visited&: Cache, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
636 }
637 };
638
639 // By now we processed all instructions in a function, except
640 // non-volatile loads and non-volatile value stores. Let's find
641 // ref edges for both of instruction sets
642 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
643 // We can add some values to the Visited set when processing load
644 // instructions which are also used by stores in NonVolatileStores.
645 // For example this can happen if we have following code:
646 //
647 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
648 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
649 //
650 // After processing loads we'll add bitcast to the Visited set, and if
651 // we use the same set while processing stores, we'll never see store
652 // to @bar and @bar will be mistakenly treated as readonly.
653 SmallPtrSet<const llvm::User *, 8> StoreCache;
654 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
655
656 // If both load and store instruction reference the same variable
657 // we won't be able to optimize it. Add all such reference edges
658 // to RefEdges set.
659 for (const auto &VI : StoreRefEdges)
660 if (LoadRefEdges.remove(X: VI))
661 RefEdges.insert(X: VI);
662
663 unsigned RefCnt = RefEdges.size();
664 // All new reference edges inserted in two loops below are either
665 // read or write only. They will be grouped in the end of RefEdges
666 // vector, so we can use a single integer value to identify them.
667 RefEdges.insert_range(R&: LoadRefEdges);
668
669 unsigned FirstWORef = RefEdges.size();
670 RefEdges.insert_range(R&: StoreRefEdges);
671
672 Refs = RefEdges.takeVector();
673 for (; RefCnt < FirstWORef; ++RefCnt)
674 Refs[RefCnt].setReadOnly();
675
676 for (; RefCnt < Refs.size(); ++RefCnt)
677 Refs[RefCnt].setWriteOnly();
678 } else {
679 Refs = RefEdges.takeVector();
680 }
681 // Explicit add hot edges to enforce importing for designated GUIDs for
682 // sample PGO, to enable the same inlines as the profiled optimized binary.
683 for (auto &I : F.getImportGUIDs())
684 CallGraphEdges[Index.getOrInsertValueInfo(GUID: I)].updateHotness(
685 OtherHotness: ForceSummaryEdgesCold == FunctionSummary::FSHT_All
686 ? CalleeInfo::HotnessType::Cold
687 : CalleeInfo::HotnessType::Critical);
688
689#ifndef NDEBUG
690 // Make sure that all calls we decided could not have memprof summaries get a
691 // false value for mayHaveMemprofSummary, to ensure that this handling remains
692 // in sync with the ThinLTO backend handling.
693 if (IsThinLTO) {
694 for (const BasicBlock &BB : F) {
695 for (const Instruction &I : BB) {
696 const auto *CB = dyn_cast<CallBase>(&I);
697 if (!CB)
698 continue;
699 // We already checked these above.
700 if (CallsThatMayHaveMemprofSummary.count(CB))
701 continue;
702 assert(!mayHaveMemprofSummary(CB));
703 }
704 }
705 }
706#endif
707
708 bool NonRenamableLocal = isNonRenamableLocal(GV: F);
709 bool NotEligibleForImport =
710 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
711 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
712 GlobalValueSummary::GVFlags Flags(
713 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
714 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
715 GlobalValueSummary::ImportKind::Definition);
716 FunctionSummary::FFlags FunFlags{
717 .ReadNone: F.doesNotAccessMemory(), .ReadOnly: F.onlyReadsMemory() && !F.doesNotAccessMemory(),
718 .NoRecurse: F.hasFnAttribute(Kind: Attribute::NoRecurse), .ReturnDoesNotAlias: F.returnDoesNotAlias(),
719 // FIXME: refactor this to use the same code that inliner is using.
720 // Don't try to import functions with noinline attribute.
721 .NoInline: F.getAttributes().hasFnAttr(Kind: Attribute::NoInline),
722 .AlwaysInline: F.hasFnAttribute(Kind: Attribute::AlwaysInline),
723 .NoUnwind: F.hasFnAttribute(Kind: Attribute::NoUnwind), .MayThrow: MayThrow, .HasUnknownCall: HasUnknownCall,
724 .MustBeUnreachable: mustBeUnreachableFunction(F)};
725 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
726 if (auto *SSI = GetSSICallback(F))
727 ParamAccesses = SSI->getParamAccesses(Index);
728 auto FuncSummary = std::make_unique<FunctionSummary>(
729 args&: Flags, args&: NumInsts, args&: FunFlags, args: std::move(Refs), args: CallGraphEdges.takeVector(),
730 args: TypeTests.takeVector(), args: TypeTestAssumeVCalls.takeVector(),
731 args: TypeCheckedLoadVCalls.takeVector(),
732 args: TypeTestAssumeConstVCalls.takeVector(),
733 args: TypeCheckedLoadConstVCalls.takeVector(), args: std::move(ParamAccesses),
734 args: std::move(Callsites), args: std::move(Allocs));
735 if (NonRenamableLocal)
736 CantBePromoted.insert(V: F.getGUID());
737 Index.addGlobalValueSummary(GV: F, Summary: std::move(FuncSummary));
738}
739
740/// Find function pointers referenced within the given vtable initializer
741/// (or subset of an initializer) \p I. The starting offset of \p I within
742/// the vtable initializer is \p StartingOffset. Any discovered function
743/// pointers are added to \p VTableFuncs along with their cumulative offset
744/// within the initializer.
745static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
746 const Module &M, ModuleSummaryIndex &Index,
747 VTableFuncList &VTableFuncs,
748 const GlobalVariable &OrigGV) {
749 // First check if this is a function pointer.
750 if (I->getType()->isPointerTy()) {
751 auto C = I->stripPointerCasts();
752 auto A = dyn_cast<GlobalAlias>(Val: C);
753 if (isa<Function>(Val: C) || (A && isa<Function>(Val: A->getAliasee()))) {
754 auto GV = dyn_cast<GlobalValue>(Val: C);
755 assert(GV);
756 // We can disregard __cxa_pure_virtual as a possible call target, as
757 // calls to pure virtuals are UB.
758 if (GV && GV->getName() != "__cxa_pure_virtual")
759 VTableFuncs.push_back(x: {Index.getOrInsertValueInfo(GV), StartingOffset});
760 return;
761 }
762 }
763
764 // Walk through the elements in the constant struct or array and recursively
765 // look for virtual function pointers.
766 const DataLayout &DL = M.getDataLayout();
767 if (auto *C = dyn_cast<ConstantStruct>(Val: I)) {
768 StructType *STy = dyn_cast<StructType>(Val: C->getType());
769 assert(STy);
770 const StructLayout *SL = DL.getStructLayout(Ty: C->getType());
771
772 for (auto EI : llvm::enumerate(First: STy->elements())) {
773 auto Offset = SL->getElementOffset(Idx: EI.index());
774 unsigned Op = SL->getElementContainingOffset(FixedOffset: Offset);
775 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i: Op)),
776 StartingOffset: StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
777 }
778 } else if (auto *C = dyn_cast<ConstantArray>(Val: I)) {
779 ArrayType *ATy = C->getType();
780 Type *EltTy = ATy->getElementType();
781 uint64_t EltSize = DL.getTypeAllocSize(Ty: EltTy);
782 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
783 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i)),
784 StartingOffset: StartingOffset + i * EltSize, M, Index, VTableFuncs,
785 OrigGV);
786 }
787 } else if (const auto *CE = dyn_cast<ConstantExpr>(Val: I)) {
788 // For relative vtables, the next sub-component should be a trunc.
789 if (CE->getOpcode() != Instruction::Trunc ||
790 !(CE = dyn_cast<ConstantExpr>(Val: CE->getOperand(i_nocapture: 0))))
791 return;
792
793 // If this constant can be reduced to the offset between a function and a
794 // global, then we know this is a valid virtual function if the RHS is the
795 // original vtable we're scanning through.
796 if (CE->getOpcode() == Instruction::Sub) {
797 GlobalValue *LHS, *RHS;
798 APSInt LHSOffset, RHSOffset;
799 if (IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 0), GV&: LHS, Offset&: LHSOffset, DL) &&
800 IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 1), GV&: RHS, Offset&: RHSOffset, DL) &&
801 RHS == &OrigGV &&
802
803 // For relative vtables, this component should point to the callable
804 // function without any offsets.
805 LHSOffset == 0 &&
806
807 // Also, the RHS should always point to somewhere within the vtable.
808 RHSOffset <=
809 static_cast<uint64_t>(DL.getTypeAllocSize(Ty: OrigGV.getInitializer()->getType()))) {
810 findFuncPointers(I: LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
811 }
812 }
813 }
814}
815
816// Identify the function pointers referenced by vtable definition \p V.
817static void computeVTableFuncs(ModuleSummaryIndex &Index,
818 const GlobalVariable &V, const Module &M,
819 VTableFuncList &VTableFuncs) {
820 if (!V.isConstant())
821 return;
822
823 findFuncPointers(I: V.getInitializer(), /*StartingOffset=*/0, M, Index,
824 VTableFuncs, OrigGV: V);
825
826#ifndef NDEBUG
827 // Validate that the VTableFuncs list is ordered by offset.
828 uint64_t PrevOffset = 0;
829 for (auto &P : VTableFuncs) {
830 // The findVFuncPointers traversal should have encountered the
831 // functions in offset order. We need to use ">=" since PrevOffset
832 // starts at 0.
833 assert(P.VTableOffset >= PrevOffset);
834 PrevOffset = P.VTableOffset;
835 }
836#endif
837}
838
839/// Record vtable definition \p V for each type metadata it references.
840static void
841recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index,
842 const GlobalVariable &V,
843 SmallVectorImpl<MDNode *> &Types) {
844 for (MDNode *Type : Types) {
845 auto TypeID = Type->getOperand(I: 1).get();
846
847 uint64_t Offset =
848 cast<ConstantInt>(
849 Val: cast<ConstantAsMetadata>(Val: Type->getOperand(I: 0))->getValue())
850 ->getZExtValue();
851
852 if (auto *TypeId = dyn_cast<MDString>(Val: TypeID))
853 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId: TypeId->getString())
854 .push_back(x: {Offset, Index.getOrInsertValueInfo(GV: &V)});
855 }
856}
857
858static void computeVariableSummary(ModuleSummaryIndex &Index,
859 const GlobalVariable &V,
860 DenseSet<GlobalValue::GUID> &CantBePromoted,
861 const Module &M,
862 SmallVectorImpl<MDNode *> &Types) {
863 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges;
864 SmallPtrSet<const User *, 8> Visited;
865 bool RefLocalIFunc = false;
866 bool HasBlockAddress =
867 findRefEdges(Index, CurUser: &V, RefEdges, Visited, RefLocalLinkageIFunc&: RefLocalIFunc);
868 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
869 bool NonRenamableLocal = isNonRenamableLocal(GV: V);
870 GlobalValueSummary::GVFlags Flags(
871 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
872 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
873 GlobalValueSummary::Definition);
874
875 VTableFuncList VTableFuncs;
876 // If splitting is not enabled, then we compute the summary information
877 // necessary for index-based whole program devirtualization.
878 if (!Index.enableSplitLTOUnit()) {
879 Types.clear();
880 V.getMetadata(KindID: LLVMContext::MD_type, MDs&: Types);
881 if (!Types.empty()) {
882 // Identify the function pointers referenced by this vtable definition.
883 computeVTableFuncs(Index, V, M, VTableFuncs);
884
885 // Record this vtable definition for each type metadata it references.
886 recordTypeIdCompatibleVtableReferences(Index, V, Types);
887 }
888 }
889
890 // Don't mark variables we won't be able to internalize as read/write-only.
891 bool CanBeInternalized =
892 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
893 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
894 bool Constant = V.isConstant();
895 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
896 Constant ? false : CanBeInternalized,
897 Constant, V.getVCallVisibility());
898 auto GVarSummary = std::make_unique<GlobalVarSummary>(args&: Flags, args&: VarFlags,
899 args: RefEdges.takeVector());
900 if (NonRenamableLocal)
901 CantBePromoted.insert(V: V.getGUID());
902 if (NotEligibleForImport)
903 GVarSummary->setNotEligibleToImport();
904 if (!VTableFuncs.empty())
905 GVarSummary->setVTableFuncs(VTableFuncs);
906 Index.addGlobalValueSummary(GV: V, Summary: std::move(GVarSummary));
907}
908
909static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
910 DenseSet<GlobalValue::GUID> &CantBePromoted) {
911 // Skip summary for indirect function aliases as summary for aliasee will not
912 // be emitted.
913 const GlobalObject *Aliasee = A.getAliaseeObject();
914 if (isa<GlobalIFunc>(Val: Aliasee))
915 return;
916 bool NonRenamableLocal = isNonRenamableLocal(GV: A);
917 GlobalValueSummary::GVFlags Flags(
918 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
919 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
920 GlobalValueSummary::Definition);
921 auto AS = std::make_unique<AliasSummary>(args&: Flags);
922 auto AliaseeVI = Index.getValueInfo(GUID: Aliasee->getGUID());
923 assert(AliaseeVI && "Alias expects aliasee summary to be available");
924 assert(AliaseeVI.getSummaryList().size() == 1 &&
925 "Expected a single entry per aliasee in per-module index");
926 AS->setAliasee(AliaseeVI, Aliasee: AliaseeVI.getSummaryList()[0].get());
927 if (NonRenamableLocal)
928 CantBePromoted.insert(V: A.getGUID());
929 Index.addGlobalValueSummary(GV: A, Summary: std::move(AS));
930}
931
932// Set LiveRoot flag on entries matching the given value name.
933static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
934 if (ValueInfo VI =
935 Index.getValueInfo(GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Name)))
936 for (const auto &Summary : VI.getSummaryList())
937 Summary->setLive(true);
938}
939
940ModuleSummaryIndex llvm::buildModuleSummaryIndex(
941 const Module &M,
942 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
943 ProfileSummaryInfo *PSI,
944 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
945 assert(PSI);
946 bool EnableSplitLTOUnit = false;
947 bool UnifiedLTO = false;
948 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
949 MD: M.getModuleFlag(Key: "EnableSplitLTOUnit")))
950 EnableSplitLTOUnit = MD->getZExtValue();
951 if (auto *MD =
952 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "UnifiedLTO")))
953 UnifiedLTO = MD->getZExtValue();
954 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
955
956 // Identify the local values in the llvm.used and llvm.compiler.used sets,
957 // which should not be exported as they would then require renaming and
958 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
959 // here because we use this information to mark functions containing inline
960 // assembly calls as not importable.
961 SmallPtrSet<GlobalValue *, 4> LocalsUsed;
962 SmallVector<GlobalValue *, 4> Used;
963 // First collect those in the llvm.used set.
964 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/false);
965 // Next collect those in the llvm.compiler.used set.
966 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/true);
967 DenseSet<GlobalValue::GUID> CantBePromoted;
968 for (auto *V : Used) {
969 if (V->hasLocalLinkage()) {
970 LocalsUsed.insert(Ptr: V);
971 CantBePromoted.insert(V: V->getGUID());
972 }
973 }
974
975 bool HasLocalInlineAsmSymbol = false;
976 if (!M.getModuleInlineAsm().empty()) {
977 // Collect the local values defined by module level asm, and set up
978 // summaries for these symbols so that they can be marked as NoRename,
979 // to prevent export of any use of them in regular IR that would require
980 // renaming within the module level asm. Note we don't need to create a
981 // summary for weak or global defs, as they don't need to be flagged as
982 // NoRename, and defs in module level asm can't be imported anyway.
983 // Also, any values used but not defined within module level asm should
984 // be listed on the llvm.used or llvm.compiler.used global and marked as
985 // referenced from there.
986 ModuleSymbolTable::CollectAsmSymbols(
987 M, AsmSymbol: [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
988 // Symbols not marked as Weak or Global are local definitions.
989 if (Flags & (object::BasicSymbolRef::SF_Weak |
990 object::BasicSymbolRef::SF_Global))
991 return;
992 HasLocalInlineAsmSymbol = true;
993 GlobalValue *GV = M.getNamedValue(Name);
994 if (!GV)
995 return;
996 assert(GV->isDeclaration() && "Def in module asm already has definition");
997 GlobalValueSummary::GVFlags GVFlags(
998 GlobalValue::InternalLinkage, GlobalValue::DefaultVisibility,
999 /* NotEligibleToImport = */ true,
1000 /* Live = */ true,
1001 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
1002 GlobalValueSummary::Definition);
1003 CantBePromoted.insert(V: GV->getGUID());
1004 // Create the appropriate summary type.
1005 if (Function *F = dyn_cast<Function>(Val: GV)) {
1006 std::unique_ptr<FunctionSummary> Summary =
1007 std::make_unique<FunctionSummary>(
1008 args&: GVFlags, /*InstCount=*/args: 0,
1009 args: FunctionSummary::FFlags{
1010 .ReadNone: F->hasFnAttribute(Kind: Attribute::ReadNone),
1011 .ReadOnly: F->hasFnAttribute(Kind: Attribute::ReadOnly),
1012 .NoRecurse: F->hasFnAttribute(Kind: Attribute::NoRecurse),
1013 .ReturnDoesNotAlias: F->returnDoesNotAlias(),
1014 /* NoInline = */ false,
1015 .AlwaysInline: F->hasFnAttribute(Kind: Attribute::AlwaysInline),
1016 .NoUnwind: F->hasFnAttribute(Kind: Attribute::NoUnwind),
1017 /* MayThrow */ true,
1018 /* HasUnknownCall */ true,
1019 /* MustBeUnreachable */ false},
1020 args: SmallVector<ValueInfo, 0>{},
1021 args: SmallVector<FunctionSummary::EdgeTy, 0>{},
1022 args: ArrayRef<GlobalValue::GUID>{},
1023 args: ArrayRef<FunctionSummary::VFuncId>{},
1024 args: ArrayRef<FunctionSummary::VFuncId>{},
1025 args: ArrayRef<FunctionSummary::ConstVCall>{},
1026 args: ArrayRef<FunctionSummary::ConstVCall>{},
1027 args: ArrayRef<FunctionSummary::ParamAccess>{},
1028 args: ArrayRef<CallsiteInfo>{}, args: ArrayRef<AllocInfo>{});
1029 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1030 } else {
1031 std::unique_ptr<GlobalVarSummary> Summary =
1032 std::make_unique<GlobalVarSummary>(
1033 args&: GVFlags,
1034 args: GlobalVarSummary::GVarFlags(
1035 false, false, cast<GlobalVariable>(Val: GV)->isConstant(),
1036 GlobalObject::VCallVisibilityPublic),
1037 args: SmallVector<ValueInfo, 0>{});
1038 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1039 }
1040 });
1041 }
1042
1043 bool IsThinLTO = true;
1044 if (auto *MD =
1045 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
1046 IsThinLTO = MD->getZExtValue();
1047
1048 // Compute summaries for all functions defined in module, and save in the
1049 // index.
1050 for (const auto &F : M) {
1051 if (F.isDeclaration())
1052 continue;
1053
1054 DominatorTree DT(const_cast<Function &>(F));
1055 BlockFrequencyInfo *BFI = nullptr;
1056 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1057 if (GetBFICallback)
1058 BFI = GetBFICallback(F);
1059 else if (F.hasProfileData()) {
1060 LoopInfo LI{DT};
1061 BranchProbabilityInfo BPI{F, LI};
1062 BFIPtr = std::make_unique<BlockFrequencyInfo>(args: F, args&: BPI, args&: LI);
1063 BFI = BFIPtr.get();
1064 }
1065
1066 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
1067 HasLocalsInUsedOrAsm: !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1068 CantBePromoted, IsThinLTO, GetSSICallback);
1069 }
1070
1071 // Compute summaries for all variables defined in module, and save in the
1072 // index.
1073 SmallVector<MDNode *, 2> Types;
1074 for (const GlobalVariable &G : M.globals()) {
1075 if (G.isDeclaration())
1076 continue;
1077 computeVariableSummary(Index, V: G, CantBePromoted, M, Types);
1078 }
1079
1080 // Compute summaries for all aliases defined in module, and save in the
1081 // index.
1082 for (const GlobalAlias &A : M.aliases())
1083 computeAliasSummary(Index, A, CantBePromoted);
1084
1085 // Iterate through ifuncs, set their resolvers all alive.
1086 for (const GlobalIFunc &I : M.ifuncs()) {
1087 I.applyAlongResolverPath(Op: [&Index](const GlobalValue &GV) {
1088 Index.getGlobalValueSummary(GV)->setLive(true);
1089 });
1090 }
1091
1092 for (auto *V : LocalsUsed) {
1093 auto *Summary = Index.getGlobalValueSummary(GV: *V);
1094 assert(Summary && "Missing summary for global value");
1095 Summary->setNotEligibleToImport();
1096 }
1097
1098 // The linker doesn't know about these LLVM produced values, so we need
1099 // to flag them as live in the index to ensure index-based dead value
1100 // analysis treats them as live roots of the analysis.
1101 setLiveRoot(Index, Name: "llvm.used");
1102 setLiveRoot(Index, Name: "llvm.compiler.used");
1103 setLiveRoot(Index, Name: "llvm.global_ctors");
1104 setLiveRoot(Index, Name: "llvm.global_dtors");
1105 setLiveRoot(Index, Name: "llvm.global.annotations");
1106
1107 for (auto &GlobalList : Index) {
1108 // Ignore entries for references that are undefined in the current module.
1109 if (GlobalList.second.getSummaryList().empty())
1110 continue;
1111
1112 assert(GlobalList.second.getSummaryList().size() == 1 &&
1113 "Expected module's index to have one summary per GUID");
1114 auto &Summary = GlobalList.second.getSummaryList()[0];
1115 if (!IsThinLTO) {
1116 Summary->setNotEligibleToImport();
1117 continue;
1118 }
1119
1120 bool AllRefsCanBeExternallyReferenced =
1121 llvm::all_of(Range: Summary->refs(), P: [&](const ValueInfo &VI) {
1122 return !CantBePromoted.count(V: VI.getGUID());
1123 });
1124 if (!AllRefsCanBeExternallyReferenced) {
1125 Summary->setNotEligibleToImport();
1126 continue;
1127 }
1128
1129 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Val: Summary.get())) {
1130 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1131 Range: FuncSummary->calls(), P: [&](const FunctionSummary::EdgeTy &Edge) {
1132 return !CantBePromoted.count(V: Edge.first.getGUID());
1133 });
1134 if (!AllCallsCanBeExternallyReferenced)
1135 Summary->setNotEligibleToImport();
1136 }
1137 }
1138
1139 if (!ModuleSummaryDotFile.empty()) {
1140 std::error_code EC;
1141 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_Text);
1142 if (EC)
1143 report_fatal_error(reason: Twine("Failed to open dot file ") +
1144 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1145 Index.exportToDot(OS&: OSDot, GUIDPreservedSymbols: {});
1146 }
1147
1148 return Index;
1149}
1150
1151AnalysisKey ModuleSummaryIndexAnalysis::Key;
1152
1153ModuleSummaryIndex
1154ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
1155 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(IR&: M);
1156 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1157 bool NeedSSI = needsParamAccessSummary(M);
1158 return buildModuleSummaryIndex(
1159 M,
1160 GetBFICallback: [&FAM](const Function &F) {
1161 return &FAM.getResult<BlockFrequencyAnalysis>(
1162 IR&: *const_cast<Function *>(&F));
1163 },
1164 PSI: &PSI,
1165 GetSSICallback: [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1166 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1167 IR&: const_cast<Function &>(F))
1168 : nullptr;
1169 });
1170}
1171
1172char ModuleSummaryIndexWrapperPass::ID = 0;
1173
1174INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1175 "Module Summary Analysis", false, true)
1176INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
1177INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
1178INITIALIZE_PASS_DEPENDENCY(StackSafetyInfoWrapperPass)
1179INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1180 "Module Summary Analysis", false, true)
1181
1182ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
1183 return new ModuleSummaryIndexWrapperPass();
1184}
1185
1186ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
1187 : ModulePass(ID) {}
1188
1189bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
1190 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1191 bool NeedSSI = needsParamAccessSummary(M);
1192 Index.emplace(args: buildModuleSummaryIndex(
1193 M,
1194 GetBFICallback: [this](const Function &F) {
1195 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1196 F&: *const_cast<Function *>(&F))
1197 .getBFI());
1198 },
1199 PSI,
1200 GetSSICallback: [&](const Function &F) -> const StackSafetyInfo * {
1201 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1202 F&: const_cast<Function &>(F))
1203 .getResult()
1204 : nullptr;
1205 }));
1206 return false;
1207}
1208
1209bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
1210 Index.reset();
1211 return false;
1212}
1213
1214void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
1215 AU.setPreservesAll();
1216 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1217 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1218 AU.addRequired<StackSafetyInfoWrapperPass>();
1219}
1220
1221char ImmutableModuleSummaryIndexWrapperPass::ID = 0;
1222
1223ImmutableModuleSummaryIndexWrapperPass::ImmutableModuleSummaryIndexWrapperPass(
1224 const ModuleSummaryIndex *Index)
1225 : ImmutablePass(ID), Index(Index) {}
1226
1227void ImmutableModuleSummaryIndexWrapperPass::getAnalysisUsage(
1228 AnalysisUsage &AU) const {
1229 AU.setPreservesAll();
1230}
1231
1232ImmutablePass *llvm::createImmutableModuleSummaryIndexWrapperPass(
1233 const ModuleSummaryIndex *Index) {
1234 return new ImmutableModuleSummaryIndexWrapperPass(Index);
1235}
1236
1237INITIALIZE_PASS(ImmutableModuleSummaryIndexWrapperPass, "module-summary-info",
1238 "Module summary info", false, true)
1239
1240bool llvm::mayHaveMemprofSummary(const CallBase *CB) {
1241 if (!CB)
1242 return false;
1243 if (CB->isDebugOrPseudoInst())
1244 return false;
1245 auto *CI = dyn_cast<CallInst>(Val: CB);
1246 auto *CalledValue = CB->getCalledOperand();
1247 auto *CalledFunction = CB->getCalledFunction();
1248 if (CalledValue && !CalledFunction) {
1249 CalledValue = CalledValue->stripPointerCasts();
1250 // Stripping pointer casts can reveal a called function.
1251 CalledFunction = dyn_cast<Function>(Val: CalledValue);
1252 }
1253 // Check if this is an alias to a function. If so, get the
1254 // called aliasee for the checks below.
1255 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
1256 assert(!CalledFunction &&
1257 "Expected null called function in callsite for alias");
1258 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
1259 }
1260 // Check if this is a direct call to a known function or a known
1261 // intrinsic, or an indirect call with profile data.
1262 if (CalledFunction) {
1263 if (CI && CalledFunction->isIntrinsic())
1264 return false;
1265 } else {
1266 // Skip indirect calls if we haven't enabled memprof ICP.
1267 if (!EnableMemProfIndirectCallSupport)
1268 return false;
1269 // Skip inline assembly calls.
1270 if (CI && CI->isInlineAsm())
1271 return false;
1272 // Skip direct calls via Constant.
1273 if (!CalledValue || isa<Constant>(Val: CalledValue))
1274 return false;
1275 return true;
1276 }
1277 return true;
1278}
1279