1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ModuleSummaryAnalysis.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/Analysis/BlockFrequencyInfo.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
27#include "llvm/Analysis/LoopInfo.h"
28#include "llvm/Analysis/MemoryProfileInfo.h"
29#include "llvm/Analysis/ProfileSummaryInfo.h"
30#include "llvm/Analysis/StackSafetyAnalysis.h"
31#include "llvm/Analysis/TypeMetadataUtils.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Dominators.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalAlias.h"
39#include "llvm/IR/GlobalValue.h"
40#include "llvm/IR/GlobalVariable.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/ModuleSummaryIndex.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
48#include "llvm/InitializePasses.h"
49#include "llvm/Object/ModuleSymbolTable.h"
50#include "llvm/Object/SymbolicFile.h"
51#include "llvm/Pass.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Compiler.h"
55#include "llvm/Support/FileSystem.h"
56#include <cassert>
57#include <cstdint>
58#include <vector>
59
60using namespace llvm;
61using namespace llvm::memprof;
62
63#define DEBUG_TYPE "module-summary-analysis"
64
65// Option to force edges cold which will block importing when the
66// -import-cold-multiplier is set to 0. Useful for debugging.
67namespace llvm {
68FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold =
69 FunctionSummary::FSHT_None;
70
71static cl::opt<FunctionSummary::ForceSummaryHotnessType, true> FSEC(
72 "force-summary-edges-cold", cl::Hidden, cl::location(L&: ForceSummaryEdgesCold),
73 cl::desc("Force all edges in the function summary to cold"),
74 cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."),
75 clEnumValN(FunctionSummary::FSHT_AllNonCritical,
76 "all-non-critical", "All non-critical edges."),
77 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
78
79static cl::opt<std::string> ModuleSummaryDotFile(
80 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
81 cl::desc("File to emit dot graph of new summary into"));
82
83static cl::opt<bool> EnableMemProfIndirectCallSupport(
84 "enable-memprof-indirect-call-support", cl::init(Val: true), cl::Hidden,
85 cl::desc(
86 "Enable MemProf support for summarizing and cloning indirect calls"));
87
88// This can be used to override the number of callees created from VP metadata
89// normally taken from the -icp-max-prom option with a larger amount, if useful
90// for analysis. Use a separate option so that we can control the number of
91// indirect callees for ThinLTO summary based analysis (e.g. for MemProf which
92// needs this information for a correct and not overly-conservative callsite
93// graph analysis, especially because allocation contexts may not be very
94// frequent), without affecting normal ICP.
95cl::opt<unsigned>
96 MaxSummaryIndirectEdges("module-summary-max-indirect-edges", cl::init(Val: 0),
97 cl::Hidden,
98 cl::desc("Max number of summary edges added from "
99 "indirect call profile metadata"));
100
101LLVM_ABI extern cl::opt<bool> ScalePartialSampleProfileWorkingSetSize;
102
103extern cl::opt<unsigned> MaxNumVTableAnnotations;
104
105extern cl::opt<bool> MemProfReportHintedSizes;
106} // namespace llvm
107
108// Walk through the operands of a given User via worklist iteration and populate
109// the set of GlobalValue references encountered. Invoked either on an
110// Instruction or a GlobalVariable (which walks its initializer).
111// Return true if any of the operands contains blockaddress. This is important
112// to know when computing summary for global var, because if global variable
113// references basic block address we can't import it separately from function
114// containing that basic block. For simplicity we currently don't import such
115// global vars at all. When importing function we aren't interested if any
116// instruction in it takes an address of any basic block, because instruction
117// can only take an address of basic block located in the same function.
118// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
119// local-linkage ifunc.
120static bool
121findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
122 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &RefEdges,
123 SmallPtrSet<const User *, 8> &Visited,
124 bool &RefLocalLinkageIFunc) {
125 bool HasBlockAddress = false;
126 SmallVector<const User *, 32> Worklist;
127 if (Visited.insert(Ptr: CurUser).second)
128 Worklist.push_back(Elt: CurUser);
129
130 while (!Worklist.empty()) {
131 const User *U = Worklist.pop_back_val();
132 const auto *CB = dyn_cast<CallBase>(Val: U);
133
134 for (const auto &OI : U->operands()) {
135 const User *Operand = dyn_cast<User>(Val: OI);
136 if (!Operand)
137 continue;
138 if (isa<BlockAddress>(Val: Operand)) {
139 HasBlockAddress = true;
140 continue;
141 }
142 if (auto *GV = dyn_cast<GlobalValue>(Val: Operand)) {
143 // We have a reference to a global value. This should be added to
144 // the reference set unless it is a callee. Callees are handled
145 // specially by WriteFunction and are added to a separate list.
146 if (!(CB && CB->isCallee(U: &OI))) {
147 // If an ifunc has local linkage, do not add it into ref edges, and
148 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
149 // for import. An ifunc doesn't have summary and ThinLTO cannot
150 // promote it; importing the referencer may cause linkage errors.
151 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: GV);
152 GI && GI->hasLocalLinkage()) {
153 RefLocalLinkageIFunc = true;
154 continue;
155 }
156 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
157 }
158 continue;
159 }
160 if (Visited.insert(Ptr: Operand).second)
161 Worklist.push_back(Elt: Operand);
162 }
163 }
164
165 const Instruction *I = dyn_cast<Instruction>(Val: CurUser);
166 if (I) {
167 uint64_t TotalCount = 0;
168 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
169 // the instruction.
170 auto ValueDataArray = getValueProfDataFromInst(
171 Inst: *I, ValueKind: IPVK_VTableTarget, MaxNumValueData: MaxNumVTableAnnotations, TotalC&: TotalCount);
172
173 for (const auto &V : ValueDataArray)
174 RefEdges.insert(X: Index.getOrInsertValueInfo(/* VTableGUID = */
175 GUID: V.Value));
176 }
177 return HasBlockAddress;
178}
179
180static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
181 ProfileSummaryInfo *PSI) {
182 if (!PSI)
183 return CalleeInfo::HotnessType::Unknown;
184 if (PSI->isHotCount(C: ProfileCount))
185 return CalleeInfo::HotnessType::Hot;
186 if (PSI->isColdCount(C: ProfileCount))
187 return CalleeInfo::HotnessType::Cold;
188 return CalleeInfo::HotnessType::None;
189}
190
191static bool isNonRenamableLocal(const GlobalValue &GV) {
192 return GV.hasSection() && GV.hasLocalLinkage();
193}
194
195/// Determine whether this call has all constant integer arguments (excluding
196/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
197static void addVCallToSet(
198 DevirtCallSite Call, GlobalValue::GUID Guid,
199 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
200 &VCalls,
201 SetVector<FunctionSummary::ConstVCall,
202 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
203 std::vector<uint64_t> Args;
204 // Start from the second argument to skip the "this" pointer.
205 for (auto &Arg : drop_begin(RangeOrContainer: Call.CB.args())) {
206 auto *CI = dyn_cast<ConstantInt>(Val&: Arg);
207 if (!CI || CI->getBitWidth() > 64) {
208 VCalls.insert(X: {.GUID: Guid, .Offset: Call.Offset});
209 return;
210 }
211 Args.push_back(x: CI->getZExtValue());
212 }
213 ConstVCalls.insert(X: {.VFunc: {.GUID: Guid, .Offset: Call.Offset}, .Args: std::move(Args)});
214}
215
216/// If this intrinsic call requires that we add information to the function
217/// summary, do so via the non-constant reference arguments.
218static void addIntrinsicToSummary(
219 const CallInst *CI,
220 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
221 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
222 &TypeTestAssumeVCalls,
223 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
224 &TypeCheckedLoadVCalls,
225 SetVector<FunctionSummary::ConstVCall,
226 std::vector<FunctionSummary::ConstVCall>>
227 &TypeTestAssumeConstVCalls,
228 SetVector<FunctionSummary::ConstVCall,
229 std::vector<FunctionSummary::ConstVCall>>
230 &TypeCheckedLoadConstVCalls,
231 DominatorTree &DT) {
232 switch (CI->getCalledFunction()->getIntrinsicID()) {
233 case Intrinsic::type_test:
234 case Intrinsic::public_type_test: {
235 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
236 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
237 if (!TypeId)
238 break;
239 GlobalValue::GUID Guid =
240 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
241
242 // Produce a summary from type.test intrinsics. We only summarize type.test
243 // intrinsics that are used other than by an llvm.assume intrinsic.
244 // Intrinsics that are assumed are relevant only to the devirtualization
245 // pass, not the type test lowering pass.
246 bool HasNonAssumeUses = llvm::any_of(Range: CI->uses(), P: [](const Use &CIU) {
247 return !isa<AssumeInst>(Val: CIU.getUser());
248 });
249 if (HasNonAssumeUses)
250 TypeTests.insert(X: Guid);
251
252 SmallVector<DevirtCallSite, 4> DevirtCalls;
253 SmallVector<CallInst *, 4> Assumes;
254 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
255 for (auto &Call : DevirtCalls)
256 addVCallToSet(Call, Guid, VCalls&: TypeTestAssumeVCalls,
257 ConstVCalls&: TypeTestAssumeConstVCalls);
258
259 break;
260 }
261
262 case Intrinsic::type_checked_load_relative:
263 case Intrinsic::type_checked_load: {
264 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2));
265 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
266 if (!TypeId)
267 break;
268 GlobalValue::GUID Guid =
269 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
270
271 SmallVector<DevirtCallSite, 4> DevirtCalls;
272 SmallVector<Instruction *, 4> LoadedPtrs;
273 SmallVector<Instruction *, 4> Preds;
274 bool HasNonCallUses = false;
275 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
276 HasNonCallUses, CI, DT);
277 // Any non-call uses of the result of llvm.type.checked.load will
278 // prevent us from optimizing away the llvm.type.test.
279 if (HasNonCallUses)
280 TypeTests.insert(X: Guid);
281 for (auto &Call : DevirtCalls)
282 addVCallToSet(Call, Guid, VCalls&: TypeCheckedLoadVCalls,
283 ConstVCalls&: TypeCheckedLoadConstVCalls);
284
285 break;
286 }
287 default:
288 break;
289 }
290}
291
292static bool isNonVolatileLoad(const Instruction *I) {
293 if (const auto *LI = dyn_cast<LoadInst>(Val: I))
294 return !LI->isVolatile();
295
296 return false;
297}
298
299static bool isNonVolatileStore(const Instruction *I) {
300 if (const auto *SI = dyn_cast<StoreInst>(Val: I))
301 return !SI->isVolatile();
302
303 return false;
304}
305
306// Returns true if the function definition must be unreachable.
307//
308// Note if this helper function returns true, `F` is guaranteed
309// to be unreachable; if it returns false, `F` might still
310// be unreachable but not covered by this helper function.
311static bool mustBeUnreachableFunction(const Function &F) {
312 // A function must be unreachable if its entry block ends with an
313 // 'unreachable'.
314 assert(!F.isDeclaration());
315 return isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator());
316}
317
318static void computeFunctionSummary(
319 ModuleSummaryIndex &Index, const Module &M, const Function &F,
320 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT,
321 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
322 bool IsThinLTO,
323 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
324 // Summary not currently supported for anonymous functions, they should
325 // have been named.
326 assert(F.hasName());
327
328 unsigned NumInsts = 0;
329 // Map from callee ValueId to profile count. Used to accumulate profile
330 // counts for all static calls to a given callee.
331 MapVector<ValueInfo, CalleeInfo, DenseMap<ValueInfo, unsigned>,
332 SmallVector<FunctionSummary::EdgeTy, 0>>
333 CallGraphEdges;
334 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges, LoadRefEdges,
335 StoreRefEdges;
336 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> TypeTests;
337 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
338 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
339 SetVector<FunctionSummary::ConstVCall,
340 std::vector<FunctionSummary::ConstVCall>>
341 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
342 ICallPromotionAnalysis ICallAnalysis;
343 SmallPtrSet<const User *, 8> Visited;
344
345 // Add personality function, prefix data and prologue data to function's ref
346 // list.
347 bool HasLocalIFuncCallOrRef = false;
348 findRefEdges(Index, CurUser: &F, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
349 std::vector<const Instruction *> NonVolatileLoads;
350 std::vector<const Instruction *> NonVolatileStores;
351
352 std::vector<CallsiteInfo> Callsites;
353 std::vector<AllocInfo> Allocs;
354
355#ifndef NDEBUG
356 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
357#endif
358
359 bool HasInlineAsmMaybeReferencingInternal = false;
360 bool HasIndirBranchToBlockAddress = false;
361 bool HasUnknownCall = false;
362 bool MayThrow = false;
363 for (const BasicBlock &BB : F) {
364 // We don't allow inlining of function with indirect branch to blockaddress.
365 // If the blockaddress escapes the function, e.g., via a global variable,
366 // inlining may lead to an invalid cross-function reference. So we shouldn't
367 // import such function either.
368 if (BB.hasAddressTaken()) {
369 for (User *U : BlockAddress::get(BB: const_cast<BasicBlock *>(&BB))->users())
370 if (!isa<CallBrInst>(Val: *U)) {
371 HasIndirBranchToBlockAddress = true;
372 break;
373 }
374 }
375
376 for (const Instruction &I : BB) {
377 if (I.isDebugOrPseudoInst())
378 continue;
379 ++NumInsts;
380
381 // Regular LTO module doesn't participate in ThinLTO import,
382 // so no reference from it can be read/writeonly, since this
383 // would require importing variable as local copy
384 if (IsThinLTO) {
385 if (isNonVolatileLoad(I: &I)) {
386 // Postpone processing of non-volatile load instructions
387 // See comments below
388 Visited.insert(Ptr: &I);
389 NonVolatileLoads.push_back(x: &I);
390 continue;
391 } else if (isNonVolatileStore(I: &I)) {
392 Visited.insert(Ptr: &I);
393 NonVolatileStores.push_back(x: &I);
394 // All references from second operand of store (destination address)
395 // can be considered write-only if they're not referenced by any
396 // non-store instruction. References from first operand of store
397 // (stored value) can't be treated either as read- or as write-only
398 // so we add them to RefEdges as we do with all other instructions
399 // except non-volatile load.
400 Value *Stored = I.getOperand(i: 0);
401 if (auto *GV = dyn_cast<GlobalValue>(Val: Stored))
402 // findRefEdges will try to examine GV operands, so instead
403 // of calling it we should add GV to RefEdges directly.
404 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
405 else if (auto *U = dyn_cast<User>(Val: Stored))
406 findRefEdges(Index, CurUser: U, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
407 continue;
408 }
409 }
410 findRefEdges(Index, CurUser: &I, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
411 const auto *CB = dyn_cast<CallBase>(Val: &I);
412 if (!CB) {
413 if (I.mayThrow())
414 MayThrow = true;
415 continue;
416 }
417
418 const auto *CI = dyn_cast<CallInst>(Val: &I);
419 // Since we don't know exactly which local values are referenced in inline
420 // assembly, conservatively mark the function as possibly referencing
421 // a local value from inline assembly to ensure we don't export a
422 // reference (which would require renaming and promotion of the
423 // referenced value).
424 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
425 HasInlineAsmMaybeReferencingInternal = true;
426
427 // Compute this once per indirect call.
428 uint32_t NumCandidates = 0;
429 uint64_t TotalCount = 0;
430 MutableArrayRef<InstrProfValueData> CandidateProfileData;
431
432 auto *CalledValue = CB->getCalledOperand();
433 auto *CalledFunction = CB->getCalledFunction();
434 if (CalledValue && !CalledFunction) {
435 CalledValue = CalledValue->stripPointerCasts();
436 // Stripping pointer casts can reveal a called function.
437 CalledFunction = dyn_cast<Function>(Val: CalledValue);
438 }
439 // Check if this is an alias to a function. If so, get the
440 // called aliasee for the checks below.
441 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
442 assert(!CalledFunction && "Expected null called function in callsite for alias");
443 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
444 }
445 // Check if this is a direct call to a known function or a known
446 // intrinsic, or an indirect call with profile data.
447 if (CalledFunction) {
448 if (CI && CalledFunction->isIntrinsic()) {
449 addIntrinsicToSummary(
450 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
451 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
452 continue;
453 }
454 // We should have named any anonymous globals
455 assert(CalledFunction->hasName());
456 auto ScaledCount = PSI->getProfileCount(CallInst: *CB, BFI);
457 auto Hotness = ScaledCount ? getHotness(ProfileCount: *ScaledCount, PSI)
458 : CalleeInfo::HotnessType::Unknown;
459 if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
460 Hotness = CalleeInfo::HotnessType::Cold;
461
462 // Use the original CalledValue, in case it was an alias. We want
463 // to record the call edge to the alias in that case. Eventually
464 // an alias summary will be created to associate the alias and
465 // aliasee.
466 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
467 GV: cast<GlobalValue>(Val: CalledValue))];
468 ValueInfo.updateHotness(OtherHotness: Hotness);
469 if (CB->isTailCall())
470 ValueInfo.setHasTailCall(true);
471 } else {
472 HasUnknownCall = true;
473 // If F is imported, a local linkage ifunc (e.g. target_clones on a
474 // static function) called by F will be cloned. Since summaries don't
475 // track ifunc, we do not know implementation functions referenced by
476 // the ifunc resolver need to be promoted in the exporter, and we will
477 // get linker errors due to cloned declarations for implementation
478 // functions. As a simple fix, just mark F as not eligible for import.
479 // Non-local ifunc is not cloned and does not have the issue.
480 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: CalledValue))
481 if (GI->hasLocalLinkage())
482 HasLocalIFuncCallOrRef = true;
483 // Skip inline assembly calls.
484 if (CI && CI->isInlineAsm())
485 continue;
486 // Skip direct calls.
487 if (!CalledValue || isa<Constant>(Val: CalledValue))
488 continue;
489
490 // Check if the instruction has a callees metadata. If so, add callees
491 // to CallGraphEdges to reflect the references from the metadata, and
492 // to enable importing for subsequent indirect call promotion and
493 // inlining.
494 if (auto *MD = I.getMetadata(KindID: LLVMContext::MD_callees)) {
495 for (const auto &Op : MD->operands()) {
496 Function *Callee = mdconst::extract_or_null<Function>(MD: Op);
497 if (Callee)
498 CallGraphEdges[Index.getOrInsertValueInfo(GV: Callee)];
499 }
500 }
501
502 CandidateProfileData =
503 ICallAnalysis.getPromotionCandidatesForInstruction(
504 I: &I, TotalCount, NumCandidates, MaxNumValueData: MaxSummaryIndirectEdges);
505 for (const auto &Candidate : CandidateProfileData)
506 CallGraphEdges[Index.getOrInsertValueInfo(GUID: Candidate.Value)]
507 .updateHotness(OtherHotness: getHotness(ProfileCount: Candidate.Count, PSI));
508 }
509
510 // Summarize memprof related metadata. This is only needed for ThinLTO.
511 if (!IsThinLTO)
512 continue;
513
514 // Skip indirect calls if we haven't enabled memprof ICP.
515 if (!CalledFunction && !EnableMemProfIndirectCallSupport)
516 continue;
517
518 // Ensure we keep this analysis in sync with the handling in the ThinLTO
519 // backend (see MemProfContextDisambiguation::applyImport). Save this call
520 // so that we can skip it in checking the reverse case later.
521 assert(mayHaveMemprofSummary(CB));
522#ifndef NDEBUG
523 CallsThatMayHaveMemprofSummary.insert(CB);
524#endif
525
526 // Compute the list of stack ids first (so we can trim them from the stack
527 // ids on any MIBs).
528 CallStack<MDNode, MDNode::op_iterator> InstCallsite(
529 I.getMetadata(KindID: LLVMContext::MD_callsite));
530 auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
531 if (MemProfMD) {
532 std::vector<MIBInfo> MIBs;
533 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
534 bool HasNonZeroContextSizeInfos = false;
535 for (auto &MDOp : MemProfMD->operands()) {
536 auto *MIBMD = cast<const MDNode>(Val: MDOp);
537 MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
538 assert(StackNode);
539 SmallVector<unsigned> StackIdIndices;
540 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
541 // Collapse out any on the allocation call (inlining).
542 for (auto ContextIter =
543 StackContext.beginAfterSharedPrefix(Other: InstCallsite);
544 ContextIter != StackContext.end(); ++ContextIter) {
545 unsigned StackIdIdx = Index.addOrGetStackIdIndex(StackId: *ContextIter);
546 // If this is a direct recursion, simply skip the duplicate
547 // entries. If this is mutual recursion, handling is left to
548 // the LTO link analysis client.
549 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
550 StackIdIndices.push_back(Elt: StackIdIdx);
551 }
552 // If we have context size information, collect it for inclusion in
553 // the summary.
554 assert(MIBMD->getNumOperands() > 2 ||
555 !metadataIncludesAllContextSizeInfo());
556 if (MIBMD->getNumOperands() > 2) {
557 std::vector<ContextTotalSize> ContextSizes;
558 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
559 MDNode *ContextSizePair = dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
560 assert(ContextSizePair->getNumOperands() == 2);
561 uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
562 MD: ContextSizePair->getOperand(I: 0))
563 ->getZExtValue();
564 uint64_t TS = mdconst::dyn_extract<ConstantInt>(
565 MD: ContextSizePair->getOperand(I: 1))
566 ->getZExtValue();
567 ContextSizes.push_back(x: {.FullStackId: FullStackId, .TotalSize: TS});
568 }
569 // Flag that we need to keep the ContextSizeInfos array for this
570 // alloc as it now contains non-zero context info sizes.
571 HasNonZeroContextSizeInfos = true;
572 ContextSizeInfos.push_back(x: std::move(ContextSizes));
573 } else {
574 // The ContextSizeInfos must be in the same relative position as the
575 // associated MIB. In some cases we only include a ContextSizeInfo
576 // for a subset of MIBs in an allocation. To handle that, eagerly
577 // fill any MIB entries that don't have context size info metadata
578 // with a pair of 0s. Later on we will only use this array if it
579 // ends up containing any non-zero entries (see where we set
580 // HasNonZeroContextSizeInfos above).
581 ContextSizeInfos.push_back(x: {{.FullStackId: 0, .TotalSize: 0}});
582 }
583 MIBs.push_back(
584 x: MIBInfo(getMIBAllocType(MIB: MIBMD), std::move(StackIdIndices)));
585 }
586 Allocs.push_back(x: AllocInfo(std::move(MIBs)));
587 assert(HasNonZeroContextSizeInfos ||
588 !metadataIncludesAllContextSizeInfo());
589 // We eagerly build the ContextSizeInfos array, but it will be filled
590 // with sub arrays of pairs of 0s if no MIBs on this alloc actually
591 // contained context size info metadata. Only save it if any MIBs had
592 // any such metadata.
593 if (HasNonZeroContextSizeInfos) {
594 assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
595 Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
596 }
597 } else if (!InstCallsite.empty()) {
598 SmallVector<unsigned> StackIdIndices;
599 for (auto StackId : InstCallsite)
600 StackIdIndices.push_back(Elt: Index.addOrGetStackIdIndex(StackId));
601 if (CalledFunction) {
602 // Use the original CalledValue, in case it was an alias. We want
603 // to record the call edge to the alias in that case. Eventually
604 // an alias summary will be created to associate the alias and
605 // aliasee.
606 auto CalleeValueInfo =
607 Index.getOrInsertValueInfo(GV: cast<GlobalValue>(Val: CalledValue));
608 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
609 } else {
610 assert(EnableMemProfIndirectCallSupport);
611 // For indirect callsites, create multiple Callsites, one per target.
612 // This enables having a different set of clone versions per target,
613 // and we will apply the cloning decisions while speculatively
614 // devirtualizing in the ThinLTO backends.
615 for (const auto &Candidate : CandidateProfileData) {
616 auto CalleeValueInfo = Index.getOrInsertValueInfo(GUID: Candidate.Value);
617 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
618 }
619 }
620 }
621 }
622 }
623
624 if (PSI->hasPartialSampleProfile() && ScalePartialSampleProfileWorkingSetSize)
625 Index.addBlockCount(C: F.size());
626
627 SmallVector<ValueInfo, 0> Refs;
628 if (IsThinLTO) {
629 auto AddRefEdges =
630 [&](const std::vector<const Instruction *> &Instrs,
631 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &Edges,
632 SmallPtrSet<const User *, 8> &Cache) {
633 for (const auto *I : Instrs) {
634 Cache.erase(Ptr: I);
635 findRefEdges(Index, CurUser: I, RefEdges&: Edges, Visited&: Cache, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
636 }
637 };
638
639 // By now we processed all instructions in a function, except
640 // non-volatile loads and non-volatile value stores. Let's find
641 // ref edges for both of instruction sets
642 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
643 // We can add some values to the Visited set when processing load
644 // instructions which are also used by stores in NonVolatileStores.
645 // For example this can happen if we have following code:
646 //
647 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
648 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
649 //
650 // After processing loads we'll add bitcast to the Visited set, and if
651 // we use the same set while processing stores, we'll never see store
652 // to @bar and @bar will be mistakenly treated as readonly.
653 SmallPtrSet<const llvm::User *, 8> StoreCache;
654 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
655
656 // If both load and store instruction reference the same variable
657 // we won't be able to optimize it. Add all such reference edges
658 // to RefEdges set.
659 for (const auto &VI : StoreRefEdges)
660 if (LoadRefEdges.remove(X: VI))
661 RefEdges.insert(X: VI);
662
663 unsigned RefCnt = RefEdges.size();
664 // All new reference edges inserted in two loops below are either
665 // read or write only. They will be grouped in the end of RefEdges
666 // vector, so we can use a single integer value to identify them.
667 RefEdges.insert_range(R&: LoadRefEdges);
668
669 unsigned FirstWORef = RefEdges.size();
670 RefEdges.insert_range(R&: StoreRefEdges);
671
672 Refs = RefEdges.takeVector();
673 for (; RefCnt < FirstWORef; ++RefCnt)
674 Refs[RefCnt].setReadOnly();
675
676 for (; RefCnt < Refs.size(); ++RefCnt)
677 Refs[RefCnt].setWriteOnly();
678 } else {
679 Refs = RefEdges.takeVector();
680 }
681 // Explicit add hot edges to enforce importing for designated GUIDs for
682 // sample PGO, to enable the same inlines as the profiled optimized binary.
683 for (auto &I : F.getImportGUIDs())
684 CallGraphEdges[Index.getOrInsertValueInfo(GUID: I)].updateHotness(
685 OtherHotness: ForceSummaryEdgesCold == FunctionSummary::FSHT_All
686 ? CalleeInfo::HotnessType::Cold
687 : CalleeInfo::HotnessType::Critical);
688
689#ifndef NDEBUG
690 // Make sure that all calls we decided could not have memprof summaries get a
691 // false value for mayHaveMemprofSummary, to ensure that this handling remains
692 // in sync with the ThinLTO backend handling.
693 if (IsThinLTO) {
694 for (const BasicBlock &BB : F) {
695 for (const Instruction &I : BB) {
696 const auto *CB = dyn_cast<CallBase>(&I);
697 if (!CB)
698 continue;
699 // We already checked these above.
700 if (CallsThatMayHaveMemprofSummary.count(CB))
701 continue;
702 assert(!mayHaveMemprofSummary(CB));
703 }
704 }
705 }
706#endif
707
708 bool NonRenamableLocal = isNonRenamableLocal(GV: F);
709 bool NotEligibleForImport =
710 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
711 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
712 GlobalValueSummary::GVFlags Flags(
713 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
714 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
715 GlobalValueSummary::ImportKind::Definition,
716 /* NoRenameOnPromotion = */ false);
717 FunctionSummary::FFlags FunFlags{
718 .ReadNone: F.doesNotAccessMemory(), .ReadOnly: F.onlyReadsMemory() && !F.doesNotAccessMemory(),
719 .NoRecurse: F.hasFnAttribute(Kind: Attribute::NoRecurse), .ReturnDoesNotAlias: F.returnDoesNotAlias(),
720 // FIXME: refactor this to use the same code that inliner is using.
721 // Don't try to import functions with noinline attribute.
722 .NoInline: F.getAttributes().hasFnAttr(Kind: Attribute::NoInline),
723 .AlwaysInline: F.hasFnAttribute(Kind: Attribute::AlwaysInline),
724 .NoUnwind: F.hasFnAttribute(Kind: Attribute::NoUnwind), .MayThrow: MayThrow, .HasUnknownCall: HasUnknownCall,
725 .MustBeUnreachable: mustBeUnreachableFunction(F)};
726 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
727 if (auto *SSI = GetSSICallback(F))
728 ParamAccesses = SSI->getParamAccesses(Index);
729 auto FuncSummary = std::make_unique<FunctionSummary>(
730 args&: Flags, args&: NumInsts, args&: FunFlags, args: std::move(Refs), args: CallGraphEdges.takeVector(),
731 args: TypeTests.takeVector(), args: TypeTestAssumeVCalls.takeVector(),
732 args: TypeCheckedLoadVCalls.takeVector(),
733 args: TypeTestAssumeConstVCalls.takeVector(),
734 args: TypeCheckedLoadConstVCalls.takeVector(), args: std::move(ParamAccesses),
735 args: std::move(Callsites), args: std::move(Allocs));
736 if (NonRenamableLocal)
737 CantBePromoted.insert(V: F.getGUID());
738 Index.addGlobalValueSummary(GV: F, Summary: std::move(FuncSummary));
739}
740
741/// Find function pointers referenced within the given vtable initializer
742/// (or subset of an initializer) \p I. The starting offset of \p I within
743/// the vtable initializer is \p StartingOffset. Any discovered function
744/// pointers are added to \p VTableFuncs along with their cumulative offset
745/// within the initializer.
746static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
747 const Module &M, ModuleSummaryIndex &Index,
748 VTableFuncList &VTableFuncs,
749 const GlobalVariable &OrigGV) {
750 // First check if this is a function pointer.
751 if (I->getType()->isPointerTy()) {
752 auto C = I->stripPointerCasts();
753 auto A = dyn_cast<GlobalAlias>(Val: C);
754 if (isa<Function>(Val: C) || (A && isa<Function>(Val: A->getAliasee()))) {
755 auto GV = dyn_cast<GlobalValue>(Val: C);
756 assert(GV);
757 // We can disregard __cxa_pure_virtual as a possible call target, as
758 // calls to pure virtuals are UB.
759 if (GV && GV->getName() != "__cxa_pure_virtual")
760 VTableFuncs.push_back(x: {Index.getOrInsertValueInfo(GV), StartingOffset});
761 return;
762 }
763 }
764
765 // Walk through the elements in the constant struct or array and recursively
766 // look for virtual function pointers.
767 const DataLayout &DL = M.getDataLayout();
768 if (auto *C = dyn_cast<ConstantStruct>(Val: I)) {
769 StructType *STy = dyn_cast<StructType>(Val: C->getType());
770 assert(STy);
771 const StructLayout *SL = DL.getStructLayout(Ty: C->getType());
772
773 for (auto EI : llvm::enumerate(First: STy->elements())) {
774 auto Offset = SL->getElementOffset(Idx: EI.index());
775 unsigned Op = SL->getElementContainingOffset(FixedOffset: Offset);
776 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i: Op)),
777 StartingOffset: StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
778 }
779 } else if (auto *C = dyn_cast<ConstantArray>(Val: I)) {
780 ArrayType *ATy = C->getType();
781 Type *EltTy = ATy->getElementType();
782 uint64_t EltSize = DL.getTypeAllocSize(Ty: EltTy);
783 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
784 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i)),
785 StartingOffset: StartingOffset + i * EltSize, M, Index, VTableFuncs,
786 OrigGV);
787 }
788 } else if (const auto *CE = dyn_cast<ConstantExpr>(Val: I)) {
789 // For relative vtables, the next sub-component should be a trunc.
790 if (CE->getOpcode() != Instruction::Trunc ||
791 !(CE = dyn_cast<ConstantExpr>(Val: CE->getOperand(i_nocapture: 0))))
792 return;
793
794 // If this constant can be reduced to the offset between a function and a
795 // global, then we know this is a valid virtual function if the RHS is the
796 // original vtable we're scanning through.
797 if (CE->getOpcode() == Instruction::Sub) {
798 GlobalValue *LHS, *RHS;
799 APSInt LHSOffset, RHSOffset;
800 if (IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 0), GV&: LHS, Offset&: LHSOffset, DL) &&
801 IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 1), GV&: RHS, Offset&: RHSOffset, DL) &&
802 RHS == &OrigGV &&
803
804 // For relative vtables, this component should point to the callable
805 // function without any offsets.
806 LHSOffset == 0 &&
807
808 // Also, the RHS should always point to somewhere within the vtable.
809 RHSOffset <=
810 static_cast<uint64_t>(DL.getTypeAllocSize(Ty: OrigGV.getInitializer()->getType()))) {
811 findFuncPointers(I: LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
812 }
813 }
814 }
815}
816
817// Identify the function pointers referenced by vtable definition \p V.
818static void computeVTableFuncs(ModuleSummaryIndex &Index,
819 const GlobalVariable &V, const Module &M,
820 VTableFuncList &VTableFuncs) {
821 if (!V.isConstant())
822 return;
823
824 findFuncPointers(I: V.getInitializer(), /*StartingOffset=*/0, M, Index,
825 VTableFuncs, OrigGV: V);
826
827#ifndef NDEBUG
828 // Validate that the VTableFuncs list is ordered by offset.
829 uint64_t PrevOffset = 0;
830 for (auto &P : VTableFuncs) {
831 // The findVFuncPointers traversal should have encountered the
832 // functions in offset order. We need to use ">=" since PrevOffset
833 // starts at 0.
834 assert(P.VTableOffset >= PrevOffset);
835 PrevOffset = P.VTableOffset;
836 }
837#endif
838}
839
840/// Record vtable definition \p V for each type metadata it references.
841static void
842recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index,
843 const GlobalVariable &V,
844 SmallVectorImpl<MDNode *> &Types) {
845 for (MDNode *Type : Types) {
846 auto TypeID = Type->getOperand(I: 1).get();
847
848 uint64_t Offset =
849 cast<ConstantInt>(
850 Val: cast<ConstantAsMetadata>(Val: Type->getOperand(I: 0))->getValue())
851 ->getZExtValue();
852
853 if (auto *TypeId = dyn_cast<MDString>(Val: TypeID))
854 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId: TypeId->getString())
855 .push_back(x: {Offset, Index.getOrInsertValueInfo(GV: &V)});
856 }
857}
858
859static void computeVariableSummary(ModuleSummaryIndex &Index,
860 const GlobalVariable &V,
861 DenseSet<GlobalValue::GUID> &CantBePromoted,
862 const Module &M,
863 SmallVectorImpl<MDNode *> &Types) {
864 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges;
865 SmallPtrSet<const User *, 8> Visited;
866 bool RefLocalIFunc = false;
867 bool HasBlockAddress =
868 findRefEdges(Index, CurUser: &V, RefEdges, Visited, RefLocalLinkageIFunc&: RefLocalIFunc);
869 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
870 bool NonRenamableLocal = isNonRenamableLocal(GV: V);
871 GlobalValueSummary::GVFlags Flags(
872 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
873 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
874 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
875
876 VTableFuncList VTableFuncs;
877 // If splitting is not enabled, then we compute the summary information
878 // necessary for index-based whole program devirtualization.
879 if (!Index.enableSplitLTOUnit()) {
880 Types.clear();
881 V.getMetadata(KindID: LLVMContext::MD_type, MDs&: Types);
882 if (!Types.empty()) {
883 // Identify the function pointers referenced by this vtable definition.
884 computeVTableFuncs(Index, V, M, VTableFuncs);
885
886 // Record this vtable definition for each type metadata it references.
887 recordTypeIdCompatibleVtableReferences(Index, V, Types);
888 }
889 }
890
891 // Don't mark variables we won't be able to internalize as read/write-only.
892 bool CanBeInternalized =
893 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
894 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
895 bool Constant = V.isConstant();
896 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
897 Constant ? false : CanBeInternalized,
898 Constant, V.getVCallVisibility());
899 auto GVarSummary = std::make_unique<GlobalVarSummary>(args&: Flags, args&: VarFlags,
900 args: RefEdges.takeVector());
901 if (NonRenamableLocal)
902 CantBePromoted.insert(V: V.getGUID());
903 if (NotEligibleForImport)
904 GVarSummary->setNotEligibleToImport();
905 if (!VTableFuncs.empty())
906 GVarSummary->setVTableFuncs(VTableFuncs);
907 Index.addGlobalValueSummary(GV: V, Summary: std::move(GVarSummary));
908}
909
910static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
911 DenseSet<GlobalValue::GUID> &CantBePromoted) {
912 // Skip summary for indirect function aliases as summary for aliasee will not
913 // be emitted.
914 const GlobalObject *Aliasee = A.getAliaseeObject();
915 if (isa<GlobalIFunc>(Val: Aliasee))
916 return;
917 bool NonRenamableLocal = isNonRenamableLocal(GV: A);
918 GlobalValueSummary::GVFlags Flags(
919 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
920 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
921 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
922 auto AS = std::make_unique<AliasSummary>(args&: Flags);
923 auto AliaseeVI = Index.getValueInfo(GUID: Aliasee->getGUID());
924 assert(AliaseeVI && "Alias expects aliasee summary to be available");
925 assert(AliaseeVI.getSummaryList().size() == 1 &&
926 "Expected a single entry per aliasee in per-module index");
927 AS->setAliasee(AliaseeVI, Aliasee: AliaseeVI.getSummaryList()[0].get());
928 if (NonRenamableLocal)
929 CantBePromoted.insert(V: A.getGUID());
930 Index.addGlobalValueSummary(GV: A, Summary: std::move(AS));
931}
932
933// Set LiveRoot flag on entries matching the given value name.
934static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
935 if (ValueInfo VI =
936 Index.getValueInfo(GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Name)))
937 for (const auto &Summary : VI.getSummaryList())
938 Summary->setLive(true);
939}
940
941ModuleSummaryIndex llvm::buildModuleSummaryIndex(
942 const Module &M,
943 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
944 ProfileSummaryInfo *PSI,
945 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
946 assert(PSI);
947 bool EnableSplitLTOUnit = false;
948 bool UnifiedLTO = false;
949 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
950 MD: M.getModuleFlag(Key: "EnableSplitLTOUnit")))
951 EnableSplitLTOUnit = MD->getZExtValue();
952 if (auto *MD =
953 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "UnifiedLTO")))
954 UnifiedLTO = MD->getZExtValue();
955 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
956
957 // Identify the local values in the llvm.used and llvm.compiler.used sets,
958 // which should not be exported as they would then require renaming and
959 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
960 // here because we use this information to mark functions containing inline
961 // assembly calls as not importable.
962 SmallPtrSet<GlobalValue *, 4> LocalsUsed;
963 SmallVector<GlobalValue *, 4> Used;
964 // First collect those in the llvm.used set.
965 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/false);
966 // Next collect those in the llvm.compiler.used set.
967 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/true);
968 DenseSet<GlobalValue::GUID> CantBePromoted;
969 for (auto *V : Used) {
970 if (V->hasLocalLinkage()) {
971 LocalsUsed.insert(Ptr: V);
972 CantBePromoted.insert(V: V->getGUID());
973 }
974 }
975
976 bool HasLocalInlineAsmSymbol = false;
977 if (!M.getModuleInlineAsm().empty()) {
978 // Collect the local values defined by module level asm, and set up
979 // summaries for these symbols so that they can be marked as NoRename,
980 // to prevent export of any use of them in regular IR that would require
981 // renaming within the module level asm. Note we don't need to create a
982 // summary for weak or global defs, as they don't need to be flagged as
983 // NoRename, and defs in module level asm can't be imported anyway.
984 // Also, any values used but not defined within module level asm should
985 // be listed on the llvm.used or llvm.compiler.used global and marked as
986 // referenced from there.
987 ModuleSymbolTable::CollectAsmSymbols(
988 M, AsmSymbol: [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
989 // Symbols not marked as Weak or Global are local definitions.
990 if (Flags & (object::BasicSymbolRef::SF_Weak |
991 object::BasicSymbolRef::SF_Global))
992 return;
993 HasLocalInlineAsmSymbol = true;
994 GlobalValue *GV = M.getNamedValue(Name);
995 if (!GV)
996 return;
997 assert(GV->isDeclaration() && "Def in module asm already has definition");
998 GlobalValueSummary::GVFlags GVFlags(
999 GlobalValue::InternalLinkage, GlobalValue::DefaultVisibility,
1000 /* NotEligibleToImport = */ true,
1001 /* Live = */ true,
1002 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
1003 GlobalValueSummary::Definition,
1004 /* NoRenameOnPromotion = */ false);
1005 CantBePromoted.insert(V: GV->getGUID());
1006 // Create the appropriate summary type.
1007 if (Function *F = dyn_cast<Function>(Val: GV)) {
1008 std::unique_ptr<FunctionSummary> Summary =
1009 std::make_unique<FunctionSummary>(
1010 args&: GVFlags, /*InstCount=*/args: 0,
1011 args: FunctionSummary::FFlags{
1012 .ReadNone: F->hasFnAttribute(Kind: Attribute::ReadNone),
1013 .ReadOnly: F->hasFnAttribute(Kind: Attribute::ReadOnly),
1014 .NoRecurse: F->hasFnAttribute(Kind: Attribute::NoRecurse),
1015 .ReturnDoesNotAlias: F->returnDoesNotAlias(),
1016 /* NoInline = */ false,
1017 .AlwaysInline: F->hasFnAttribute(Kind: Attribute::AlwaysInline),
1018 .NoUnwind: F->hasFnAttribute(Kind: Attribute::NoUnwind),
1019 /* MayThrow */ true,
1020 /* HasUnknownCall */ true,
1021 /* MustBeUnreachable */ false},
1022 args: SmallVector<ValueInfo, 0>{},
1023 args: SmallVector<FunctionSummary::EdgeTy, 0>{},
1024 args: ArrayRef<GlobalValue::GUID>{},
1025 args: ArrayRef<FunctionSummary::VFuncId>{},
1026 args: ArrayRef<FunctionSummary::VFuncId>{},
1027 args: ArrayRef<FunctionSummary::ConstVCall>{},
1028 args: ArrayRef<FunctionSummary::ConstVCall>{},
1029 args: ArrayRef<FunctionSummary::ParamAccess>{},
1030 args: ArrayRef<CallsiteInfo>{}, args: ArrayRef<AllocInfo>{});
1031 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1032 } else {
1033 std::unique_ptr<GlobalVarSummary> Summary =
1034 std::make_unique<GlobalVarSummary>(
1035 args&: GVFlags,
1036 args: GlobalVarSummary::GVarFlags(
1037 false, false, cast<GlobalVariable>(Val: GV)->isConstant(),
1038 GlobalObject::VCallVisibilityPublic),
1039 args: SmallVector<ValueInfo, 0>{});
1040 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1041 }
1042 });
1043 }
1044
1045 bool IsThinLTO = true;
1046 if (auto *MD =
1047 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
1048 IsThinLTO = MD->getZExtValue();
1049
1050 // Compute summaries for all functions defined in module, and save in the
1051 // index.
1052 for (const auto &F : M) {
1053 if (F.isDeclaration())
1054 continue;
1055
1056 DominatorTree DT(const_cast<Function &>(F));
1057 BlockFrequencyInfo *BFI = nullptr;
1058 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1059 if (GetBFICallback)
1060 BFI = GetBFICallback(F);
1061 else if (F.hasProfileData()) {
1062 LoopInfo LI{DT};
1063 BranchProbabilityInfo BPI{F, LI};
1064 BFIPtr = std::make_unique<BlockFrequencyInfo>(args: F, args&: BPI, args&: LI);
1065 BFI = BFIPtr.get();
1066 }
1067
1068 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
1069 HasLocalsInUsedOrAsm: !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1070 CantBePromoted, IsThinLTO, GetSSICallback);
1071 }
1072
1073 // Compute summaries for all variables defined in module, and save in the
1074 // index.
1075 SmallVector<MDNode *, 2> Types;
1076 for (const GlobalVariable &G : M.globals()) {
1077 if (G.isDeclaration())
1078 continue;
1079 computeVariableSummary(Index, V: G, CantBePromoted, M, Types);
1080 }
1081
1082 // Compute summaries for all aliases defined in module, and save in the
1083 // index.
1084 for (const GlobalAlias &A : M.aliases())
1085 computeAliasSummary(Index, A, CantBePromoted);
1086
1087 // Iterate through ifuncs, set their resolvers all alive.
1088 for (const GlobalIFunc &I : M.ifuncs()) {
1089 I.applyAlongResolverPath(Op: [&Index](const GlobalValue &GV) {
1090 Index.getGlobalValueSummary(GV)->setLive(true);
1091 });
1092 }
1093
1094 for (auto *V : LocalsUsed) {
1095 auto *Summary = Index.getGlobalValueSummary(GV: *V);
1096 assert(Summary && "Missing summary for global value");
1097 Summary->setNotEligibleToImport();
1098 }
1099
1100 // The linker doesn't know about these LLVM produced values, so we need
1101 // to flag them as live in the index to ensure index-based dead value
1102 // analysis treats them as live roots of the analysis.
1103 setLiveRoot(Index, Name: "llvm.used");
1104 setLiveRoot(Index, Name: "llvm.compiler.used");
1105 setLiveRoot(Index, Name: "llvm.global_ctors");
1106 setLiveRoot(Index, Name: "llvm.global_dtors");
1107 setLiveRoot(Index, Name: "llvm.global.annotations");
1108
1109 for (auto &GlobalList : Index) {
1110 // Ignore entries for references that are undefined in the current module.
1111 if (GlobalList.second.getSummaryList().empty())
1112 continue;
1113
1114 assert(GlobalList.second.getSummaryList().size() == 1 &&
1115 "Expected module's index to have one summary per GUID");
1116 auto &Summary = GlobalList.second.getSummaryList()[0];
1117 if (!IsThinLTO) {
1118 Summary->setNotEligibleToImport();
1119 continue;
1120 }
1121
1122 bool AllRefsCanBeExternallyReferenced =
1123 llvm::all_of(Range: Summary->refs(), P: [&](const ValueInfo &VI) {
1124 return !CantBePromoted.count(V: VI.getGUID());
1125 });
1126 if (!AllRefsCanBeExternallyReferenced) {
1127 Summary->setNotEligibleToImport();
1128 continue;
1129 }
1130
1131 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Val: Summary.get())) {
1132 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1133 Range: FuncSummary->calls(), P: [&](const FunctionSummary::EdgeTy &Edge) {
1134 return !CantBePromoted.count(V: Edge.first.getGUID());
1135 });
1136 if (!AllCallsCanBeExternallyReferenced)
1137 Summary->setNotEligibleToImport();
1138 }
1139 }
1140
1141 if (!ModuleSummaryDotFile.empty()) {
1142 std::error_code EC;
1143 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_Text);
1144 if (EC)
1145 report_fatal_error(reason: Twine("Failed to open dot file ") +
1146 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1147 Index.exportToDot(OS&: OSDot, GUIDPreservedSymbols: {});
1148 }
1149
1150 return Index;
1151}
1152
1153AnalysisKey ModuleSummaryIndexAnalysis::Key;
1154
1155ModuleSummaryIndex
1156ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
1157 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(IR&: M);
1158 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1159 bool NeedSSI = needsParamAccessSummary(M);
1160 return buildModuleSummaryIndex(
1161 M,
1162 GetBFICallback: [&FAM](const Function &F) {
1163 return &FAM.getResult<BlockFrequencyAnalysis>(
1164 IR&: *const_cast<Function *>(&F));
1165 },
1166 PSI: &PSI,
1167 GetSSICallback: [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1168 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1169 IR&: const_cast<Function &>(F))
1170 : nullptr;
1171 });
1172}
1173
1174char ModuleSummaryIndexWrapperPass::ID = 0;
1175
1176INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1177 "Module Summary Analysis", false, true)
1178INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
1179INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
1180INITIALIZE_PASS_DEPENDENCY(StackSafetyInfoWrapperPass)
1181INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1182 "Module Summary Analysis", false, true)
1183
1184ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
1185 return new ModuleSummaryIndexWrapperPass();
1186}
1187
1188ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
1189 : ModulePass(ID) {}
1190
1191bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
1192 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1193 bool NeedSSI = needsParamAccessSummary(M);
1194 Index.emplace(args: buildModuleSummaryIndex(
1195 M,
1196 GetBFICallback: [this](const Function &F) {
1197 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1198 F&: *const_cast<Function *>(&F))
1199 .getBFI());
1200 },
1201 PSI,
1202 GetSSICallback: [&](const Function &F) -> const StackSafetyInfo * {
1203 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1204 F&: const_cast<Function &>(F))
1205 .getResult()
1206 : nullptr;
1207 }));
1208 return false;
1209}
1210
1211bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
1212 Index.reset();
1213 return false;
1214}
1215
1216void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
1217 AU.setPreservesAll();
1218 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1219 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1220 AU.addRequired<StackSafetyInfoWrapperPass>();
1221}
1222
1223char ImmutableModuleSummaryIndexWrapperPass::ID = 0;
1224
1225ImmutableModuleSummaryIndexWrapperPass::ImmutableModuleSummaryIndexWrapperPass(
1226 const ModuleSummaryIndex *Index)
1227 : ImmutablePass(ID), Index(Index) {}
1228
1229void ImmutableModuleSummaryIndexWrapperPass::getAnalysisUsage(
1230 AnalysisUsage &AU) const {
1231 AU.setPreservesAll();
1232}
1233
1234ImmutablePass *llvm::createImmutableModuleSummaryIndexWrapperPass(
1235 const ModuleSummaryIndex *Index) {
1236 return new ImmutableModuleSummaryIndexWrapperPass(Index);
1237}
1238
1239INITIALIZE_PASS(ImmutableModuleSummaryIndexWrapperPass, "module-summary-info",
1240 "Module summary info", false, true)
1241
1242bool llvm::mayHaveMemprofSummary(const CallBase *CB) {
1243 if (!CB)
1244 return false;
1245 if (CB->isDebugOrPseudoInst())
1246 return false;
1247 auto *CI = dyn_cast<CallInst>(Val: CB);
1248 auto *CalledValue = CB->getCalledOperand();
1249 auto *CalledFunction = CB->getCalledFunction();
1250 if (CalledValue && !CalledFunction) {
1251 CalledValue = CalledValue->stripPointerCasts();
1252 // Stripping pointer casts can reveal a called function.
1253 CalledFunction = dyn_cast<Function>(Val: CalledValue);
1254 }
1255 // Check if this is an alias to a function. If so, get the
1256 // called aliasee for the checks below.
1257 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
1258 assert(!CalledFunction &&
1259 "Expected null called function in callsite for alias");
1260 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
1261 }
1262 // Check if this is a direct call to a known function or a known
1263 // intrinsic, or an indirect call with profile data.
1264 if (CalledFunction) {
1265 if (CI && CalledFunction->isIntrinsic())
1266 return false;
1267 } else {
1268 // Skip indirect calls if we haven't enabled memprof ICP.
1269 if (!EnableMemProfIndirectCallSupport)
1270 return false;
1271 // Skip inline assembly calls.
1272 if (CI && CI->isInlineAsm())
1273 return false;
1274 // Skip direct calls via Constant.
1275 if (!CalledValue || isa<Constant>(Val: CalledValue))
1276 return false;
1277 return true;
1278 }
1279 return true;
1280}
1281