1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ModuleSummaryAnalysis.h"
15#include "AnalysisOptions.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseSet.h"
18#include "llvm/ADT/MapVector.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Analysis/BlockFrequencyInfo.h"
25#include "llvm/Analysis/BranchProbabilityInfo.h"
26#include "llvm/Analysis/ConstantFolding.h"
27#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
28#include "llvm/Analysis/LoopInfo.h"
29#include "llvm/Analysis/MemoryProfileInfo.h"
30#include "llvm/Analysis/ProfileSummaryInfo.h"
31#include "llvm/Analysis/StackSafetyAnalysis.h"
32#include "llvm/Analysis/TypeMetadataUtils.h"
33#include "llvm/IR/Attributes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/CycleInfo.h"
38#include "llvm/IR/Dominators.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalAlias.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/GlobalVariable.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/ModuleSummaryIndex.h"
49#include "llvm/IR/Use.h"
50#include "llvm/IR/User.h"
51#include "llvm/InitializePasses.h"
52#include "llvm/Object/ModuleSymbolTable.h"
53#include "llvm/Object/SymbolicFile.h"
54#include "llvm/Pass.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/Compiler.h"
57#include "llvm/Support/FileSystem.h"
58#include <cassert>
59#include <cstdint>
60#include <vector>
61
62using namespace llvm;
63using namespace llvm::memprof;
64
65#define DEBUG_TYPE "module-summary-analysis"
66
67unsigned llvm::getMaxSummaryIndirectEdges(const LLVMContext &C) {
68 return getAnalysisOptions(C).module_summary_max_indirect_edges;
69}
70
71// Walk through the operands of a given User via worklist iteration and populate
72// the set of GlobalValue references encountered. Invoked either on an
73// Instruction or a GlobalVariable (which walks its initializer).
74// Return true if any of the operands contains blockaddress. This is important
75// to know when computing summary for global var, because if global variable
76// references basic block address we can't import it separately from function
77// containing that basic block. For simplicity we currently don't import such
78// global vars at all. When importing function we aren't interested if any
79// instruction in it takes an address of any basic block, because instruction
80// can only take an address of basic block located in the same function.
81// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
82// local-linkage ifunc.
83static bool
84findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
85 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &RefEdges,
86 SmallPtrSet<const User *, 8> &Visited,
87 bool &RefLocalLinkageIFunc) {
88 bool HasBlockAddress = false;
89 SmallVector<const User *, 32> Worklist;
90 if (Visited.insert(Ptr: CurUser).second)
91 Worklist.push_back(Elt: CurUser);
92
93 while (!Worklist.empty()) {
94 const User *U = Worklist.pop_back_val();
95 const auto *CB = dyn_cast<CallBase>(Val: U);
96
97 for (const auto &OI : U->operands()) {
98 const User *Operand = dyn_cast<User>(Val: OI);
99 if (!Operand)
100 continue;
101 if (isa<BlockAddress>(Val: Operand)) {
102 HasBlockAddress = true;
103 continue;
104 }
105 if (auto *GV = dyn_cast<GlobalValue>(Val: Operand)) {
106 // We have a reference to a global value. This should be added to
107 // the reference set unless it is a callee. Callees are handled
108 // specially by WriteFunction and are added to a separate list.
109 if (!(CB && CB->isCallee(U: &OI))) {
110 // If an ifunc has local linkage, do not add it into ref edges, and
111 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
112 // for import. An ifunc doesn't have summary and ThinLTO cannot
113 // promote it; importing the referencer may cause linkage errors.
114 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: GV);
115 GI && GI->hasLocalLinkage()) {
116 RefLocalLinkageIFunc = true;
117 continue;
118 }
119 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
120 }
121 continue;
122 }
123 if (Visited.insert(Ptr: Operand).second)
124 Worklist.push_back(Elt: Operand);
125 }
126 }
127
128 const Instruction *I = dyn_cast<Instruction>(Val: CurUser);
129 if (I) {
130 uint64_t TotalCount = 0;
131 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
132 // the instruction.
133 auto ValueDataArray = getValueProfDataFromInst(
134 Inst: *I, ValueKind: IPVK_VTableTarget, MaxNumValueData: getMaxNumVTableAnnotations(C: I->getContext()),
135 TotalC&: TotalCount);
136
137 for (const auto &V : ValueDataArray)
138 RefEdges.insert(X: Index.getOrInsertValueInfo(/* VTableGUID = */
139 GUID: V.Value));
140 }
141 return HasBlockAddress;
142}
143
144/// Collect globals referenced via !implicit.ref metadata on a function
145/// and add them as reference edges in the module summary. This ensures
146/// ThinLTO liveness analysis treats them as live when the function is
147/// live, and imports them alongside the function during cross-module
148/// importing.
149static void findImplicitRefEdges(
150 ModuleSummaryIndex &Index, const Function &F,
151 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &RefEdges) {
152 if (!F.hasMetadata(KindID: LLVMContext::MD_implicit_ref))
153 return;
154 SmallVector<MDNode *, 4> MDs;
155 F.getMetadata(KindID: LLVMContext::MD_implicit_ref, MDs);
156 for (MDNode *MD : MDs) {
157 for (const MDOperand &Op : MD->operands()) {
158 if (auto *VAM = dyn_cast_or_null<ValueAsMetadata>(Val: Op.get()))
159 if (auto *GV = dyn_cast<GlobalValue>(Val: VAM->getValue()))
160 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
161 }
162 }
163}
164
165static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
166 ProfileSummaryInfo *PSI) {
167 if (!PSI)
168 return CalleeInfo::HotnessType::Unknown;
169 if (PSI->isHotCount(C: ProfileCount))
170 return CalleeInfo::HotnessType::Hot;
171 if (PSI->isColdCount(C: ProfileCount))
172 return CalleeInfo::HotnessType::Cold;
173 return CalleeInfo::HotnessType::None;
174}
175
176static bool isNonRenamableLocal(const GlobalValue &GV) {
177 return GV.hasSection() && GV.hasLocalLinkage();
178}
179
180/// Determine whether this call has all constant integer arguments (excluding
181/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
182static void addVCallToSet(
183 DevirtCallSite Call, GlobalValue::GUID Guid,
184 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
185 &VCalls,
186 SetVector<FunctionSummary::ConstVCall,
187 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
188 std::vector<uint64_t> Args;
189 // Start from the second argument to skip the "this" pointer.
190 for (auto &Arg : drop_begin(RangeOrContainer: Call.CB.args())) {
191 auto *CI = dyn_cast<ConstantInt>(Val&: Arg);
192 if (!CI || CI->getBitWidth() > 64) {
193 VCalls.insert(X: {.GUID: Guid, .Offset: Call.Offset});
194 return;
195 }
196 Args.push_back(x: CI->getZExtValue());
197 }
198 ConstVCalls.insert(X: {.VFunc: {.GUID: Guid, .Offset: Call.Offset}, .Args: std::move(Args)});
199}
200
201/// If this intrinsic call requires that we add information to the function
202/// summary, do so via the non-constant reference arguments.
203static void addIntrinsicToSummary(
204 const CallInst *CI,
205 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
206 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
207 &TypeTestAssumeVCalls,
208 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
209 &TypeCheckedLoadVCalls,
210 SetVector<FunctionSummary::ConstVCall,
211 std::vector<FunctionSummary::ConstVCall>>
212 &TypeTestAssumeConstVCalls,
213 SetVector<FunctionSummary::ConstVCall,
214 std::vector<FunctionSummary::ConstVCall>>
215 &TypeCheckedLoadConstVCalls,
216 DominatorTree &DT) {
217 switch (CI->getCalledFunction()->getIntrinsicID()) {
218 case Intrinsic::type_test:
219 case Intrinsic::public_type_test: {
220 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
221 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
222 if (!TypeId)
223 break;
224 GlobalValue::GUID Guid =
225 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
226
227 // Produce a summary from type.test intrinsics. We only summarize type.test
228 // intrinsics that are used other than by an llvm.assume intrinsic.
229 // Intrinsics that are assumed are relevant only to the devirtualization
230 // pass, not the type test lowering pass.
231 bool HasNonAssumeUses = llvm::any_of(Range: CI->uses(), P: [](const Use &CIU) {
232 return !isa<AssumeInst>(Val: CIU.getUser());
233 });
234 if (HasNonAssumeUses)
235 TypeTests.insert(X: Guid);
236
237 SmallVector<DevirtCallSite, 4> DevirtCalls;
238 SmallVector<CallInst *, 4> Assumes;
239 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
240 for (auto &Call : DevirtCalls)
241 addVCallToSet(Call, Guid, VCalls&: TypeTestAssumeVCalls,
242 ConstVCalls&: TypeTestAssumeConstVCalls);
243
244 break;
245 }
246
247 case Intrinsic::type_checked_load_relative:
248 case Intrinsic::type_checked_load: {
249 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2));
250 auto *TypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
251 if (!TypeId)
252 break;
253 GlobalValue::GUID Guid =
254 GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: TypeId->getString());
255
256 SmallVector<DevirtCallSite, 4> DevirtCalls;
257 SmallVector<Instruction *, 4> LoadedPtrs;
258 SmallVector<Instruction *, 4> Preds;
259 bool HasNonCallUses = false;
260 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
261 HasNonCallUses, CI, DT);
262 // Any non-call uses of the result of llvm.type.checked.load will
263 // prevent us from optimizing away the llvm.type.test.
264 if (HasNonCallUses)
265 TypeTests.insert(X: Guid);
266 for (auto &Call : DevirtCalls)
267 addVCallToSet(Call, Guid, VCalls&: TypeCheckedLoadVCalls,
268 ConstVCalls&: TypeCheckedLoadConstVCalls);
269
270 break;
271 }
272 default:
273 break;
274 }
275}
276
277static bool isNonVolatileLoad(const Instruction *I) {
278 if (const auto *LI = dyn_cast<LoadInst>(Val: I))
279 return !LI->isVolatile();
280
281 return false;
282}
283
284static bool isNonVolatileStore(const Instruction *I) {
285 if (const auto *SI = dyn_cast<StoreInst>(Val: I))
286 return !SI->isVolatile();
287
288 return false;
289}
290
291// Returns true if the function definition must be unreachable.
292//
293// Note if this helper function returns true, `F` is guaranteed
294// to be unreachable; if it returns false, `F` might still
295// be unreachable but not covered by this helper function.
296static bool mustBeUnreachableFunction(const Function &F) {
297 // A function must be unreachable if its entry block ends with an
298 // 'unreachable'.
299 assert(!F.isDeclaration());
300 return isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator());
301}
302
303static void computeFunctionSummary(
304 const AnalysisOptions &Opts, ModuleSummaryIndex &Index, const Module &M,
305 const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
306 DominatorTree &DT, bool HasLocalsInUsedOrAsm,
307 DenseSet<GlobalValue::GUID> &CantBePromoted, bool IsThinLTO,
308 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
309 // Summary not currently supported for anonymous functions, they should
310 // have been named.
311 assert(F.hasName());
312
313 unsigned NumInsts = 0;
314 // Map from callee ValueId to profile count. Used to accumulate profile
315 // counts for all static calls to a given callee.
316 MapVector<ValueInfo, CalleeInfo, DenseMap<ValueInfo, unsigned>,
317 SmallVector<FunctionSummary::EdgeTy, 0>>
318 CallGraphEdges;
319 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges, LoadRefEdges,
320 StoreRefEdges;
321 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> TypeTests;
322 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
323 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
324 SetVector<FunctionSummary::ConstVCall,
325 std::vector<FunctionSummary::ConstVCall>>
326 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
327 ICallPromotionAnalysis ICallAnalysis;
328 SmallPtrSet<const User *, 8> Visited;
329
330 // Add personality function, prefix data and prologue data to function's ref
331 // list.
332 bool HasLocalIFuncCallOrRef = false;
333 findRefEdges(Index, CurUser: &F, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
334 findImplicitRefEdges(Index, F, RefEdges);
335
336 std::vector<const Instruction *> NonVolatileLoads;
337 std::vector<const Instruction *> NonVolatileStores;
338
339 std::vector<CallsiteInfo> Callsites;
340 std::vector<AllocInfo> Allocs;
341
342#ifndef NDEBUG
343 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
344#endif
345
346 bool HasInlineAsmMaybeReferencingInternal = false;
347 bool HasIndirBranchToBlockAddress = false;
348 bool HasUnknownCall = false;
349 bool MayThrow = false;
350 for (const BasicBlock &BB : F) {
351 // We don't allow inlining of function with indirect branch to blockaddress.
352 // If the blockaddress escapes the function, e.g., via a global variable,
353 // inlining may lead to an invalid cross-function reference. So we shouldn't
354 // import such function either.
355 if (BB.hasAddressTaken()) {
356 for (User *U : BlockAddress::get(BB: const_cast<BasicBlock *>(&BB))->users())
357 if (!isa<CallBrInst>(Val: *U)) {
358 HasIndirBranchToBlockAddress = true;
359 break;
360 }
361 }
362
363 for (const Instruction &I : BB) {
364 if (isa<PseudoProbeInst>(Val: I))
365 continue;
366 ++NumInsts;
367
368 // Regular LTO module doesn't participate in ThinLTO import,
369 // so no reference from it can be read/writeonly, since this
370 // would require importing variable as local copy
371 if (IsThinLTO) {
372 if (isNonVolatileLoad(I: &I)) {
373 // Postpone processing of non-volatile load instructions
374 // See comments below
375 Visited.insert(Ptr: &I);
376 NonVolatileLoads.push_back(x: &I);
377 continue;
378 } else if (isNonVolatileStore(I: &I)) {
379 Visited.insert(Ptr: &I);
380 NonVolatileStores.push_back(x: &I);
381 // All references from second operand of store (destination address)
382 // can be considered write-only if they're not referenced by any
383 // non-store instruction. References from first operand of store
384 // (stored value) can't be treated either as read- or as write-only
385 // so we add them to RefEdges as we do with all other instructions
386 // except non-volatile load.
387 Value *Stored = I.getOperand(i: 0);
388 if (auto *GV = dyn_cast<GlobalValue>(Val: Stored))
389 // findRefEdges will try to examine GV operands, so instead
390 // of calling it we should add GV to RefEdges directly.
391 RefEdges.insert(X: Index.getOrInsertValueInfo(GV));
392 else if (auto *U = dyn_cast<User>(Val: Stored))
393 findRefEdges(Index, CurUser: U, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
394 continue;
395 }
396 }
397 findRefEdges(Index, CurUser: &I, RefEdges, Visited, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
398 const auto *CB = dyn_cast<CallBase>(Val: &I);
399 if (!CB) {
400 if (I.mayThrow())
401 MayThrow = true;
402 continue;
403 }
404
405 const auto *CI = dyn_cast<CallInst>(Val: &I);
406 // Since we don't know exactly which local values are referenced in inline
407 // assembly, conservatively mark the function as possibly referencing
408 // a local value from inline assembly to ensure we don't export a
409 // reference (which would require renaming and promotion of the
410 // referenced value).
411 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
412 HasInlineAsmMaybeReferencingInternal = true;
413
414 // Compute this once per indirect call.
415 uint32_t NumCandidates = 0;
416 uint64_t TotalCount = 0;
417 MutableArrayRef<InstrProfValueData> CandidateProfileData;
418
419 auto *CalledValue = CB->getCalledOperand();
420 auto *CalledFunction = CB->getCalledFunction();
421 if (CalledValue && !CalledFunction) {
422 CalledValue = CalledValue->stripPointerCasts();
423 // Stripping pointer casts can reveal a called function.
424 CalledFunction = dyn_cast<Function>(Val: CalledValue);
425 }
426 // Check if this is an alias to a function. If so, get the
427 // called aliasee for the checks below.
428 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
429 assert(!CalledFunction && "Expected null called function in callsite for alias");
430 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
431 }
432 // Check if this is a direct call to a known function or a known
433 // intrinsic, or an indirect call with profile data.
434 if (CalledFunction) {
435 if (CI && CalledFunction->isIntrinsic()) {
436 addIntrinsicToSummary(
437 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
438 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
439 continue;
440 }
441 // We should have named any anonymous globals
442 assert(CalledFunction->hasName());
443 auto ScaledCount = PSI->getProfileCount(CallInst: *CB, BFI);
444 auto Hotness = ScaledCount ? getHotness(ProfileCount: *ScaledCount, PSI)
445 : CalleeInfo::HotnessType::Unknown;
446 if (Opts.force_summary_edges_cold != ForceSummaryHotness::None)
447 Hotness = CalleeInfo::HotnessType::Cold;
448
449 // Use the original CalledValue, in case it was an alias. We want
450 // to record the call edge to the alias in that case. Eventually
451 // an alias summary will be created to associate the alias and
452 // aliasee.
453 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
454 GV: cast<GlobalValue>(Val: CalledValue))];
455 ValueInfo.updateHotness(OtherHotness: Hotness);
456 if (CB->isTailCall())
457 ValueInfo.setHasTailCall(true);
458 } else {
459 HasUnknownCall = true;
460 // If F is imported, a local linkage ifunc (e.g. target_clones on a
461 // static function) called by F will be cloned. Since summaries don't
462 // track ifunc, we do not know implementation functions referenced by
463 // the ifunc resolver need to be promoted in the exporter, and we will
464 // get linker errors due to cloned declarations for implementation
465 // functions. As a simple fix, just mark F as not eligible for import.
466 // Non-local ifunc is not cloned and does not have the issue.
467 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(Val: CalledValue))
468 if (GI->hasLocalLinkage())
469 HasLocalIFuncCallOrRef = true;
470 // Skip inline assembly calls.
471 if (CI && CI->isInlineAsm())
472 continue;
473 // Skip direct calls.
474 if (!CalledValue || isa<Constant>(Val: CalledValue))
475 continue;
476
477 // Check if the instruction has a callees metadata. If so, add callees
478 // to CallGraphEdges to reflect the references from the metadata, and
479 // to enable importing for subsequent indirect call promotion and
480 // inlining.
481 if (auto *MD = I.getMetadata(KindID: LLVMContext::MD_callees)) {
482 for (const auto &Op : MD->operands()) {
483 Function *Callee = mdconst::extract_or_null<Function>(MD: Op);
484 if (Callee)
485 CallGraphEdges[Index.getOrInsertValueInfo(GV: Callee)]
486 .setHasIndirectCall(true);
487 }
488 }
489
490 CandidateProfileData =
491 ICallAnalysis.getPromotionCandidatesForInstruction(
492 I: &I, TotalCount, NumCandidates,
493 MaxNumValueData: Opts.module_summary_max_indirect_edges);
494 for (const auto &Candidate : CandidateProfileData) {
495 auto &Edge =
496 CallGraphEdges[Index.getOrInsertValueInfo(GUID: Candidate.Value)];
497 Edge.updateHotness(OtherHotness: getHotness(ProfileCount: Candidate.Count, PSI));
498 Edge.setHasIndirectCall(true);
499 // Record that the profiled target was reached via a tail call, so
500 // that MemProf context disambiguation can identify frames missing
501 // from profiled contexts due to indirect tail calls (see
502 // findProfiledCalleeThroughTailCalls).
503 if (CB->isTailCall())
504 Edge.setHasTailCall(true);
505 }
506 }
507
508 // Summarize memprof related metadata. This is only needed for ThinLTO.
509 if (!IsThinLTO)
510 continue;
511
512 // Skip indirect calls if we haven't enabled memprof ICP.
513 if (!CalledFunction && !Opts.enable_memprof_indirect_call_support)
514 continue;
515
516 // Ensure we keep this analysis in sync with the handling in the ThinLTO
517 // backend (see MemProfContextDisambiguation::applyImport). Save this call
518 // so that we can skip it in checking the reverse case later.
519 assert(mayHaveMemprofSummary(CB));
520#ifndef NDEBUG
521 CallsThatMayHaveMemprofSummary.insert(CB);
522#endif
523
524 // Compute the list of stack ids first (so we can trim them from the stack
525 // ids on any MIBs).
526 CallStack<MDNode, MDNode::op_iterator> InstCallsite(
527 I.getMetadata(KindID: LLVMContext::MD_callsite));
528 auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
529 if (MemProfMD) {
530 std::vector<MIBInfo> MIBs;
531 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
532 bool HasNonZeroContextSizeInfos = false;
533 for (auto &MDOp : MemProfMD->operands()) {
534 auto *MIBMD = cast<const MDNode>(Val: MDOp);
535 MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
536 assert(StackNode);
537 SmallVector<unsigned> StackIdIndices;
538 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
539 // Collapse out any on the allocation call (inlining).
540 for (auto ContextIter =
541 StackContext.beginAfterSharedPrefix(Other: InstCallsite);
542 ContextIter != StackContext.end(); ++ContextIter) {
543 unsigned StackIdIdx = Index.addOrGetStackIdIndex(StackId: *ContextIter);
544 // If this is a direct recursion, simply skip the duplicate
545 // entries. If this is mutual recursion, handling is left to
546 // the LTO link analysis client.
547 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
548 StackIdIndices.push_back(Elt: StackIdIdx);
549 }
550 // If we have context size information, collect it for inclusion in
551 // the summary.
552 assert(MIBMD->getNumOperands() > 2 ||
553 !metadataIncludesAllContextSizeInfo());
554 if (MIBMD->getNumOperands() > 2) {
555 std::vector<ContextTotalSize> ContextSizes;
556 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
557 MDNode *ContextSizePair = dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
558 assert(ContextSizePair->getNumOperands() == 2);
559 uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
560 MD: ContextSizePair->getOperand(I: 0))
561 ->getZExtValue();
562 uint64_t TS = mdconst::dyn_extract<ConstantInt>(
563 MD: ContextSizePair->getOperand(I: 1))
564 ->getZExtValue();
565 ContextSizes.push_back(x: {.FullStackId: FullStackId, .TotalSize: TS});
566 }
567 // Flag that we need to keep the ContextSizeInfos array for this
568 // alloc as it now contains non-zero context info sizes.
569 HasNonZeroContextSizeInfos = true;
570 ContextSizeInfos.push_back(x: std::move(ContextSizes));
571 } else {
572 // The ContextSizeInfos must be in the same relative position as the
573 // associated MIB. In some cases we only include a ContextSizeInfo
574 // for a subset of MIBs in an allocation. To handle that, eagerly
575 // fill any MIB entries that don't have context size info metadata
576 // with a pair of 0s. Later on we will only use this array if it
577 // ends up containing any non-zero entries (see where we set
578 // HasNonZeroContextSizeInfos above).
579 ContextSizeInfos.push_back(x: {{.FullStackId: 0, .TotalSize: 0}});
580 }
581 MIBs.push_back(
582 x: MIBInfo(getMIBAllocType(MIB: MIBMD), std::move(StackIdIndices)));
583 }
584 Allocs.push_back(x: AllocInfo(std::move(MIBs)));
585 assert(HasNonZeroContextSizeInfos ||
586 !metadataIncludesAllContextSizeInfo());
587 // We eagerly build the ContextSizeInfos array, but it will be filled
588 // with sub arrays of pairs of 0s if no MIBs on this alloc actually
589 // contained context size info metadata. Only save it if any MIBs had
590 // any such metadata.
591 if (HasNonZeroContextSizeInfos) {
592 assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
593 Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
594 }
595 } else if (!InstCallsite.empty()) {
596 SmallVector<unsigned> StackIdIndices;
597 for (auto StackId : InstCallsite)
598 StackIdIndices.push_back(Elt: Index.addOrGetStackIdIndex(StackId));
599 if (CalledFunction) {
600 // Use the original CalledValue, in case it was an alias. We want
601 // to record the call edge to the alias in that case. Eventually
602 // an alias summary will be created to associate the alias and
603 // aliasee.
604 auto CalleeValueInfo =
605 Index.getOrInsertValueInfo(GV: cast<GlobalValue>(Val: CalledValue));
606 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
607 } else {
608 assert(Opts.enable_memprof_indirect_call_support);
609 // For indirect callsites, create multiple Callsites, one per target.
610 // This enables having a different set of clone versions per target,
611 // and we will apply the cloning decisions while speculatively
612 // devirtualizing in the ThinLTO backends.
613 for (const auto &Candidate : CandidateProfileData) {
614 auto CalleeValueInfo = Index.getOrInsertValueInfo(GUID: Candidate.Value);
615 Callsites.push_back(x: {CalleeValueInfo, StackIdIndices});
616 }
617 }
618 }
619 }
620 }
621
622 if (PSI->hasPartialSampleProfile() &&
623 Opts.scale_partial_sample_profile_working_set_size)
624 Index.addBlockCount(C: F.size());
625
626 SmallVector<ValueInfo, 0> Refs;
627 if (IsThinLTO) {
628 auto AddRefEdges =
629 [&](const std::vector<const Instruction *> &Instrs,
630 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &Edges,
631 SmallPtrSet<const User *, 8> &Cache) {
632 for (const auto *I : Instrs) {
633 Cache.erase(Ptr: I);
634 findRefEdges(Index, CurUser: I, RefEdges&: Edges, Visited&: Cache, RefLocalLinkageIFunc&: HasLocalIFuncCallOrRef);
635 }
636 };
637
638 // By now we processed all instructions in a function, except
639 // non-volatile loads and non-volatile value stores. Let's find
640 // ref edges for both of instruction sets
641 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
642 // We can add some values to the Visited set when processing load
643 // instructions which are also used by stores in NonVolatileStores.
644 // For example this can happen if we have following code:
645 //
646 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
647 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
648 //
649 // After processing loads we'll add bitcast to the Visited set, and if
650 // we use the same set while processing stores, we'll never see store
651 // to @bar and @bar will be mistakenly treated as readonly.
652 SmallPtrSet<const llvm::User *, 8> StoreCache;
653 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
654
655 // If both load and store instruction reference the same variable
656 // we won't be able to optimize it. Add all such reference edges
657 // to RefEdges set.
658 for (const auto &VI : StoreRefEdges)
659 if (LoadRefEdges.remove(X: VI))
660 RefEdges.insert(X: VI);
661
662 unsigned RefCnt = RefEdges.size();
663 // All new reference edges inserted in two loops below are either
664 // read or write only. They will be grouped in the end of RefEdges
665 // vector, so we can use a single integer value to identify them.
666 RefEdges.insert_range(R&: LoadRefEdges);
667
668 unsigned FirstWORef = RefEdges.size();
669 RefEdges.insert_range(R&: StoreRefEdges);
670
671 Refs = RefEdges.takeVector();
672 for (; RefCnt < FirstWORef; ++RefCnt)
673 Refs[RefCnt].setReadOnly();
674
675 for (; RefCnt < Refs.size(); ++RefCnt)
676 Refs[RefCnt].setWriteOnly();
677 } else {
678 Refs = RefEdges.takeVector();
679 }
680 // Explicit add hot edges to enforce importing for designated GUIDs for
681 // sample PGO, to enable the same inlines as the profiled optimized binary.
682 for (auto &I : F.getImportGUIDs())
683 CallGraphEdges[Index.getOrInsertValueInfo(GUID: I)].updateHotness(
684 OtherHotness: Opts.force_summary_edges_cold == ForceSummaryHotness::All
685 ? CalleeInfo::HotnessType::Cold
686 : CalleeInfo::HotnessType::Critical);
687
688#ifndef NDEBUG
689 // Make sure that all calls we decided could not have memprof summaries get a
690 // false value for mayHaveMemprofSummary, to ensure that this handling remains
691 // in sync with the ThinLTO backend handling.
692 if (IsThinLTO) {
693 for (const BasicBlock &BB : F) {
694 for (const Instruction &I : BB) {
695 const auto *CB = dyn_cast<CallBase>(&I);
696 if (!CB)
697 continue;
698 // We already checked these above.
699 if (CallsThatMayHaveMemprofSummary.count(CB))
700 continue;
701 assert(!mayHaveMemprofSummary(CB));
702 }
703 }
704 }
705#endif
706
707 bool NonRenamableLocal = isNonRenamableLocal(GV: F);
708 bool NotEligibleForImport =
709 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
710 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
711 GlobalValueSummary::GVFlags Flags(
712 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
713 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
714 GlobalValueSummary::ImportKind::Definition,
715 /* NoRenameOnPromotion = */ false);
716 FunctionSummary::FFlags FunFlags{
717 .ReadNone: F.doesNotAccessMemory(), .ReadOnly: F.onlyReadsMemory() && !F.doesNotAccessMemory(),
718 .NoRecurse: F.hasFnAttribute(Kind: Attribute::NoRecurse), .ReturnDoesNotAlias: F.returnDoesNotAlias(),
719 // FIXME: refactor this to use the same code that inliner is using.
720 // Don't try to import functions with noinline attribute.
721 .NoInline: F.isNoInline(), .AlwaysInline: F.hasFnAttribute(Kind: Attribute::AlwaysInline),
722 .NoUnwind: F.hasFnAttribute(Kind: Attribute::NoUnwind), .MayThrow: MayThrow, .HasUnknownCall: HasUnknownCall,
723 .MustBeUnreachable: mustBeUnreachableFunction(F)};
724 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
725 if (auto *SSI = GetSSICallback(F))
726 ParamAccesses = SSI->getParamAccesses(Index);
727 auto FuncSummary = std::make_unique<FunctionSummary>(
728 args&: Flags, args&: NumInsts, args&: FunFlags, args: std::move(Refs), args: CallGraphEdges.takeVector(),
729 args: TypeTests.takeVector(), args: TypeTestAssumeVCalls.takeVector(),
730 args: TypeCheckedLoadVCalls.takeVector(),
731 args: TypeTestAssumeConstVCalls.takeVector(),
732 args: TypeCheckedLoadConstVCalls.takeVector(), args: std::move(ParamAccesses),
733 args: std::move(Callsites), args: std::move(Allocs));
734 if (NonRenamableLocal)
735 CantBePromoted.insert(V: F.getGUID());
736 Index.addGlobalValueSummary(GV: F, Summary: std::move(FuncSummary));
737}
738
739/// Find function pointers referenced within the given vtable initializer
740/// (or subset of an initializer) \p I. The starting offset of \p I within
741/// the vtable initializer is \p StartingOffset. Any discovered function
742/// pointers are added to \p VTableFuncs along with their cumulative offset
743/// within the initializer.
744static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
745 const Module &M, ModuleSummaryIndex &Index,
746 VTableFuncList &VTableFuncs,
747 const GlobalVariable &OrigGV) {
748 // First check if this is a function pointer.
749 if (I->getType()->isPointerTy()) {
750 auto C = I->stripPointerCasts();
751 auto A = dyn_cast<GlobalAlias>(Val: C);
752 if (isa<Function>(Val: C) || (A && isa<Function>(Val: A->getAliasee()))) {
753 auto GV = dyn_cast<GlobalValue>(Val: C);
754 assert(GV);
755 // We can disregard __cxa_pure_virtual as a possible call target, as
756 // calls to pure virtuals are UB.
757 if (GV && GV->getName() != "__cxa_pure_virtual")
758 VTableFuncs.push_back(x: {Index.getOrInsertValueInfo(GV), StartingOffset});
759 return;
760 }
761 }
762
763 // Walk through the elements in the constant struct or array and recursively
764 // look for virtual function pointers.
765 const DataLayout &DL = M.getDataLayout();
766 if (auto *C = dyn_cast<ConstantStruct>(Val: I)) {
767 StructType *STy = C->getType();
768 assert(STy);
769 const StructLayout *SL = DL.getStructLayout(Ty: C->getType());
770
771 for (auto EI : llvm::enumerate(First: STy->elements())) {
772 auto Offset = SL->getElementOffset(Idx: EI.index());
773 unsigned Op = SL->getElementContainingOffset(FixedOffset: Offset);
774 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i: Op)),
775 StartingOffset: StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
776 }
777 } else if (auto *C = dyn_cast<ConstantArray>(Val: I)) {
778 ArrayType *ATy = C->getType();
779 Type *EltTy = ATy->getElementType();
780 uint64_t EltSize = DL.getTypeAllocSize(Ty: EltTy);
781 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
782 findFuncPointers(I: cast<Constant>(Val: I->getOperand(i)),
783 StartingOffset: StartingOffset + i * EltSize, M, Index, VTableFuncs,
784 OrigGV);
785 }
786 } else if (const auto *CE = dyn_cast<ConstantExpr>(Val: I)) {
787 // For relative vtables, the next sub-component should be a trunc.
788 if (CE->getOpcode() != Instruction::Trunc ||
789 !(CE = dyn_cast<ConstantExpr>(Val: CE->getOperand(i_nocapture: 0))))
790 return;
791
792 // If this constant can be reduced to the offset between a function and a
793 // global, then we know this is a valid virtual function if the RHS is the
794 // original vtable we're scanning through.
795 if (CE->getOpcode() == Instruction::Sub) {
796 GlobalValue *LHS, *RHS;
797 APSInt LHSOffset, RHSOffset;
798 if (IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 0), GV&: LHS, Offset&: LHSOffset, DL) &&
799 IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 1), GV&: RHS, Offset&: RHSOffset, DL) &&
800 RHS == &OrigGV &&
801
802 // For relative vtables, this component should point to the callable
803 // function without any offsets.
804 LHSOffset == 0 &&
805
806 // Also, the RHS should always point to somewhere within the vtable.
807 RHSOffset <=
808 static_cast<uint64_t>(DL.getTypeAllocSize(Ty: OrigGV.getInitializer()->getType()))) {
809 findFuncPointers(I: LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
810 }
811 }
812 }
813}
814
815// Identify the function pointers referenced by vtable definition \p V.
816static void computeVTableFuncs(ModuleSummaryIndex &Index,
817 const GlobalVariable &V, const Module &M,
818 VTableFuncList &VTableFuncs) {
819 if (!V.isConstant())
820 return;
821
822 findFuncPointers(I: V.getInitializer(), /*StartingOffset=*/0, M, Index,
823 VTableFuncs, OrigGV: V);
824
825#ifndef NDEBUG
826 // Validate that the VTableFuncs list is ordered by offset.
827 uint64_t PrevOffset = 0;
828 for (auto &P : VTableFuncs) {
829 // The findVFuncPointers traversal should have encountered the
830 // functions in offset order. We need to use ">=" since PrevOffset
831 // starts at 0.
832 assert(P.VTableOffset >= PrevOffset);
833 PrevOffset = P.VTableOffset;
834 }
835#endif
836}
837
838/// Record vtable definition \p V for each type metadata it references.
839static void
840recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index,
841 const GlobalVariable &V,
842 SmallVectorImpl<MDNode *> &Types) {
843 for (MDNode *Type : Types) {
844 auto TypeID = Type->getOperand(I: 1).get();
845
846 uint64_t Offset =
847 cast<ConstantInt>(
848 Val: cast<ConstantAsMetadata>(Val: Type->getOperand(I: 0))->getValue())
849 ->getZExtValue();
850
851 if (auto *TypeId = dyn_cast<MDString>(Val: TypeID))
852 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId: TypeId->getString())
853 .push_back(x: {Offset, Index.getOrInsertValueInfo(GV: &V)});
854 }
855}
856
857static void computeVariableSummary(ModuleSummaryIndex &Index,
858 const GlobalVariable &V,
859 DenseSet<GlobalValue::GUID> &CantBePromoted,
860 const Module &M,
861 SmallVectorImpl<MDNode *> &Types) {
862 SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges;
863 SmallPtrSet<const User *, 8> Visited;
864 bool RefLocalIFunc = false;
865 bool HasBlockAddress =
866 findRefEdges(Index, CurUser: &V, RefEdges, Visited, RefLocalLinkageIFunc&: RefLocalIFunc);
867 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
868 bool NonRenamableLocal = isNonRenamableLocal(GV: V);
869 GlobalValueSummary::GVFlags Flags(
870 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
871 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
872 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
873
874 VTableFuncList VTableFuncs;
875 // If splitting is not enabled, then we compute the summary information
876 // necessary for index-based whole program devirtualization.
877 if (!Index.enableSplitLTOUnit()) {
878 Types.clear();
879 V.getMetadata(KindID: LLVMContext::MD_type, MDs&: Types);
880 if (!Types.empty()) {
881 // Identify the function pointers referenced by this vtable definition.
882 computeVTableFuncs(Index, V, M, VTableFuncs);
883
884 // Record this vtable definition for each type metadata it references.
885 recordTypeIdCompatibleVtableReferences(Index, V, Types);
886 }
887 }
888
889 // Don't mark variables we won't be able to internalize as read/write-only.
890 bool CanBeInternalized =
891 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
892 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
893 bool Constant = V.isConstant();
894 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
895 Constant ? false : CanBeInternalized,
896 Constant, V.getVCallVisibility());
897 auto GVarSummary = std::make_unique<GlobalVarSummary>(args&: Flags, args&: VarFlags,
898 args: RefEdges.takeVector());
899 if (NonRenamableLocal)
900 CantBePromoted.insert(V: V.getGUID());
901 if (NotEligibleForImport)
902 GVarSummary->setNotEligibleToImport();
903 if (!VTableFuncs.empty())
904 GVarSummary->setVTableFuncs(VTableFuncs);
905 Index.addGlobalValueSummary(GV: V, Summary: std::move(GVarSummary));
906}
907
908static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
909 DenseSet<GlobalValue::GUID> &CantBePromoted) {
910 // Skip summary for indirect function aliases as summary for aliasee will not
911 // be emitted.
912 const GlobalObject *Aliasee = A.getAliaseeObject();
913 if (isa<GlobalIFunc>(Val: Aliasee))
914 return;
915 bool NonRenamableLocal = isNonRenamableLocal(GV: A);
916 GlobalValueSummary::GVFlags Flags(
917 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
918 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
919 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
920 auto AS = std::make_unique<AliasSummary>(args&: Flags);
921 auto AliaseeVI = Index.getValueInfo(GUID: Aliasee->getGUID());
922 assert(AliaseeVI && "Alias expects aliasee summary to be available");
923 assert(AliaseeVI.getSummaryList().size() == 1 &&
924 "Expected a single entry per aliasee in per-module index");
925 AS->setAliasee(AliaseeVI, Aliasee: AliaseeVI.getSummaryList()[0].get());
926 if (NonRenamableLocal)
927 CantBePromoted.insert(V: A.getGUID());
928 Index.addGlobalValueSummary(GV: A, Summary: std::move(AS));
929}
930
931// Set LiveRoot flag on entries matching the given value name.
932static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
933 if (ValueInfo VI =
934 Index.getValueInfo(GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: Name)))
935 for (const auto &Summary : VI.getSummaryList())
936 Summary->setLive(true);
937}
938
939ModuleSummaryIndex llvm::buildModuleSummaryIndex(
940 const Module &M,
941 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
942 ProfileSummaryInfo *PSI,
943 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
944 assert(PSI);
945 const AnalysisOptions &Opts = getAnalysisOptions(M);
946 bool EnableSplitLTOUnit = false;
947 bool UnifiedLTO = false;
948 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
949 MD: M.getModuleFlag(Key: "EnableSplitLTOUnit")))
950 EnableSplitLTOUnit = MD->getZExtValue();
951 if (auto *MD =
952 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "UnifiedLTO")))
953 UnifiedLTO = MD->getZExtValue();
954 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
955
956 // Identify the local values in the llvm.used and llvm.compiler.used sets,
957 // which should not be exported as they would then require renaming and
958 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
959 // here because we use this information to mark functions containing inline
960 // assembly calls as not importable.
961 SmallPtrSet<GlobalValue *, 4> LocalsUsed;
962 SmallVector<GlobalValue *, 4> Used;
963 // First collect those in the llvm.used set.
964 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/false);
965 // Next collect those in the llvm.compiler.used set.
966 collectUsedGlobalVariables(M, Vec&: Used, /*CompilerUsed=*/true);
967 DenseSet<GlobalValue::GUID> CantBePromoted;
968 for (auto *V : Used) {
969 if (V->hasLocalLinkage()) {
970 LocalsUsed.insert(Ptr: V);
971 CantBePromoted.insert(V: V->getGUID());
972 }
973 }
974
975 bool HasLocalInlineAsmSymbol = false;
976 if (!M.getModuleInlineAsm().empty()) {
977 // Collect the local values defined by module level asm, and set up
978 // summaries for these symbols so that they can be marked as NoRename,
979 // to prevent export of any use of them in regular IR that would require
980 // renaming within the module level asm. Note we don't need to create a
981 // summary for weak or global defs, as they don't need to be flagged as
982 // NoRename, and defs in module level asm can't be imported anyway.
983 // Also, any values used but not defined within module level asm should
984 // be listed on the llvm.used or llvm.compiler.used global and marked as
985 // referenced from there.
986 ModuleSymbolTable::CollectAsmSymbols(
987 M, AsmSymbol: [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
988 // Symbols not marked as Weak or Global are local definitions.
989 if (Flags & (object::BasicSymbolRef::SF_Weak |
990 object::BasicSymbolRef::SF_Global))
991 return;
992 HasLocalInlineAsmSymbol = true;
993 GlobalValue *GV = M.getNamedValue(Name);
994 if (!GV)
995 return;
996 assert(GV->isDeclaration() && "Def in module asm already has definition");
997 GlobalValueSummary::GVFlags GVFlags(
998 GlobalValue::InternalLinkage, GlobalValue::DefaultVisibility,
999 /* NotEligibleToImport = */ true,
1000 /* Live = */ true,
1001 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
1002 GlobalValueSummary::Definition,
1003 /* NoRenameOnPromotion = */ false);
1004 CantBePromoted.insert(V: GV->getGUID());
1005 // Create the appropriate summary type.
1006 if (Function *F = dyn_cast<Function>(Val: GV)) {
1007 std::unique_ptr<FunctionSummary> Summary =
1008 std::make_unique<FunctionSummary>(
1009 args&: GVFlags, /*InstCount=*/args: 0,
1010 args: FunctionSummary::FFlags{
1011 .ReadNone: F->hasFnAttribute(Kind: Attribute::ReadNone),
1012 .ReadOnly: F->hasFnAttribute(Kind: Attribute::ReadOnly),
1013 .NoRecurse: F->hasFnAttribute(Kind: Attribute::NoRecurse),
1014 .ReturnDoesNotAlias: F->returnDoesNotAlias(),
1015 /* NoInline = */ false,
1016 .AlwaysInline: F->hasFnAttribute(Kind: Attribute::AlwaysInline),
1017 .NoUnwind: F->hasFnAttribute(Kind: Attribute::NoUnwind),
1018 /* MayThrow */ true,
1019 /* HasUnknownCall */ true,
1020 /* MustBeUnreachable */ false},
1021 args: SmallVector<ValueInfo, 0>{},
1022 args: SmallVector<FunctionSummary::EdgeTy, 0>{},
1023 args: ArrayRef<GlobalValue::GUID>{},
1024 args: ArrayRef<FunctionSummary::VFuncId>{},
1025 args: ArrayRef<FunctionSummary::VFuncId>{},
1026 args: ArrayRef<FunctionSummary::ConstVCall>{},
1027 args: ArrayRef<FunctionSummary::ConstVCall>{},
1028 args: ArrayRef<FunctionSummary::ParamAccess>{},
1029 args: ArrayRef<CallsiteInfo>{}, args: ArrayRef<AllocInfo>{});
1030 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1031 } else {
1032 std::unique_ptr<GlobalVarSummary> Summary =
1033 std::make_unique<GlobalVarSummary>(
1034 args&: GVFlags,
1035 args: GlobalVarSummary::GVarFlags(
1036 false, false, cast<GlobalVariable>(Val: GV)->isConstant(),
1037 GlobalObject::VCallVisibilityPublic),
1038 args: SmallVector<ValueInfo, 0>{});
1039 Index.addGlobalValueSummary(GV: *GV, Summary: std::move(Summary));
1040 }
1041 });
1042 }
1043
1044 bool IsThinLTO = true;
1045 if (auto *MD =
1046 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
1047 IsThinLTO = MD->getZExtValue();
1048
1049 // Compute summaries for all functions defined in module, and save in the
1050 // index.
1051 for (const auto &F : M) {
1052 if (F.isDeclaration())
1053 continue;
1054
1055 DominatorTree DT(const_cast<Function &>(F));
1056 BlockFrequencyInfo *BFI = nullptr;
1057 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1058 if (F.hasProfileData()) {
1059 if (GetBFICallback) {
1060 BFI = GetBFICallback(F);
1061 } else {
1062 CycleInfo CI;
1063 CI.compute(F&: const_cast<Function &>(F));
1064 BranchProbabilityInfo BPI{F, CI};
1065 BFIPtr = std::make_unique<BlockFrequencyInfo>(args: F, args&: BPI, args&: CI);
1066 BFI = BFIPtr.get();
1067 }
1068 }
1069
1070 computeFunctionSummary(Opts, Index, M, F, BFI, PSI, DT,
1071 HasLocalsInUsedOrAsm: !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1072 CantBePromoted, IsThinLTO, GetSSICallback);
1073 }
1074
1075 // Compute summaries for all variables defined in module, and save in the
1076 // index.
1077 SmallVector<MDNode *, 2> Types;
1078 for (const GlobalVariable &G : M.globals()) {
1079 if (G.isDeclaration())
1080 continue;
1081 computeVariableSummary(Index, V: G, CantBePromoted, M, Types);
1082 }
1083
1084 // Compute summaries for all aliases defined in module, and save in the
1085 // index.
1086 for (const GlobalAlias &A : M.aliases())
1087 computeAliasSummary(Index, A, CantBePromoted);
1088
1089 // Iterate through ifuncs, set their resolvers all alive.
1090 for (const GlobalIFunc &I : M.ifuncs()) {
1091 I.applyAlongResolverPath(Op: [&Index](const GlobalValue &GV) {
1092 Index.getGlobalValueSummary(GV)->setLive(true);
1093 });
1094 }
1095
1096 for (auto *V : LocalsUsed) {
1097 auto *Summary = Index.getGlobalValueSummary(GV: *V);
1098 assert(Summary && "Missing summary for global value");
1099 Summary->setNotEligibleToImport();
1100 }
1101
1102 // The linker doesn't know about these LLVM produced values, so we need
1103 // to flag them as live in the index to ensure index-based dead value
1104 // analysis treats them as live roots of the analysis.
1105 setLiveRoot(Index, Name: "llvm.used");
1106 setLiveRoot(Index, Name: "llvm.compiler.used");
1107 setLiveRoot(Index, Name: "llvm.global_ctors");
1108 setLiveRoot(Index, Name: "llvm.global_dtors");
1109 setLiveRoot(Index, Name: "llvm.global.annotations");
1110
1111 for (auto &GlobalList : Index) {
1112 // Ignore entries for references that are undefined in the current module.
1113 if (GlobalList.second.getSummaryList().empty())
1114 continue;
1115
1116 assert(GlobalList.second.getSummaryList().size() == 1 &&
1117 "Expected module's index to have one summary per GUID");
1118 auto &Summary = GlobalList.second.getSummaryList()[0];
1119 if (!IsThinLTO) {
1120 Summary->setNotEligibleToImport();
1121 continue;
1122 }
1123
1124 bool AllRefsCanBeExternallyReferenced =
1125 llvm::all_of(Range: Summary->refs(), P: [&](const ValueInfo &VI) {
1126 return !CantBePromoted.count(V: VI.getGUID());
1127 });
1128 if (!AllRefsCanBeExternallyReferenced) {
1129 Summary->setNotEligibleToImport();
1130 continue;
1131 }
1132
1133 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Val: Summary.get())) {
1134 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1135 Range: FuncSummary->calls(), P: [&](const FunctionSummary::EdgeTy &Edge) {
1136 return !CantBePromoted.count(V: Edge.first.getGUID());
1137 });
1138 if (!AllCallsCanBeExternallyReferenced)
1139 Summary->setNotEligibleToImport();
1140 }
1141 }
1142
1143 if (!Opts.module_summary_dot_file.empty()) {
1144 std::error_code EC;
1145 raw_fd_ostream OSDot(Opts.module_summary_dot_file, EC,
1146 sys::fs::OpenFlags::OF_Text);
1147 if (EC)
1148 report_fatal_error(reason: Twine("Failed to open dot file ") +
1149 Opts.module_summary_dot_file + ": " + EC.message() +
1150 "\n");
1151 Index.exportToDot(OS&: OSDot, GUIDPreservedSymbols: {});
1152 }
1153
1154 return Index;
1155}
1156
1157AnalysisKey ModuleSummaryIndexAnalysis::Key;
1158AnalysisKey ImmutableModuleSummaryIndexAnalysis::Key;
1159
1160ModuleSummaryIndex
1161ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
1162 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(IR&: M);
1163 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1164 bool NeedSSI = needsParamAccessSummary(M);
1165 return buildModuleSummaryIndex(
1166 M,
1167 GetBFICallback: [&FAM](const Function &F) {
1168 return &FAM.getResult<BlockFrequencyAnalysis>(
1169 IR&: *const_cast<Function *>(&F));
1170 },
1171 PSI: &PSI,
1172 GetSSICallback: [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1173 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1174 IR&: const_cast<Function &>(F))
1175 : nullptr;
1176 });
1177}
1178
1179char ModuleSummaryIndexWrapperPass::ID = 0;
1180
1181INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1182 "Module Summary Analysis", false, true)
1183INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
1184INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
1185INITIALIZE_PASS_DEPENDENCY(StackSafetyInfoWrapperPass)
1186INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
1187 "Module Summary Analysis", false, true)
1188
1189ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
1190 return new ModuleSummaryIndexWrapperPass();
1191}
1192
1193ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
1194 : ModulePass(ID) {}
1195
1196bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
1197 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1198 bool NeedSSI = needsParamAccessSummary(M);
1199 Index.emplace(args: buildModuleSummaryIndex(
1200 M,
1201 GetBFICallback: [this](const Function &F) {
1202 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1203 F&: *const_cast<Function *>(&F))
1204 .getBFI());
1205 },
1206 PSI,
1207 GetSSICallback: [&](const Function &F) -> const StackSafetyInfo * {
1208 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1209 F&: const_cast<Function &>(F))
1210 .getResult()
1211 : nullptr;
1212 }));
1213 return false;
1214}
1215
1216bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
1217 Index.reset();
1218 return false;
1219}
1220
1221void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
1222 AU.setPreservesAll();
1223 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1224 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1225 AU.addRequired<StackSafetyInfoWrapperPass>();
1226}
1227
1228char ImmutableModuleSummaryIndexWrapperPass::ID = 0;
1229
1230ImmutableModuleSummaryIndexWrapperPass::ImmutableModuleSummaryIndexWrapperPass(
1231 const ModuleSummaryIndex *Index)
1232 : ImmutablePass(ID), Index(Index) {}
1233
1234void ImmutableModuleSummaryIndexWrapperPass::getAnalysisUsage(
1235 AnalysisUsage &AU) const {
1236 AU.setPreservesAll();
1237}
1238
1239ImmutablePass *llvm::createImmutableModuleSummaryIndexWrapperPass(
1240 const ModuleSummaryIndex *Index) {
1241 return new ImmutableModuleSummaryIndexWrapperPass(Index);
1242}
1243
1244INITIALIZE_PASS(ImmutableModuleSummaryIndexWrapperPass, "module-summary-info",
1245 "Module summary info", false, true)
1246
1247bool llvm::mayHaveMemprofSummary(const CallBase *CB) {
1248 if (!CB)
1249 return false;
1250 auto *CI = dyn_cast<CallInst>(Val: CB);
1251 auto *CalledValue = CB->getCalledOperand();
1252 auto *CalledFunction = CB->getCalledFunction();
1253 if (CalledValue && !CalledFunction) {
1254 CalledValue = CalledValue->stripPointerCasts();
1255 // Stripping pointer casts can reveal a called function.
1256 CalledFunction = dyn_cast<Function>(Val: CalledValue);
1257 }
1258 // Check if this is an alias to a function. If so, get the
1259 // called aliasee for the checks below.
1260 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
1261 assert(!CalledFunction &&
1262 "Expected null called function in callsite for alias");
1263 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
1264 }
1265 // Check if this is a direct call to a known function or a known
1266 // intrinsic, or an indirect call with profile data.
1267 if (CalledFunction) {
1268 if (CI && CalledFunction->isIntrinsic())
1269 return false;
1270 } else {
1271 // Skip indirect calls if we haven't enabled memprof ICP.
1272 if (!getAnalysisOptions(C: CB->getContext())
1273 .enable_memprof_indirect_call_support)
1274 return false;
1275 // Skip inline assembly calls.
1276 if (CI && CI->isInlineAsm())
1277 return false;
1278 // Skip direct calls via Constant.
1279 if (!CalledValue || isa<Constant>(Val: CalledValue))
1280 return false;
1281 return true;
1282 }
1283 return true;
1284}
1285