1//===- IndirectCallPromotion.cpp - Optimizations based on value profiling -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the transformation that promotes indirect calls to
10// conditional direct calls when the indirect-call value profile metadata is
11// available.
12//
13//===----------------------------------------------------------------------===//
14
15#include "InstrumentationOptions.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
21#include "llvm/Analysis/IndirectCallVisitor.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/Analysis/ProfileSummaryInfo.h"
24#include "llvm/Analysis/TypeMetadataUtils.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/InstrTypes.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/ModuleSummaryIndex.h"
33#include "llvm/IR/PassManager.h"
34#include "llvm/IR/ProfDataUtils.h"
35#include "llvm/IR/Value.h"
36#include "llvm/ProfileData/InstrProf.h"
37#include "llvm/ProfileData/ProfileCommon.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/Debug.h"
41#include "llvm/Support/Error.h"
42#include "llvm/Support/raw_ostream.h"
43#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
44#include "llvm/Transforms/Utils/CallPromotionUtils.h"
45#include "llvm/Transforms/Utils/Instrumentation.h"
46#include <cassert>
47#include <cstdint>
48#include <set>
49#include <string>
50#include <utility>
51#include <vector>
52
53using namespace llvm;
54
55#define DEBUG_TYPE "pgo-icall-prom"
56
57STATISTIC(NumOfPGOICallPromotion, "Number of indirect call promotions.");
58STATISTIC(NumOfPGOICallsites, "Number of indirect call candidate sites.");
59
60namespace llvm {
61extern cl::opt<bool> EnableVTableProfileUse;
62} // namespace llvm
63
64namespace {
65
66// The key is a vtable global variable, and the value is a map.
67// In the inner map, the key represents address point offsets and the value is a
68// constant for this address point.
69using VTableAddressPointOffsetValMap =
70 SmallDenseMap<const GlobalVariable *, DenseMap<int, Constant *>>;
71
72// A struct to collect type information for a virtual call site.
73struct VirtualCallSiteInfo {
74 // The offset from the address point to virtual function in the vtable.
75 uint64_t FunctionOffset;
76 // The instruction that computes the address point of vtable.
77 Instruction *VPtr;
78 // The compatible type used in LLVM type intrinsics.
79 StringRef CompatibleTypeStr;
80};
81
82// The key is a virtual call, and value is its type information.
83using VirtualCallSiteTypeInfoMap =
84 SmallDenseMap<const CallBase *, VirtualCallSiteInfo>;
85
86// The key is vtable GUID, and value is its value profile count.
87using VTableGUIDCountsMap = SmallDenseMap<uint64_t, uint64_t, 16>;
88
89// Return the address point offset of the given compatible type.
90//
91// Type metadata of a vtable specifies the types that can contain a pointer to
92// this vtable, for example, `Base*` can be a pointer to an derived type
93// but not vice versa. See also https://llvm.org/docs/TypeMetadata.html
94static std::optional<uint64_t>
95getAddressPointOffset(const GlobalVariable &VTableVar,
96 StringRef CompatibleType) {
97 SmallVector<MDNode *> Types;
98 VTableVar.getMetadata(KindID: LLVMContext::MD_type, MDs&: Types);
99
100 for (MDNode *Type : Types)
101 if (auto *TypeId = dyn_cast<MDString>(Val: Type->getOperand(I: 1).get());
102 TypeId && TypeId->getString() == CompatibleType)
103 return cast<ConstantInt>(
104 Val: cast<ConstantAsMetadata>(Val: Type->getOperand(I: 0))->getValue())
105 ->getZExtValue();
106
107 return std::nullopt;
108}
109
110// Return a constant representing the vtable's address point specified by the
111// offset.
112static Constant *getVTableAddressPointOffset(GlobalVariable *VTable,
113 uint32_t AddressPointOffset) {
114 Module &M = *VTable->getParent();
115 LLVMContext &Context = M.getContext();
116 assert(AddressPointOffset < VTable->getGlobalSize(M.getDataLayout()) &&
117 "Out-of-bound access");
118
119 return ConstantExpr::getInBoundsPtrAdd(
120 Ptr: VTable,
121 Offset: llvm::ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: AddressPointOffset));
122}
123
124// Return the basic block in which Use `U` is used via its `UserInst`.
125static BasicBlock *getUserBasicBlock(Use &U, Instruction *UserInst) {
126 if (PHINode *PN = dyn_cast<PHINode>(Val: UserInst))
127 return PN->getIncomingBlock(U);
128
129 return UserInst->getParent();
130}
131
132// `DestBB` is a suitable basic block to sink `Inst` into when `Inst` have users
133// and all users are in `DestBB`. The caller guarantees that `Inst->getParent()`
134// is the sole predecessor of `DestBB` and `DestBB` is dominated by
135// `Inst->getParent()`.
136static bool isDestBBSuitableForSink(Instruction *Inst, BasicBlock *DestBB) {
137 // 'BB' is used only by assert.
138 [[maybe_unused]] BasicBlock *BB = Inst->getParent();
139
140 assert(BB != DestBB && BB->getTerminator()->getNumSuccessors() == 2 &&
141 DestBB->getUniquePredecessor() == BB &&
142 "Guaranteed by ICP transformation");
143
144 BasicBlock *UserBB = nullptr;
145 for (Use &Use : Inst->uses()) {
146 User *User = Use.getUser();
147 // Do checked cast since IR verifier guarantees that the user of an
148 // instruction must be an instruction. See `Verifier::visitInstruction`.
149 Instruction *UserInst = cast<Instruction>(Val: User);
150 UserBB = getUserBasicBlock(U&: Use, UserInst);
151 // Do not sink if Inst is used in a basic block that is not DestBB.
152 // TODO: Sink to the common dominator of all user blocks.
153 if (UserBB != DestBB)
154 return false;
155 }
156 return UserBB != nullptr;
157}
158
159// For the virtual call dispatch sequence, try to sink vtable load instructions
160// to the cold indirect call fallback.
161// FIXME: Move the sink eligibility check below to a utility function in
162// Transforms/Utils/ directory.
163static bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
164 if (!isDestBBSuitableForSink(Inst: I, DestBB: DestBlock))
165 return false;
166
167 // Do not move control-flow-involving, volatile loads, vaarg, alloca
168 // instructions, etc.
169 if (isa<PHINode>(Val: I) || I->isEHPad() || I->mayThrow() || !I->willReturn() ||
170 isa<AllocaInst>(Val: I))
171 return false;
172
173 // Do not sink convergent call instructions.
174 if (const auto *C = dyn_cast<CallBase>(Val: I))
175 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())
176 return false;
177
178 // Do not move an instruction that may write to memory.
179 if (I->mayWriteToMemory())
180 return false;
181
182 // We can only sink load instructions if there is nothing between the load and
183 // the end of block that could change the value.
184 if (I->mayReadFromMemory()) {
185 // We already know that SrcBlock is the unique predecessor of DestBlock.
186 for (BasicBlock::iterator Scan = std::next(x: I->getIterator()),
187 E = I->getParent()->end();
188 Scan != E; ++Scan) {
189 // Note analysis analysis can tell whether two pointers can point to the
190 // same object in memory or not thereby find further opportunities to
191 // sink.
192 if (Scan->mayWriteToMemory())
193 return false;
194 }
195 }
196
197 BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
198 I->moveBefore(BB&: *DestBlock, I: InsertPos);
199
200 // TODO: Sink debug intrinsic users of I to 'DestBlock'.
201 // 'InstCombinerImpl::tryToSinkInstructionDbgValues' and
202 // 'InstCombinerImpl::tryToSinkInstructionDbgVariableRecords' already have
203 // the core logic to do this.
204 return true;
205}
206
207// Try to sink instructions after VPtr to the indirect call fallback.
208// Return the number of sunk IR instructions.
209static int tryToSinkInstructions(BasicBlock *OriginalBB,
210 BasicBlock *IndirectCallBB) {
211 int SinkCount = 0;
212 // Do not sink across a critical edge for simplicity.
213 if (IndirectCallBB->getUniquePredecessor() != OriginalBB)
214 return SinkCount;
215 // Sink all eligible instructions in OriginalBB in reverse order.
216 for (Instruction &I :
217 llvm::make_early_inc_range(Range: llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: *OriginalBB))))
218 if (tryToSinkInstruction(I: &I, DestBlock: IndirectCallBB))
219 SinkCount++;
220
221 return SinkCount;
222}
223
224// Promote indirect calls to conditional direct calls, keeping track of
225// thresholds.
226class IndirectCallPromoter {
227private:
228 const InstrumentationOptions &Opts;
229 Function &F;
230 Module &M;
231
232 // Symtab that maps indirect call profile values to function names and
233 // defines.
234 InstrProfSymtab *const Symtab;
235
236 const bool SamplePGO;
237
238 // A map from a virtual call to its type information.
239 const VirtualCallSiteTypeInfoMap &VirtualCSInfo;
240
241 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal;
242
243 OptimizationRemarkEmitter &ORE;
244
245 const DenseSet<StringRef> &IgnoredBaseTypes;
246
247 // The combined summary index of a ThinLTO backend compile, or null.
248 const ModuleSummaryIndex *const ImportSummary;
249
250 // A struct that records the direct target and it's call count.
251 struct PromotionCandidate {
252 Function *const TargetFunction;
253 const uint64_t Count;
254 const uint32_t Index;
255
256 // The following fields only exists for promotion candidates with vtable
257 // information.
258 //
259 // Due to class inheritance, one virtual call candidate can come from
260 // multiple vtables. `VTableGUIDAndCounts` tracks the vtable GUIDs and
261 // counts for 'TargetFunction'. `AddressPoints` stores the vtable address
262 // points for comparison.
263 VTableGUIDCountsMap VTableGUIDAndCounts;
264 SmallVector<Constant *> AddressPoints;
265
266 PromotionCandidate(Function *F, uint64_t C, uint32_t I)
267 : TargetFunction(F), Count(C), Index(I) {}
268 };
269
270 // Check if the indirect-call call site should be promoted. Return the number
271 // of promotions. Inst is the candidate indirect call, ValueDataRef
272 // contains the array of value profile data for profiled targets,
273 // TotalCount is the total profiled count of call executions, and
274 // NumCandidates is the number of candidate entries in ValueDataRef.
275 std::vector<PromotionCandidate> getPromotionCandidatesForCallSite(
276 const CallBase &CB, ArrayRef<InstrProfValueData> ValueDataRef,
277 uint64_t TotalCount, uint32_t NumCandidates);
278
279 // Promote a list of targets for one indirect-call callsite by comparing
280 // indirect callee with functions. Return true if there are IR
281 // transformations and false otherwise.
282 bool tryToPromoteWithFuncCmp(
283 CallBase &CB, Instruction *VPtr, ArrayRef<PromotionCandidate> Candidates,
284 uint64_t TotalCount, MutableArrayRef<InstrProfValueData> ICallProfDataRef,
285 uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts);
286
287 // Promote a list of targets for one indirect call by comparing vtables with
288 // functions. Return true if there are IR transformations and false
289 // otherwise.
290 bool tryToPromoteWithVTableCmp(
291 CallBase &CB, Instruction *VPtr, ArrayRef<PromotionCandidate> Candidates,
292 uint64_t TotalFuncCount, uint32_t NumCandidates,
293 MutableArrayRef<InstrProfValueData> ICallProfDataRef,
294 VTableGUIDCountsMap &VTableGUIDCounts);
295
296 // Return true if it's profitable to compare vtables for the callsite.
297 bool isProfitableToCompareVTables(const CallBase &CB,
298 ArrayRef<PromotionCandidate> Candidates);
299
300 // Return true if the vtable corresponding to VTableGUID should be skipped
301 // for vtable-based comparison.
302 bool shouldSkipVTable(uint64_t VTableGUID);
303
304 // Given an indirect callsite and the list of function candidates, compute
305 // the following vtable information in output parameters and return vtable
306 // pointer if type profiles exist.
307 // - Populate `VTableGUIDCounts` with <vtable-guid, count> using !prof
308 // metadata attached on the vtable pointer.
309 // - For each function candidate, finds out the vtables from which it gets
310 // called and stores the <vtable-guid, count> in promotion candidate.
311 Instruction *computeVTableInfos(const CallBase *CB,
312 VTableGUIDCountsMap &VTableGUIDCounts,
313 std::vector<PromotionCandidate> &Candidates);
314
315 Constant *getOrCreateVTableAddressPointVar(GlobalVariable *GV,
316 uint64_t AddressPointOffset);
317
318 void updateFuncValueProfiles(CallBase &CB,
319 MutableArrayRef<InstrProfValueData> VDs,
320 uint64_t Sum, uint32_t MaxMDCount);
321
322 void updateVPtrValueProfiles(Instruction *VPtr,
323 VTableGUIDCountsMap &VTableGUIDCounts);
324
325 bool isValidTarget(uint64_t, Function *, const CallBase &, uint64_t);
326
327 // Return true if the ThinLTO index has a summary for the declared target
328 // indicating that it is live, so that a direct call to it from this module
329 // will resolve.
330 bool isDeclaredTargetLiveInIndex(const Function &TargetFunction) const;
331
332public:
333 IndirectCallPromoter(
334 const InstrumentationOptions &Opts, Function &Func, Module &M,
335 InstrProfSymtab *Symtab, bool SamplePGO,
336 const VirtualCallSiteTypeInfoMap &VirtualCSInfo,
337 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal,
338 const DenseSet<StringRef> &IgnoredBaseTypes,
339 OptimizationRemarkEmitter &ORE, const ModuleSummaryIndex *ImportSummary)
340 : Opts(Opts), F(Func), M(M), Symtab(Symtab), SamplePGO(SamplePGO),
341 VirtualCSInfo(VirtualCSInfo),
342 VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE),
343 IgnoredBaseTypes(IgnoredBaseTypes), ImportSummary(ImportSummary) {}
344 IndirectCallPromoter(const IndirectCallPromoter &) = delete;
345 IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete;
346
347 bool processFunction(ProfileSummaryInfo *PSI);
348};
349
350} // end anonymous namespace
351
352bool IndirectCallPromoter::isDeclaredTargetLiveInIndex(
353 const Function &TargetFunction) const {
354 assert(TargetFunction.isDeclaration());
355 // In a ThinLTO backend, a direct call to a declaration only resolves if the
356 // index has a summary for the target indicating that it is live: dead
357 // definitions are dropped in their defining module while declarations of
358 // them elsewhere are not, so a declaration left behind by dead code
359 // elimination, or named by a stale profile, would otherwise become an
360 // undefined reference (see llvm/test/ThinLTO/X86/icp-allow-decls-index.ll).
361 // In distributed ThinLTO the per-module index only carries the module's own
362 // and imported GUIDs, including declarations imported for indirect call
363 // targets, so a target without a summary is rejected as well. The thin link
364 // exports every target it imports a declaration for, and every target
365 // declared in more than one module, so a live target is never internalized.
366 ValueInfo VI = ImportSummary->getValueInfo(GUID: TargetFunction.getGUID());
367 return VI && llvm::any_of(Range: VI.getSummaryList(),
368 P: [](const std::unique_ptr<GlobalValueSummary> &S) {
369 return S->isLive();
370 });
371}
372
373bool IndirectCallPromoter::isValidTarget(uint64_t Target,
374 Function *TargetFunction,
375 const CallBase &CB, uint64_t Count) {
376 // Don't promote if the symbol is not defined in the module. This avoids
377 // creating a reference to a symbol that doesn't exist in the module
378 // This can happen when we compile with a sample profile collected from
379 // one binary but used for another, which may have profiled targets that
380 // aren't used in the new binary. We might have a declaration initially in
381 // the case where the symbol is globally dead in the binary and removed by
382 // ThinLTO.
383 using namespace ore;
384 if (TargetFunction == nullptr) {
385 LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");
386 ORE.emit(RemarkBuilder: [&]() {
387 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", &CB)
388 << "Cannot promote indirect call: target with md5sum "
389 << NV("target md5sum", Target)
390 << " not found (count=" << NV("Count", Count) << ")";
391 });
392 return false;
393 }
394 if (TargetFunction->isDeclaration() &&
395 (!Opts.icp_allow_decls ||
396 (ImportSummary && !isDeclaredTargetLiveInIndex(TargetFunction: *TargetFunction)))) {
397 LLVM_DEBUG(dbgs() << " Not promote: target definition is not available\n");
398 ORE.emit(RemarkBuilder: [&]() {
399 return OptimizationRemarkMissed(DEBUG_TYPE, "NoTargetDef", &CB)
400 << "Do not promote indirect call: target with md5sum "
401 << NV("target md5sum", Target)
402 << " definition not available (count=" << ore::NV("Count", Count)
403 << ")";
404 });
405 return false;
406 }
407
408 const char *Reason = nullptr;
409 if (!isLegalToPromote(CB, Callee: TargetFunction, FailureReason: &Reason)) {
410
411 ORE.emit(RemarkBuilder: [&]() {
412 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", &CB)
413 << "Cannot promote indirect call to "
414 << NV("TargetFunction", TargetFunction)
415 << " (count=" << NV("Count", Count) << "): " << Reason;
416 });
417 return false;
418 }
419 return true;
420}
421
422// Indirect-call promotion heuristic. The direct targets are sorted based on
423// the count. Stop at the first target that is not promoted.
424std::vector<IndirectCallPromoter::PromotionCandidate>
425IndirectCallPromoter::getPromotionCandidatesForCallSite(
426 const CallBase &CB, ArrayRef<InstrProfValueData> ValueDataRef,
427 uint64_t TotalCount, uint32_t NumCandidates) {
428 std::vector<PromotionCandidate> Ret;
429
430 LLVM_DEBUG(dbgs() << " \nWork on callsite #" << NumOfPGOICallsites << CB
431 << " Num_targets: " << ValueDataRef.size()
432 << " Num_candidates: " << NumCandidates << "\n");
433 NumOfPGOICallsites++;
434 if (Opts.icp_csskip != 0 && NumOfPGOICallsites <= Opts.icp_csskip) {
435 LLVM_DEBUG(dbgs() << " Skip: User options.\n");
436 return Ret;
437 }
438
439 for (uint32_t I = 0; I < NumCandidates; I++) {
440 uint64_t Count = ValueDataRef[I].Count;
441 assert(Count <= TotalCount);
442 (void)TotalCount;
443 uint64_t Target = ValueDataRef[I].Value;
444 LLVM_DEBUG(dbgs() << " Candidate " << I << " Count=" << Count
445 << " Target_func: " << Target << "\n");
446
447 if (Opts.icp_invoke_only && isa<CallInst>(Val: CB)) {
448 LLVM_DEBUG(dbgs() << " Not promote: User options.\n");
449 ORE.emit(RemarkBuilder: [&]() {
450 return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", &CB)
451 << " Not promote: User options";
452 });
453 break;
454 }
455 if (Opts.icp_call_only && isa<InvokeInst>(Val: CB)) {
456 LLVM_DEBUG(dbgs() << " Not promote: User option.\n");
457 ORE.emit(RemarkBuilder: [&]() {
458 return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", &CB)
459 << " Not promote: User options";
460 });
461 break;
462 }
463 if (Opts.icp_cutoff != 0 && NumOfPGOICallPromotion >= Opts.icp_cutoff) {
464 LLVM_DEBUG(dbgs() << " Not promote: Cutoff reached.\n");
465 ORE.emit(RemarkBuilder: [&]() {
466 return OptimizationRemarkMissed(DEBUG_TYPE, "CutOffReached", &CB)
467 << " Not promote: Cutoff reached";
468 });
469 break;
470 }
471
472 Function *TargetFunction = Symtab->getFunction(FuncMD5Hash: Target);
473 if (!isValidTarget(Target, TargetFunction, CB, Count)) {
474 if (Opts.icp_allow_candidate_skip)
475 continue;
476 else
477 break;
478 }
479
480 Ret.push_back(x: PromotionCandidate(TargetFunction, Count, I));
481 TotalCount -= Count;
482 }
483 return Ret;
484}
485
486Constant *IndirectCallPromoter::getOrCreateVTableAddressPointVar(
487 GlobalVariable *GV, uint64_t AddressPointOffset) {
488 auto [Iter, Inserted] =
489 VTableAddressPointOffsetVal[GV].try_emplace(Key: AddressPointOffset, Args: nullptr);
490 if (Inserted)
491 Iter->second = getVTableAddressPointOffset(VTable: GV, AddressPointOffset);
492 return Iter->second;
493}
494
495Instruction *IndirectCallPromoter::computeVTableInfos(
496 const CallBase *CB, VTableGUIDCountsMap &GUIDCountsMap,
497 std::vector<PromotionCandidate> &Candidates) {
498 if (!EnableVTableProfileUse)
499 return nullptr;
500
501 // Take the following code sequence as an example, here is how the code works
502 // @vtable1 = {[n x ptr] [... ptr @func1]}
503 // @vtable2 = {[m x ptr] [... ptr @func2]}
504 //
505 // %vptr = load ptr, ptr %d, !prof !0
506 // %0 = tail call i1 @llvm.type.test(ptr %vptr, metadata !"vtable1")
507 // tail call void @llvm.assume(i1 %0)
508 // %vfn = getelementptr inbounds ptr, ptr %vptr, i64 1
509 // %1 = load ptr, ptr %vfn
510 // call void %1(ptr %d), !prof !1
511 //
512 // !0 = !{!"VP", i32 2, i64 100, i64 123, i64 50, i64 456, i64 50}
513 // !1 = !{!"VP", i32 0, i64 100, i64 789, i64 50, i64 579, i64 50}
514 //
515 // Step 1. Find out the %vptr instruction for indirect call and use its !prof
516 // to populate `GUIDCountsMap`.
517 // Step 2. For each vtable-guid, look up its definition from symtab. LTO can
518 // make vtable definitions visible across modules.
519 // Step 3. Compute the byte offset of the virtual call, by adding vtable
520 // address point offset and function's offset relative to vtable address
521 // point. For each function candidate, this step tells us the vtable from
522 // which it comes from, and the vtable address point to compare %vptr with.
523
524 // Only virtual calls have virtual call site info.
525 auto Iter = VirtualCSInfo.find(Val: CB);
526 if (Iter == VirtualCSInfo.end())
527 return nullptr;
528
529 LLVM_DEBUG(dbgs() << "\nComputing vtable infos for callsite #"
530 << NumOfPGOICallsites << "\n");
531
532 const auto &VirtualCallInfo = Iter->second;
533 Instruction *VPtr = VirtualCallInfo.VPtr;
534
535 SmallDenseMap<Function *, int, 4> CalleeIndexMap;
536 for (size_t I = 0; I < Candidates.size(); I++)
537 CalleeIndexMap[Candidates[I].TargetFunction] = I;
538
539 uint64_t TotalVTableCount = 0;
540 auto VTableValueDataArray = getValueProfDataFromInst(
541 Inst: *VirtualCallInfo.VPtr, ValueKind: IPVK_VTableTarget,
542 MaxNumValueData: getMaxNumVTableAnnotations(C: VPtr->getContext()), TotalC&: TotalVTableCount);
543 if (VTableValueDataArray.empty())
544 return VPtr;
545
546 // Compute the functions and counts from by each vtable.
547 for (const auto &V : VTableValueDataArray) {
548 uint64_t VTableVal = V.Value;
549 GUIDCountsMap[VTableVal] = V.Count;
550 GlobalVariable *VTableVar = Symtab->getGlobalVariable(MD5Hash: VTableVal);
551 if (!VTableVar) {
552 LLVM_DEBUG(dbgs() << " Cannot find vtable definition for " << VTableVal
553 << "; maybe the vtable isn't imported\n");
554 continue;
555 }
556
557 std::optional<uint64_t> MaybeAddressPointOffset =
558 getAddressPointOffset(VTableVar: *VTableVar, CompatibleType: VirtualCallInfo.CompatibleTypeStr);
559 if (!MaybeAddressPointOffset)
560 continue;
561
562 const uint64_t AddressPointOffset = *MaybeAddressPointOffset;
563
564 Function *Callee = nullptr;
565 std::tie(args&: Callee, args: std::ignore) = getFunctionAtVTableOffset(
566 GV: VTableVar, Offset: AddressPointOffset + VirtualCallInfo.FunctionOffset, M);
567 if (!Callee)
568 continue;
569 auto CalleeIndexIter = CalleeIndexMap.find(Val: Callee);
570 if (CalleeIndexIter == CalleeIndexMap.end())
571 continue;
572
573 auto &Candidate = Candidates[CalleeIndexIter->second];
574 // There should never be duplicate GUIDs in one !prof metdata, as this is
575 // an IR invariant enforced by the verifier. Assigning counters directly
576 // won't cause overwrite or counter loss.
577 Candidate.VTableGUIDAndCounts[VTableVal] = V.Count;
578 Candidate.AddressPoints.push_back(
579 Elt: getOrCreateVTableAddressPointVar(GV: VTableVar, AddressPointOffset));
580 }
581
582 return VPtr;
583}
584
585// Creates 'branch_weights' prof metadata using TrueWeight and FalseWeight.
586// Scales uint64_t counters down to uint32_t if necessary to prevent overflow.
587static MDNode *createBranchWeights(LLVMContext &Context, uint64_t TrueWeight,
588 uint64_t FalseWeight) {
589 MDBuilder MDB(Context);
590 uint64_t Scale = calculateCountScale(MaxCount: std::max(a: TrueWeight, b: FalseWeight));
591 return MDB.createBranchWeights(TrueWeight: scaleBranchCount(Count: TrueWeight, Scale),
592 FalseWeight: scaleBranchCount(Count: FalseWeight, Scale));
593}
594
595CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
596 uint64_t Count, uint64_t TotalCount,
597 bool AttachProfToDirectCall,
598 OptimizationRemarkEmitter *ORE) {
599 CallBase &NewInst = promoteCallWithIfThenElse(
600 CB, Callee: DirectCallee,
601 BranchWeights: createBranchWeights(Context&: CB.getContext(), TrueWeight: Count, FalseWeight: TotalCount - Count));
602
603 if (AttachProfToDirectCall)
604 setFittedBranchWeights(I&: NewInst, Weights: {Count},
605 /*IsExpected=*/false);
606
607 using namespace ore;
608
609 if (ORE)
610 ORE->emit(RemarkBuilder: [&]() {
611 return OptimizationRemark(DEBUG_TYPE, "Promoted", &CB)
612 << "Promote indirect call to " << NV("DirectCallee", DirectCallee)
613 << " with count " << NV("Count", Count) << " out of "
614 << NV("TotalCount", TotalCount);
615 });
616 return NewInst;
617}
618
619// Promote indirect-call to conditional direct-call for one callsite.
620bool IndirectCallPromoter::tryToPromoteWithFuncCmp(
621 CallBase &CB, Instruction *VPtr, ArrayRef<PromotionCandidate> Candidates,
622 uint64_t TotalCount, MutableArrayRef<InstrProfValueData> ICallProfDataRef,
623 uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts) {
624 uint32_t NumPromoted = 0;
625
626 for (const auto &C : Candidates) {
627 uint64_t FuncCount = C.Count;
628 pgo::promoteIndirectCall(CB, DirectCallee: C.TargetFunction, Count: FuncCount, TotalCount,
629 AttachProfToDirectCall: SamplePGO, ORE: &ORE);
630 assert(TotalCount >= FuncCount);
631 TotalCount -= FuncCount;
632 NumOfPGOICallPromotion++;
633 NumPromoted++;
634
635 // Update the count and this entry will be erased later.
636 ICallProfDataRef[C.Index].Count = 0;
637 if (!EnableVTableProfileUse || C.VTableGUIDAndCounts.empty())
638 continue;
639
640 // After a virtual call candidate gets promoted, update the vtable's counts
641 // proportionally. Each vtable-guid in `C.VTableGUIDAndCounts` represents
642 // a vtable from which the virtual call is loaded. Compute the sum and use
643 // 128-bit APInt to improve accuracy.
644 uint64_t SumVTableCount = 0;
645 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts)
646 SumVTableCount += VTableCount;
647
648 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts) {
649 APInt APFuncCount((unsigned)128, FuncCount, false /*signed*/);
650 APFuncCount *= VTableCount;
651 VTableGUIDCounts[GUID] -= APFuncCount.udiv(RHS: SumVTableCount).getZExtValue();
652 }
653 }
654 if (NumPromoted == 0)
655 return false;
656
657 assert(NumPromoted <= ICallProfDataRef.size() &&
658 "Number of promoted functions should not be greater than the number "
659 "of values in profile metadata");
660
661 updateFuncValueProfiles(CB, VDs: ICallProfDataRef, Sum: TotalCount, MaxMDCount: NumCandidates);
662 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);
663 return true;
664}
665
666void IndirectCallPromoter::updateFuncValueProfiles(
667 CallBase &CB, MutableArrayRef<InstrProfValueData> CallVDs,
668 uint64_t TotalCount, uint32_t MaxMDCount) {
669 // First clear the existing !prof.
670 CB.setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
671
672 // Sort value profiles by count in descending order.
673 llvm::stable_sort(Range&: CallVDs, C: [](const InstrProfValueData &LHS,
674 const InstrProfValueData &RHS) {
675 return LHS.Count > RHS.Count;
676 });
677 // Drop the <target-value, count> pair if count is zero.
678 ArrayRef<InstrProfValueData> VDs(
679 CallVDs.begin(),
680 llvm::upper_bound(Range&: CallVDs, Value: 0U,
681 C: [](uint64_t Count, const InstrProfValueData &ProfData) {
682 return ProfData.Count <= Count;
683 }));
684
685 // Annotate the remaining value profiles if counter is not zero.
686 if (TotalCount != 0)
687 annotateValueSite(M, Inst&: CB, VDs, Sum: TotalCount, ValueKind: IPVK_IndirectCallTarget,
688 MaxMDCount);
689}
690
691void IndirectCallPromoter::updateVPtrValueProfiles(
692 Instruction *VPtr, VTableGUIDCountsMap &VTableGUIDCounts) {
693 if (!EnableVTableProfileUse || VPtr == nullptr ||
694 !VPtr->getMetadata(KindID: LLVMContext::MD_prof))
695 return;
696 VPtr->setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
697 std::vector<InstrProfValueData> VTableValueProfiles;
698 uint64_t TotalVTableCount = 0;
699 for (auto [GUID, Count] : VTableGUIDCounts) {
700 if (Count == 0)
701 continue;
702
703 VTableValueProfiles.push_back(x: {.Value: GUID, .Count: Count});
704 TotalVTableCount += Count;
705 }
706 llvm::sort(C&: VTableValueProfiles,
707 Comp: [](const InstrProfValueData &LHS, const InstrProfValueData &RHS) {
708 return LHS.Count > RHS.Count;
709 });
710
711 annotateValueSite(M, Inst&: *VPtr, VDs: VTableValueProfiles, Sum: TotalVTableCount,
712 ValueKind: IPVK_VTableTarget, MaxMDCount: VTableValueProfiles.size());
713}
714
715bool IndirectCallPromoter::tryToPromoteWithVTableCmp(
716 CallBase &CB, Instruction *VPtr, ArrayRef<PromotionCandidate> Candidates,
717 uint64_t TotalFuncCount, uint32_t NumCandidates,
718 MutableArrayRef<InstrProfValueData> ICallProfDataRef,
719 VTableGUIDCountsMap &VTableGUIDCounts) {
720 SmallVector<std::pair<uint32_t, uint64_t>, 4> PromotedFuncCount;
721
722 for (const auto &Candidate : Candidates) {
723 for (auto &[GUID, Count] : Candidate.VTableGUIDAndCounts)
724 VTableGUIDCounts[GUID] -= Count;
725
726 // 'OriginalBB' is the basic block of indirect call. After each candidate
727 // is promoted, a new basic block is created for the indirect fallback basic
728 // block and indirect call `CB` is moved into this new BB.
729 BasicBlock *OriginalBB = CB.getParent();
730 promoteCallWithVTableCmp(
731 CB, VPtr, Callee: Candidate.TargetFunction, AddressPoints: Candidate.AddressPoints,
732 BranchWeights: createBranchWeights(Context&: CB.getContext(), TrueWeight: Candidate.Count,
733 FalseWeight: TotalFuncCount - Candidate.Count));
734
735 int SinkCount = tryToSinkInstructions(OriginalBB, IndirectCallBB: CB.getParent());
736
737 ORE.emit(RemarkBuilder: [&]() {
738 OptimizationRemark Remark(DEBUG_TYPE, "Promoted", &CB);
739
740 const auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;
741 Remark << "Promote indirect call to "
742 << ore::NV("DirectCallee", Candidate.TargetFunction)
743 << " with count " << ore::NV("Count", Candidate.Count)
744 << " out of " << ore::NV("TotalCount", TotalFuncCount) << ", sink "
745 << ore::NV("SinkCount", SinkCount)
746 << " instruction(s) and compare "
747 << ore::NV("VTable", VTableGUIDAndCounts.size())
748 << " vtable(s): {";
749
750 // Sort GUIDs so remark message is deterministic.
751 std::set<uint64_t> GUIDSet;
752 for (auto [GUID, Count] : VTableGUIDAndCounts)
753 GUIDSet.insert(x: GUID);
754 for (auto Iter = GUIDSet.begin(); Iter != GUIDSet.end(); Iter++) {
755 if (Iter != GUIDSet.begin())
756 Remark << ", ";
757 Remark << ore::NV("VTable", Symtab->getGlobalVariable(MD5Hash: *Iter));
758 }
759
760 Remark << "}";
761
762 return Remark;
763 });
764
765 PromotedFuncCount.push_back(Elt: {Candidate.Index, Candidate.Count});
766
767 assert(TotalFuncCount >= Candidate.Count &&
768 "Within one prof metadata, total count is the sum of counts from "
769 "individual <target, count> pairs");
770 // Use std::min since 'TotalFuncCount' is the saturated sum of individual
771 // counts, see
772 // https://github.com/llvm/llvm-project/blob/abedb3b8356d5d56f1c575c4f7682fba2cb19787/llvm/lib/ProfileData/InstrProf.cpp#L1281-L1288
773 TotalFuncCount -= std::min(a: TotalFuncCount, b: Candidate.Count);
774 NumOfPGOICallPromotion++;
775 }
776
777 if (PromotedFuncCount.empty())
778 return false;
779
780 // Update value profiles for 'CB' and 'VPtr', assuming that each 'CB' has a
781 // a distinct 'VPtr'.
782 // FIXME: When Clang `-fstrict-vtable-pointers` is enabled, a vtable might be
783 // used to load multiple virtual functions. The vtable profiles needs to be
784 // updated properly in that case (e.g, for each indirect call annotate both
785 // type profiles and function profiles in one !prof).
786 for (size_t I = 0; I < PromotedFuncCount.size(); I++) {
787 uint32_t Index = PromotedFuncCount[I].first;
788 ICallProfDataRef[Index].Count -=
789 std::max(a: PromotedFuncCount[I].second, b: ICallProfDataRef[Index].Count);
790 }
791 updateFuncValueProfiles(CB, CallVDs: ICallProfDataRef, TotalCount: TotalFuncCount, MaxMDCount: NumCandidates);
792 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);
793 return true;
794}
795
796// Traverse all the indirect-call callsite and get the value profile
797// annotation to perform indirect-call promotion.
798bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) {
799 bool Changed = false;
800 ICallPromotionAnalysis ICallAnalysis;
801 for (auto *CB : findIndirectCalls(F)) {
802 uint32_t NumCandidates;
803 uint64_t TotalCount;
804 auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction(
805 I: CB, TotalCount, NumCandidates);
806 if (!NumCandidates)
807 continue;
808 if (PSI && PSI->hasProfileSummary()) {
809 // Don't promote cold candidates.
810 if (PSI->isColdCount(C: TotalCount)) {
811 LLVM_DEBUG(dbgs() << "Don't promote the cold candidate: TotalCount="
812 << TotalCount << "\n");
813 continue;
814 }
815 // Only promote hot if -icp-allow-hot-only is true. ICP has its own cutoff
816 // threshold for hotness, which defaults to ProfileSummaryCutoffHot if
817 // unspecified.
818 if (Opts.icp_allow_hot_only &&
819 !PSI->isHotCountNthPercentile(PercentileCutoff: Opts.hot_func_cutoff_for_icp == -1
820 ? ProfileSummaryCutoffHot
821 : Opts.hot_func_cutoff_for_icp,
822 C: TotalCount)) {
823 LLVM_DEBUG(dbgs() << "Don't promote the non-hot candidate: TotalCount="
824 << TotalCount << "\n");
825 continue;
826 }
827 }
828
829 auto PromotionCandidates = getPromotionCandidatesForCallSite(
830 CB: *CB, ValueDataRef: ICallProfDataRef, TotalCount, NumCandidates);
831
832 VTableGUIDCountsMap VTableGUIDCounts;
833 Instruction *VPtr =
834 computeVTableInfos(CB, GUIDCountsMap&: VTableGUIDCounts, Candidates&: PromotionCandidates);
835
836 if (isProfitableToCompareVTables(CB: *CB, Candidates: PromotionCandidates))
837 Changed |= tryToPromoteWithVTableCmp(CB&: *CB, VPtr, Candidates: PromotionCandidates,
838 TotalFuncCount: TotalCount, NumCandidates,
839 ICallProfDataRef, VTableGUIDCounts);
840 else
841 Changed |= tryToPromoteWithFuncCmp(CB&: *CB, VPtr, Candidates: PromotionCandidates,
842 TotalCount, ICallProfDataRef,
843 NumCandidates, VTableGUIDCounts);
844 }
845 return Changed;
846}
847
848// TODO: Return false if the function addressing and vtable load instructions
849// cannot sink to indirect fallback.
850bool IndirectCallPromoter::isProfitableToCompareVTables(
851 const CallBase &CB, ArrayRef<PromotionCandidate> Candidates) {
852 if (!EnableVTableProfileUse || Candidates.empty())
853 return false;
854 LLVM_DEBUG(dbgs() << "\nEvaluating vtable profitability for callsite #"
855 << NumOfPGOICallsites << CB << "\n");
856 const size_t CandidateSize = Candidates.size();
857 for (size_t I = 0; I < CandidateSize; I++) {
858 auto &Candidate = Candidates[I];
859 auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;
860
861 LLVM_DEBUG({
862 dbgs() << " Candidate " << I << " FunctionCount: " << Candidate.Count
863 << ", VTableCounts:";
864 for (const auto &[GUID, Count] : VTableGUIDAndCounts)
865 dbgs() << " {" << Symtab->getGlobalVariable(GUID)->getName() << ", "
866 << Count << "}";
867 dbgs() << "\n";
868 });
869
870 uint64_t CandidateVTableCount = 0;
871
872 for (auto &[GUID, Count] : VTableGUIDAndCounts) {
873 CandidateVTableCount += Count;
874
875 if (shouldSkipVTable(VTableGUID: GUID))
876 return false;
877 }
878
879 if (CandidateVTableCount <
880 Candidate.Count * Opts.icp_vtable_percentage_threshold) {
881 LLVM_DEBUG(
882 dbgs() << " function count " << Candidate.Count
883 << " and its vtable sum count " << CandidateVTableCount
884 << " have discrepancies. Bail out vtable comparison.\n");
885 return false;
886 }
887
888 // 'MaxNumVTable' limits the number of vtables to make vtable comparison
889 // profitable. Comparing multiple vtables for one function candidate will
890 // insert additional instructions on the hot path, and allowing more than
891 // one vtable for non last candidates may or may not elongate the dependency
892 // chain for the subsequent candidates. Set its value to 1 for non-last
893 // candidate and allow option to override it for the last candidate.
894 int MaxNumVTable = 1;
895 if (I == CandidateSize - 1)
896 MaxNumVTable = Opts.icp_max_num_vtable_last_candidate;
897
898 if ((int)Candidate.AddressPoints.size() > MaxNumVTable) {
899 LLVM_DEBUG(dbgs() << " allow at most " << MaxNumVTable << " and got "
900 << Candidate.AddressPoints.size()
901 << " vtables. Bail out for vtable comparison.\n");
902 return false;
903 }
904 }
905
906 return true;
907}
908
909bool IndirectCallPromoter::shouldSkipVTable(uint64_t VTableGUID) {
910 if (IgnoredBaseTypes.empty())
911 return false;
912
913 auto *VTableVar = Symtab->getGlobalVariable(MD5Hash: VTableGUID);
914
915 assert(VTableVar && "VTableVar must exist for GUID in VTableGUIDAndCounts");
916
917 SmallVector<MDNode *, 2> Types;
918 VTableVar->getMetadata(KindID: LLVMContext::MD_type, MDs&: Types);
919
920 for (auto *Type : Types)
921 if (auto *TypeId = dyn_cast<MDString>(Val: Type->getOperand(I: 1).get()))
922 if (IgnoredBaseTypes.contains(V: TypeId->getString())) {
923 LLVM_DEBUG(dbgs() << " vtable profiles should be ignored. Bail "
924 "out of vtable comparison.");
925 return true;
926 }
927 return false;
928}
929
930// For virtual calls in the module, collect per-callsite information which will
931// be used to associate an ICP candidate with a vtable and a specific function
932// in the vtable. With type intrinsics (llvm.type.test), we can find virtual
933// calls in a compile-time efficient manner (by iterating its users) and more
934// importantly use the compatible type later to figure out the function byte
935// offset relative to the start of vtables.
936static void
937computeVirtualCallSiteTypeInfoMap(Module &M, ModuleAnalysisManager &MAM,
938 VirtualCallSiteTypeInfoMap &VirtualCSInfo) {
939 // Right now only llvm.type.test is used to find out virtual call sites.
940 // With ThinLTO and whole-program-devirtualization, llvm.type.test and
941 // llvm.public.type.test are emitted, and llvm.public.type.test is either
942 // refined to llvm.type.test or dropped before indirect-call-promotion pass.
943 //
944 // FIXME: For fullLTO with VFE, `llvm.type.checked.load intrinsic` is emitted.
945 // Find out virtual calls by looking at users of llvm.type.checked.load in
946 // that case.
947 Function *TypeTestFunc =
948 Intrinsic::getDeclarationIfExists(M: &M, id: Intrinsic::type_test);
949 if (!TypeTestFunc || TypeTestFunc->use_empty())
950 return;
951
952 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
953 auto LookupDomTree = [&FAM](Function &F) -> DominatorTree & {
954 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
955 };
956 // Iterate all type.test calls to find all indirect calls.
957 for (Use &U : llvm::make_early_inc_range(Range: TypeTestFunc->uses())) {
958 auto *CI = dyn_cast<CallInst>(Val: U.getUser());
959 if (!CI)
960 continue;
961 auto *TypeMDVal = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
962 if (!TypeMDVal)
963 continue;
964 auto *CompatibleTypeId = dyn_cast<MDString>(Val: TypeMDVal->getMetadata());
965 if (!CompatibleTypeId)
966 continue;
967
968 // Find out all devirtualizable call sites given a llvm.type.test
969 // intrinsic call.
970 SmallVector<DevirtCallSite, 1> DevirtCalls;
971 SmallVector<CallInst *, 1> Assumes;
972 auto &DT = LookupDomTree(*CI->getFunction());
973 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
974
975 for (auto &DevirtCall : DevirtCalls) {
976 CallBase &CB = DevirtCall.CB;
977 // Given an indirect call, try find the instruction which loads a
978 // pointer to virtual table.
979 Instruction *VTablePtr =
980 PGOIndirectCallVisitor::tryGetVTableInstruction(CB: &CB);
981 if (!VTablePtr)
982 continue;
983 VirtualCSInfo[&CB] = {.FunctionOffset: DevirtCall.Offset, .VPtr: VTablePtr,
984 .CompatibleTypeStr: CompatibleTypeId->getString()};
985 }
986 }
987}
988
989// A wrapper function that does the actual work.
990static bool promoteIndirectCalls(const InstrumentationOptions &Opts, Module &M,
991 ProfileSummaryInfo *PSI, bool InLTO,
992 bool SamplePGO, ModuleAnalysisManager &MAM,
993 const ModuleSummaryIndex *ImportSummary) {
994 if (Opts.disable_icp)
995 return false;
996 InstrProfSymtab Symtab;
997 if (Error E = Symtab.create(M, InLTO)) {
998 std::string SymtabFailure = toString(E: std::move(E));
999 M.getContext().emitError(ErrorStr: "Failed to create symtab: " + SymtabFailure);
1000 return false;
1001 }
1002 bool Changed = false;
1003 VirtualCallSiteTypeInfoMap VirtualCSInfo;
1004
1005 DenseSet<StringRef> IgnoredBaseTypes;
1006
1007 if (EnableVTableProfileUse) {
1008 computeVirtualCallSiteTypeInfoMap(M, MAM, VirtualCSInfo);
1009
1010 IgnoredBaseTypes.insert_range(R: Opts.icp_ignored_base_types);
1011 }
1012
1013 // VTableAddressPointOffsetVal stores the vtable address points. The vtable
1014 // address point of a given <vtable, address point offset> is static (doesn't
1015 // change after being computed once).
1016 // IndirectCallPromoter::getOrCreateVTableAddressPointVar creates the map
1017 // entry the first time a <vtable, offset> pair is seen, as
1018 // promoteIndirectCalls processes an IR module and calls IndirectCallPromoter
1019 // repeatedly on each function.
1020 VTableAddressPointOffsetValMap VTableAddressPointOffsetVal;
1021
1022 for (auto &F : M) {
1023 if (F.isDeclaration() || F.hasOptNone())
1024 continue;
1025
1026 auto &FAM =
1027 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1028 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
1029
1030 IndirectCallPromoter CallPromoter(
1031 Opts, F, M, &Symtab, SamplePGO, VirtualCSInfo,
1032 VTableAddressPointOffsetVal, IgnoredBaseTypes, ORE, ImportSummary);
1033 bool FuncChanged = CallPromoter.processFunction(PSI);
1034 if (Opts.icp_dumpafter && FuncChanged) {
1035 LLVM_DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
1036 LLVM_DEBUG(dbgs() << "\n");
1037 }
1038 Changed |= FuncChanged;
1039 if (Opts.icp_cutoff != 0 && NumOfPGOICallPromotion >= Opts.icp_cutoff) {
1040 LLVM_DEBUG(dbgs() << " Stop: Cutoff reached.\n");
1041 break;
1042 }
1043 }
1044 return Changed;
1045}
1046
1047PreservedAnalyses PGOIndirectCallPromotion::run(Module &M,
1048 ModuleAnalysisManager &MAM) {
1049 ProfileSummaryInfo *PSI = &MAM.getResult<ProfileSummaryAnalysis>(IR&: M);
1050
1051 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
1052 if (!promoteIndirectCalls(Opts, M, PSI, InLTO: InLTO | Opts.icp_lto,
1053 SamplePGO: SamplePGO | Opts.icp_samplepgo, MAM, ImportSummary))
1054 return PreservedAnalyses::all();
1055
1056 return PreservedAnalyses::none();
1057}
1058