1//===- SampleProfileProbe.cpp - Pseudo probe Instrumentation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SampleProfileProber transformation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/SampleProfileProbe.h"
14#include "IPOOptions.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/ADT/StringSet.h"
19#include "llvm/Analysis/BlockFrequencyInfo.h"
20#include "llvm/Analysis/EHUtils.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/IR/BasicBlock.h"
23#include "llvm/IR/DebugInfoMetadata.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/Module.h"
30#include "llvm/IR/PassInstrumentation.h"
31#include "llvm/IR/PseudoProbe.h"
32#include "llvm/ProfileData/SampleProf.h"
33#include "llvm/Support/CRC.h"
34#include "llvm/Target/TargetMachine.h"
35#include "llvm/Transforms/Utils/Instrumentation.h"
36#include "llvm/Transforms/Utils/ModuleUtils.h"
37#include <vector>
38
39using namespace llvm;
40#define DEBUG_TYPE "pseudo-probe"
41
42STATISTIC(ArtificialDbgLine,
43 "Number of probes that have an artificial debug line");
44
45static uint64_t getCallStackHash(const DILocation *DIL) {
46 uint64_t Hash = 0;
47 const DILocation *InlinedAt = DIL ? DIL->getInlinedAt() : nullptr;
48 while (InlinedAt) {
49 Hash ^= MD5Hash(Str: std::to_string(val: InlinedAt->getLine()));
50 Hash ^= MD5Hash(Str: std::to_string(val: InlinedAt->getColumn()));
51 auto Name = InlinedAt->getSubprogramLinkageName();
52 Hash ^= MD5Hash(Str: Name);
53 InlinedAt = InlinedAt->getInlinedAt();
54 }
55 return Hash;
56}
57
58static uint64_t computeCallStackHash(const Instruction &Inst) {
59 return getCallStackHash(DIL: Inst.getDebugLoc());
60}
61
62bool PseudoProbeVerifier::shouldVerifyFunction(const Function *F) {
63 // Skip function declaration.
64 if (F->isDeclaration())
65 return false;
66 // Skip function that will not be emitted into object file. The prevailing
67 // defintion will be verified instead.
68 if (F->hasAvailableExternallyLinkage())
69 return false;
70 return VerifyFuncNames.empty() || VerifyFuncNames.contains(key: F->getName());
71}
72
73PseudoProbeVerifier::PseudoProbeVerifier(const LLVMContext &C)
74 : Enabled(getIPOOptions(C).verify_pseudo_probe),
75 VerifyFuncNames(llvm::from_range,
76 getIPOOptions(C).verify_pseudo_probe_funcs) {}
77
78void PseudoProbeVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
79 if (Enabled) {
80 PIC.registerAfterPassCallback(
81 C: [this](StringRef P, IRUnitRef IR, const PreservedAnalyses &) {
82 this->runAfterPass(PassID: P, IR);
83 });
84 }
85}
86
87// Callback to run after each transformation for the new pass manager.
88void PseudoProbeVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
89 std::string Banner =
90 "\n*** Pseudo Probe Verification After " + PassID.str() + " ***\n";
91 dbgs() << Banner;
92 if (const auto *M = dyn_cast<Module>(Val&: IR))
93 runAfterPass(M);
94 else if (const auto *F = dyn_cast<Function>(Val&: IR))
95 runAfterPass(F);
96 else if (const auto *C = dyn_cast<LazyCallGraph::SCC>(Val&: IR))
97 runAfterPass(C);
98 else if (const auto *L = dyn_cast<Loop>(Val&: IR))
99 runAfterPass(L);
100 else
101 llvm_unreachable("Unknown IR unit");
102}
103
104void PseudoProbeVerifier::runAfterPass(const Module *M) {
105 for (const Function &F : *M)
106 runAfterPass(F: &F);
107}
108
109void PseudoProbeVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
110 for (const LazyCallGraph::Node &N : *C)
111 runAfterPass(F: &N.getFunction());
112}
113
114void PseudoProbeVerifier::runAfterPass(const Function *F) {
115 if (!shouldVerifyFunction(F))
116 return;
117 ProbeFactorMap ProbeFactors;
118 for (const auto &BB : *F)
119 collectProbeFactors(BB: &BB, ProbeFactors);
120 verifyProbeFactors(F, ProbeFactors);
121}
122
123void PseudoProbeVerifier::runAfterPass(const Loop *L) {
124 const Function *F = L->getHeader()->getParent();
125 runAfterPass(F);
126}
127
128void PseudoProbeVerifier::collectProbeFactors(const BasicBlock *Block,
129 ProbeFactorMap &ProbeFactors) {
130 for (const auto &I : *Block) {
131 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
132 uint64_t Hash = computeCallStackHash(Inst: I);
133 ProbeFactors[{Probe->Id, Hash}] += Probe->Factor;
134 }
135 }
136}
137
138void PseudoProbeVerifier::verifyProbeFactors(
139 const Function *F, const ProbeFactorMap &ProbeFactors) {
140 bool BannerPrinted = false;
141 auto &PrevProbeFactors = FunctionProbeFactors[F->getName()];
142 for (const auto &I : ProbeFactors) {
143 float CurProbeFactor = I.second;
144 auto [It, Inserted] = PrevProbeFactors.try_emplace(Key: I.first);
145 if (!Inserted) {
146 float PrevProbeFactor = It->second;
147 if (std::abs(x: CurProbeFactor - PrevProbeFactor) >
148 DistributionFactorVariance) {
149 if (!BannerPrinted) {
150 dbgs() << "Function " << F->getName() << ":\n";
151 BannerPrinted = true;
152 }
153 dbgs() << "Probe " << I.first.first << "\tprevious factor "
154 << format(Fmt: "%0.2f", Vals: PrevProbeFactor) << "\tcurrent factor "
155 << format(Fmt: "%0.2f", Vals: CurProbeFactor) << "\n";
156 }
157 }
158
159 // Update
160 It->second = I.second;
161 }
162}
163
164SampleProfileProber::SampleProfileProber(Function &Func) : F(&Func) {
165 BlockProbeIds.clear();
166 CallProbeIds.clear();
167 LastProbeId = (uint32_t)PseudoProbeReservedId::Last;
168
169 DenseSet<BasicBlock *> BlocksToIgnore;
170 DenseSet<BasicBlock *> BlocksAndCallsToIgnore;
171 computeBlocksToIgnore(BlocksToIgnore, BlocksAndCallsToIgnore);
172
173 computeProbeId(BlocksToIgnore, BlocksAndCallsToIgnore);
174 computeCFGHash(BlocksToIgnore);
175}
176
177// Two purposes to compute the blocks to ignore:
178// 1. Reduce the IR size.
179// 2. Make the instrumentation(checksum) stable. e.g. the frondend may
180// generate unstable IR while optimizing nounwind attribute, some versions are
181// optimized with the call-to-invoke conversion, while other versions do not.
182// This discrepancy in probe ID could cause profile mismatching issues.
183// Note that those ignored blocks are either cold blocks or new split blocks
184// whose original blocks are instrumented, so it shouldn't degrade the profile
185// quality.
186void SampleProfileProber::computeBlocksToIgnore(
187 DenseSet<BasicBlock *> &BlocksToIgnore,
188 DenseSet<BasicBlock *> &BlocksAndCallsToIgnore) {
189 // Ignore the cold EH and unreachable blocks and calls.
190 computeEHOnlyBlocks(F&: *F, EHBlocks&: BlocksAndCallsToIgnore);
191 findUnreachableBlocks(BlocksToIgnore&: BlocksAndCallsToIgnore);
192
193 BlocksToIgnore.insert_range(R&: BlocksAndCallsToIgnore);
194
195 // Handle the call-to-invoke conversion case: make sure that the probe id and
196 // callsite id are consistent before and after the block split. For block
197 // probe, we only keep the head block probe id and ignore the block ids of the
198 // normal dests. For callsite probe, it's different to block probe, there is
199 // no additional callsite in the normal dests, so we don't ignore the
200 // callsites.
201 findInvokeNormalDests(InvokeNormalDests&: BlocksToIgnore);
202}
203
204// Unreachable blocks and calls are always cold, ignore them.
205void SampleProfileProber::findUnreachableBlocks(
206 DenseSet<BasicBlock *> &BlocksToIgnore) {
207 for (auto &BB : *F) {
208 if (&BB != &F->getEntryBlock() && pred_size(BB: &BB) == 0)
209 BlocksToIgnore.insert(V: &BB);
210 }
211}
212
213// Follow invoke normal-dest edges and record blocks that sit on a cycle.
214static void
215findInvokeNormalDestCycles(const Function &F,
216 DenseSet<const BasicBlock *> &CycleBlocks) {
217 DenseSet<const BasicBlock *> Processed;
218 DenseSet<const BasicBlock *> OnCurrentPath;
219 SmallVector<const BasicBlock *, 16> CurrentPath;
220
221 for (const BasicBlock &Start : F) {
222 if (Processed.contains(V: &Start))
223 continue;
224
225 CurrentPath.clear();
226 OnCurrentPath.clear();
227 const BasicBlock *Cur = &Start;
228 while (Cur) {
229 if (OnCurrentPath.contains(V: Cur)) {
230 // Back-edge onto CurrentPath: the cycle is the suffix starting at Cur.
231 auto CycleStart = llvm::find(Range&: CurrentPath, Val: Cur);
232 assert(CycleStart != CurrentPath.end() &&
233 "OnCurrentPath must hold exactly the blocks in CurrentPath");
234 CycleBlocks.insert(I: CycleStart, E: CurrentPath.end());
235 break;
236 }
237 if (Processed.contains(V: Cur))
238 break;
239 OnCurrentPath.insert(V: Cur);
240 CurrentPath.push_back(Elt: Cur);
241 if (const auto *II = dyn_cast<InvokeInst>(Val: Cur->getTerminator()))
242 Cur = II->getNormalDest();
243 else
244 Cur = nullptr;
245 }
246 for (const BasicBlock *B : CurrentPath)
247 Processed.insert(V: B);
248 }
249}
250
251// In call-to-invoke conversion, basic block can be split into multiple blocks,
252// only instrument probe in the head block, ignore the normal dests.
253void SampleProfileProber::findInvokeNormalDests(
254 DenseSet<BasicBlock *> &InvokeNormalDests) {
255 DenseSet<const BasicBlock *> CycleBlocks;
256 findInvokeNormalDestCycles(F: *F, CycleBlocks);
257
258 for (auto &BB : *F) {
259 auto *TI = BB.getTerminator();
260 if (auto *II = dyn_cast<InvokeInst>(Val: TI)) {
261 auto *ND = II->getNormalDest();
262 // Cycle members are original loop blocks, not split continuations.
263 if (!CycleBlocks.contains(V: ND))
264 InvokeNormalDests.insert(V: ND);
265
266 // The normal dest and the try/catch block are connected by an
267 // unconditional branch.
268 while (pred_size(BB: ND) == 1) {
269 auto *Pred = *pred_begin(BB: ND);
270 if (succ_size(BB: Pred) == 1) {
271 InvokeNormalDests.insert(V: Pred);
272 ND = Pred;
273 } else
274 break;
275 }
276 }
277 }
278}
279
280// The call-to-invoke conversion splits the original block into a list of block,
281// we need to compute the hash using the original block's successors to keep the
282// CFG Hash consistent. For a given head block, we keep searching the
283// succesor(normal dest or unconditional branch dest) to find the tail block,
284// the tail block's successors are the original block's successors.
285const Instruction *SampleProfileProber::getOriginalTerminator(
286 const BasicBlock *Head, const DenseSet<BasicBlock *> &BlocksToIgnore) {
287 // Follow invoke dests and ignored blocks to the original terminator. Stop
288 // if a block repeats; a cycle of invokes has no unique tail.
289 DenseSet<const BasicBlock *> Visited;
290 const BasicBlock *BB = Head;
291 Visited.insert(V: BB);
292 while (true) {
293 auto *TI = BB->getTerminator();
294 const BasicBlock *Next = nullptr;
295 if (const auto *II = dyn_cast<InvokeInst>(Val: TI))
296 Next = II->getNormalDest();
297 else if (succ_size(BB) == 1 && BlocksToIgnore.contains(V: *succ_begin(BB)))
298 Next = *succ_begin(BB);
299 else
300 return TI;
301
302 // A cycle has no tail block whose terminator represents the original
303 // block. Stop at the terminator that closes the cycle.
304 if (!Visited.insert(V: Next).second)
305 return TI;
306
307 BB = Next;
308 }
309}
310
311// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
312// value of each BB in the CFG. The higher 32 bits record the number of edges
313// preceded by the number of indirect calls.
314// This is derived from FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash().
315void SampleProfileProber::computeCFGHash(
316 const DenseSet<BasicBlock *> &BlocksToIgnore) {
317 std::vector<uint8_t> Indexes;
318 JamCRC JC;
319 for (auto &BB : *F) {
320 if (BlocksToIgnore.contains(V: &BB))
321 continue;
322
323 auto *TI = getOriginalTerminator(Head: &BB, BlocksToIgnore);
324 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
325 auto *Succ = TI->getSuccessor(Idx: I);
326 auto Index = getBlockId(BB: Succ);
327 // Ingore ignored-block(zero ID) to avoid unstable checksum.
328 if (Index == 0)
329 continue;
330 for (int J = 0; J < 4; J++)
331 Indexes.push_back(x: (uint8_t)(Index >> (J * 8)));
332 }
333 }
334
335 JC.update(Data: Indexes);
336
337 FunctionHash = (uint64_t)CallProbeIds.size() << 48 |
338 (uint64_t)Indexes.size() << 32 | JC.getCRC();
339 // Reserve bit 60-63 for other information purpose.
340 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
341 assert(FunctionHash && "Function checksum should not be zero");
342 LLVM_DEBUG(dbgs() << "\nFunction Hash Computation for " << F->getName()
343 << ":\n"
344 << " CRC = " << JC.getCRC() << ", Edges = "
345 << Indexes.size() << ", ICSites = " << CallProbeIds.size()
346 << ", Hash = " << FunctionHash << "\n");
347}
348
349void SampleProfileProber::computeProbeId(
350 const DenseSet<BasicBlock *> &BlocksToIgnore,
351 const DenseSet<BasicBlock *> &BlocksAndCallsToIgnore) {
352 LLVMContext &Ctx = F->getContext();
353 Module *M = F->getParent();
354
355 for (auto &BB : *F) {
356 if (!BlocksToIgnore.contains(V: &BB))
357 BlockProbeIds[&BB] = ++LastProbeId;
358
359 if (BlocksAndCallsToIgnore.contains(V: &BB))
360 continue;
361 for (auto &I : BB) {
362 if (!isa<CallBase>(Val: I) || isa<IntrinsicInst>(Val: &I))
363 continue;
364
365 // The current implementation uses the lower 16 bits of the discriminator
366 // so anything larger than 0xFFFF will be ignored.
367 if (LastProbeId >= 0xFFFF) {
368 std::string Msg = "Pseudo instrumentation incomplete for " +
369 std::string(F->getName()) + " because it's too large";
370 Ctx.diagnose(
371 DI: DiagnosticInfoSampleProfile(M->getName().data(), Msg, DS_Warning));
372 return;
373 }
374
375 CallProbeIds[&I] = ++LastProbeId;
376 }
377 }
378}
379
380uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const {
381 auto I = BlockProbeIds.find(Val: const_cast<BasicBlock *>(BB));
382 return I == BlockProbeIds.end() ? 0 : I->second;
383}
384
385uint32_t SampleProfileProber::getCallsiteId(const Instruction *Call) const {
386 auto Iter = CallProbeIds.find(Val: const_cast<Instruction *>(Call));
387 return Iter == CallProbeIds.end() ? 0 : Iter->second;
388}
389
390void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
391 Module *M = F.getParent();
392 MDBuilder MDB(F.getContext());
393 // Since the GUID from probe desc and inline stack are computed separately, we
394 // need to make sure their names are consistent, so here also use the name
395 // from debug info.
396 StringRef FName = F.getName();
397 if (auto *SP = F.getSubprogram()) {
398 FName = SP->getLinkageName();
399 if (FName.empty())
400 FName = SP->getName();
401 }
402 uint64_t Guid = Function::getGUIDAssumingExternalLinkage(GlobalName: FName);
403
404 // Assign an artificial debug line to a probe that doesn't come with a real
405 // line. A probe not having a debug line will get an incomplete inline
406 // context. This will cause samples collected on the probe to be counted
407 // into the base profile instead of a context profile. The line number
408 // itself is not important though.
409 auto AssignDebugLoc = [&](Instruction *I) {
410 assert((isa<PseudoProbeInst>(I) || isa<CallBase>(I)) &&
411 "Expecting pseudo probe or call instructions");
412 if (!I->getDebugLoc()) {
413 if (auto *SP = F.getSubprogram()) {
414 auto DIL = DILocation::get(Context&: SP->getContext(), Line: 0, Column: 0, Scope: SP);
415 I->setDebugLoc(DIL);
416 ArtificialDbgLine++;
417 LLVM_DEBUG({
418 dbgs() << "\nIn Function " << F.getName()
419 << " Probe gets an artificial debug line\n";
420 I->dump();
421 });
422 }
423 }
424 };
425
426 // Probe basic blocks.
427 for (auto &I : BlockProbeIds) {
428 BasicBlock *BB = I.first;
429 uint32_t Index = I.second;
430 // Insert a probe before an instruction with a valid debug line number which
431 // will be assigned to the probe. The line number will be used later to
432 // model the inline context when the probe is inlined into other functions.
433 // Debug instructions, phi nodes and lifetime markers do not have an valid
434 // line number. Real instructions generated by optimizations may not come
435 // with a line number either.
436 auto HasValidDbgLine = [](Instruction *J) {
437 return !isa<PHINode>(Val: J) && !J->isLifetimeStartOrEnd() && J->getDebugLoc();
438 };
439
440 Instruction *J = &*BB->getFirstInsertionPt();
441 while (J != BB->getTerminator() && !HasValidDbgLine(J)) {
442 J = J->getNextNode();
443 }
444
445 // A pseudo probe must not be inserted between a `musttail` or
446 // `llvm.experimental.deoptimize` call and its following `ret`, as this
447 // produces invalid IR. Such a call is required to immediately precede the
448 // block's `ret`, so only that position needs to be checked. Insert the
449 // probe before the call instead.
450 if (auto *Ret = dyn_cast<ReturnInst>(Val: BB->getTerminator()))
451 if (auto *CI = dyn_cast_or_null<CallInst>(Val: Ret->getPrevNode()))
452 if ((CI->isMustTailCall() ||
453 CI->getIntrinsicID() == Intrinsic::experimental_deoptimize) &&
454 !J->comesBefore(Other: CI))
455 J = CI;
456
457 IRBuilder<> Builder(J);
458 assert(Builder.GetInsertPoint() != BB->end() &&
459 "Cannot get the probing point");
460 Function *ProbeFn =
461 llvm::Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::pseudoprobe);
462 Value *Args[] = {Builder.getInt64(C: Guid), Builder.getInt64(C: Index),
463 Builder.getInt32(C: 0),
464 Builder.getInt64(C: PseudoProbeFullDistributionFactor)};
465 auto *Probe = Builder.CreateCall(Callee: ProbeFn, Args);
466 AssignDebugLoc(Probe);
467 // Reset the dwarf discriminator if the debug location comes with any. The
468 // discriminator field may be used by FS-AFDO later in the pipeline.
469 if (auto DIL = Probe->getDebugLoc()) {
470 if (DIL->getDiscriminator()) {
471 DIL = DIL->cloneWithDiscriminator(Discriminator: 0);
472 Probe->setDebugLoc(DIL);
473 }
474 }
475 }
476
477 // Probe both direct calls and indirect calls. Direct calls are probed so that
478 // their probe ID can be used as an call site identifier to represent a
479 // calling context.
480 for (auto &I : CallProbeIds) {
481 auto *Call = I.first;
482 uint32_t Index = I.second;
483 uint32_t Type = cast<CallBase>(Val: Call)->getCalledFunction()
484 ? (uint32_t)PseudoProbeType::DirectCall
485 : (uint32_t)PseudoProbeType::IndirectCall;
486 AssignDebugLoc(Call);
487 if (auto DIL = Call->getDebugLoc()) {
488 // Levarge the 32-bit discriminator field of debug data to store the ID
489 // and type of a callsite probe. This gets rid of the dependency on
490 // plumbing a customized metadata through the codegen pipeline.
491 uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
492 Index, Type, Flags: 0, Factor: PseudoProbeDwarfDiscriminator::FullDistributionFactor,
493 DwarfBaseDiscriminator: DIL->getBaseDiscriminator());
494 DIL = DIL->cloneWithDiscriminator(Discriminator: V);
495 Call->setDebugLoc(DIL);
496 }
497 }
498
499 // Create module-level metadata that contains function info necessary to
500 // synthesize probe-based sample counts, which are
501 // - FunctionGUID
502 // - FunctionHash.
503 // - FunctionName
504 auto Hash = getFunctionHash();
505 auto *MD = MDB.createPseudoProbeDesc(GUID: Guid, Hash, FName);
506 auto *NMD = M->getNamedMetadata(Name: PseudoProbeDescMetadataName);
507 assert(NMD && "llvm.pseudo_probe_desc should be pre-created");
508 NMD->addOperand(M: MD);
509}
510
511PreservedAnalyses SampleProfileProbePass::run(Module &M,
512 ModuleAnalysisManager &AM) {
513 // Create the pseudo probe desc metadata beforehand.
514 // Note that modules with only data but no functions will require this to
515 // be set up so that they will be known as probed later.
516 M.getOrInsertNamedMetadata(Name: PseudoProbeDescMetadataName);
517
518 for (auto &F : M) {
519 if (F.isDeclaration())
520 continue;
521 SampleProfileProber ProbeManager(F);
522 ProbeManager.instrumentOneFunc(F, TM);
523 }
524
525 return PreservedAnalyses::none();
526}
527
528void PseudoProbeUpdatePass::runOnFunction(Function &F,
529 FunctionAnalysisManager &FAM) {
530 BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
531 auto BBProfileCount = [&BFI](BasicBlock *BB) {
532 return BFI.getBlockProfileCount(BB).value_or(u: 0);
533 };
534
535 // Collect the sum of execution weight for each probe.
536 ProbeFactorMap ProbeFactors;
537 for (auto &Block : F) {
538 for (auto &I : Block) {
539 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
540 uint64_t Hash = computeCallStackHash(Inst: I);
541 ProbeFactors[{Probe->Id, Hash}] += BBProfileCount(&Block);
542 }
543 }
544 }
545
546 // Fix up over-counted probes.
547 for (auto &Block : F) {
548 for (auto &I : Block) {
549 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
550 uint64_t Hash = computeCallStackHash(Inst: I);
551 float Sum = ProbeFactors[{Probe->Id, Hash}];
552 if (Sum != 0)
553 setProbeDistributionFactor(Inst&: I, Factor: BBProfileCount(&Block) / Sum);
554 }
555 }
556 }
557}
558
559PreservedAnalyses PseudoProbeUpdatePass::run(Module &M,
560 ModuleAnalysisManager &AM) {
561 if (getIPOOptions(M).update_pseudo_probe) {
562 for (auto &F : M) {
563 if (F.isDeclaration())
564 continue;
565 FunctionAnalysisManager &FAM =
566 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
567 runOnFunction(F, FAM);
568 }
569 }
570 return PreservedAnalyses::none();
571}
572