1//===- SampleProfileProbe.cpp - Pseudo probe Instrumentation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SampleProfileProber transformation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/IPO/SampleProfileProbe.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/ADT/StringSet.h"
18#include "llvm/Analysis/BlockFrequencyInfo.h"
19#include "llvm/Analysis/EHUtils.h"
20#include "llvm/Analysis/LoopInfo.h"
21#include "llvm/IR/BasicBlock.h"
22#include "llvm/IR/DebugInfoMetadata.h"
23#include "llvm/IR/DiagnosticInfo.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/Instruction.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/MDBuilder.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/PassInstrumentation.h"
30#include "llvm/IR/PseudoProbe.h"
31#include "llvm/ProfileData/SampleProf.h"
32#include "llvm/Support/CRC.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Target/TargetMachine.h"
35#include "llvm/Transforms/Utils/Instrumentation.h"
36#include "llvm/Transforms/Utils/ModuleUtils.h"
37#include <vector>
38
39using namespace llvm;
40#define DEBUG_TYPE "pseudo-probe"
41
42STATISTIC(ArtificialDbgLine,
43 "Number of probes that have an artificial debug line");
44
45static cl::opt<bool>
46 VerifyPseudoProbe("verify-pseudo-probe", cl::init(Val: false), cl::Hidden,
47 cl::desc("Do pseudo probe verification"));
48
49static cl::list<std::string> VerifyPseudoProbeFuncList(
50 "verify-pseudo-probe-funcs", cl::Hidden,
51 cl::desc("The option to specify the name of the functions to verify."));
52
53static cl::opt<bool>
54 UpdatePseudoProbe("update-pseudo-probe", cl::init(Val: true), cl::Hidden,
55 cl::desc("Update pseudo probe distribution factor"));
56
57static uint64_t getCallStackHash(const DILocation *DIL) {
58 uint64_t Hash = 0;
59 const DILocation *InlinedAt = DIL ? DIL->getInlinedAt() : nullptr;
60 while (InlinedAt) {
61 Hash ^= MD5Hash(Str: std::to_string(val: InlinedAt->getLine()));
62 Hash ^= MD5Hash(Str: std::to_string(val: InlinedAt->getColumn()));
63 auto Name = InlinedAt->getSubprogramLinkageName();
64 Hash ^= MD5Hash(Str: Name);
65 InlinedAt = InlinedAt->getInlinedAt();
66 }
67 return Hash;
68}
69
70static uint64_t computeCallStackHash(const Instruction &Inst) {
71 return getCallStackHash(DIL: Inst.getDebugLoc());
72}
73
74bool PseudoProbeVerifier::shouldVerifyFunction(const Function *F) {
75 // Skip function declaration.
76 if (F->isDeclaration())
77 return false;
78 // Skip function that will not be emitted into object file. The prevailing
79 // defintion will be verified instead.
80 if (F->hasAvailableExternallyLinkage())
81 return false;
82 // Do a name matching.
83 static const StringSet<> VerifyFuncNames(llvm::from_range,
84 VerifyPseudoProbeFuncList);
85 return VerifyFuncNames.empty() || VerifyFuncNames.contains(key: F->getName());
86}
87
88void PseudoProbeVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
89 if (VerifyPseudoProbe) {
90 PIC.registerAfterPassCallback(
91 C: [this](StringRef P, IRUnitRef IR, const PreservedAnalyses &) {
92 this->runAfterPass(PassID: P, IR);
93 });
94 }
95}
96
97// Callback to run after each transformation for the new pass manager.
98void PseudoProbeVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
99 std::string Banner =
100 "\n*** Pseudo Probe Verification After " + PassID.str() + " ***\n";
101 dbgs() << Banner;
102 if (const auto *M = dyn_cast<Module>(Val&: IR))
103 runAfterPass(M);
104 else if (const auto *F = dyn_cast<Function>(Val&: IR))
105 runAfterPass(F);
106 else if (const auto *C = dyn_cast<LazyCallGraph::SCC>(Val&: IR))
107 runAfterPass(C);
108 else if (const auto *L = dyn_cast<Loop>(Val&: IR))
109 runAfterPass(L);
110 else
111 llvm_unreachable("Unknown IR unit");
112}
113
114void PseudoProbeVerifier::runAfterPass(const Module *M) {
115 for (const Function &F : *M)
116 runAfterPass(F: &F);
117}
118
119void PseudoProbeVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
120 for (const LazyCallGraph::Node &N : *C)
121 runAfterPass(F: &N.getFunction());
122}
123
124void PseudoProbeVerifier::runAfterPass(const Function *F) {
125 if (!shouldVerifyFunction(F))
126 return;
127 ProbeFactorMap ProbeFactors;
128 for (const auto &BB : *F)
129 collectProbeFactors(BB: &BB, ProbeFactors);
130 verifyProbeFactors(F, ProbeFactors);
131}
132
133void PseudoProbeVerifier::runAfterPass(const Loop *L) {
134 const Function *F = L->getHeader()->getParent();
135 runAfterPass(F);
136}
137
138void PseudoProbeVerifier::collectProbeFactors(const BasicBlock *Block,
139 ProbeFactorMap &ProbeFactors) {
140 for (const auto &I : *Block) {
141 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
142 uint64_t Hash = computeCallStackHash(Inst: I);
143 ProbeFactors[{Probe->Id, Hash}] += Probe->Factor;
144 }
145 }
146}
147
148void PseudoProbeVerifier::verifyProbeFactors(
149 const Function *F, const ProbeFactorMap &ProbeFactors) {
150 bool BannerPrinted = false;
151 auto &PrevProbeFactors = FunctionProbeFactors[F->getName()];
152 for (const auto &I : ProbeFactors) {
153 float CurProbeFactor = I.second;
154 auto [It, Inserted] = PrevProbeFactors.try_emplace(Key: I.first);
155 if (!Inserted) {
156 float PrevProbeFactor = It->second;
157 if (std::abs(x: CurProbeFactor - PrevProbeFactor) >
158 DistributionFactorVariance) {
159 if (!BannerPrinted) {
160 dbgs() << "Function " << F->getName() << ":\n";
161 BannerPrinted = true;
162 }
163 dbgs() << "Probe " << I.first.first << "\tprevious factor "
164 << format(Fmt: "%0.2f", Vals: PrevProbeFactor) << "\tcurrent factor "
165 << format(Fmt: "%0.2f", Vals: CurProbeFactor) << "\n";
166 }
167 }
168
169 // Update
170 It->second = I.second;
171 }
172}
173
174SampleProfileProber::SampleProfileProber(Function &Func) : F(&Func) {
175 BlockProbeIds.clear();
176 CallProbeIds.clear();
177 LastProbeId = (uint32_t)PseudoProbeReservedId::Last;
178
179 DenseSet<BasicBlock *> BlocksToIgnore;
180 DenseSet<BasicBlock *> BlocksAndCallsToIgnore;
181 computeBlocksToIgnore(BlocksToIgnore, BlocksAndCallsToIgnore);
182
183 computeProbeId(BlocksToIgnore, BlocksAndCallsToIgnore);
184 computeCFGHash(BlocksToIgnore);
185}
186
187// Two purposes to compute the blocks to ignore:
188// 1. Reduce the IR size.
189// 2. Make the instrumentation(checksum) stable. e.g. the frondend may
190// generate unstable IR while optimizing nounwind attribute, some versions are
191// optimized with the call-to-invoke conversion, while other versions do not.
192// This discrepancy in probe ID could cause profile mismatching issues.
193// Note that those ignored blocks are either cold blocks or new split blocks
194// whose original blocks are instrumented, so it shouldn't degrade the profile
195// quality.
196void SampleProfileProber::computeBlocksToIgnore(
197 DenseSet<BasicBlock *> &BlocksToIgnore,
198 DenseSet<BasicBlock *> &BlocksAndCallsToIgnore) {
199 // Ignore the cold EH and unreachable blocks and calls.
200 computeEHOnlyBlocks(F&: *F, EHBlocks&: BlocksAndCallsToIgnore);
201 findUnreachableBlocks(BlocksToIgnore&: BlocksAndCallsToIgnore);
202
203 BlocksToIgnore.insert_range(R&: BlocksAndCallsToIgnore);
204
205 // Handle the call-to-invoke conversion case: make sure that the probe id and
206 // callsite id are consistent before and after the block split. For block
207 // probe, we only keep the head block probe id and ignore the block ids of the
208 // normal dests. For callsite probe, it's different to block probe, there is
209 // no additional callsite in the normal dests, so we don't ignore the
210 // callsites.
211 findInvokeNormalDests(InvokeNormalDests&: BlocksToIgnore);
212}
213
214// Unreachable blocks and calls are always cold, ignore them.
215void SampleProfileProber::findUnreachableBlocks(
216 DenseSet<BasicBlock *> &BlocksToIgnore) {
217 for (auto &BB : *F) {
218 if (&BB != &F->getEntryBlock() && pred_size(BB: &BB) == 0)
219 BlocksToIgnore.insert(V: &BB);
220 }
221}
222
223// Follow invoke normal-dest edges and record blocks that sit on a cycle.
224static void
225findInvokeNormalDestCycles(const Function &F,
226 DenseSet<const BasicBlock *> &CycleBlocks) {
227 DenseSet<const BasicBlock *> Processed;
228 DenseSet<const BasicBlock *> OnCurrentPath;
229 SmallVector<const BasicBlock *, 16> CurrentPath;
230
231 for (const BasicBlock &Start : F) {
232 if (Processed.contains(V: &Start))
233 continue;
234
235 CurrentPath.clear();
236 OnCurrentPath.clear();
237 const BasicBlock *Cur = &Start;
238 while (Cur) {
239 if (OnCurrentPath.contains(V: Cur)) {
240 // Back-edge onto CurrentPath: the cycle is the suffix starting at Cur.
241 auto CycleStart = llvm::find(Range&: CurrentPath, Val: Cur);
242 assert(CycleStart != CurrentPath.end() &&
243 "OnCurrentPath must hold exactly the blocks in CurrentPath");
244 CycleBlocks.insert(I: CycleStart, E: CurrentPath.end());
245 break;
246 }
247 if (Processed.contains(V: Cur))
248 break;
249 OnCurrentPath.insert(V: Cur);
250 CurrentPath.push_back(Elt: Cur);
251 if (const auto *II = dyn_cast<InvokeInst>(Val: Cur->getTerminator()))
252 Cur = II->getNormalDest();
253 else
254 Cur = nullptr;
255 }
256 for (const BasicBlock *B : CurrentPath)
257 Processed.insert(V: B);
258 }
259}
260
261// In call-to-invoke conversion, basic block can be split into multiple blocks,
262// only instrument probe in the head block, ignore the normal dests.
263void SampleProfileProber::findInvokeNormalDests(
264 DenseSet<BasicBlock *> &InvokeNormalDests) {
265 DenseSet<const BasicBlock *> CycleBlocks;
266 findInvokeNormalDestCycles(F: *F, CycleBlocks);
267
268 for (auto &BB : *F) {
269 auto *TI = BB.getTerminator();
270 if (auto *II = dyn_cast<InvokeInst>(Val: TI)) {
271 auto *ND = II->getNormalDest();
272 // Cycle members are original loop blocks, not split continuations.
273 if (!CycleBlocks.contains(V: ND))
274 InvokeNormalDests.insert(V: ND);
275
276 // The normal dest and the try/catch block are connected by an
277 // unconditional branch.
278 while (pred_size(BB: ND) == 1) {
279 auto *Pred = *pred_begin(BB: ND);
280 if (succ_size(BB: Pred) == 1) {
281 InvokeNormalDests.insert(V: Pred);
282 ND = Pred;
283 } else
284 break;
285 }
286 }
287 }
288}
289
290// The call-to-invoke conversion splits the original block into a list of block,
291// we need to compute the hash using the original block's successors to keep the
292// CFG Hash consistent. For a given head block, we keep searching the
293// succesor(normal dest or unconditional branch dest) to find the tail block,
294// the tail block's successors are the original block's successors.
295const Instruction *SampleProfileProber::getOriginalTerminator(
296 const BasicBlock *Head, const DenseSet<BasicBlock *> &BlocksToIgnore) {
297 // Follow invoke dests and ignored blocks to the original terminator. Stop
298 // if a block repeats; a cycle of invokes has no unique tail.
299 DenseSet<const BasicBlock *> Visited;
300 const BasicBlock *BB = Head;
301 Visited.insert(V: BB);
302 while (true) {
303 auto *TI = BB->getTerminator();
304 const BasicBlock *Next = nullptr;
305 if (const auto *II = dyn_cast<InvokeInst>(Val: TI))
306 Next = II->getNormalDest();
307 else if (succ_size(BB) == 1 && BlocksToIgnore.contains(V: *succ_begin(BB)))
308 Next = *succ_begin(BB);
309 else
310 return TI;
311
312 // A cycle has no tail block whose terminator represents the original
313 // block. Stop at the terminator that closes the cycle.
314 if (!Visited.insert(V: Next).second)
315 return TI;
316
317 BB = Next;
318 }
319}
320
321// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
322// value of each BB in the CFG. The higher 32 bits record the number of edges
323// preceded by the number of indirect calls.
324// This is derived from FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash().
325void SampleProfileProber::computeCFGHash(
326 const DenseSet<BasicBlock *> &BlocksToIgnore) {
327 std::vector<uint8_t> Indexes;
328 JamCRC JC;
329 for (auto &BB : *F) {
330 if (BlocksToIgnore.contains(V: &BB))
331 continue;
332
333 auto *TI = getOriginalTerminator(Head: &BB, BlocksToIgnore);
334 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
335 auto *Succ = TI->getSuccessor(Idx: I);
336 auto Index = getBlockId(BB: Succ);
337 // Ingore ignored-block(zero ID) to avoid unstable checksum.
338 if (Index == 0)
339 continue;
340 for (int J = 0; J < 4; J++)
341 Indexes.push_back(x: (uint8_t)(Index >> (J * 8)));
342 }
343 }
344
345 JC.update(Data: Indexes);
346
347 FunctionHash = (uint64_t)CallProbeIds.size() << 48 |
348 (uint64_t)Indexes.size() << 32 | JC.getCRC();
349 // Reserve bit 60-63 for other information purpose.
350 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
351 assert(FunctionHash && "Function checksum should not be zero");
352 LLVM_DEBUG(dbgs() << "\nFunction Hash Computation for " << F->getName()
353 << ":\n"
354 << " CRC = " << JC.getCRC() << ", Edges = "
355 << Indexes.size() << ", ICSites = " << CallProbeIds.size()
356 << ", Hash = " << FunctionHash << "\n");
357}
358
359void SampleProfileProber::computeProbeId(
360 const DenseSet<BasicBlock *> &BlocksToIgnore,
361 const DenseSet<BasicBlock *> &BlocksAndCallsToIgnore) {
362 LLVMContext &Ctx = F->getContext();
363 Module *M = F->getParent();
364
365 for (auto &BB : *F) {
366 if (!BlocksToIgnore.contains(V: &BB))
367 BlockProbeIds[&BB] = ++LastProbeId;
368
369 if (BlocksAndCallsToIgnore.contains(V: &BB))
370 continue;
371 for (auto &I : BB) {
372 if (!isa<CallBase>(Val: I) || isa<IntrinsicInst>(Val: &I))
373 continue;
374
375 // The current implementation uses the lower 16 bits of the discriminator
376 // so anything larger than 0xFFFF will be ignored.
377 if (LastProbeId >= 0xFFFF) {
378 std::string Msg = "Pseudo instrumentation incomplete for " +
379 std::string(F->getName()) + " because it's too large";
380 Ctx.diagnose(
381 DI: DiagnosticInfoSampleProfile(M->getName().data(), Msg, DS_Warning));
382 return;
383 }
384
385 CallProbeIds[&I] = ++LastProbeId;
386 }
387 }
388}
389
390uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const {
391 auto I = BlockProbeIds.find(Val: const_cast<BasicBlock *>(BB));
392 return I == BlockProbeIds.end() ? 0 : I->second;
393}
394
395uint32_t SampleProfileProber::getCallsiteId(const Instruction *Call) const {
396 auto Iter = CallProbeIds.find(Val: const_cast<Instruction *>(Call));
397 return Iter == CallProbeIds.end() ? 0 : Iter->second;
398}
399
400void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
401 Module *M = F.getParent();
402 MDBuilder MDB(F.getContext());
403 // Since the GUID from probe desc and inline stack are computed separately, we
404 // need to make sure their names are consistent, so here also use the name
405 // from debug info.
406 StringRef FName = F.getName();
407 if (auto *SP = F.getSubprogram()) {
408 FName = SP->getLinkageName();
409 if (FName.empty())
410 FName = SP->getName();
411 }
412 uint64_t Guid = Function::getGUIDAssumingExternalLinkage(GlobalName: FName);
413
414 // Assign an artificial debug line to a probe that doesn't come with a real
415 // line. A probe not having a debug line will get an incomplete inline
416 // context. This will cause samples collected on the probe to be counted
417 // into the base profile instead of a context profile. The line number
418 // itself is not important though.
419 auto AssignDebugLoc = [&](Instruction *I) {
420 assert((isa<PseudoProbeInst>(I) || isa<CallBase>(I)) &&
421 "Expecting pseudo probe or call instructions");
422 if (!I->getDebugLoc()) {
423 if (auto *SP = F.getSubprogram()) {
424 auto DIL = DILocation::get(Context&: SP->getContext(), Line: 0, Column: 0, Scope: SP);
425 I->setDebugLoc(DIL);
426 ArtificialDbgLine++;
427 LLVM_DEBUG({
428 dbgs() << "\nIn Function " << F.getName()
429 << " Probe gets an artificial debug line\n";
430 I->dump();
431 });
432 }
433 }
434 };
435
436 // Probe basic blocks.
437 for (auto &I : BlockProbeIds) {
438 BasicBlock *BB = I.first;
439 uint32_t Index = I.second;
440 // Insert a probe before an instruction with a valid debug line number which
441 // will be assigned to the probe. The line number will be used later to
442 // model the inline context when the probe is inlined into other functions.
443 // Debug instructions, phi nodes and lifetime markers do not have an valid
444 // line number. Real instructions generated by optimizations may not come
445 // with a line number either.
446 auto HasValidDbgLine = [](Instruction *J) {
447 return !isa<PHINode>(Val: J) && !J->isLifetimeStartOrEnd() && J->getDebugLoc();
448 };
449
450 Instruction *J = &*BB->getFirstInsertionPt();
451 while (J != BB->getTerminator() && !HasValidDbgLine(J)) {
452 J = J->getNextNode();
453 }
454
455 // A pseudo probe must not be inserted between a `musttail` or
456 // `llvm.experimental.deoptimize` call and its following `ret`, as this
457 // produces invalid IR. Such a call is required to immediately precede the
458 // block's `ret`, so only that position needs to be checked. Insert the
459 // probe before the call instead.
460 if (auto *Ret = dyn_cast<ReturnInst>(Val: BB->getTerminator()))
461 if (auto *CI = dyn_cast_or_null<CallInst>(Val: Ret->getPrevNode()))
462 if ((CI->isMustTailCall() ||
463 CI->getIntrinsicID() == Intrinsic::experimental_deoptimize) &&
464 !J->comesBefore(Other: CI))
465 J = CI;
466
467 IRBuilder<> Builder(J);
468 assert(Builder.GetInsertPoint() != BB->end() &&
469 "Cannot get the probing point");
470 Function *ProbeFn =
471 llvm::Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::pseudoprobe);
472 Value *Args[] = {Builder.getInt64(C: Guid), Builder.getInt64(C: Index),
473 Builder.getInt32(C: 0),
474 Builder.getInt64(C: PseudoProbeFullDistributionFactor)};
475 auto *Probe = Builder.CreateCall(Callee: ProbeFn, Args);
476 AssignDebugLoc(Probe);
477 // Reset the dwarf discriminator if the debug location comes with any. The
478 // discriminator field may be used by FS-AFDO later in the pipeline.
479 if (auto DIL = Probe->getDebugLoc()) {
480 if (DIL->getDiscriminator()) {
481 DIL = DIL->cloneWithDiscriminator(Discriminator: 0);
482 Probe->setDebugLoc(DIL);
483 }
484 }
485 }
486
487 // Probe both direct calls and indirect calls. Direct calls are probed so that
488 // their probe ID can be used as an call site identifier to represent a
489 // calling context.
490 for (auto &I : CallProbeIds) {
491 auto *Call = I.first;
492 uint32_t Index = I.second;
493 uint32_t Type = cast<CallBase>(Val: Call)->getCalledFunction()
494 ? (uint32_t)PseudoProbeType::DirectCall
495 : (uint32_t)PseudoProbeType::IndirectCall;
496 AssignDebugLoc(Call);
497 if (auto DIL = Call->getDebugLoc()) {
498 // Levarge the 32-bit discriminator field of debug data to store the ID
499 // and type of a callsite probe. This gets rid of the dependency on
500 // plumbing a customized metadata through the codegen pipeline.
501 uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
502 Index, Type, Flags: 0, Factor: PseudoProbeDwarfDiscriminator::FullDistributionFactor,
503 DwarfBaseDiscriminator: DIL->getBaseDiscriminator());
504 DIL = DIL->cloneWithDiscriminator(Discriminator: V);
505 Call->setDebugLoc(DIL);
506 }
507 }
508
509 // Create module-level metadata that contains function info necessary to
510 // synthesize probe-based sample counts, which are
511 // - FunctionGUID
512 // - FunctionHash.
513 // - FunctionName
514 auto Hash = getFunctionHash();
515 auto *MD = MDB.createPseudoProbeDesc(GUID: Guid, Hash, FName);
516 auto *NMD = M->getNamedMetadata(Name: PseudoProbeDescMetadataName);
517 assert(NMD && "llvm.pseudo_probe_desc should be pre-created");
518 NMD->addOperand(M: MD);
519}
520
521PreservedAnalyses SampleProfileProbePass::run(Module &M,
522 ModuleAnalysisManager &AM) {
523 // Create the pseudo probe desc metadata beforehand.
524 // Note that modules with only data but no functions will require this to
525 // be set up so that they will be known as probed later.
526 M.getOrInsertNamedMetadata(Name: PseudoProbeDescMetadataName);
527
528 for (auto &F : M) {
529 if (F.isDeclaration())
530 continue;
531 SampleProfileProber ProbeManager(F);
532 ProbeManager.instrumentOneFunc(F, TM);
533 }
534
535 return PreservedAnalyses::none();
536}
537
538void PseudoProbeUpdatePass::runOnFunction(Function &F,
539 FunctionAnalysisManager &FAM) {
540 BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
541 auto BBProfileCount = [&BFI](BasicBlock *BB) {
542 return BFI.getBlockProfileCount(BB).value_or(u: 0);
543 };
544
545 // Collect the sum of execution weight for each probe.
546 ProbeFactorMap ProbeFactors;
547 for (auto &Block : F) {
548 for (auto &I : Block) {
549 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
550 uint64_t Hash = computeCallStackHash(Inst: I);
551 ProbeFactors[{Probe->Id, Hash}] += BBProfileCount(&Block);
552 }
553 }
554 }
555
556 // Fix up over-counted probes.
557 for (auto &Block : F) {
558 for (auto &I : Block) {
559 if (std::optional<PseudoProbe> Probe = extractProbe(Inst: I)) {
560 uint64_t Hash = computeCallStackHash(Inst: I);
561 float Sum = ProbeFactors[{Probe->Id, Hash}];
562 if (Sum != 0)
563 setProbeDistributionFactor(Inst&: I, Factor: BBProfileCount(&Block) / Sum);
564 }
565 }
566 }
567}
568
569PreservedAnalyses PseudoProbeUpdatePass::run(Module &M,
570 ModuleAnalysisManager &AM) {
571 if (UpdatePseudoProbe) {
572 for (auto &F : M) {
573 if (F.isDeclaration())
574 continue;
575 FunctionAnalysisManager &FAM =
576 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
577 runOnFunction(F, FAM);
578 }
579 }
580 return PreservedAnalyses::none();
581}
582