1//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements PseudoProbeInserter pass, which inserts pseudo probe
10// annotations for call instructions with a pseudo-probe-specific dwarf
11// discriminator. such discriminator indicates that the call instruction comes
12// with a pseudo probe, and the discriminator value holds information to
13// identify the corresponding counter.
14//===----------------------------------------------------------------------===//
15
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineFunctionPass.h"
18#include "llvm/CodeGen/MachineInstr.h"
19#include "llvm/CodeGen/TargetInstrInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/PseudoProbe.h"
23#include "llvm/InitializePasses.h"
24
25#define DEBUG_TYPE "pseudo-probe-inserter"
26
27using namespace llvm;
28
29namespace {
30class PseudoProbeInserter : public MachineFunctionPass {
31public:
32 static char ID;
33
34 PseudoProbeInserter() : MachineFunctionPass(ID) {
35 initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
36 }
37
38 StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
39
40 void getAnalysisUsage(AnalysisUsage &AU) const override {
41 AU.setPreservesAll();
42 MachineFunctionPass::getAnalysisUsage(AU);
43 }
44
45 bool doInitialization(Module &M) override {
46 ShouldRun = M.getNamedMetadata(Name: PseudoProbeDescMetadataName);
47 return false;
48 }
49
50 bool runOnMachineFunction(MachineFunction &MF) override {
51 if (!ShouldRun)
52 return false;
53 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
54 bool Changed = false;
55 for (MachineBasicBlock &MBB : MF) {
56 MachineInstr *FirstInstr = nullptr;
57 for (MachineInstr &MI : MBB) {
58 if (!MI.isPseudo())
59 FirstInstr = &MI;
60 if (MI.isCall()) {
61 if (DILocation *DL = MI.getDebugLoc()) {
62 auto Value = DL->getDiscriminator();
63 if (DILocation::isPseudoProbeDiscriminator(Discriminator: Value)) {
64 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PSEUDO_PROBE))
65 .addImm(Val: getFuncGUID(M: MF.getFunction().getParent(), DL))
66 .addImm(
67 Val: PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
68 .addImm(
69 Val: PseudoProbeDwarfDiscriminator::extractProbeType(Value))
70 .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeAttributes(
71 Value));
72 Changed = true;
73 }
74 }
75 }
76 }
77
78 // Walk the block backwards, move PSEUDO_PROBE before the first real
79 // instruction to fix out-of-order probes. There is a problem with probes
80 // as the terminator of the block. During the offline counts processing,
81 // the samples collected on the first physical instruction following a
82 // probe will be counted towards the probe. This logically equals to
83 // treating the instruction next to a probe as if it is from the same
84 // block of the probe. This is accurate most of the time unless the
85 // instruction can be reached from multiple flows, which means it actually
86 // starts a new block. Samples collected on such probes may cause
87 // imprecision with the counts inference algorithm. Fortunately, if
88 // there are still other native instructions preceding the probe we can
89 // use them as a place holder to collect samples for the probe.
90 if (FirstInstr) {
91 auto MII = MBB.rbegin();
92 while (MII != MBB.rend()) {
93 // Skip all pseudo probes followed by a real instruction since they
94 // are not dangling.
95 if (!MII->isPseudo())
96 break;
97 auto Cur = MII++;
98 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
99 continue;
100 // Move the dangling probe before FirstInstr.
101 auto *ProbeInstr = &*Cur;
102 MBB.remove(I: ProbeInstr);
103 MBB.insert(I: FirstInstr, MI: ProbeInstr);
104 Changed = true;
105 }
106 } else {
107 // Probes not surrounded by any real instructions in the same block are
108 // called dangling probes. Since there's no good way to pick up a sample
109 // collection point for dangling probes at compile time, they are being
110 // removed so that the profile correlation tool will not report any
111 // samples collected for them and it's up to the counts inference tool
112 // to get them a reasonable count.
113 SmallVector<MachineInstr *, 4> ToBeRemoved;
114 for (MachineInstr &MI : MBB) {
115 if (MI.isPseudoProbe())
116 ToBeRemoved.push_back(Elt: &MI);
117 }
118
119 for (auto *MI : ToBeRemoved)
120 MI->eraseFromParent();
121
122 Changed |= !ToBeRemoved.empty();
123 }
124 }
125
126 return Changed;
127 }
128
129private:
130 uint64_t getFuncGUID(Module *M, DILocation *DL) {
131 auto Name = DL->getSubprogramLinkageName();
132 return Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
133 }
134
135 bool ShouldRun = false;
136};
137} // namespace
138
139char PseudoProbeInserter::ID = 0;
140INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
141 "Insert pseudo probe annotations for value profiling",
142 false, false)
143INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
144INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
145 "Insert pseudo probe annotations for value profiling",
146 false, false)
147
148FunctionPass *llvm::createPseudoProbeInserter() {
149 return new PseudoProbeInserter();
150}
151