1//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements PseudoProbeInserter pass, which inserts pseudo probe
10// annotations for call instructions with a pseudo-probe-specific dwarf
11// discriminator. such discriminator indicates that the call instruction comes
12// with a pseudo probe, and the discriminator value holds information to
13// identify the corresponding counter.
14//===----------------------------------------------------------------------===//
15
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineFunctionPass.h"
18#include "llvm/CodeGen/MachineInstr.h"
19#include "llvm/CodeGen/TargetInstrInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/PseudoProbe.h"
23#include "llvm/InitializePasses.h"
24#include "llvm/ProfileData/SampleProf.h"
25
26#define DEBUG_TYPE "pseudo-probe-inserter"
27
28using namespace llvm;
29
30namespace {
31
32// A real instruction is a non-meta, non-pseudo instruction. Calls (including
33// call pseudos like tail-call returns) are also treated as real because they
34// expand to a branch and any preceding call probe must be preserved.
35static bool isCallOrRealInstruction(const MachineInstr &MI) {
36 return MI.isCall() || (!MI.isPseudo() && !MI.isMetaInstruction());
37}
38
39class PseudoProbeInserter : public MachineFunctionPass {
40public:
41 static char ID;
42
43 PseudoProbeInserter() : MachineFunctionPass(ID) {}
44
45 StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
46
47 void getAnalysisUsage(AnalysisUsage &AU) const override {
48 AU.setPreservesAll();
49 MachineFunctionPass::getAnalysisUsage(AU);
50 }
51
52 bool doInitialization(Module &M) override {
53 ShouldRun = M.getNamedMetadata(Name: PseudoProbeDescMetadataName);
54 return false;
55 }
56
57 bool runOnMachineFunction(MachineFunction &MF) override {
58 if (!ShouldRun)
59 return false;
60 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
61 bool Changed = false;
62 for (MachineBasicBlock &MBB : MF) {
63 MachineInstr *FirstInstr = nullptr;
64 for (MachineInstr &MI : MBB) {
65 // Pseudo instructions like TCRETURNdi results in a branch instruction
66 // and the call probe for that tail call should be preserved.
67 if (isCallOrRealInstruction(MI))
68 FirstInstr = &MI;
69 if (MI.isCall()) {
70 if (DILocation *DL = MI.getDebugLoc()) {
71 auto Value = DL->getDiscriminator();
72 if (DILocation::isPseudoProbeDiscriminator(Discriminator: Value)) {
73 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PSEUDO_PROBE))
74 .addImm(Val: getFuncGUID(M: MF.getFunction().getParent(), DL))
75 .addImm(
76 Val: PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
77 .addImm(
78 Val: PseudoProbeDwarfDiscriminator::extractProbeType(Value))
79 .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeAttributes(
80 Value));
81 Changed = true;
82 }
83 }
84 }
85 }
86
87 // Walk the block backwards, move PSEUDO_PROBE before the first real
88 // instruction to fix out-of-order probes. There is a problem with probes
89 // as the terminator of the block. During the offline counts processing,
90 // the samples collected on the first physical instruction following a
91 // probe will be counted towards the probe. This logically equals to
92 // treating the instruction next to a probe as if it is from the same
93 // block of the probe. This is accurate most of the time unless the
94 // instruction can be reached from multiple flows, which means it actually
95 // starts a new block. Samples collected on such probes may cause
96 // imprecision with the counts inference algorithm. Fortunately, if
97 // there are still other native instructions preceding the probe we can
98 // use them as a place holder to collect samples for the probe.
99 if (FirstInstr) {
100 auto MII = MBB.rbegin();
101 while (MII != MBB.rend()) {
102 // Skip all pseudo probes followed by a real instruction since they
103 // are not dangling. Treat call pseudos (e.g. tail-call returns)
104 // as real instructions to keep this consistent with the forward
105 // scan above.
106 if (isCallOrRealInstruction(MI: *MII))
107 break;
108 auto Cur = MII++;
109 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
110 continue;
111 // Move the dangling probe before FirstInstr.
112 auto *ProbeInstr = &*Cur;
113 MBB.remove(I: ProbeInstr);
114 MBB.insert(I: FirstInstr, MI: ProbeInstr);
115 Changed = true;
116 }
117 } else {
118 // Probes not surrounded by any real instructions in the same block are
119 // called dangling probes. Since there's no good way to pick up a sample
120 // collection point for dangling probes at compile time, they are being
121 // removed so that the profile correlation tool will not report any
122 // samples collected for them and it's up to the counts inference tool
123 // to get them a reasonable count.
124 SmallVector<MachineInstr *, 4> ToBeRemoved;
125 for (MachineInstr &MI : MBB) {
126 if (MI.isPseudoProbe())
127 ToBeRemoved.push_back(Elt: &MI);
128 }
129
130 for (auto *MI : ToBeRemoved)
131 MI->eraseFromParent();
132
133 Changed |= !ToBeRemoved.empty();
134 }
135 }
136
137 return Changed;
138 }
139
140private:
141 uint64_t getFuncGUID(Module *M, DILocation *DL) {
142 auto Name = DL->getSubprogramLinkageName();
143 // CoroSplit Pass will change the debug info with suffixes i.e. `.resume`,
144 // `.destroy`, `.cleanup`. Strip these suffixes to make the GUID consistent
145 // with the pseudo probe
146 Name = FunctionSamples::getCanonicalCoroFnName(FnName: Name);
147 return Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
148 }
149
150 bool ShouldRun = false;
151};
152} // namespace
153
154char PseudoProbeInserter::ID = 0;
155INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
156 "Insert pseudo probe annotations for value profiling",
157 false, false)
158INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
159INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
160 "Insert pseudo probe annotations for value profiling",
161 false, false)
162
163FunctionPass *llvm::createPseudoProbeInserter() {
164 return new PseudoProbeInserter();
165}
166