1//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements PseudoProbeInserter pass, which inserts pseudo probe
10// annotations for call instructions with a pseudo-probe-specific dwarf
11// discriminator. such discriminator indicates that the call instruction comes
12// with a pseudo probe, and the discriminator value holds information to
13// identify the corresponding counter.
14//===----------------------------------------------------------------------===//
15
16#include "llvm/CodeGen/PseudoProbeInserter.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
19#include "llvm/CodeGen/MachineFunctionPass.h"
20#include "llvm/CodeGen/MachineInstr.h"
21#include "llvm/CodeGen/MachinePassManager.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/IR/Analysis.h"
24#include "llvm/IR/DebugInfoMetadata.h"
25#include "llvm/IR/Module.h"
26#include "llvm/IR/PseudoProbe.h"
27#include "llvm/InitializePasses.h"
28#include "llvm/ProfileData/SampleProf.h"
29
30#define DEBUG_TYPE "pseudo-probe-inserter"
31
32using namespace llvm;
33
34namespace {
35
36// A real instruction is a non-meta, non-pseudo instruction. Calls (including
37// call pseudos like tail-call returns) are also treated as real because they
38// expand to a branch and any preceding call probe must be preserved.
39static bool isCallOrRealInstruction(const MachineInstr &MI) {
40 return MI.isCall() || (!MI.isPseudo() && !MI.isMetaInstruction());
41}
42
43static uint64_t getFuncGUID(Module *M, DILocation *DL) {
44 auto Name = DL->getSubprogramLinkageName();
45 // CoroSplit Pass will change the debug info with suffixes i.e. `.resume`,
46 // `.destroy`, `.cleanup`. Strip these suffixes to make the GUID consistent
47 // with the pseudo probe
48 Name = FunctionSamples::getCanonicalCoroFnName(FnName: Name);
49 return Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
50}
51
52static bool insertPseudoProbesInMF(MachineFunction &MF) {
53 if (!MF.getFunction().getParent()->getNamedMetadata(
54 Name: PseudoProbeDescMetadataName))
55 return false;
56 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
57 bool Changed = false;
58 for (MachineBasicBlock &MBB : MF) {
59 MachineInstr *FirstInstr = nullptr;
60 for (MachineInstr &MI : MBB) {
61 // Pseudo instructions like TCRETURNdi results in a branch instruction
62 // and the call probe for that tail call should be preserved.
63 if (isCallOrRealInstruction(MI))
64 FirstInstr = &MI;
65 if (MI.isCall()) {
66 if (DILocation *DL = MI.getDebugLoc()) {
67 auto Value = DL->getDiscriminator();
68 if (DILocation::isPseudoProbeDiscriminator(Discriminator: Value)) {
69 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PSEUDO_PROBE))
70 .addImm(Val: getFuncGUID(M: MF.getFunction().getParent(), DL))
71 .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
72 .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeType(Value))
73 .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeAttributes(
74 Value));
75 Changed = true;
76 }
77 }
78 }
79 }
80
81 // Walk the block backwards, move PSEUDO_PROBE before the first real
82 // instruction to fix out-of-order probes. There is a problem with probes
83 // as the terminator of the block. During the offline counts processing,
84 // the samples collected on the first physical instruction following a
85 // probe will be counted towards the probe. This logically equals to
86 // treating the instruction next to a probe as if it is from the same
87 // block of the probe. This is accurate most of the time unless the
88 // instruction can be reached from multiple flows, which means it actually
89 // starts a new block. Samples collected on such probes may cause
90 // imprecision with the counts inference algorithm. Fortunately, if
91 // there are still other native instructions preceding the probe we can
92 // use them as a place holder to collect samples for the probe.
93 if (FirstInstr) {
94 auto MII = MBB.rbegin();
95 while (MII != MBB.rend()) {
96 // Skip all pseudo probes followed by a real instruction since they
97 // are not dangling. Treat call pseudos (e.g. tail-call returns)
98 // as real instructions to keep this consistent with the forward
99 // scan above.
100 if (isCallOrRealInstruction(MI: *MII))
101 break;
102 auto Cur = MII++;
103 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
104 continue;
105 // Move the dangling probe before FirstInstr.
106 auto *ProbeInstr = &*Cur;
107 MBB.remove(I: ProbeInstr);
108 MBB.insert(I: FirstInstr, MI: ProbeInstr);
109 Changed = true;
110 }
111 } else {
112 // Probes not surrounded by any real instructions in the same block are
113 // called dangling probes. Since there's no good way to pick up a sample
114 // collection point for dangling probes at compile time, they are being
115 // removed so that the profile correlation tool will not report any
116 // samples collected for them and it's up to the counts inference tool
117 // to get them a reasonable count.
118 SmallVector<MachineInstr *, 4> ToBeRemoved;
119 for (MachineInstr &MI : MBB) {
120 if (MI.isPseudoProbe())
121 ToBeRemoved.push_back(Elt: &MI);
122 }
123
124 for (auto *MI : ToBeRemoved)
125 MI->eraseFromParent();
126
127 Changed |= !ToBeRemoved.empty();
128 }
129 }
130
131 return Changed;
132}
133
134class PseudoProbeInserterLegacy : public MachineFunctionPass {
135public:
136 static char ID;
137
138 PseudoProbeInserterLegacy() : MachineFunctionPass(ID) {}
139
140 StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
141
142 void getAnalysisUsage(AnalysisUsage &AU) const override {
143 AU.setPreservesAll();
144 MachineFunctionPass::getAnalysisUsage(AU);
145 }
146
147 bool runOnMachineFunction(MachineFunction &MF) override {
148 return insertPseudoProbesInMF(MF);
149 }
150};
151} // namespace
152
153PreservedAnalyses
154PseudoProbeInserterPass::run(MachineFunction &MF,
155 MachineFunctionAnalysisManager &MFAM) {
156 return insertPseudoProbesInMF(MF) ? getMachineFunctionPassPreservedAnalyses()
157 .preserveSet<CFGAnalyses>()
158 : PreservedAnalyses::all();
159}
160
161char PseudoProbeInserterLegacy::ID = 0;
162INITIALIZE_PASS_BEGIN(PseudoProbeInserterLegacy, DEBUG_TYPE,
163 "Insert pseudo probe annotations for value profiling",
164 false, false)
165INITIALIZE_PASS_END(PseudoProbeInserterLegacy, DEBUG_TYPE,
166 "Insert pseudo probe annotations for value profiling",
167 false, false)
168
169FunctionPass *llvm::createPseudoProbeInserterLegacyPass() {
170 return new PseudoProbeInserterLegacy();
171}
172