| 1 | //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements PseudoProbeInserter pass, which inserts pseudo probe |
| 10 | // annotations for call instructions with a pseudo-probe-specific dwarf |
| 11 | // discriminator. such discriminator indicates that the call instruction comes |
| 12 | // with a pseudo probe, and the discriminator value holds information to |
| 13 | // identify the corresponding counter. |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
| 16 | #include "llvm/CodeGen/PseudoProbeInserter.h" |
| 17 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 18 | #include "llvm/CodeGen/MachineFunctionAnalysisManager.h" |
| 19 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 20 | #include "llvm/CodeGen/MachineInstr.h" |
| 21 | #include "llvm/CodeGen/MachinePassManager.h" |
| 22 | #include "llvm/CodeGen/TargetInstrInfo.h" |
| 23 | #include "llvm/IR/Analysis.h" |
| 24 | #include "llvm/IR/DebugInfoMetadata.h" |
| 25 | #include "llvm/IR/Module.h" |
| 26 | #include "llvm/IR/PseudoProbe.h" |
| 27 | #include "llvm/InitializePasses.h" |
| 28 | #include "llvm/ProfileData/SampleProf.h" |
| 29 | |
| 30 | #define DEBUG_TYPE "pseudo-probe-inserter" |
| 31 | |
| 32 | using namespace llvm; |
| 33 | |
| 34 | namespace { |
| 35 | |
| 36 | // A real instruction is a non-meta, non-pseudo instruction. Calls (including |
| 37 | // call pseudos like tail-call returns) are also treated as real because they |
| 38 | // expand to a branch and any preceding call probe must be preserved. |
| 39 | static bool isCallOrRealInstruction(const MachineInstr &MI) { |
| 40 | return MI.isCall() || (!MI.isPseudo() && !MI.isMetaInstruction()); |
| 41 | } |
| 42 | |
| 43 | static uint64_t getFuncGUID(Module *M, DILocation *DL) { |
| 44 | auto Name = DL->getSubprogramLinkageName(); |
| 45 | // CoroSplit Pass will change the debug info with suffixes i.e. `.resume`, |
| 46 | // `.destroy`, `.cleanup`. Strip these suffixes to make the GUID consistent |
| 47 | // with the pseudo probe |
| 48 | Name = FunctionSamples::getCanonicalCoroFnName(FnName: Name); |
| 49 | return Function::getGUIDAssumingExternalLinkage(GlobalName: Name); |
| 50 | } |
| 51 | |
| 52 | static bool insertPseudoProbesInMF(MachineFunction &MF) { |
| 53 | if (!MF.getFunction().getParent()->getNamedMetadata( |
| 54 | Name: PseudoProbeDescMetadataName)) |
| 55 | return false; |
| 56 | const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); |
| 57 | bool Changed = false; |
| 58 | for (MachineBasicBlock &MBB : MF) { |
| 59 | MachineInstr *FirstInstr = nullptr; |
| 60 | for (MachineInstr &MI : MBB) { |
| 61 | // Pseudo instructions like TCRETURNdi results in a branch instruction |
| 62 | // and the call probe for that tail call should be preserved. |
| 63 | if (isCallOrRealInstruction(MI)) |
| 64 | FirstInstr = &MI; |
| 65 | if (MI.isCall()) { |
| 66 | if (DILocation *DL = MI.getDebugLoc()) { |
| 67 | auto Value = DL->getDiscriminator(); |
| 68 | if (DILocation::isPseudoProbeDiscriminator(Discriminator: Value)) { |
| 69 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PSEUDO_PROBE)) |
| 70 | .addImm(Val: getFuncGUID(M: MF.getFunction().getParent(), DL)) |
| 71 | .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeIndex(Value)) |
| 72 | .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeType(Value)) |
| 73 | .addImm(Val: PseudoProbeDwarfDiscriminator::extractProbeAttributes( |
| 74 | Value)); |
| 75 | Changed = true; |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // Walk the block backwards, move PSEUDO_PROBE before the first real |
| 82 | // instruction to fix out-of-order probes. There is a problem with probes |
| 83 | // as the terminator of the block. During the offline counts processing, |
| 84 | // the samples collected on the first physical instruction following a |
| 85 | // probe will be counted towards the probe. This logically equals to |
| 86 | // treating the instruction next to a probe as if it is from the same |
| 87 | // block of the probe. This is accurate most of the time unless the |
| 88 | // instruction can be reached from multiple flows, which means it actually |
| 89 | // starts a new block. Samples collected on such probes may cause |
| 90 | // imprecision with the counts inference algorithm. Fortunately, if |
| 91 | // there are still other native instructions preceding the probe we can |
| 92 | // use them as a place holder to collect samples for the probe. |
| 93 | if (FirstInstr) { |
| 94 | auto MII = MBB.rbegin(); |
| 95 | while (MII != MBB.rend()) { |
| 96 | // Skip all pseudo probes followed by a real instruction since they |
| 97 | // are not dangling. Treat call pseudos (e.g. tail-call returns) |
| 98 | // as real instructions to keep this consistent with the forward |
| 99 | // scan above. |
| 100 | if (isCallOrRealInstruction(MI: *MII)) |
| 101 | break; |
| 102 | auto Cur = MII++; |
| 103 | if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE) |
| 104 | continue; |
| 105 | // Move the dangling probe before FirstInstr. |
| 106 | auto *ProbeInstr = &*Cur; |
| 107 | MBB.remove(I: ProbeInstr); |
| 108 | MBB.insert(I: FirstInstr, MI: ProbeInstr); |
| 109 | Changed = true; |
| 110 | } |
| 111 | } else { |
| 112 | // Probes not surrounded by any real instructions in the same block are |
| 113 | // called dangling probes. Since there's no good way to pick up a sample |
| 114 | // collection point for dangling probes at compile time, they are being |
| 115 | // removed so that the profile correlation tool will not report any |
| 116 | // samples collected for them and it's up to the counts inference tool |
| 117 | // to get them a reasonable count. |
| 118 | SmallVector<MachineInstr *, 4> ToBeRemoved; |
| 119 | for (MachineInstr &MI : MBB) { |
| 120 | if (MI.isPseudoProbe()) |
| 121 | ToBeRemoved.push_back(Elt: &MI); |
| 122 | } |
| 123 | |
| 124 | for (auto *MI : ToBeRemoved) |
| 125 | MI->eraseFromParent(); |
| 126 | |
| 127 | Changed |= !ToBeRemoved.empty(); |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | return Changed; |
| 132 | } |
| 133 | |
| 134 | class PseudoProbeInserterLegacy : public MachineFunctionPass { |
| 135 | public: |
| 136 | static char ID; |
| 137 | |
| 138 | PseudoProbeInserterLegacy() : MachineFunctionPass(ID) {} |
| 139 | |
| 140 | StringRef getPassName() const override { return "Pseudo Probe Inserter" ; } |
| 141 | |
| 142 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 143 | AU.setPreservesAll(); |
| 144 | MachineFunctionPass::getAnalysisUsage(AU); |
| 145 | } |
| 146 | |
| 147 | bool runOnMachineFunction(MachineFunction &MF) override { |
| 148 | return insertPseudoProbesInMF(MF); |
| 149 | } |
| 150 | }; |
| 151 | } // namespace |
| 152 | |
| 153 | PreservedAnalyses |
| 154 | PseudoProbeInserterPass::run(MachineFunction &MF, |
| 155 | MachineFunctionAnalysisManager &MFAM) { |
| 156 | return insertPseudoProbesInMF(MF) ? getMachineFunctionPassPreservedAnalyses() |
| 157 | .preserveSet<CFGAnalyses>() |
| 158 | : PreservedAnalyses::all(); |
| 159 | } |
| 160 | |
| 161 | char PseudoProbeInserterLegacy::ID = 0; |
| 162 | INITIALIZE_PASS_BEGIN(PseudoProbeInserterLegacy, DEBUG_TYPE, |
| 163 | "Insert pseudo probe annotations for value profiling" , |
| 164 | false, false) |
| 165 | INITIALIZE_PASS_END(PseudoProbeInserterLegacy, DEBUG_TYPE, |
| 166 | "Insert pseudo probe annotations for value profiling" , |
| 167 | false, false) |
| 168 | |
| 169 | FunctionPass *llvm::createPseudoProbeInserterLegacyPass() { |
| 170 | return new PseudoProbeInserterLegacy(); |
| 171 | } |
| 172 | |