1//===- StaticDataSplitter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass uses branch profile data to assign hotness based section qualifiers
10// for the following types of static data:
11// - Jump tables
12// - Module-internal global variables
13// - Constant pools
14//
15// For the original RFC of this pass please see
16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/ProfileSummaryInfo.h"
20#include "llvm/Analysis/StaticDataProfileInfo.h"
21#include "llvm/CodeGen/MBFIWrapper.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
24#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineJumpTableInfo.h"
29#include "llvm/CodeGen/Passes.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Pass.h"
33#include "llvm/Target/TargetLoweringObjectFile.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "static-data-splitter"
38
39STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
40STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
41STATISTIC(NumUnknownJumpTables,
42 "Number of jump tables with unknown hotness. They are from functions "
43 "without profile information.");
44
45class StaticDataSplitter : public MachineFunctionPass {
46 const MachineBranchProbabilityInfo *MBPI = nullptr;
47 const MachineBlockFrequencyInfo *MBFI = nullptr;
48 const ProfileSummaryInfo *PSI = nullptr;
49 StaticDataProfileInfo *SDPI = nullptr;
50
51 // If the global value is a local linkage global variable, return it.
52 // Otherwise, return nullptr.
53 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);
54
55 // Returns true if the global variable is in one of {.rodata, .bss, .data,
56 // .data.rel.ro} sections.
57 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);
58
59 // Returns the constant if the operand refers to a global variable or constant
60 // that gets lowered to static data sections. Otherwise, return nullptr.
61 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,
62 const MachineConstantPool *MCP);
63
64 // Use profiles to partition static data.
65 bool partitionStaticDataWithProfiles(MachineFunction &MF);
66
67 // Update LLVM statistics for a machine function with profiles.
68 void updateStatsWithProfiles(const MachineFunction &MF);
69
70 // Update LLVM statistics for a machine function without profiles.
71 void updateStatsWithoutProfiles(const MachineFunction &MF);
72
73 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);
74
75public:
76 static char ID;
77
78 StaticDataSplitter() : MachineFunctionPass(ID) {
79 initializeStaticDataSplitterPass(*PassRegistry::getPassRegistry());
80 }
81
82 StringRef getPassName() const override { return "Static Data Splitter"; }
83
84 void getAnalysisUsage(AnalysisUsage &AU) const override {
85 MachineFunctionPass::getAnalysisUsage(AU);
86 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
87 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
88 AU.addRequired<ProfileSummaryInfoWrapperPass>();
89 AU.addRequired<StaticDataProfileInfoWrapperPass>();
90 // This pass does not modify any required analysis results except
91 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass
92 // is made an immutable pass that it won't be re-scheduled by pass manager
93 // anyway. So mark setPreservesAll() here for faster compile time.
94 AU.setPreservesAll();
95 }
96
97 bool runOnMachineFunction(MachineFunction &MF) override;
98};
99
100bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) {
101 MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
102 MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
103 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
104
105 SDPI = &getAnalysis<StaticDataProfileInfoWrapperPass>()
106 .getStaticDataProfileInfo();
107
108 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
109 MF.getFunction().hasProfileData();
110
111 if (!ProfileAvailable) {
112 annotateStaticDataWithoutProfiles(MF);
113 updateStatsWithoutProfiles(MF);
114 return false;
115 }
116
117 bool Changed = partitionStaticDataWithProfiles(MF);
118
119 updateStatsWithProfiles(MF);
120 return Changed;
121}
122
123const Constant *
124StaticDataSplitter::getConstant(const MachineOperand &Op,
125 const TargetMachine &TM,
126 const MachineConstantPool *MCP) {
127 if (!Op.isGlobal() && !Op.isCPI())
128 return nullptr;
129
130 if (Op.isGlobal()) {
131 // Find global variables with local linkage.
132 const GlobalVariable *GV = getLocalLinkageGlobalVariable(GV: Op.getGlobal());
133 // Skip 'llvm.'-prefixed global variables conservatively because they are
134 // often handled specially, and skip those not in static data
135 // sections.
136 if (!GV || GV->getName().starts_with(Prefix: "llvm.") ||
137 !inStaticDataSection(GV: *GV, TM))
138 return nullptr;
139 return GV;
140 }
141 assert(Op.isCPI() && "Op must be constant pool index in this branch");
142 int CPI = Op.getIndex();
143 if (CPI == -1)
144 return nullptr;
145
146 assert(MCP != nullptr && "Constant pool info is not available.");
147 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];
148
149 if (CPE.isMachineConstantPoolEntry())
150 return nullptr;
151
152 return CPE.Val.ConstVal;
153}
154
155bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) {
156 // If any of the static data (jump tables, global variables, constant pools)
157 // are captured by the analysis, set `Changed` to true. Note this pass won't
158 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main
159 // purpose of tracking and conveying the change (to pass manager) is
160 // informative as opposed to invalidating any analysis results. As an example
161 // of where this information is useful, `PMDataManager::dumpPassInfo` will
162 // only dump pass info if a local change happens, otherwise a pass appears as
163 // "skipped".
164 bool Changed = false;
165
166 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
167
168 // Jump table could be used by either terminating instructions or
169 // non-terminating ones, so we walk all instructions and use
170 // `MachineOperand::isJTI()` to identify jump table operands.
171 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool
172 // usages in the same loop.
173 for (const auto &MBB : MF) {
174 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(MBB: &MBB);
175 for (const MachineInstr &I : MBB) {
176 for (const MachineOperand &Op : I.operands()) {
177 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())
178 continue;
179
180 if (Op.isJTI()) {
181 assert(MJTI != nullptr && "Jump table info is not available.");
182 const int JTI = Op.getIndex();
183 // This is not a source block of jump table.
184 if (JTI == -1)
185 continue;
186
187 auto Hotness = MachineFunctionDataHotness::Hot;
188
189 // Hotness is based on source basic block hotness.
190 // TODO: PSI APIs are about instruction hotness. Introduce API for
191 // data access hotness.
192 if (Count && PSI->isColdCount(C: *Count))
193 Hotness = MachineFunctionDataHotness::Cold;
194
195 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);
196 } else if (const Constant *C =
197 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool())) {
198 SDPI->addConstantProfileCount(C, Count);
199 Changed = true;
200 }
201 }
202 }
203 }
204 return Changed;
205}
206
207const GlobalVariable *
208StaticDataSplitter::getLocalLinkageGlobalVariable(const GlobalValue *GV) {
209 // LLVM IR Verifier requires that a declaration must have valid declaration
210 // linkage, and local linkages are not among the valid ones. So there is no
211 // need to check GV is not a declaration here.
212 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(Val: GV) : nullptr;
213}
214
215bool StaticDataSplitter::inStaticDataSection(const GlobalVariable &GV,
216 const TargetMachine &TM) {
217
218 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(GO: &GV, TM);
219 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||
220 Kind.isBSS();
221}
222
223void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) {
224 if (!AreStatisticsEnabled())
225 return;
226
227 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
228 for (const auto &JumpTable : MJTI->getJumpTables()) {
229 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
230 ++NumHotJumpTables;
231 } else {
232 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
233 "A jump table is either hot or cold when profile information is "
234 "available.");
235 ++NumColdJumpTables;
236 }
237 }
238 }
239}
240
241void StaticDataSplitter::annotateStaticDataWithoutProfiles(
242 const MachineFunction &MF) {
243 for (const auto &MBB : MF)
244 for (const MachineInstr &I : MBB)
245 for (const MachineOperand &Op : I.operands())
246 if (const Constant *C =
247 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool()))
248 SDPI->addConstantProfileCount(C, Count: std::nullopt);
249}
250
251void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) {
252 if (!AreStatisticsEnabled())
253 return;
254
255 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
256 NumUnknownJumpTables += MJTI->getJumpTables().size();
257 }
258}
259
260char StaticDataSplitter::ID = 0;
261
262INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data",
263 false, false)
264INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
265INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
266INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
267INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass)
268INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false,
269 false)
270
271MachineFunctionPass *llvm::createStaticDataSplitterPass() {
272 return new StaticDataSplitter();
273}
274