1//===- StaticDataSplitter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass uses branch profile data to assign hotness based section qualifiers
10// for the following types of static data:
11// - Jump tables
12// - Module-internal global variables
13// - Constant pools
14//
15// For the original RFC of this pass please see
16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17
18#include "llvm/CodeGen/StaticDataSplitter.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/Analysis/ProfileSummaryInfo.h"
21#include "llvm/Analysis/StaticDataProfileInfo.h"
22#include "llvm/CodeGen/MBFIWrapper.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
25#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
29#include "llvm/CodeGen/MachineFunctionPass.h"
30#include "llvm/CodeGen/MachineJumpTableInfo.h"
31#include "llvm/CodeGen/MachinePassManager.h"
32#include "llvm/CodeGen/Passes.h"
33#include "llvm/IR/Analysis.h"
34#include "llvm/IR/GlobalVariable.h"
35#include "llvm/InitializePasses.h"
36#include "llvm/Pass.h"
37#include "llvm/Target/TargetLoweringObjectFile.h"
38
39using namespace llvm;
40
41#define DEBUG_TYPE "static-data-splitter"
42
43STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
44STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
45STATISTIC(NumUnknownJumpTables,
46 "Number of jump tables with unknown hotness. They are from functions "
47 "without profile information.");
48
49class StaticDataSplitterImpl {
50 const MachineBlockFrequencyInfo *MBFI = nullptr;
51 const ProfileSummaryInfo *PSI = nullptr;
52 StaticDataProfileInfo *SDPI = nullptr;
53
54 // If the global value is a local linkage global variable, return it.
55 // Otherwise, return nullptr.
56 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);
57
58 // Returns true if the global variable is in one of {.rodata, .bss, .data,
59 // .data.rel.ro} sections.
60 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);
61
62 // Returns the constant if the operand refers to a global variable or constant
63 // that gets lowered to static data sections. Otherwise, return nullptr.
64 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,
65 const MachineConstantPool *MCP);
66
67 // Use profiles to partition static data.
68 bool partitionStaticDataWithProfiles(MachineFunction &MF);
69
70 // Update LLVM statistics for a machine function with profiles.
71 void updateStatsWithProfiles(const MachineFunction &MF);
72
73 // Update LLVM statistics for a machine function without profiles.
74 void updateStatsWithoutProfiles(const MachineFunction &MF);
75
76 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);
77
78public:
79 explicit StaticDataSplitterImpl(MachineBlockFrequencyInfo *MBFI,
80 ProfileSummaryInfo *PSI,
81 StaticDataProfileInfo *SDPI)
82 : MBFI(MBFI), PSI(PSI), SDPI(SDPI) {}
83 bool runOnMachineFunction(MachineFunction &MF);
84};
85
86class StaticDataSplitterLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 StaticDataSplitterLegacy() : MachineFunctionPass(ID) {}
91
92 StringRef getPassName() const override { return "Static Data Splitter"; }
93
94 void getAnalysisUsage(AnalysisUsage &AU) const override {
95 MachineFunctionPass::getAnalysisUsage(AU);
96 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
97 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
98 AU.addRequired<ProfileSummaryInfoWrapperPass>();
99 AU.addRequired<StaticDataProfileInfoWrapperPass>();
100 // This pass does not modify any required analysis results except
101 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass
102 // is made an immutable pass that it won't be re-scheduled by pass manager
103 // anyway. So mark setPreservesAll() here for faster compile time.
104 AU.setPreservesAll();
105 }
106
107 bool runOnMachineFunction(MachineFunction &MF) override;
108};
109
110bool StaticDataSplitterImpl::runOnMachineFunction(MachineFunction &MF) {
111 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
112 MF.getFunction().hasProfileData();
113
114 if (!ProfileAvailable) {
115 annotateStaticDataWithoutProfiles(MF);
116 updateStatsWithoutProfiles(MF);
117 return false;
118 }
119
120 bool Changed = partitionStaticDataWithProfiles(MF);
121
122 updateStatsWithProfiles(MF);
123 return Changed;
124}
125
126const Constant *
127StaticDataSplitterImpl::getConstant(const MachineOperand &Op,
128 const TargetMachine &TM,
129 const MachineConstantPool *MCP) {
130 if (!Op.isGlobal() && !Op.isCPI())
131 return nullptr;
132
133 if (Op.isGlobal()) {
134 // Find global variables with local linkage.
135 const GlobalVariable *GV = getLocalLinkageGlobalVariable(GV: Op.getGlobal());
136 // Skip those not eligible for annotation or not in static data sections.
137 if (!GV || !llvm::memprof::IsAnnotationOK(GV: *GV) ||
138 !inStaticDataSection(GV: *GV, TM))
139 return nullptr;
140 return GV;
141 }
142 assert(Op.isCPI() && "Op must be constant pool index in this branch");
143 int CPI = Op.getIndex();
144 if (CPI == -1)
145 return nullptr;
146
147 assert(MCP != nullptr && "Constant pool info is not available.");
148 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];
149
150 if (CPE.isMachineConstantPoolEntry())
151 return nullptr;
152
153 return CPE.Val.ConstVal;
154}
155
156bool StaticDataSplitterImpl::partitionStaticDataWithProfiles(
157 MachineFunction &MF) {
158 // If any of the static data (jump tables, global variables, constant pools)
159 // are captured by the analysis, set `Changed` to true. Note this pass won't
160 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main
161 // purpose of tracking and conveying the change (to pass manager) is
162 // informative as opposed to invalidating any analysis results. As an example
163 // of where this information is useful, `PMDataManager::dumpPassInfo` will
164 // only dump pass info if a local change happens, otherwise a pass appears as
165 // "skipped".
166 bool Changed = false;
167
168 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
169
170 // Jump table could be used by either terminating instructions or
171 // non-terminating ones, so we walk all instructions and use
172 // `MachineOperand::isJTI()` to identify jump table operands.
173 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool
174 // usages in the same loop.
175 for (const auto &MBB : MF) {
176 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(MBB: &MBB);
177 for (const MachineInstr &I : MBB) {
178 for (const MachineOperand &Op : I.operands()) {
179 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())
180 continue;
181
182 if (Op.isJTI()) {
183 assert(MJTI != nullptr && "Jump table info is not available.");
184 const int JTI = Op.getIndex();
185 // This is not a source block of jump table.
186 if (JTI == -1)
187 continue;
188
189 auto Hotness = MachineFunctionDataHotness::Hot;
190
191 // Hotness is based on source basic block hotness.
192 // TODO: PSI APIs are about instruction hotness. Introduce API for
193 // data access hotness.
194 if (Count && PSI->isColdCount(C: *Count))
195 Hotness = MachineFunctionDataHotness::Cold;
196
197 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);
198 } else if (const Constant *C =
199 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool())) {
200 SDPI->addConstantProfileCount(C, Count);
201 Changed = true;
202 }
203 }
204 }
205 }
206 return Changed;
207}
208
209const GlobalVariable *
210StaticDataSplitterImpl::getLocalLinkageGlobalVariable(const GlobalValue *GV) {
211 // LLVM IR Verifier requires that a declaration must have valid declaration
212 // linkage, and local linkages are not among the valid ones. So there is no
213 // need to check GV is not a declaration here.
214 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(Val: GV) : nullptr;
215}
216
217bool StaticDataSplitterImpl::inStaticDataSection(const GlobalVariable &GV,
218 const TargetMachine &TM) {
219
220 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(GO: &GV, TM);
221 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||
222 Kind.isBSS();
223}
224
225void StaticDataSplitterImpl::updateStatsWithProfiles(
226 const MachineFunction &MF) {
227 if (!AreStatisticsEnabled())
228 return;
229
230 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
231 for (const auto &JumpTable : MJTI->getJumpTables()) {
232 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
233 ++NumHotJumpTables;
234 } else {
235 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
236 "A jump table is either hot or cold when profile information is "
237 "available.");
238 ++NumColdJumpTables;
239 }
240 }
241 }
242}
243
244void StaticDataSplitterImpl::annotateStaticDataWithoutProfiles(
245 const MachineFunction &MF) {
246 for (const auto &MBB : MF)
247 for (const MachineInstr &I : MBB)
248 for (const MachineOperand &Op : I.operands())
249 if (const Constant *C =
250 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool()))
251 SDPI->addConstantProfileCount(C, Count: std::nullopt);
252}
253
254void StaticDataSplitterImpl::updateStatsWithoutProfiles(
255 const MachineFunction &MF) {
256 if (!AreStatisticsEnabled())
257 return;
258
259 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
260 NumUnknownJumpTables += MJTI->getJumpTables().size();
261 }
262}
263
264char StaticDataSplitterLegacy::ID = 0;
265
266INITIALIZE_PASS_BEGIN(StaticDataSplitterLegacy, DEBUG_TYPE, "Split static data",
267 false, false)
268INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
269INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
270INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
271INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass)
272INITIALIZE_PASS_END(StaticDataSplitterLegacy, DEBUG_TYPE, "Split static data",
273 false, false)
274
275MachineFunctionPass *llvm::createStaticDataSplitterLegacyPass() {
276 return new StaticDataSplitterLegacy();
277}
278
279bool StaticDataSplitterLegacy::runOnMachineFunction(MachineFunction &MF) {
280 MachineBlockFrequencyInfo *MBFI =
281 &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
282 ProfileSummaryInfo *PSI =
283 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
284 StaticDataProfileInfo *SDPI = &getAnalysis<StaticDataProfileInfoWrapperPass>()
285 .getStaticDataProfileInfo();
286 StaticDataSplitterImpl Impl(MBFI, PSI, SDPI);
287 return Impl.runOnMachineFunction(MF);
288}
289
290PreservedAnalyses
291StaticDataSplitterPass::run(MachineFunction &MF,
292 MachineFunctionAnalysisManager &MFAM) {
293 MachineBlockFrequencyInfo *MBFI =
294 &MFAM.getResult<MachineBlockFrequencyAnalysis>(IR&: MF);
295 auto &ModuleAnalysisManagerProxy =
296 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
297 ProfileSummaryInfo *PSI =
298 ModuleAnalysisManagerProxy.getCachedResult<ProfileSummaryAnalysis>(
299 IR&: *MF.getFunction().getParent());
300 StaticDataProfileInfo *SDPI =
301 &ModuleAnalysisManagerProxy
302 .getCachedResult<StaticDataProfileInfoAnalysis>(
303 IR&: *MF.getFunction().getParent())
304 ->getStaticDataProfileInfo();
305 StaticDataSplitterImpl Impl(MBFI, PSI, SDPI);
306 return Impl.runOnMachineFunction(MF)
307 ? getMachineFunctionPassPreservedAnalyses()
308 .preserveSet<CFGAnalyses>()
309 : PreservedAnalyses::all();
310}
311