1//===- StaticDataSplitter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass uses branch profile data to assign hotness based section qualifiers
10// for the following types of static data:
11// - Jump tables
12// - Module-internal global variables
13// - Constant pools
14//
15// For the original RFC of this pass please see
16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17
18#include "llvm/CodeGen/StaticDataSplitter.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/Analysis/ProfileSummaryInfo.h"
21#include "llvm/Analysis/StaticDataProfileInfo.h"
22#include "llvm/CodeGen/MBFIWrapper.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/MachineJumpTableInfo.h"
30#include "llvm/CodeGen/MachinePassManager.h"
31#include "llvm/CodeGen/Passes.h"
32#include "llvm/IR/Analysis.h"
33#include "llvm/IR/GlobalVariable.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/Pass.h"
36#include "llvm/Target/TargetLoweringObjectFile.h"
37
38using namespace llvm;
39
40#define DEBUG_TYPE "static-data-splitter"
41
42STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
43STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
44STATISTIC(NumUnknownJumpTables,
45 "Number of jump tables with unknown hotness. They are from functions "
46 "without profile information.");
47
48class StaticDataSplitterImpl {
49 const MachineBlockFrequencyInfo *MBFI = nullptr;
50 const ProfileSummaryInfo *PSI = nullptr;
51 StaticDataProfileInfo *SDPI = nullptr;
52
53 // If the global value is a local linkage global variable, return it.
54 // Otherwise, return nullptr.
55 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);
56
57 // Returns true if the global variable is in one of {.rodata, .bss, .data,
58 // .data.rel.ro} sections.
59 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);
60
61 // Returns the constant if the operand refers to a global variable or constant
62 // that gets lowered to static data sections. Otherwise, return nullptr.
63 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,
64 const MachineConstantPool *MCP);
65
66 // Use profiles to partition static data.
67 bool partitionStaticDataWithProfiles(MachineFunction &MF);
68
69 // Update LLVM statistics for a machine function with profiles.
70 void updateStatsWithProfiles(const MachineFunction &MF);
71
72 // Update LLVM statistics for a machine function without profiles.
73 void updateStatsWithoutProfiles(const MachineFunction &MF);
74
75 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);
76
77public:
78 explicit StaticDataSplitterImpl(MachineBlockFrequencyInfo *MBFI,
79 ProfileSummaryInfo *PSI,
80 StaticDataProfileInfo *SDPI)
81 : MBFI(MBFI), PSI(PSI), SDPI(SDPI) {}
82 bool runOnMachineFunction(MachineFunction &MF);
83};
84
85class StaticDataSplitterLegacy : public MachineFunctionPass {
86public:
87 static char ID;
88
89 StaticDataSplitterLegacy() : MachineFunctionPass(ID) {}
90
91 StringRef getPassName() const override { return "Static Data Splitter"; }
92
93 void getAnalysisUsage(AnalysisUsage &AU) const override {
94 MachineFunctionPass::getAnalysisUsage(AU);
95 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
96 AU.addRequired<ProfileSummaryInfoWrapperPass>();
97 AU.addRequired<StaticDataProfileInfoWrapperPass>();
98 // This pass does not modify any required analysis results except
99 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass
100 // is made an immutable pass that it won't be re-scheduled by pass manager
101 // anyway. So mark setPreservesAll() here for faster compile time.
102 AU.setPreservesAll();
103 }
104
105 bool runOnMachineFunction(MachineFunction &MF) override;
106};
107
108bool StaticDataSplitterImpl::runOnMachineFunction(MachineFunction &MF) {
109 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
110 MF.getFunction().hasProfileData();
111
112 if (!ProfileAvailable) {
113 annotateStaticDataWithoutProfiles(MF);
114 updateStatsWithoutProfiles(MF);
115 return false;
116 }
117
118 bool Changed = partitionStaticDataWithProfiles(MF);
119
120 updateStatsWithProfiles(MF);
121 return Changed;
122}
123
124const Constant *
125StaticDataSplitterImpl::getConstant(const MachineOperand &Op,
126 const TargetMachine &TM,
127 const MachineConstantPool *MCP) {
128 if (!Op.isGlobal() && !Op.isCPI())
129 return nullptr;
130
131 if (Op.isGlobal()) {
132 // Find global variables with local linkage.
133 const GlobalVariable *GV = getLocalLinkageGlobalVariable(GV: Op.getGlobal());
134 // Skip those not eligible for annotation or not in static data sections.
135 if (!GV || !llvm::memprof::IsAnnotationOK(GV: *GV) ||
136 !inStaticDataSection(GV: *GV, TM))
137 return nullptr;
138 return GV;
139 }
140 assert(Op.isCPI() && "Op must be constant pool index in this branch");
141 int CPI = Op.getIndex();
142 if (CPI == -1)
143 return nullptr;
144
145 assert(MCP != nullptr && "Constant pool info is not available.");
146 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];
147
148 if (CPE.isMachineConstantPoolEntry())
149 return nullptr;
150
151 return CPE.Val.ConstVal;
152}
153
154bool StaticDataSplitterImpl::partitionStaticDataWithProfiles(
155 MachineFunction &MF) {
156 // If any of the static data (jump tables, global variables, constant pools)
157 // are captured by the analysis, set `Changed` to true. Note this pass won't
158 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main
159 // purpose of tracking and conveying the change (to pass manager) is
160 // informative as opposed to invalidating any analysis results. As an example
161 // of where this information is useful, `PMDataManager::dumpPassInfo` will
162 // only dump pass info if a local change happens, otherwise a pass appears as
163 // "skipped".
164 bool Changed = false;
165
166 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
167
168 // Jump table could be used by either terminating instructions or
169 // non-terminating ones, so we walk all instructions and use
170 // `MachineOperand::isJTI()` to identify jump table operands.
171 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool
172 // usages in the same loop.
173 for (const auto &MBB : MF) {
174 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(MBB: &MBB);
175 for (const MachineInstr &I : MBB) {
176 for (const MachineOperand &Op : I.operands()) {
177 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())
178 continue;
179
180 if (Op.isJTI()) {
181 assert(MJTI != nullptr && "Jump table info is not available.");
182 const int JTI = Op.getIndex();
183 // This is not a source block of jump table.
184 if (JTI == -1)
185 continue;
186
187 auto Hotness = MachineFunctionDataHotness::Hot;
188
189 // Hotness is based on source basic block hotness.
190 // TODO: PSI APIs are about instruction hotness. Introduce API for
191 // data access hotness.
192 if (Count && PSI->isColdCount(C: *Count))
193 Hotness = MachineFunctionDataHotness::Cold;
194
195 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);
196 } else if (const Constant *C =
197 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool())) {
198 SDPI->addConstantProfileCount(C, Count);
199 Changed = true;
200 }
201 }
202 }
203 }
204 return Changed;
205}
206
207const GlobalVariable *
208StaticDataSplitterImpl::getLocalLinkageGlobalVariable(const GlobalValue *GV) {
209 // LLVM IR Verifier requires that a declaration must have valid declaration
210 // linkage, and local linkages are not among the valid ones. So there is no
211 // need to check GV is not a declaration here.
212 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(Val: GV) : nullptr;
213}
214
215bool StaticDataSplitterImpl::inStaticDataSection(const GlobalVariable &GV,
216 const TargetMachine &TM) {
217
218 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(GO: &GV, TM);
219 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||
220 Kind.isBSS();
221}
222
223void StaticDataSplitterImpl::updateStatsWithProfiles(
224 const MachineFunction &MF) {
225 if (!AreStatisticsEnabled())
226 return;
227
228 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
229 for (const auto &JumpTable : MJTI->getJumpTables()) {
230 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
231 ++NumHotJumpTables;
232 } else {
233 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
234 "A jump table is either hot or cold when profile information is "
235 "available.");
236 ++NumColdJumpTables;
237 }
238 }
239 }
240}
241
242void StaticDataSplitterImpl::annotateStaticDataWithoutProfiles(
243 const MachineFunction &MF) {
244 for (const auto &MBB : MF)
245 for (const MachineInstr &I : MBB)
246 for (const MachineOperand &Op : I.operands())
247 if (const Constant *C =
248 getConstant(Op, TM: MF.getTarget(), MCP: MF.getConstantPool()))
249 SDPI->addConstantProfileCount(C, Count: std::nullopt);
250}
251
252void StaticDataSplitterImpl::updateStatsWithoutProfiles(
253 const MachineFunction &MF) {
254 if (!AreStatisticsEnabled())
255 return;
256
257 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
258 NumUnknownJumpTables += MJTI->getJumpTables().size();
259 }
260}
261
262char StaticDataSplitterLegacy::ID = 0;
263
264INITIALIZE_PASS_BEGIN(StaticDataSplitterLegacy, DEBUG_TYPE, "Split static data",
265 false, false)
266INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
267INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
268INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
269INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass)
270INITIALIZE_PASS_END(StaticDataSplitterLegacy, DEBUG_TYPE, "Split static data",
271 false, false)
272
273MachineFunctionPass *llvm::createStaticDataSplitterLegacyPass() {
274 return new StaticDataSplitterLegacy();
275}
276
277bool StaticDataSplitterLegacy::runOnMachineFunction(MachineFunction &MF) {
278 MachineBlockFrequencyInfo *MBFI =
279 &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
280 ProfileSummaryInfo *PSI =
281 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
282 StaticDataProfileInfo *SDPI = &getAnalysis<StaticDataProfileInfoWrapperPass>()
283 .getStaticDataProfileInfo();
284 StaticDataSplitterImpl Impl(MBFI, PSI, SDPI);
285 return Impl.runOnMachineFunction(MF);
286}
287
288PreservedAnalyses
289StaticDataSplitterPass::run(MachineFunction &MF,
290 MachineFunctionAnalysisManager &MFAM) {
291 MachineBlockFrequencyInfo *MBFI =
292 &MFAM.getResult<MachineBlockFrequencyAnalysis>(IR&: MF);
293 auto &ModuleAnalysisManagerProxy =
294 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
295 ProfileSummaryInfo *PSI =
296 ModuleAnalysisManagerProxy.getCachedResult<ProfileSummaryAnalysis>(
297 IR&: *MF.getFunction().getParent());
298 StaticDataProfileInfo *SDPI =
299 &ModuleAnalysisManagerProxy
300 .getCachedResult<StaticDataProfileInfoAnalysis>(
301 IR&: *MF.getFunction().getParent())
302 ->getStaticDataProfileInfo();
303 StaticDataSplitterImpl Impl(MBFI, PSI, SDPI);
304 return Impl.runOnMachineFunction(MF)
305 ? getMachineFunctionPassPreservedAnalyses()
306 .preserveSet<CFGAnalyses>()
307 : PreservedAnalyses::all();
308}
309