1#include "llvm/Analysis/StaticDataProfileInfo.h"
2#include "llvm/Analysis/ProfileSummaryInfo.h"
3#include "llvm/IR/Constant.h"
4#include "llvm/IR/Constants.h"
5#include "llvm/IR/GlobalVariable.h"
6#include "llvm/IR/Module.h"
7#include "llvm/IR/PassManager.h"
8#include "llvm/InitializePasses.h"
9#include "llvm/ProfileData/InstrProf.h"
10
11#define DEBUG_TYPE "static-data-profile-info"
12
13using namespace llvm;
14
15cl::opt<bool> PreserveHotDataSectionPrefix(
16 "preserve-hot-data-section-prefix", cl::Hidden, cl::init(Val: true),
17 cl::desc("If true, hot data section prefixes are preserved"));
18
19namespace llvm {
20// FIXME: This option is added for incremental rollout purposes.
21// After the option, string literal partitioning should be implied by
22// AnnotateStaticDataSectionPrefix in MemProfUse.cpp and this option should be
23// cleaned up.
24cl::opt<bool> AnnotateStringLiteralSectionPrefix(
25 "memprof-annotate-string-literal-section-prefix", cl::init(Val: true),
26 cl::Hidden,
27 cl::desc("If true, annotate the string literal data section prefix"));
28namespace memprof {
29// Returns true iff the global variable has custom section either by
30// __attribute__((section("name")))
31// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)
32// or #pragma clang section directives
33// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).
34static bool hasExplicitSectionName(const GlobalVariable &GVar) {
35 if (GVar.hasSection())
36 return true;
37
38 auto Attrs = GVar.getAttributes();
39 if (Attrs.hasAttribute(Kind: "bss-section") || Attrs.hasAttribute(Kind: "data-section") ||
40 Attrs.hasAttribute(Kind: "relro-section") ||
41 Attrs.hasAttribute(Kind: "rodata-section"))
42 return true;
43 return false;
44}
45
46AnnotationKind getAnnotationKind(const GlobalVariable &GV) {
47 if (GV.isDeclarationForLinker())
48 return AnnotationKind::DeclForLinker;
49 // Skip 'llvm.'-prefixed global variables conservatively because they are
50 // often handled specially,
51 StringRef Name = GV.getName();
52 if (Name.starts_with(Prefix: "llvm."))
53 return AnnotationKind::ReservedName;
54 // Respect user-specified custom data sections.
55 if (hasExplicitSectionName(GVar: GV))
56 return AnnotationKind::ExplicitSection;
57 return AnnotationKind::AnnotationOK;
58}
59
60bool IsAnnotationOK(const GlobalVariable &GV) {
61 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;
62}
63} // namespace memprof
64} // namespace llvm
65
66void StaticDataProfileInfo::addConstantProfileCount(
67 const Constant *C, std::optional<uint64_t> Count) {
68 if (!Count) {
69 ConstantWithoutCounts.insert(V: C);
70 return;
71 }
72 uint64_t &OriginalCount = ConstantProfileCounts[C];
73 OriginalCount = llvm::SaturatingAdd(X: *Count, Y: OriginalCount);
74 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
75 // large values for special use.
76 if (OriginalCount > getInstrMaxCountValue())
77 OriginalCount = getInstrMaxCountValue();
78}
79
80StaticDataProfileInfo::StaticDataHotness
81StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
82 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
83 // The accummulated counter shows the constant is hot. Return enum 'hot'
84 // whether this variable is seen by unprofiled functions or not.
85 if (PSI->isHotCount(C: Count))
86 return StaticDataHotness::Hot;
87 // The constant is not hot, and seen by unprofiled functions. We don't want to
88 // assign it to unlikely sections, even if the counter says 'cold'. So return
89 // enum 'LukewarmOrUnknown'.
90 if (ConstantWithoutCounts.count(V: C))
91 return StaticDataHotness::LukewarmOrUnknown;
92 // The accummulated counter shows the constant is cold so return enum 'cold'.
93 if (PSI->isColdCount(C: Count))
94 return StaticDataHotness::Cold;
95
96 return StaticDataHotness::LukewarmOrUnknown;
97}
98
99StaticDataProfileInfo::StaticDataHotness
100StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
101 std::optional<StringRef> MaybeSectionPrefix) const {
102 if (!MaybeSectionPrefix)
103 return StaticDataHotness::LukewarmOrUnknown;
104 StringRef Prefix = *MaybeSectionPrefix;
105 assert((Prefix == "hot" || Prefix == "unlikely") &&
106 "Expect section_prefix to be one of hot or unlikely");
107 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
108}
109
110StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
111 switch (Hotness) {
112 case StaticDataHotness::Cold:
113 return "unlikely";
114 case StaticDataHotness::Hot:
115 return PreserveHotDataSectionPrefix ? "hot" : "";
116 default:
117 return "";
118 }
119}
120
121std::optional<uint64_t>
122StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
123 auto I = ConstantProfileCounts.find(Val: C);
124 if (I == ConstantProfileCounts.end())
125 return std::nullopt;
126 return I->second;
127}
128
129StringRef StaticDataProfileInfo::getConstantSectionPrefix(
130 const Constant *C, const ProfileSummaryInfo *PSI) const {
131 std::optional<uint64_t> Count = getConstantProfileCount(C);
132
133#ifndef NDEBUG
134 auto DbgPrintPrefix = [](StringRef Prefix) {
135 return Prefix.empty() ? "<empty>" : Prefix;
136 };
137#endif
138
139 if (EnableDataAccessProf) {
140 // Both data access profiles and PGO counters are available. Use the
141 // hotter one to be conservative. Basically, we want the non-unlikely
142 // sections to have max coverage of accessed symbols and meanwhile can
143 // tolerant some cold symbols in it, and the unlikely section variant to not
144 // have potentially hot symbols if possible, to avoid the penalty of access
145 // cold pages.
146 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C);
147 GV && llvm::memprof::IsAnnotationOK(GV: *GV) &&
148 (AnnotateStringLiteralSectionPrefix ||
149 !GV->getName().starts_with(Prefix: ".str"))) {
150 // Note a global var is covered by data access profiles iff the
151 // symbol name is preserved in the symbol table; most notably, a string
152 // literal with private linkage (e.g., those not externalized by ThinLTO
153 // and with insignificant address) won't have an entry in the symbol
154 // table (unless there is another string with identical content that
155 // gets a symbol table entry). For the private-linkage string literals,
156 // their hotness will be at least lukewarm (i.e., empty prefix).
157 auto HotnessFromDataAccessProf =
158 getSectionHotnessUsingDataAccessProfile(MaybeSectionPrefix: GV->getSectionPrefix());
159
160 if (!Count) {
161 StringRef Prefix = hotnessToStr(Hotness: HotnessFromDataAccessProf);
162 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
163 << DbgPrintPrefix(Prefix)
164 << ", solely from data access profiles\n");
165 return Prefix;
166 }
167
168 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, Count: *Count);
169 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
170 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
171 HotnessFromPGO == StaticDataHotness::Hot) {
172 GlobalVarHotness = StaticDataHotness::Hot;
173 } else if (HotnessFromDataAccessProf ==
174 StaticDataHotness::LukewarmOrUnknown ||
175 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
176 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
177 } else {
178 GlobalVarHotness = StaticDataHotness::Cold;
179 }
180 StringRef Prefix = hotnessToStr(Hotness: GlobalVarHotness);
181 LLVM_DEBUG(
182 dbgs() << GV->getName() << " has section prefix "
183 << DbgPrintPrefix(Prefix)
184 << ", the max from data access profiles as "
185 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
186 << " and PGO counters as "
187 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
188 return Prefix;
189 }
190 }
191 if (!Count)
192 return "";
193 return hotnessToStr(Hotness: getConstantHotnessUsingProfileCount(C, PSI, Count: *Count));
194}
195
196static std::unique_ptr<StaticDataProfileInfo>
197computeStaticDataProfileInfo(Module &M) {
198 bool EnableDataAccessProf = false;
199 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
200 MD: M.getModuleFlag(Key: "EnableDataAccessProf")))
201 EnableDataAccessProf = MD->getZExtValue();
202 return std::make_unique<StaticDataProfileInfo>(args&: EnableDataAccessProf);
203}
204
205bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
206 Info = computeStaticDataProfileInfo(M);
207 return false;
208}
209
210bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
211 Info.reset();
212 return false;
213}
214
215INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
216 "Static Data Profile Info", false, true)
217
218StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
219 : ImmutablePass(ID) {}
220
221char StaticDataProfileInfoWrapperPass::ID = 0;
222
223StaticDataProfileInfoAnalysis::Result
224StaticDataProfileInfoAnalysis::run(Module &M, ModuleAnalysisManager &) {
225 return StaticDataProfileInfoAnalysis::Result(computeStaticDataProfileInfo(M));
226}
227
228AnalysisKey llvm::StaticDataProfileInfoAnalysis::Key;
229