1#include "llvm/Analysis/StaticDataProfileInfo.h"
2#include "llvm/Analysis/ProfileSummaryInfo.h"
3#include "llvm/IR/Constant.h"
4#include "llvm/IR/Constants.h"
5#include "llvm/IR/GlobalVariable.h"
6#include "llvm/IR/Module.h"
7#include "llvm/IR/PassManager.h"
8#include "llvm/InitializePasses.h"
9#include "llvm/ProfileData/InstrProf.h"
10
11#define DEBUG_TYPE "static-data-profile-info"
12
13using namespace llvm;
14
15namespace llvm {
16// FIXME: This option is added for incremental rollout purposes.
17// After the option, string literal partitioning should be implied by
18// AnnotateStaticDataSectionPrefix in MemProfUse.cpp and this option should be
19// cleaned up.
20cl::opt<bool> AnnotateStringLiteralSectionPrefix(
21 "memprof-annotate-string-literal-section-prefix", cl::init(Val: false),
22 cl::Hidden,
23 cl::desc("If true, annotate the string literal data section prefix"));
24namespace memprof {
25// Returns true iff the global variable has custom section either by
26// __attribute__((section("name")))
27// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)
28// or #pragma clang section directives
29// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).
30static bool hasExplicitSectionName(const GlobalVariable &GVar) {
31 if (GVar.hasSection())
32 return true;
33
34 auto Attrs = GVar.getAttributes();
35 if (Attrs.hasAttribute(Kind: "bss-section") || Attrs.hasAttribute(Kind: "data-section") ||
36 Attrs.hasAttribute(Kind: "relro-section") ||
37 Attrs.hasAttribute(Kind: "rodata-section"))
38 return true;
39 return false;
40}
41
42AnnotationKind getAnnotationKind(const GlobalVariable &GV) {
43 if (GV.isDeclarationForLinker())
44 return AnnotationKind::DeclForLinker;
45 // Skip 'llvm.'-prefixed global variables conservatively because they are
46 // often handled specially,
47 StringRef Name = GV.getName();
48 if (Name.starts_with(Prefix: "llvm."))
49 return AnnotationKind::ReservedName;
50 // Respect user-specified custom data sections.
51 if (hasExplicitSectionName(GVar: GV))
52 return AnnotationKind::ExplicitSection;
53 return AnnotationKind::AnnotationOK;
54}
55
56bool IsAnnotationOK(const GlobalVariable &GV) {
57 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;
58}
59} // namespace memprof
60} // namespace llvm
61
62void StaticDataProfileInfo::addConstantProfileCount(
63 const Constant *C, std::optional<uint64_t> Count) {
64 if (!Count) {
65 ConstantWithoutCounts.insert(V: C);
66 return;
67 }
68 uint64_t &OriginalCount = ConstantProfileCounts[C];
69 OriginalCount = llvm::SaturatingAdd(X: *Count, Y: OriginalCount);
70 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
71 // large values for special use.
72 if (OriginalCount > getInstrMaxCountValue())
73 OriginalCount = getInstrMaxCountValue();
74}
75
76StaticDataProfileInfo::StaticDataHotness
77StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
78 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
79 // The accummulated counter shows the constant is hot. Return enum 'hot'
80 // whether this variable is seen by unprofiled functions or not.
81 if (PSI->isHotCount(C: Count))
82 return StaticDataHotness::Hot;
83 // The constant is not hot, and seen by unprofiled functions. We don't want to
84 // assign it to unlikely sections, even if the counter says 'cold'. So return
85 // enum 'LukewarmOrUnknown'.
86 if (ConstantWithoutCounts.count(V: C))
87 return StaticDataHotness::LukewarmOrUnknown;
88 // The accummulated counter shows the constant is cold so return enum 'cold'.
89 if (PSI->isColdCount(C: Count))
90 return StaticDataHotness::Cold;
91
92 return StaticDataHotness::LukewarmOrUnknown;
93}
94
95StaticDataProfileInfo::StaticDataHotness
96StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
97 std::optional<StringRef> MaybeSectionPrefix) const {
98 if (!MaybeSectionPrefix)
99 return StaticDataHotness::LukewarmOrUnknown;
100 StringRef Prefix = *MaybeSectionPrefix;
101 assert((Prefix == "hot" || Prefix == "unlikely") &&
102 "Expect section_prefix to be one of hot or unlikely");
103 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
104}
105
106StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
107 switch (Hotness) {
108 case StaticDataHotness::Cold:
109 return "unlikely";
110 case StaticDataHotness::Hot:
111 return "hot";
112 default:
113 return "";
114 }
115}
116
117std::optional<uint64_t>
118StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
119 auto I = ConstantProfileCounts.find(Val: C);
120 if (I == ConstantProfileCounts.end())
121 return std::nullopt;
122 return I->second;
123}
124
125StringRef StaticDataProfileInfo::getConstantSectionPrefix(
126 const Constant *C, const ProfileSummaryInfo *PSI) const {
127 std::optional<uint64_t> Count = getConstantProfileCount(C);
128
129#ifndef NDEBUG
130 auto DbgPrintPrefix = [](StringRef Prefix) {
131 return Prefix.empty() ? "<empty>" : Prefix;
132 };
133#endif
134
135 if (EnableDataAccessProf) {
136 // Both data access profiles and PGO counters are available. Use the
137 // hotter one to be conservative. Basically, we want the non-unlikely
138 // sections to have max coverage of accessed symbols and meanwhile can
139 // tolerant some cold symbols in it, and the unlikely section variant to not
140 // have potentially hot symbols if possible, to avoid the penalty of access
141 // cold pages.
142 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C);
143 GV && llvm::memprof::IsAnnotationOK(GV: *GV) &&
144 (AnnotateStringLiteralSectionPrefix ||
145 !GV->getName().starts_with(Prefix: ".str"))) {
146 // Note a global var is covered by data access profiles iff the
147 // symbol name is preserved in the symbol table; most notably, a string
148 // literal with private linkage (e.g., those not externalized by ThinLTO
149 // and with insignificant address) won't have an entry in the symbol
150 // table (unless there is another string with identical content that
151 // gets a symbol table entry). For the private-linkage string literals,
152 // their hotness will be at least lukewarm (i.e., empty prefix).
153 auto HotnessFromDataAccessProf =
154 getSectionHotnessUsingDataAccessProfile(MaybeSectionPrefix: GV->getSectionPrefix());
155
156 if (!Count) {
157 StringRef Prefix = hotnessToStr(Hotness: HotnessFromDataAccessProf);
158 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
159 << DbgPrintPrefix(Prefix)
160 << ", solely from data access profiles\n");
161 return Prefix;
162 }
163
164 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, Count: *Count);
165 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
166 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
167 HotnessFromPGO == StaticDataHotness::Hot) {
168 GlobalVarHotness = StaticDataHotness::Hot;
169 } else if (HotnessFromDataAccessProf ==
170 StaticDataHotness::LukewarmOrUnknown ||
171 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
172 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
173 } else {
174 GlobalVarHotness = StaticDataHotness::Cold;
175 }
176 StringRef Prefix = hotnessToStr(Hotness: GlobalVarHotness);
177 LLVM_DEBUG(
178 dbgs() << GV->getName() << " has section prefix "
179 << DbgPrintPrefix(Prefix)
180 << ", the max from data access profiles as "
181 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
182 << " and PGO counters as "
183 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
184 return Prefix;
185 }
186 }
187 if (!Count)
188 return "";
189 return hotnessToStr(Hotness: getConstantHotnessUsingProfileCount(C, PSI, Count: *Count));
190}
191
192static std::unique_ptr<StaticDataProfileInfo>
193computeStaticDataProfileInfo(Module &M) {
194 bool EnableDataAccessProf = false;
195 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
196 MD: M.getModuleFlag(Key: "EnableDataAccessProf")))
197 EnableDataAccessProf = MD->getZExtValue();
198 return std::make_unique<StaticDataProfileInfo>(args&: EnableDataAccessProf);
199}
200
201bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
202 Info = computeStaticDataProfileInfo(M);
203 return false;
204}
205
206bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
207 Info.reset();
208 return false;
209}
210
211INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
212 "Static Data Profile Info", false, true)
213
214StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
215 : ImmutablePass(ID) {}
216
217char StaticDataProfileInfoWrapperPass::ID = 0;
218
219StaticDataProfileInfoAnalysis::Result
220StaticDataProfileInfoAnalysis::run(Module &M, ModuleAnalysisManager &) {
221 return StaticDataProfileInfoAnalysis::Result(computeStaticDataProfileInfo(M));
222}
223
224AnalysisKey llvm::StaticDataProfileInfoAnalysis::Key;
225