1#include "llvm/Analysis/StaticDataProfileInfo.h"
2#include "llvm/Analysis/ProfileSummaryInfo.h"
3#include "llvm/IR/Constant.h"
4#include "llvm/IR/Constants.h"
5#include "llvm/IR/GlobalVariable.h"
6#include "llvm/IR/Module.h"
7#include "llvm/InitializePasses.h"
8#include "llvm/ProfileData/InstrProf.h"
9
10#define DEBUG_TYPE "static-data-profile-info"
11
12using namespace llvm;
13
14namespace llvm {
15// FIXME: This option is added for incremental rollout purposes.
16// After the option, string literal partitioning should be implied by
17// AnnotateStaticDataSectionPrefix in MemProfUse.cpp and this option should be
18// cleaned up.
19cl::opt<bool> AnnotateStringLiteralSectionPrefix(
20 "memprof-annotate-string-literal-section-prefix", cl::init(Val: false),
21 cl::Hidden,
22 cl::desc("If true, annotate the string literal data section prefix"));
23namespace memprof {
24// Returns true iff the global variable has custom section either by
25// __attribute__((section("name")))
26// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)
27// or #pragma clang section directives
28// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).
29static bool hasExplicitSectionName(const GlobalVariable &GVar) {
30 if (GVar.hasSection())
31 return true;
32
33 auto Attrs = GVar.getAttributes();
34 if (Attrs.hasAttribute(Kind: "bss-section") || Attrs.hasAttribute(Kind: "data-section") ||
35 Attrs.hasAttribute(Kind: "relro-section") ||
36 Attrs.hasAttribute(Kind: "rodata-section"))
37 return true;
38 return false;
39}
40
41AnnotationKind getAnnotationKind(const GlobalVariable &GV) {
42 if (GV.isDeclarationForLinker())
43 return AnnotationKind::DeclForLinker;
44 // Skip 'llvm.'-prefixed global variables conservatively because they are
45 // often handled specially,
46 StringRef Name = GV.getName();
47 if (Name.starts_with(Prefix: "llvm."))
48 return AnnotationKind::ReservedName;
49 // Respect user-specified custom data sections.
50 if (hasExplicitSectionName(GVar: GV))
51 return AnnotationKind::ExplicitSection;
52 return AnnotationKind::AnnotationOK;
53}
54
55bool IsAnnotationOK(const GlobalVariable &GV) {
56 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;
57}
58} // namespace memprof
59} // namespace llvm
60
61void StaticDataProfileInfo::addConstantProfileCount(
62 const Constant *C, std::optional<uint64_t> Count) {
63 if (!Count) {
64 ConstantWithoutCounts.insert(V: C);
65 return;
66 }
67 uint64_t &OriginalCount = ConstantProfileCounts[C];
68 OriginalCount = llvm::SaturatingAdd(X: *Count, Y: OriginalCount);
69 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
70 // large values for special use.
71 if (OriginalCount > getInstrMaxCountValue())
72 OriginalCount = getInstrMaxCountValue();
73}
74
75StaticDataProfileInfo::StaticDataHotness
76StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
77 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
78 // The accummulated counter shows the constant is hot. Return enum 'hot'
79 // whether this variable is seen by unprofiled functions or not.
80 if (PSI->isHotCount(C: Count))
81 return StaticDataHotness::Hot;
82 // The constant is not hot, and seen by unprofiled functions. We don't want to
83 // assign it to unlikely sections, even if the counter says 'cold'. So return
84 // enum 'LukewarmOrUnknown'.
85 if (ConstantWithoutCounts.count(V: C))
86 return StaticDataHotness::LukewarmOrUnknown;
87 // The accummulated counter shows the constant is cold so return enum 'cold'.
88 if (PSI->isColdCount(C: Count))
89 return StaticDataHotness::Cold;
90
91 return StaticDataHotness::LukewarmOrUnknown;
92}
93
94StaticDataProfileInfo::StaticDataHotness
95StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
96 std::optional<StringRef> MaybeSectionPrefix) const {
97 if (!MaybeSectionPrefix)
98 return StaticDataHotness::LukewarmOrUnknown;
99 StringRef Prefix = *MaybeSectionPrefix;
100 assert((Prefix == "hot" || Prefix == "unlikely") &&
101 "Expect section_prefix to be one of hot or unlikely");
102 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
103}
104
105StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
106 switch (Hotness) {
107 case StaticDataHotness::Cold:
108 return "unlikely";
109 case StaticDataHotness::Hot:
110 return "hot";
111 default:
112 return "";
113 }
114}
115
116std::optional<uint64_t>
117StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
118 auto I = ConstantProfileCounts.find(Val: C);
119 if (I == ConstantProfileCounts.end())
120 return std::nullopt;
121 return I->second;
122}
123
124StringRef StaticDataProfileInfo::getConstantSectionPrefix(
125 const Constant *C, const ProfileSummaryInfo *PSI) const {
126 std::optional<uint64_t> Count = getConstantProfileCount(C);
127
128#ifndef NDEBUG
129 auto DbgPrintPrefix = [](StringRef Prefix) {
130 return Prefix.empty() ? "<empty>" : Prefix;
131 };
132#endif
133
134 if (EnableDataAccessProf) {
135 // Both data access profiles and PGO counters are available. Use the
136 // hotter one to be conservative. Basically, we want the non-unlikely
137 // sections to have max coverage of accessed symbols and meanwhile can
138 // tolerant some cold symbols in it, and the unlikely section variant to not
139 // have potentially hot symbols if possible, to avoid the penalty of access
140 // cold pages.
141 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C);
142 GV && llvm::memprof::IsAnnotationOK(GV: *GV) &&
143 (AnnotateStringLiteralSectionPrefix ||
144 !GV->getName().starts_with(Prefix: ".str"))) {
145 // Note a global var is covered by data access profiles iff the
146 // symbol name is preserved in the symbol table; most notably, a string
147 // literal with private linkage (e.g., those not externalized by ThinLTO
148 // and with insignificant address) won't have an entry in the symbol
149 // table (unless there is another string with identical content that
150 // gets a symbol table entry). For the private-linkage string literals,
151 // their hotness will be at least lukewarm (i.e., empty prefix).
152 auto HotnessFromDataAccessProf =
153 getSectionHotnessUsingDataAccessProfile(MaybeSectionPrefix: GV->getSectionPrefix());
154
155 if (!Count) {
156 StringRef Prefix = hotnessToStr(Hotness: HotnessFromDataAccessProf);
157 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
158 << DbgPrintPrefix(Prefix)
159 << ", solely from data access profiles\n");
160 return Prefix;
161 }
162
163 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, Count: *Count);
164 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
165 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
166 HotnessFromPGO == StaticDataHotness::Hot) {
167 GlobalVarHotness = StaticDataHotness::Hot;
168 } else if (HotnessFromDataAccessProf ==
169 StaticDataHotness::LukewarmOrUnknown ||
170 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
171 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
172 } else {
173 GlobalVarHotness = StaticDataHotness::Cold;
174 }
175 StringRef Prefix = hotnessToStr(Hotness: GlobalVarHotness);
176 LLVM_DEBUG(
177 dbgs() << GV->getName() << " has section prefix "
178 << DbgPrintPrefix(Prefix)
179 << ", the max from data access profiles as "
180 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
181 << " and PGO counters as "
182 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
183 return Prefix;
184 }
185 }
186 if (!Count)
187 return "";
188 return hotnessToStr(Hotness: getConstantHotnessUsingProfileCount(C, PSI, Count: *Count));
189}
190
191bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
192 bool EnableDataAccessProf = false;
193 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
194 MD: M.getModuleFlag(Key: "EnableDataAccessProf")))
195 EnableDataAccessProf = MD->getZExtValue();
196 Info.reset(p: new StaticDataProfileInfo(EnableDataAccessProf));
197 return false;
198}
199
200bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
201 Info.reset();
202 return false;
203}
204
205INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
206 "Static Data Profile Info", false, true)
207
208StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
209 : ImmutablePass(ID) {}
210
211char StaticDataProfileInfoWrapperPass::ID = 0;
212