1#include "llvm/Analysis/StaticDataProfileInfo.h"
2#include "AnalysisOptions.h"
3#include "llvm/Analysis/ProfileSummaryInfo.h"
4#include "llvm/IR/Constant.h"
5#include "llvm/IR/Constants.h"
6#include "llvm/IR/GlobalVariable.h"
7#include "llvm/IR/Module.h"
8#include "llvm/IR/PassManager.h"
9#include "llvm/InitializePasses.h"
10#include "llvm/ProfileData/InstrProf.h"
11
12#define DEBUG_TYPE "static-data-profile-info"
13
14using namespace llvm;
15
16bool StaticDataProfileInfo::shouldPreserveHotDataSectionPrefix(
17 const LLVMContext &C) {
18 return getAnalysisOptions(C).preserve_hot_data_section_prefix;
19}
20
21bool llvm::memprof::shouldAnnotateStringLiteralSectionPrefix(
22 const LLVMContext &C) {
23 return getAnalysisOptions(C).memprof_annotate_string_literal_section_prefix;
24}
25
26namespace llvm {
27namespace memprof {
28// Returns true iff the global variable has custom section either by
29// __attribute__((section("name")))
30// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)
31// or #pragma clang section directives
32// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).
33static bool hasExplicitSectionName(const GlobalVariable &GVar) {
34 if (GVar.hasSection())
35 return true;
36
37 auto Attrs = GVar.getAttributes();
38 if (Attrs.hasAttribute(Kind: "bss-section") || Attrs.hasAttribute(Kind: "data-section") ||
39 Attrs.hasAttribute(Kind: "relro-section") ||
40 Attrs.hasAttribute(Kind: "rodata-section"))
41 return true;
42 return false;
43}
44
45AnnotationKind getAnnotationKind(const GlobalVariable &GV) {
46 if (GV.isDeclarationForLinker())
47 return AnnotationKind::DeclForLinker;
48 // Skip 'llvm.'-prefixed global variables conservatively because they are
49 // often handled specially,
50 StringRef Name = GV.getName();
51 if (Name.starts_with(Prefix: "llvm."))
52 return AnnotationKind::ReservedName;
53 // Respect user-specified custom data sections.
54 if (hasExplicitSectionName(GVar: GV))
55 return AnnotationKind::ExplicitSection;
56 return AnnotationKind::AnnotationOK;
57}
58
59bool IsAnnotationOK(const GlobalVariable &GV) {
60 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;
61}
62} // namespace memprof
63} // namespace llvm
64
65void StaticDataProfileInfo::addConstantProfileCount(
66 const Constant *C, std::optional<uint64_t> Count) {
67 if (!Count) {
68 ConstantWithoutCounts.insert(V: C);
69 return;
70 }
71 uint64_t &OriginalCount = ConstantProfileCounts[C];
72 OriginalCount = llvm::SaturatingAdd(X: *Count, Y: OriginalCount);
73 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
74 // large values for special use.
75 if (OriginalCount > getInstrMaxCountValue())
76 OriginalCount = getInstrMaxCountValue();
77}
78
79StaticDataProfileInfo::StaticDataHotness
80StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
81 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
82 // The accummulated counter shows the constant is hot. Return enum 'hot'
83 // whether this variable is seen by unprofiled functions or not.
84 if (PSI->isHotCount(C: Count))
85 return StaticDataHotness::Hot;
86 // The constant is not hot, and seen by unprofiled functions. We don't want to
87 // assign it to unlikely sections, even if the counter says 'cold'. So return
88 // enum 'LukewarmOrUnknown'.
89 if (ConstantWithoutCounts.count(V: C))
90 return StaticDataHotness::LukewarmOrUnknown;
91 // The accummulated counter shows the constant is cold so return enum 'cold'.
92 if (PSI->isColdCount(C: Count))
93 return StaticDataHotness::Cold;
94
95 return StaticDataHotness::LukewarmOrUnknown;
96}
97
98StaticDataProfileInfo::StaticDataHotness
99StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
100 std::optional<StringRef> MaybeSectionPrefix) const {
101 if (!MaybeSectionPrefix)
102 return StaticDataHotness::LukewarmOrUnknown;
103 StringRef Prefix = *MaybeSectionPrefix;
104 assert((Prefix == "hot" || Prefix == "unlikely") &&
105 "Expect section_prefix to be one of hot or unlikely");
106 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
107}
108
109StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
110 switch (Hotness) {
111 case StaticDataHotness::Cold:
112 return "unlikely";
113 case StaticDataHotness::Hot:
114 return Opts.preserve_hot_data_section_prefix ? "hot" : "";
115 default:
116 return "";
117 }
118}
119
120std::optional<uint64_t>
121StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
122 auto I = ConstantProfileCounts.find(Val: C);
123 if (I == ConstantProfileCounts.end())
124 return std::nullopt;
125 return I->second;
126}
127
128StringRef StaticDataProfileInfo::getConstantSectionPrefix(
129 const Constant *C, const ProfileSummaryInfo *PSI) const {
130 std::optional<uint64_t> Count = getConstantProfileCount(C);
131
132#ifndef NDEBUG
133 auto DbgPrintPrefix = [](StringRef Prefix) {
134 return Prefix.empty() ? "<empty>" : Prefix;
135 };
136#endif
137
138 if (EnableDataAccessProf) {
139 // Both data access profiles and PGO counters are available. Use the
140 // hotter one to be conservative. Basically, we want the non-unlikely
141 // sections to have max coverage of accessed symbols and meanwhile can
142 // tolerant some cold symbols in it, and the unlikely section variant to not
143 // have potentially hot symbols if possible, to avoid the penalty of access
144 // cold pages.
145 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C);
146 GV && llvm::memprof::IsAnnotationOK(GV: *GV) &&
147 (Opts.memprof_annotate_string_literal_section_prefix ||
148 !GV->getName().starts_with(Prefix: ".str"))) {
149 // Note a global var is covered by data access profiles iff the
150 // symbol name is preserved in the symbol table; most notably, a string
151 // literal with private linkage (e.g., those not externalized by ThinLTO
152 // and with insignificant address) won't have an entry in the symbol
153 // table (unless there is another string with identical content that
154 // gets a symbol table entry). For the private-linkage string literals,
155 // their hotness will be at least lukewarm (i.e., empty prefix).
156 auto HotnessFromDataAccessProf =
157 getSectionHotnessUsingDataAccessProfile(MaybeSectionPrefix: GV->getSectionPrefix());
158
159 if (!Count) {
160 StringRef Prefix = hotnessToStr(Hotness: HotnessFromDataAccessProf);
161 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
162 << DbgPrintPrefix(Prefix)
163 << ", solely from data access profiles\n");
164 return Prefix;
165 }
166
167 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, Count: *Count);
168 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
169 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
170 HotnessFromPGO == StaticDataHotness::Hot) {
171 GlobalVarHotness = StaticDataHotness::Hot;
172 } else if (HotnessFromDataAccessProf ==
173 StaticDataHotness::LukewarmOrUnknown ||
174 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
175 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
176 } else {
177 GlobalVarHotness = StaticDataHotness::Cold;
178 }
179 StringRef Prefix = hotnessToStr(Hotness: GlobalVarHotness);
180 LLVM_DEBUG(
181 dbgs() << GV->getName() << " has section prefix "
182 << DbgPrintPrefix(Prefix)
183 << ", the max from data access profiles as "
184 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
185 << " and PGO counters as "
186 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
187 return Prefix;
188 }
189 }
190 if (!Count)
191 return "";
192 return hotnessToStr(Hotness: getConstantHotnessUsingProfileCount(C, PSI, Count: *Count));
193}
194
195static std::unique_ptr<StaticDataProfileInfo>
196computeStaticDataProfileInfo(Module &M) {
197 bool EnableDataAccessProf = false;
198 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
199 MD: M.getModuleFlag(Key: "EnableDataAccessProf")))
200 EnableDataAccessProf = MD->getZExtValue();
201 return std::make_unique<StaticDataProfileInfo>(args: getAnalysisOptions(M),
202 args&: EnableDataAccessProf);
203}
204
205bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
206 Info = computeStaticDataProfileInfo(M);
207 return false;
208}
209
210bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
211 Info.reset();
212 return false;
213}
214
215INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
216 "Static Data Profile Info", false, true)
217
218StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
219 : ImmutablePass(ID) {}
220
221char StaticDataProfileInfoWrapperPass::ID = 0;
222
223StaticDataProfileInfoAnalysis::Result
224StaticDataProfileInfoAnalysis::run(Module &M, ModuleAnalysisManager &) {
225 return StaticDataProfileInfoAnalysis::Result(computeStaticDataProfileInfo(M));
226}
227
228AnalysisKey llvm::StaticDataProfileInfoAnalysis::Key;
229