1#include "llvm/Analysis/StaticDataProfileInfo.h"
2#include "llvm/Analysis/ProfileSummaryInfo.h"
3#include "llvm/IR/Constant.h"
4#include "llvm/IR/Constants.h"
5#include "llvm/IR/GlobalVariable.h"
6#include "llvm/IR/Module.h"
7#include "llvm/InitializePasses.h"
8#include "llvm/ProfileData/InstrProf.h"
9
10#define DEBUG_TYPE "static-data-profile-info"
11
12using namespace llvm;
13
14namespace llvm {
15namespace memprof {
16// Returns true iff the global variable has custom section either by
17// __attribute__((section("name")))
18// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)
19// or #pragma clang section directives
20// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).
21static bool hasExplicitSectionName(const GlobalVariable &GVar) {
22 if (GVar.hasSection())
23 return true;
24
25 auto Attrs = GVar.getAttributes();
26 if (Attrs.hasAttribute(Kind: "bss-section") || Attrs.hasAttribute(Kind: "data-section") ||
27 Attrs.hasAttribute(Kind: "relro-section") ||
28 Attrs.hasAttribute(Kind: "rodata-section"))
29 return true;
30 return false;
31}
32
33AnnotationKind getAnnotationKind(const GlobalVariable &GV) {
34 if (GV.isDeclarationForLinker())
35 return AnnotationKind::DeclForLinker;
36 // Skip 'llvm.'-prefixed global variables conservatively because they are
37 // often handled specially,
38 StringRef Name = GV.getName();
39 if (Name.starts_with(Prefix: "llvm."))
40 return AnnotationKind::ReservedName;
41 // Respect user-specified custom data sections.
42 if (hasExplicitSectionName(GVar: GV))
43 return AnnotationKind::ExplicitSection;
44 return AnnotationKind::AnnotationOK;
45}
46
47bool IsAnnotationOK(const GlobalVariable &GV) {
48 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;
49}
50} // namespace memprof
51} // namespace llvm
52
53void StaticDataProfileInfo::addConstantProfileCount(
54 const Constant *C, std::optional<uint64_t> Count) {
55 if (!Count) {
56 ConstantWithoutCounts.insert(V: C);
57 return;
58 }
59 uint64_t &OriginalCount = ConstantProfileCounts[C];
60 OriginalCount = llvm::SaturatingAdd(X: *Count, Y: OriginalCount);
61 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
62 // large values for special use.
63 if (OriginalCount > getInstrMaxCountValue())
64 OriginalCount = getInstrMaxCountValue();
65}
66
67StaticDataProfileInfo::StaticDataHotness
68StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
69 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
70 // The accummulated counter shows the constant is hot. Return enum 'hot'
71 // whether this variable is seen by unprofiled functions or not.
72 if (PSI->isHotCount(C: Count))
73 return StaticDataHotness::Hot;
74 // The constant is not hot, and seen by unprofiled functions. We don't want to
75 // assign it to unlikely sections, even if the counter says 'cold'. So return
76 // enum 'LukewarmOrUnknown'.
77 if (ConstantWithoutCounts.count(V: C))
78 return StaticDataHotness::LukewarmOrUnknown;
79 // The accummulated counter shows the constant is cold so return enum 'cold'.
80 if (PSI->isColdCount(C: Count))
81 return StaticDataHotness::Cold;
82
83 return StaticDataHotness::LukewarmOrUnknown;
84}
85
86StaticDataProfileInfo::StaticDataHotness
87StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
88 std::optional<StringRef> MaybeSectionPrefix) const {
89 if (!MaybeSectionPrefix)
90 return StaticDataHotness::LukewarmOrUnknown;
91 StringRef Prefix = *MaybeSectionPrefix;
92 assert((Prefix == "hot" || Prefix == "unlikely") &&
93 "Expect section_prefix to be one of hot or unlikely");
94 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
95}
96
97StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
98 switch (Hotness) {
99 case StaticDataHotness::Cold:
100 return "unlikely";
101 case StaticDataHotness::Hot:
102 return "hot";
103 default:
104 return "";
105 }
106}
107
108std::optional<uint64_t>
109StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
110 auto I = ConstantProfileCounts.find(Val: C);
111 if (I == ConstantProfileCounts.end())
112 return std::nullopt;
113 return I->second;
114}
115
116StringRef StaticDataProfileInfo::getConstantSectionPrefix(
117 const Constant *C, const ProfileSummaryInfo *PSI) const {
118 std::optional<uint64_t> Count = getConstantProfileCount(C);
119
120#ifndef NDEBUG
121 auto DbgPrintPrefix = [](StringRef Prefix) {
122 return Prefix.empty() ? "<empty>" : Prefix;
123 };
124#endif
125
126 if (EnableDataAccessProf) {
127 // Module flag `HasDataAccessProf` is 1 -> empty section prefix means
128 // unknown hotness except for string literals.
129 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C);
130 GV && llvm::memprof::IsAnnotationOK(GV: *GV) &&
131 !GV->getName().starts_with(Prefix: ".str")) {
132 auto HotnessFromDataAccessProf =
133 getSectionHotnessUsingDataAccessProfile(MaybeSectionPrefix: GV->getSectionPrefix());
134
135 if (!Count) {
136 StringRef Prefix = hotnessToStr(Hotness: HotnessFromDataAccessProf);
137 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
138 << DbgPrintPrefix(Prefix)
139 << ", solely from data access profiles\n");
140 return Prefix;
141 }
142
143 // Both data access profiles and PGO counters are available. Use the
144 // hotter one.
145 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, Count: *Count);
146 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
147 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
148 HotnessFromPGO == StaticDataHotness::Hot) {
149 GlobalVarHotness = StaticDataHotness::Hot;
150 } else if (HotnessFromDataAccessProf ==
151 StaticDataHotness::LukewarmOrUnknown ||
152 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
153 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
154 } else {
155 GlobalVarHotness = StaticDataHotness::Cold;
156 }
157 StringRef Prefix = hotnessToStr(Hotness: GlobalVarHotness);
158 LLVM_DEBUG(
159 dbgs() << GV->getName() << " has section prefix "
160 << DbgPrintPrefix(Prefix)
161 << ", the max from data access profiles as "
162 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
163 << " and PGO counters as "
164 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
165 return Prefix;
166 }
167 }
168 if (!Count)
169 return "";
170 return hotnessToStr(Hotness: getConstantHotnessUsingProfileCount(C, PSI, Count: *Count));
171}
172
173bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
174 bool EnableDataAccessProf = false;
175 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
176 MD: M.getModuleFlag(Key: "EnableDataAccessProf")))
177 EnableDataAccessProf = MD->getZExtValue();
178 Info.reset(p: new StaticDataProfileInfo(EnableDataAccessProf));
179 return false;
180}
181
182bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
183 Info.reset();
184 return false;
185}
186
187INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
188 "Static Data Profile Info", false, true)
189
190StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
191 : ImmutablePass(ID) {}
192
193char StaticDataProfileInfoWrapperPass::ID = 0;
194