1#ifndef MEMPROF_DATA_INC
2#define MEMPROF_DATA_INC
3/*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\
4|*
5|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6|* See https://llvm.org/LICENSE.txt for license information.
7|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8|*
9\*===----------------------------------------------------------------------===*/
10/*
11 * This is the main file that defines all the data structure, signature,
12 * constant literals that are shared across profiling runtime library,
13 * and host tools (reader/writer).
14 *
15 * This file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/include/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21\*===----------------------------------------------------------------------===*/
22#include <string.h>
23
24#ifdef _MSC_VER
25#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
26#else
27#define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
28#endif
29
30// A 64-bit magic number to uniquely identify the raw binary memprof profile file.
31#define MEMPROF_RAW_MAGIC_64 \
32 ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | \
33 (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
34
35// The version number of the raw binary format.
36#define MEMPROF_RAW_VERSION 5ULL
37
38// Currently supported versions.
39#define MEMPROF_RAW_SUPPORTED_VERSIONS {3ULL, 4ULL, 5ULL}
40
41#define MEMPROF_V3_MIB_SIZE 132ULL;
42
43#define MEMPROF_BUILDID_MAX_SIZE 32ULL
44
45namespace llvm {
46namespace memprof {
47// A struct describing the header used for the raw binary memprof profile format.
48PACKED(struct Header {
49 uint64_t Magic;
50 uint64_t Version;
51 uint64_t TotalSize;
52 uint64_t SegmentOffset;
53 uint64_t MIBOffset;
54 uint64_t StackOffset;
55});
56
57// A struct describing the information necessary to describe a /proc/maps
58// segment entry for a particular binary/library identified by its build id.
59PACKED(struct SegmentEntry {
60 uint64_t Start;
61 uint64_t End;
62 uint64_t Offset;
63 uint64_t BuildIdSize;
64 uint8_t BuildId[MEMPROF_BUILDID_MAX_SIZE] = {0};
65
66 // This constructor is only used in tests so don't set the BuildId.
67 SegmentEntry(uint64_t S, uint64_t E, uint64_t O)
68 : Start(S), End(E), Offset(O), BuildIdSize(0) {}
69
70 SegmentEntry(const SegmentEntry& S) {
71 Start = S.Start;
72 End = S.End;
73 Offset = S.Offset;
74 BuildIdSize = S.BuildIdSize;
75 memcpy(BuildId, S.BuildId, S.BuildIdSize);
76 }
77
78 SegmentEntry& operator=(const SegmentEntry& S) {
79 Start = S.Start;
80 End = S.End;
81 Offset = S.Offset;
82 BuildIdSize = S.BuildIdSize;
83 memcpy(BuildId, S.BuildId, S.BuildIdSize);
84 return *this;
85 }
86
87 bool operator==(const SegmentEntry& S) const {
88 return Start == S.Start && End == S.End && Offset == S.Offset &&
89 BuildIdSize == S.BuildIdSize &&
90 memcmp(BuildId, S.BuildId, S.BuildIdSize) == 0;
91 }
92});
93
94// Packed struct definition for MSVC. We can't use the PACKED macro defined in
95// MemProfData.inc since it would mean we are embedding a directive (the
96// #include for MIBEntryDef) into the macros which is undefined behaviour.
97#ifdef _MSC_VER
98__pragma(pack(push,1))
99#endif
100
101// A struct representing the heap allocation characteristics of a particular
102// runtime context. This struct is shared between the compiler-rt runtime and
103// the raw profile reader. The indexed format uses a separate, self-describing
104// backwards compatible format.
105struct MemInfoBlock{
106
107#define MIBEntryDef(NameTag, Name, Type) Type Name;
108#include "MIBEntryDef.inc"
109#undef MIBEntryDef
110
111bool operator==(const MemInfoBlock& Other) const {
112 bool IsEqual = true;
113#define MIBEntryDef(NameTag, Name, Type) \
114 IsEqual = (IsEqual && Name == Other.Name);
115#include "MIBEntryDef.inc"
116#undef MIBEntryDef
117 return IsEqual;
118}
119
120MemInfoBlock() {
121#define MIBEntryDef(NameTag, Name, Type) Name = Type();
122#include "MIBEntryDef.inc"
123#undef MIBEntryDef
124}
125
126MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs,
127 uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu,
128 uintptr_t Histogram, uint32_t HistogramSize)
129 : MemInfoBlock() {
130 AllocCount = 1U;
131 TotalAccessCount = AccessCount;
132 MinAccessCount = AccessCount;
133 MaxAccessCount = AccessCount;
134 TotalSize = Size;
135 MinSize = Size;
136 MaxSize = Size;
137 AllocTimestamp = AllocTs;
138 DeallocTimestamp = DeallocTs;
139 TotalLifetime = DeallocTimestamp - AllocTimestamp;
140 MinLifetime = TotalLifetime;
141 MaxLifetime = TotalLifetime;
142 // Access density is accesses per byte. Multiply by 100 to include the
143 // fractional part.
144 TotalAccessDensity = AccessCount * 100 / Size;
145 MinAccessDensity = TotalAccessDensity;
146 MaxAccessDensity = TotalAccessDensity;
147 // Lifetime access density is the access density per second of lifetime.
148 // Multiply by 1000 to convert denominator lifetime to seconds (using a
149 // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first
150 // to reduce truncations to 0.
151 TotalLifetimeAccessDensity =
152 TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1);
153 MinLifetimeAccessDensity = TotalLifetimeAccessDensity;
154 MaxLifetimeAccessDensity = TotalLifetimeAccessDensity;
155 AllocCpuId = AllocCpu;
156 DeallocCpuId = DeallocCpu;
157 NumMigratedCpu = AllocCpuId != DeallocCpuId;
158 AccessHistogramSize = HistogramSize;
159 AccessHistogram = Histogram;
160}
161
162void Merge(const MemInfoBlock &newMIB) {
163 AllocCount += newMIB.AllocCount;
164
165 TotalAccessCount += newMIB.TotalAccessCount;
166 MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount;
167 MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount;
168
169 TotalSize += newMIB.TotalSize;
170 MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize;
171 MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize;
172
173 TotalLifetime += newMIB.TotalLifetime;
174 MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime;
175 MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime;
176
177 TotalAccessDensity += newMIB.TotalAccessDensity;
178 MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity
179 ? newMIB.MinAccessDensity
180 : MinAccessDensity;
181 MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity
182 ? newMIB.MaxAccessDensity
183 : MaxAccessDensity;
184
185 TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity;
186 MinLifetimeAccessDensity =
187 newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity
188 ? newMIB.MinLifetimeAccessDensity
189 : MinLifetimeAccessDensity;
190 MaxLifetimeAccessDensity =
191 newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity
192 ? newMIB.MaxLifetimeAccessDensity
193 : MaxLifetimeAccessDensity;
194
195 // We know newMIB was deallocated later, so just need to check if it was
196 // allocated before last one deallocated.
197 NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp;
198 AllocTimestamp = newMIB.AllocTimestamp;
199 DeallocTimestamp = newMIB.DeallocTimestamp;
200
201 NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId;
202 NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId;
203 AllocCpuId = newMIB.AllocCpuId;
204 DeallocCpuId = newMIB.DeallocCpuId;
205
206 // For merging histograms, we always keep the longer histogram, and add
207 // values of shorter histogram to larger one.
208 uintptr_t ShorterHistogram;
209 uint32_t ShorterHistogramSize;
210 if (newMIB.AccessHistogramSize > AccessHistogramSize) {
211 ShorterHistogram = AccessHistogram;
212 ShorterHistogramSize = AccessHistogramSize;
213 // Swap histogram of current to larger histogram
214 AccessHistogram = newMIB.AccessHistogram;
215 AccessHistogramSize = newMIB.AccessHistogramSize;
216 } else {
217 ShorterHistogram = newMIB.AccessHistogram;
218 ShorterHistogramSize = newMIB.AccessHistogramSize;
219 }
220 for (size_t i = 0; i < ShorterHistogramSize; ++i) {
221 ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i];
222 }
223}
224
225#ifdef _MSC_VER
226} __pragma(pack(pop));
227#else
228} __attribute__((__packed__));
229#endif
230
231constexpr int MantissaBits = 12;
232constexpr int ExponentBits = 4;
233constexpr uint16_t MaxMantissa = (1U << MantissaBits) - 1;
234constexpr uint16_t MaxExponent = (1U << ExponentBits) - 1;
235constexpr uint64_t MaxRepresentableValue = static_cast<uint64_t>(MaxMantissa)
236 << MaxExponent;
237
238// Encodes a 64-bit unsigned integer into a 16-bit scaled integer format.
239inline uint16_t encodeHistogramCount(uint64_t Count) {
240 if (Count == 0)
241 return 0;
242
243 if (Count > MaxRepresentableValue)
244 Count = MaxRepresentableValue;
245
246 if (Count <= MaxMantissa)
247 return Count;
248
249 uint64_t M = Count;
250 uint16_t E = 0;
251 while (M > MaxMantissa) {
252 M = (M + 1) >> 1;
253 E++;
254 }
255 return (E << MantissaBits) | static_cast<uint16_t>(M);
256}
257
258// Decodes a 16-bit scaled integer and returns the
259// decoded 64-bit unsigned integer.
260inline uint64_t decodeHistogramCount(uint16_t EncodedValue) {
261 const uint16_t E = EncodedValue >> MantissaBits;
262 const uint16_t M = EncodedValue & MaxMantissa;
263 return static_cast<uint64_t>(M) << E;
264}
265
266} // namespace memprof
267} // namespace llvm
268
269#endif
270