1#include <stdint.h>
2#include <stdlib.h>
3#include <string.h>
4
5#include "memprof_rawprofile.h"
6#include "profile/MemProfData.inc"
7#include "sanitizer_common/sanitizer_allocator_internal.h"
8#include "sanitizer_common/sanitizer_array_ref.h"
9#include "sanitizer_common/sanitizer_common.h"
10#include "sanitizer_common/sanitizer_stackdepot.h"
11#include "sanitizer_common/sanitizer_stacktrace.h"
12#include "sanitizer_common/sanitizer_vector.h"
13
14namespace __memprof {
15using ::__sanitizer::Vector;
16using ::llvm::memprof::MemInfoBlock;
17using SegmentEntry = ::llvm::memprof::SegmentEntry;
18using Header = ::llvm::memprof::Header;
19using ::llvm::memprof::encodeHistogramCount;
20
21namespace {
22template <class T> char *WriteBytes(const T &Pod, char *Buffer) {
23 static_assert(is_trivially_copyable<T>::value, "T must be POD");
24 const uint8_t *Src = reinterpret_cast<const uint8_t *>(&Pod);
25
26 for (size_t I = 0; I < sizeof(T); ++I)
27#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
28 // Reverse byte order since reader is little-endian.
29 Buffer[I] = Src[sizeof(T) - 1 - I];
30#else
31 Buffer[I] = Src[I];
32#endif
33 return Buffer + sizeof(T);
34}
35
36void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,
37 void *Arg) {
38 // No need to touch the MIB value here since we are only recording the key.
39 auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);
40 StackIds->PushBack(v: Key);
41}
42
43u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) {
44 u64 NumSegmentsToRecord = 0;
45 for (const auto &Module : Modules) {
46 for (const auto &Segment : Module.ranges()) {
47 if (Segment.executable)
48 NumSegmentsToRecord++;
49 }
50 }
51
52 return sizeof(u64) // A header which stores the number of records.
53 + sizeof(SegmentEntry) * NumSegmentsToRecord;
54}
55
56// The segment section uses the following format:
57// ---------- Segment Info
58// Num Entries
59// ---------- Segment Entry
60// Start
61// End
62// Offset
63// UuidSize
64// Uuid 32B
65// ----------
66// ...
67void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules,
68 const u64 ExpectedNumBytes, char *&Buffer) {
69 char *Ptr = Buffer;
70 // Reserve space for the final count.
71 Ptr += sizeof(u64);
72
73 u64 NumSegmentsRecorded = 0;
74
75 for (const auto &Module : Modules) {
76 for (const auto &Segment : Module.ranges()) {
77 if (Segment.executable) {
78 SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address());
79 CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE);
80 Entry.BuildIdSize = Module.uuid_size();
81 memcpy(dest: Entry.BuildId, src: Module.uuid(), n: Module.uuid_size());
82 memcpy(dest: Ptr, src: &Entry, n: sizeof(SegmentEntry));
83 Ptr += sizeof(SegmentEntry);
84 NumSegmentsRecorded++;
85 }
86 }
87 }
88 // Store the number of segments we recorded in the space we reserved.
89 *((u64 *)Buffer) = NumSegmentsRecorded;
90 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
91 "Expected num bytes != actual bytes written");
92}
93
94u64 StackSizeBytes(const Vector<u64> &StackIds) {
95 u64 NumBytesToWrite = sizeof(u64);
96
97 const u64 NumIds = StackIds.Size();
98 for (unsigned k = 0; k < NumIds; ++k) {
99 const u64 Id = StackIds[k];
100 // One entry for the id and then one more for the number of stack pcs.
101 NumBytesToWrite += 2 * sizeof(u64);
102 const StackTrace St = StackDepotGet(id: Id);
103
104 CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");
105 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
106 NumBytesToWrite += sizeof(u64);
107 }
108 }
109 return NumBytesToWrite;
110}
111
112// The stack info section uses the following format:
113//
114// ---------- Stack Info
115// Num Entries
116// ---------- Stack Entry
117// Num Stacks
118// PC1
119// PC2
120// ...
121// ----------
122void SerializeStackToBuffer(const Vector<u64> &StackIds,
123 const u64 ExpectedNumBytes, char *&Buffer) {
124 const u64 NumIds = StackIds.Size();
125 char *Ptr = Buffer;
126 Ptr = WriteBytes(Pod: static_cast<u64>(NumIds), Buffer: Ptr);
127
128 for (unsigned k = 0; k < NumIds; ++k) {
129 const u64 Id = StackIds[k];
130 Ptr = WriteBytes(Pod: Id, Buffer: Ptr);
131 Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.
132 u64 Count = 0;
133 const StackTrace St = StackDepotGet(id: Id);
134 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
135 // PCs in stack traces are actually the return addresses, that is,
136 // addresses of the next instructions after the call.
137 uptr pc = StackTrace::GetPreviousInstructionPc(pc: St.trace[i]);
138 Ptr = WriteBytes(Pod: static_cast<u64>(pc), Buffer: Ptr);
139 ++Count;
140 }
141 // Store the count in the space we reserved earlier.
142 *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;
143 }
144
145 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
146 "Expected num bytes != actual bytes written");
147}
148
149// The MIB section has the following format:
150// ---------- MIB Info
151// Num Entries
152// ---------- MIB Entry 0
153// Alloc Count
154// ...
155// ---- AccessHistogram Entry 0
156// ...
157// ---- AccessHistogram Entry AccessHistogramSize - 1
158// ---------- MIB Entry 1
159// Alloc Count
160// ...
161// ---- AccessHistogram Entry 0
162// ...
163// ---- AccessHistogram Entry AccessHistogramSize - 1
164// ----------
165void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
166 const u64 ExpectedNumBytes, char *&Buffer) {
167 char *Ptr = Buffer;
168 const u64 NumEntries = StackIds.Size();
169 Ptr = WriteBytes(Pod: NumEntries, Buffer: Ptr);
170 for (u64 i = 0; i < NumEntries; i++) {
171 const u64 Key = StackIds[i];
172 MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);
173 CHECK(h.exists());
174 Ptr = WriteBytes(Pod: Key, Buffer: Ptr);
175 // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a
176 // serialization schema will fix this issue. See also FIXME in
177 // deserialization.
178 auto &MIB = (*h)->mib;
179 Ptr = WriteBytes(Pod: MIB, Buffer: Ptr);
180 for (u64 j = 0; j < MIB.AccessHistogramSize; ++j) {
181 u16 HistogramEntry =
182 encodeHistogramCount(Count: ((u64 *)(MIB.AccessHistogram))[j]);
183 Ptr = WriteBytes(Pod: HistogramEntry, Buffer: Ptr);
184 }
185 if (MIB.AccessHistogramSize > 0) {
186 InternalFree(p: (void *)MIB.AccessHistogram);
187 }
188 }
189 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
190 "Expected num bytes != actual bytes written");
191}
192} // namespace
193
194// Format
195// ---------- Header
196// Magic
197// Version
198// Total Size
199// Segment Offset
200// MIB Info Offset
201// Stack Offset
202// ---------- Segment Info
203// Num Entries
204// ---------- Segment Entry
205// Start
206// End
207// Offset
208// BuildID 32B
209// ----------
210// ...
211// ----------
212// Optional Padding Bytes
213// ---------- MIB Info
214// Num Entries
215// ---------- MIB Entry
216// Alloc Count
217// ...
218// ---- AccessHistogram Entry 0
219// ...
220// ---- AccessHistogram Entry AccessHistogramSize - 1
221// ---------- MIB Entry 1
222// Alloc Count
223// ...
224// ---- AccessHistogram Entry 0
225// ...
226// ---- AccessHistogram Entry AccessHistogramSize - 1
227// Optional Padding Bytes
228// ---------- Stack Info
229// Num Entries
230// ---------- Stack Entry
231// Num Stacks
232// PC1
233// PC2
234// ...
235// ----------
236// Optional Padding Bytes
237// ...
238u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,
239 char *&Buffer) {
240 // Each section size is rounded up to 8b since the first entry in each section
241 // is a u64 which holds the number of entries in the section by convention.
242 const u64 NumSegmentBytes = RoundUpTo(size: SegmentSizeBytes(Modules), boundary: 8);
243
244 Vector<u64> StackIds;
245 MIBMap.ForEach(cb: RecordStackId, arg: reinterpret_cast<void *>(&StackIds));
246 // The first 8b are for the total number of MIB records. Each MIB record is
247 // preceded by a 8b stack id which is associated with stack frames in the next
248 // section.
249 const u64 NumMIBInfoBytes = RoundUpTo(
250 size: sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), boundary: 8);
251
252 // Get Number of AccessHistogram entries in total
253 u64 TotalAccessHistogramEntries = 0;
254 MIBMap.ForEach(
255 cb: [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) {
256 u64 *TotalAccessHistogramEntries = (u64 *)Arg;
257 *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize;
258 },
259 arg: reinterpret_cast<void *>(&TotalAccessHistogramEntries));
260 const u64 NumHistogramBytes =
261 RoundUpTo(size: TotalAccessHistogramEntries * sizeof(uint16_t), boundary: 8);
262
263 const u64 NumStackBytes = RoundUpTo(size: StackSizeBytes(StackIds), boundary: 8);
264
265 // Ensure that the profile is 8b aligned. We allow for some optional padding
266 // at the end so that any subsequent profile serialized to the same file does
267 // not incur unaligned accesses.
268 const u64 TotalSizeBytes =
269 RoundUpTo(size: sizeof(Header) + NumSegmentBytes + NumStackBytes +
270 NumMIBInfoBytes + NumHistogramBytes,
271 boundary: 8);
272
273 // Allocate the memory for the entire buffer incl. info blocks.
274 Buffer = (char *)InternalAlloc(size: TotalSizeBytes);
275 char *Ptr = Buffer;
276
277 Header header{MEMPROF_RAW_MAGIC_64,
278 MEMPROF_RAW_VERSION,
279 .TotalSize: static_cast<u64>(TotalSizeBytes),
280 .SegmentOffset: sizeof(Header),
281 .MIBOffset: sizeof(Header) + NumSegmentBytes,
282 .StackOffset: sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes +
283 NumHistogramBytes};
284 Ptr = WriteBytes(Pod: header, Buffer: Ptr);
285
286 SerializeSegmentsToBuffer(Modules, ExpectedNumBytes: NumSegmentBytes, Buffer&: Ptr);
287 Ptr += NumSegmentBytes;
288
289 SerializeMIBInfoToBuffer(MIBMap, StackIds,
290 ExpectedNumBytes: NumMIBInfoBytes + NumHistogramBytes, Buffer&: Ptr);
291 Ptr += NumMIBInfoBytes + NumHistogramBytes;
292
293 SerializeStackToBuffer(StackIds, ExpectedNumBytes: NumStackBytes, Buffer&: Ptr);
294
295 return TotalSizeBytes;
296}
297} // namespace __memprof
298