1#include <stdint.h>
2#include <stdlib.h>
3#include <string.h>
4
5#include "memprof_rawprofile.h"
6#include "profile/MemProfData.inc"
7#include "sanitizer_common/sanitizer_allocator_internal.h"
8#include "sanitizer_common/sanitizer_array_ref.h"
9#include "sanitizer_common/sanitizer_common.h"
10#include "sanitizer_common/sanitizer_linux.h"
11#include "sanitizer_common/sanitizer_procmaps.h"
12#include "sanitizer_common/sanitizer_stackdepot.h"
13#include "sanitizer_common/sanitizer_stackdepotbase.h"
14#include "sanitizer_common/sanitizer_stacktrace.h"
15#include "sanitizer_common/sanitizer_vector.h"
16
17namespace __memprof {
18using ::__sanitizer::Vector;
19using ::llvm::memprof::MemInfoBlock;
20using SegmentEntry = ::llvm::memprof::SegmentEntry;
21using Header = ::llvm::memprof::Header;
22
23namespace {
24template <class T> char *WriteBytes(const T &Pod, char *Buffer) {
25 *(T *)Buffer = Pod;
26 return Buffer + sizeof(T);
27}
28
29void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,
30 void *Arg) {
31 // No need to touch the MIB value here since we are only recording the key.
32 auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);
33 StackIds->PushBack(v: Key);
34}
35} // namespace
36
37u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) {
38 u64 NumSegmentsToRecord = 0;
39 for (const auto &Module : Modules) {
40 for (const auto &Segment : Module.ranges()) {
41 if (Segment.executable)
42 NumSegmentsToRecord++;
43 }
44 }
45
46 return sizeof(u64) // A header which stores the number of records.
47 + sizeof(SegmentEntry) * NumSegmentsToRecord;
48}
49
50// The segment section uses the following format:
51// ---------- Segment Info
52// Num Entries
53// ---------- Segment Entry
54// Start
55// End
56// Offset
57// UuidSize
58// Uuid 32B
59// ----------
60// ...
61void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules,
62 const u64 ExpectedNumBytes, char *&Buffer) {
63 char *Ptr = Buffer;
64 // Reserve space for the final count.
65 Ptr += sizeof(u64);
66
67 u64 NumSegmentsRecorded = 0;
68
69 for (const auto &Module : Modules) {
70 for (const auto &Segment : Module.ranges()) {
71 if (Segment.executable) {
72 SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address());
73 CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE);
74 Entry.BuildIdSize = Module.uuid_size();
75 memcpy(dest: Entry.BuildId, src: Module.uuid(), n: Module.uuid_size());
76 memcpy(dest: Ptr, src: &Entry, n: sizeof(SegmentEntry));
77 Ptr += sizeof(SegmentEntry);
78 NumSegmentsRecorded++;
79 }
80 }
81 }
82 // Store the number of segments we recorded in the space we reserved.
83 *((u64 *)Buffer) = NumSegmentsRecorded;
84 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
85 "Expected num bytes != actual bytes written");
86}
87
88u64 StackSizeBytes(const Vector<u64> &StackIds) {
89 u64 NumBytesToWrite = sizeof(u64);
90
91 const u64 NumIds = StackIds.Size();
92 for (unsigned k = 0; k < NumIds; ++k) {
93 const u64 Id = StackIds[k];
94 // One entry for the id and then one more for the number of stack pcs.
95 NumBytesToWrite += 2 * sizeof(u64);
96 const StackTrace St = StackDepotGet(id: Id);
97
98 CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");
99 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
100 NumBytesToWrite += sizeof(u64);
101 }
102 }
103 return NumBytesToWrite;
104}
105
106// The stack info section uses the following format:
107//
108// ---------- Stack Info
109// Num Entries
110// ---------- Stack Entry
111// Num Stacks
112// PC1
113// PC2
114// ...
115// ----------
116void SerializeStackToBuffer(const Vector<u64> &StackIds,
117 const u64 ExpectedNumBytes, char *&Buffer) {
118 const u64 NumIds = StackIds.Size();
119 char *Ptr = Buffer;
120 Ptr = WriteBytes(Pod: static_cast<u64>(NumIds), Buffer: Ptr);
121
122 for (unsigned k = 0; k < NumIds; ++k) {
123 const u64 Id = StackIds[k];
124 Ptr = WriteBytes(Pod: Id, Buffer: Ptr);
125 Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.
126 u64 Count = 0;
127 const StackTrace St = StackDepotGet(id: Id);
128 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
129 // PCs in stack traces are actually the return addresses, that is,
130 // addresses of the next instructions after the call.
131 uptr pc = StackTrace::GetPreviousInstructionPc(pc: St.trace[i]);
132 Ptr = WriteBytes(Pod: static_cast<u64>(pc), Buffer: Ptr);
133 ++Count;
134 }
135 // Store the count in the space we reserved earlier.
136 *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;
137 }
138
139 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
140 "Expected num bytes != actual bytes written");
141}
142
143// The MIB section has the following format:
144// ---------- MIB Info
145// Num Entries
146// ---------- MIB Entry 0
147// Alloc Count
148// ...
149// ---- AccessHistogram Entry 0
150// ...
151// ---- AccessHistogram Entry AccessHistogramSize - 1
152// ---------- MIB Entry 1
153// Alloc Count
154// ...
155// ---- AccessHistogram Entry 0
156// ...
157// ---- AccessHistogram Entry AccessHistogramSize - 1
158// ----------
159void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
160 const u64 ExpectedNumBytes, char *&Buffer) {
161 char *Ptr = Buffer;
162 const u64 NumEntries = StackIds.Size();
163 Ptr = WriteBytes(Pod: NumEntries, Buffer: Ptr);
164 for (u64 i = 0; i < NumEntries; i++) {
165 const u64 Key = StackIds[i];
166 MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);
167 CHECK(h.exists());
168 Ptr = WriteBytes(Pod: Key, Buffer: Ptr);
169 // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a
170 // serialization schema will fix this issue. See also FIXME in
171 // deserialization.
172 Ptr = WriteBytes(Pod: (*h)->mib, Buffer: Ptr);
173 for (u64 j = 0; j < (*h)->mib.AccessHistogramSize; ++j) {
174 u64 HistogramEntry = ((u64 *)((*h)->mib.AccessHistogram))[j];
175 Ptr = WriteBytes(Pod: HistogramEntry, Buffer: Ptr);
176 }
177 if ((*h)->mib.AccessHistogramSize > 0) {
178 InternalFree(p: (void *)((*h)->mib.AccessHistogram));
179 }
180 }
181 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
182 "Expected num bytes != actual bytes written");
183}
184
185// Format
186// ---------- Header
187// Magic
188// Version
189// Total Size
190// Segment Offset
191// MIB Info Offset
192// Stack Offset
193// ---------- Segment Info
194// Num Entries
195// ---------- Segment Entry
196// Start
197// End
198// Offset
199// BuildID 32B
200// ----------
201// ...
202// ----------
203// Optional Padding Bytes
204// ---------- MIB Info
205// Num Entries
206// ---------- MIB Entry
207// Alloc Count
208// ...
209// ---- AccessHistogram Entry 0
210// ...
211// ---- AccessHistogram Entry AccessHistogramSize - 1
212// ---------- MIB Entry 1
213// Alloc Count
214// ...
215// ---- AccessHistogram Entry 0
216// ...
217// ---- AccessHistogram Entry AccessHistogramSize - 1
218// Optional Padding Bytes
219// ---------- Stack Info
220// Num Entries
221// ---------- Stack Entry
222// Num Stacks
223// PC1
224// PC2
225// ...
226// ----------
227// Optional Padding Bytes
228// ...
229u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,
230 char *&Buffer) {
231 // Each section size is rounded up to 8b since the first entry in each section
232 // is a u64 which holds the number of entries in the section by convention.
233 const u64 NumSegmentBytes = RoundUpTo(size: SegmentSizeBytes(Modules), boundary: 8);
234
235 Vector<u64> StackIds;
236 MIBMap.ForEach(cb: RecordStackId, arg: reinterpret_cast<void *>(&StackIds));
237 // The first 8b are for the total number of MIB records. Each MIB record is
238 // preceded by a 8b stack id which is associated with stack frames in the next
239 // section.
240 const u64 NumMIBInfoBytes = RoundUpTo(
241 size: sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), boundary: 8);
242
243 // Get Number of AccessHistogram entries in total
244 u64 TotalAccessHistogramEntries = 0;
245 MIBMap.ForEach(
246 cb: [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) {
247 u64 *TotalAccessHistogramEntries = (u64 *)Arg;
248 *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize;
249 },
250 arg: reinterpret_cast<void *>(&TotalAccessHistogramEntries));
251 const u64 NumHistogramBytes =
252 RoundUpTo(size: TotalAccessHistogramEntries * sizeof(uint64_t), boundary: 8);
253
254 const u64 NumStackBytes = RoundUpTo(size: StackSizeBytes(StackIds), boundary: 8);
255
256 // Ensure that the profile is 8b aligned. We allow for some optional padding
257 // at the end so that any subsequent profile serialized to the same file does
258 // not incur unaligned accesses.
259 const u64 TotalSizeBytes =
260 RoundUpTo(size: sizeof(Header) + NumSegmentBytes + NumStackBytes +
261 NumMIBInfoBytes + NumHistogramBytes,
262 boundary: 8);
263
264 // Allocate the memory for the entire buffer incl. info blocks.
265 Buffer = (char *)InternalAlloc(size: TotalSizeBytes);
266 char *Ptr = Buffer;
267
268 Header header{MEMPROF_RAW_MAGIC_64,
269 MEMPROF_RAW_VERSION,
270 .TotalSize: static_cast<u64>(TotalSizeBytes),
271 .SegmentOffset: sizeof(Header),
272 .MIBOffset: sizeof(Header) + NumSegmentBytes,
273 .StackOffset: sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes +
274 NumHistogramBytes};
275 Ptr = WriteBytes(Pod: header, Buffer: Ptr);
276
277 SerializeSegmentsToBuffer(Modules, ExpectedNumBytes: NumSegmentBytes, Buffer&: Ptr);
278 Ptr += NumSegmentBytes;
279
280 SerializeMIBInfoToBuffer(MIBMap, StackIds,
281 ExpectedNumBytes: NumMIBInfoBytes + NumHistogramBytes, Buffer&: Ptr);
282 Ptr += NumMIBInfoBytes + NumHistogramBytes;
283
284 SerializeStackToBuffer(StackIds, ExpectedNumBytes: NumStackBytes, Buffer&: Ptr);
285
286 return TotalSizeBytes;
287}
288
289} // namespace __memprof
290