1/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9// Note: This is linked into the Darwin kernel, and must remain compatible
10// with freestanding compilation. See `darwin_add_builtin_libraries`.
11
12#ifdef _MSC_VER
13/* For _alloca */
14#include <malloc.h>
15#endif
16#if defined(__FreeBSD__) && defined(_KERNEL)
17#include <sys/systm.h>
18#else
19#include <string.h>
20#endif
21
22#include "InstrProfiling.h"
23#include "InstrProfilingInternal.h"
24#include "InstrProfilingPort.h"
25
26#define INSTR_PROF_VALUE_PROF_DATA
27#include "profile/InstrProfData.inc"
28
29COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
30static ProfBufferIO TheBufferIO;
31#define VP_BUFFER_SIZE 8 * 1024
32static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
33static InstrProfValueData VPDataArray[16];
34static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
35
36COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
37COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
38
39/* The buffer writer is responsible in keeping writer state
40 * across the call.
41 */
42COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
43 ProfDataIOVec *IOVecs,
44 uint32_t NumIOVecs) {
45 uint32_t I;
46 char **Buffer = (char **)&This->WriterCtx;
47 for (I = 0; I < NumIOVecs; I++) {
48 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
49 if (IOVecs[I].Data)
50 memcpy(dest: *Buffer, src: IOVecs[I].Data, n: Length);
51 else if (IOVecs[I].UseZeroPadding) {
52 /* Allocating the buffer should zero fill. */
53 }
54 *Buffer += Length;
55 }
56 return 0;
57}
58
59static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
60 uint8_t *Buffer, uint32_t BufferSz) {
61 BufferIO->FileWriter = FileWriter;
62 BufferIO->OwnFileWriter = 0;
63 BufferIO->BufferStart = Buffer;
64 BufferIO->BufferSz = BufferSz;
65 BufferIO->CurOffset = 0;
66}
67
68COMPILER_RT_VISIBILITY ProfBufferIO *
69lprofCreateBufferIO(ProfDataWriter *FileWriter) {
70 uint8_t *Buffer = DynamicBufferIOBuffer;
71 uint32_t BufferSize = VPBufferSize;
72 if (!Buffer) {
73 Buffer = &BufferIOBuffer[0];
74 BufferSize = sizeof(BufferIOBuffer);
75 }
76 llvmInitBufferIO(BufferIO: &TheBufferIO, FileWriter, Buffer, BufferSz: BufferSize);
77 return &TheBufferIO;
78}
79
80COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
81 if (BufferIO->OwnFileWriter)
82 FreeHook(BufferIO->FileWriter);
83 if (DynamicBufferIOBuffer) {
84 FreeHook(DynamicBufferIOBuffer);
85 DynamicBufferIOBuffer = 0;
86 VPBufferSize = 0;
87 }
88}
89
90COMPILER_RT_VISIBILITY int
91lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
92 /* Buffer is not large enough, it is time to flush. */
93 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
94 if (lprofBufferIOFlush(BufferIO) != 0)
95 return -1;
96 }
97 /* Special case, bypass the buffer completely. */
98 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
99 if (Size > BufferIO->BufferSz) {
100 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
101 return -1;
102 } else {
103 /* Write the data to buffer */
104 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
105 ProfDataWriter BufferWriter;
106 initBufferWriter(BufferWriter: &BufferWriter, Buffer: (char *)Buffer);
107 lprofBufferWriter(This: &BufferWriter, IOVecs: IO, NumIOVecs: 1);
108 BufferIO->CurOffset =
109 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
110 }
111 return 0;
112}
113
114COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
115 if (BufferIO->CurOffset) {
116 ProfDataIOVec IO[] = {
117 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
118 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
119 return -1;
120 BufferIO->CurOffset = 0;
121 }
122 return 0;
123}
124
125/* Write out value profile data for function specified with \c Data.
126 * The implementation does not use the method \c serializeValueProfData
127 * which depends on dynamic memory allocation. In this implementation,
128 * value profile data is written out to \c BufferIO piecemeal.
129 */
130static int writeOneValueProfData(ProfBufferIO *BufferIO,
131 VPDataReaderType *VPDataReader,
132 const __llvm_profile_data *Data) {
133 unsigned I, NumValueKinds = 0;
134 ValueProfData VPHeader;
135 uint8_t *SiteCountArray[IPVK_Last + 1];
136
137 for (I = 0; I <= IPVK_Last; I++) {
138 if (!Data->NumValueSites[I])
139 SiteCountArray[I] = 0;
140 else {
141 uint32_t Sz =
142 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
143 offsetof(ValueProfRecord, SiteCountArray);
144 /* Only use alloca for this small byte array to avoid excessive
145 * stack growth. */
146 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
147 memset(s: SiteCountArray[I], c: 0, n: Sz);
148 }
149 }
150
151 /* If NumValueKinds returned is 0, there is nothing to write, report
152 success and return. This should match the raw profile reader's behavior. */
153 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
154 return 0;
155
156 /* First write the header structure. */
157 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
158 VPHeader.NumValueKinds = NumValueKinds;
159 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&VPHeader,
160 Size: sizeof(ValueProfData)))
161 return -1;
162
163 /* Make sure nothing else needs to be written before value profile
164 * records. */
165 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
166 (void *)(&VPHeader + 1))
167 return -1;
168
169 /* Write out the value profile record for each value kind
170 * one by one. */
171 for (I = 0; I <= IPVK_Last; I++) {
172 uint32_t J;
173 ValueProfRecord RecordHeader;
174 /* The size of the value prof record header without counting the
175 * site count array .*/
176 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
177 uint32_t SiteCountArraySize;
178
179 if (!Data->NumValueSites[I])
180 continue;
181
182 /* Write out the record header. */
183 RecordHeader.Kind = I;
184 RecordHeader.NumValueSites = Data->NumValueSites[I];
185 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&RecordHeader,
186 Size: RecordHeaderSize))
187 return -1;
188
189 /* Write out the site value count array including padding space. */
190 SiteCountArraySize =
191 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
192 RecordHeaderSize;
193 if (lprofBufferIOWrite(BufferIO, Data: SiteCountArray[I], Size: SiteCountArraySize))
194 return -1;
195
196 /* Write out the value profile data for each value site. */
197 for (J = 0; J < Data->NumValueSites[I]; J++) {
198 uint32_t NRead, NRemain;
199 ValueProfNode *NextStartNode = 0;
200 NRemain = VPDataReader->GetNumValueDataForSite(I, J);
201 if (!NRemain)
202 continue;
203 /* Read and write out value data in small chunks till it is done. */
204 do {
205 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
206 NextStartNode =
207 VPDataReader->GetValueData(I, /* ValueKind */
208 J, /* Site */
209 &VPDataArray[0], NextStartNode, NRead);
210 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&VPDataArray[0],
211 Size: NRead * sizeof(InstrProfValueData)))
212 return -1;
213 NRemain -= NRead;
214 } while (NRemain != 0);
215 }
216 }
217 /* All done report success. */
218 return 0;
219}
220
221static int writeValueProfData(ProfDataWriter *Writer,
222 VPDataReaderType *VPDataReader,
223 const __llvm_profile_data *DataBegin,
224 const __llvm_profile_data *DataEnd) {
225 ProfBufferIO *BufferIO;
226 const __llvm_profile_data *DI = 0;
227
228 if (!VPDataReader)
229 return 0;
230
231 BufferIO = lprofCreateBufferIO(FileWriter: Writer);
232
233 for (DI = DataBegin; DI < DataEnd; DI++) {
234 if (writeOneValueProfData(BufferIO, VPDataReader, Data: DI))
235 return -1;
236 }
237
238 if (lprofBufferIOFlush(BufferIO) != 0)
239 return -1;
240 lprofDeleteBufferIO(BufferIO);
241
242 return 0;
243}
244
245COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
246 VPDataReaderType *VPDataReader,
247 int SkipNameDataWrite) {
248 /* Match logic in __llvm_profile_write_buffer(). */
249 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
250 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
251 const char *CountersBegin = __llvm_profile_begin_counters();
252 const char *CountersEnd = __llvm_profile_end_counters();
253 const char *BitmapBegin = __llvm_profile_begin_bitmap();
254 const char *BitmapEnd = __llvm_profile_end_bitmap();
255 const char *NamesBegin = __llvm_profile_begin_names();
256 const char *NamesEnd = __llvm_profile_end_names();
257 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
258 const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
259 const char *VNamesBegin = __llvm_profile_begin_vtabnames();
260 const char *VNamesEnd = __llvm_profile_end_vtabnames();
261 uint64_t Version = __llvm_profile_get_version();
262 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
263 CountersEnd, BitmapBegin, BitmapEnd,
264 /*UniformCountersBegin=*/NULL,
265 /*UniformCountersEnd=*/NULL, VPDataReader,
266 NamesBegin, NamesEnd, VTableBegin, VTableEnd,
267 VNamesBegin, VNamesEnd, SkipNameDataWrite, Version);
268}
269
270COMPILER_RT_VISIBILITY int lprofWriteDataImpl(
271 ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
272 const __llvm_profile_data *DataEnd, const char *CountersBegin,
273 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
274 const char *UniformCountersBegin, const char *UniformCountersEnd,
275 VPDataReaderType *VPDataReader, const char *NamesBegin,
276 const char *NamesEnd, const VTableProfData *VTableBegin,
277 const VTableProfData *VTableEnd, const char *VNamesBegin,
278 const char *VNamesEnd, int SkipNameDataWrite, uint64_t Version) {
279 /* Calculate size of sections. */
280 const uint64_t DataSectionSize =
281 __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
282 const uint64_t NumData = __llvm_profile_get_num_data(Begin: DataBegin, End: DataEnd);
283 const uint64_t CountersSectionSize =
284 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
285 const uint64_t NumCounters =
286 __llvm_profile_get_num_counters(Begin: CountersBegin, End: CountersEnd);
287 const uint64_t NumBitmapBytes =
288 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
289 const uint64_t NamesSize = __llvm_profile_get_name_size(Begin: NamesBegin, End: NamesEnd);
290 const uint64_t NumVTables =
291 __llvm_profile_get_num_vtable(Begin: VTableBegin, End: VTableEnd);
292 const uint64_t VTableSectionSize =
293 __llvm_profile_get_vtable_section_size(Begin: VTableBegin, End: VTableEnd);
294 const uint64_t VNamesSize =
295 __llvm_profile_get_name_size(Begin: VNamesBegin, End: VNamesEnd);
296 const uint64_t NumUniformCounters =
297 (UniformCountersBegin && UniformCountersEnd &&
298 UniformCountersEnd > UniformCountersBegin)
299 ? (UniformCountersEnd - UniformCountersBegin) / sizeof(uint64_t)
300 : 0;
301 const uint64_t UniformCountersSectionSize =
302 NumUniformCounters * sizeof(uint64_t);
303
304 /* Create the header. */
305 __llvm_profile_header Header;
306
307 /* Determine how much padding is needed before/after the counters and after
308 * the names. */
309 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
310 PaddingBytesAfterBitmapBytes, PaddingBytesAfterUniformCounters,
311 PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
312 if (__llvm_profile_get_padding_sizes_for_counters(
313 DataSize: DataSectionSize, CountersSize: CountersSectionSize, NumBitmapBytes,
314 NumUniformCounters, NamesSize, VTableSize: VTableSectionSize, VNameSize: VNamesSize,
315 PaddingBytesBeforeCounters: &PaddingBytesBeforeCounters, PaddingBytesAfterCounters: &PaddingBytesAfterCounters,
316 PaddingBytesAfterBitmap: &PaddingBytesAfterBitmapBytes, PaddingBytesAfterUniformCounters: &PaddingBytesAfterUniformCounters,
317 PaddingBytesAfterNames: &PaddingBytesAfterNames, PaddingBytesAfterVTable: &PaddingBytesAfterVTable,
318 PaddingBytesAfterVNames: &PaddingBytesAfterVNames) == -1)
319 return -1;
320
321 {
322/* Initialize header structure. */
323#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
324#include "profile/InstrProfData.inc"
325 }
326 Header.Version = Version;
327
328 /* On WIN64, label differences are truncated 32-bit values. Truncate
329 * CountersDelta to match. */
330#ifdef _WIN64
331 Header.CountersDelta = (uint32_t)Header.CountersDelta;
332 Header.BitmapDelta = (uint32_t)Header.BitmapDelta;
333 Header.UniformCountersDelta = (uint32_t)Header.UniformCountersDelta;
334#endif
335
336 /* Recompute UniformCountersDelta from file layout. The macro initializer
337 uses in-memory pointer arithmetic which is wrong when sections are in
338 separate allocations (e.g., custom profiles from device PGO). */
339 if (NumUniformCounters > 0)
340 Header.UniformCountersDelta = DataSectionSize + PaddingBytesBeforeCounters +
341 CountersSectionSize +
342 PaddingBytesAfterCounters + NumBitmapBytes +
343 PaddingBytesAfterBitmapBytes;
344 else
345 Header.UniformCountersDelta = 0;
346
347 /* The data and names sections are omitted in lightweight mode. */
348 if (NumData == 0 && NamesSize == 0) {
349 Header.CountersDelta = 0;
350 Header.BitmapDelta = 0;
351 Header.NamesDelta = 0;
352 Header.UniformCountersDelta = 0;
353 }
354
355 /* Write the profile header. */
356 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
357 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
358 return -1;
359
360 /* Write the binary id lengths and data. */
361 if (__llvm_write_binary_ids(Writer) == -1)
362 return -1;
363
364 /* Write the profile data. */
365 ProfDataIOVec IOVecData[] = {
366 {DataBegin, sizeof(uint8_t), DataSectionSize, 0},
367 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
368 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
369 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
370 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
371 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
372 {UniformCountersBegin, sizeof(uint8_t), UniformCountersSectionSize, 0},
373 {NULL, sizeof(uint8_t), PaddingBytesAfterUniformCounters, 1},
374 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
375 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
376 {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
377 {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
378 {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0},
379 {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
380 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
381 return -1;
382
383 /* Value profiling is not yet supported in continuous mode and profile
384 * correlation mode. */
385 if (__llvm_profile_is_continuous_mode_enabled() ||
386 (NumData == 0 && NamesSize == 0))
387 return 0;
388
389 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
390}
391
392/*
393 * Write binary id length and then its data, because binary id does not
394 * have a fixed length.
395 */
396COMPILER_RT_VISIBILITY
397int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
398 const uint8_t *BinaryIdData,
399 uint64_t BinaryIdPadding) {
400 ProfDataIOVec BinaryIdIOVec[] = {
401 {&BinaryIdLen, sizeof(uint64_t), 1, 0},
402 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
403 {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
404 };
405 if (Writer->Write(Writer, BinaryIdIOVec,
406 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
407 return -1;
408
409 /* Successfully wrote binary id, report success. */
410 return 0;
411}
412