1/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9// Note: This is linked into the Darwin kernel, and must remain compatible
10// with freestanding compilation. See `darwin_add_builtin_libraries`.
11
12#ifdef _MSC_VER
13/* For _alloca */
14#include <malloc.h>
15#endif
16#include <string.h>
17
18#include "InstrProfiling.h"
19#include "InstrProfilingInternal.h"
20#include "InstrProfilingPort.h"
21
22#define INSTR_PROF_VALUE_PROF_DATA
23#include "profile/InstrProfData.inc"
24
25COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
26static ProfBufferIO TheBufferIO;
27#define VP_BUFFER_SIZE 8 * 1024
28static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
29static InstrProfValueData VPDataArray[16];
30static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
31
32COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
33COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
34
35/* The buffer writer is responsible in keeping writer state
36 * across the call.
37 */
38COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
39 ProfDataIOVec *IOVecs,
40 uint32_t NumIOVecs) {
41 uint32_t I;
42 char **Buffer = (char **)&This->WriterCtx;
43 for (I = 0; I < NumIOVecs; I++) {
44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
45 if (IOVecs[I].Data)
46 memcpy(dest: *Buffer, src: IOVecs[I].Data, n: Length);
47 else if (IOVecs[I].UseZeroPadding) {
48 /* Allocating the buffer should zero fill. */
49 }
50 *Buffer += Length;
51 }
52 return 0;
53}
54
55static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
56 uint8_t *Buffer, uint32_t BufferSz) {
57 BufferIO->FileWriter = FileWriter;
58 BufferIO->OwnFileWriter = 0;
59 BufferIO->BufferStart = Buffer;
60 BufferIO->BufferSz = BufferSz;
61 BufferIO->CurOffset = 0;
62}
63
64COMPILER_RT_VISIBILITY ProfBufferIO *
65lprofCreateBufferIO(ProfDataWriter *FileWriter) {
66 uint8_t *Buffer = DynamicBufferIOBuffer;
67 uint32_t BufferSize = VPBufferSize;
68 if (!Buffer) {
69 Buffer = &BufferIOBuffer[0];
70 BufferSize = sizeof(BufferIOBuffer);
71 }
72 llvmInitBufferIO(BufferIO: &TheBufferIO, FileWriter, Buffer, BufferSz: BufferSize);
73 return &TheBufferIO;
74}
75
76COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
77 if (BufferIO->OwnFileWriter)
78 FreeHook(BufferIO->FileWriter);
79 if (DynamicBufferIOBuffer) {
80 FreeHook(DynamicBufferIOBuffer);
81 DynamicBufferIOBuffer = 0;
82 VPBufferSize = 0;
83 }
84}
85
86COMPILER_RT_VISIBILITY int
87lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
88 /* Buffer is not large enough, it is time to flush. */
89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
90 if (lprofBufferIOFlush(BufferIO) != 0)
91 return -1;
92 }
93 /* Special case, bypass the buffer completely. */
94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
95 if (Size > BufferIO->BufferSz) {
96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
97 return -1;
98 } else {
99 /* Write the data to buffer */
100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
101 ProfDataWriter BufferWriter;
102 initBufferWriter(BufferWriter: &BufferWriter, Buffer: (char *)Buffer);
103 lprofBufferWriter(This: &BufferWriter, IOVecs: IO, NumIOVecs: 1);
104 BufferIO->CurOffset =
105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
106 }
107 return 0;
108}
109
110COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
111 if (BufferIO->CurOffset) {
112 ProfDataIOVec IO[] = {
113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
115 return -1;
116 BufferIO->CurOffset = 0;
117 }
118 return 0;
119}
120
121/* Write out value profile data for function specified with \c Data.
122 * The implementation does not use the method \c serializeValueProfData
123 * which depends on dynamic memory allocation. In this implementation,
124 * value profile data is written out to \c BufferIO piecemeal.
125 */
126static int writeOneValueProfData(ProfBufferIO *BufferIO,
127 VPDataReaderType *VPDataReader,
128 const __llvm_profile_data *Data) {
129 unsigned I, NumValueKinds = 0;
130 ValueProfData VPHeader;
131 uint8_t *SiteCountArray[IPVK_Last + 1];
132
133 for (I = 0; I <= IPVK_Last; I++) {
134 if (!Data->NumValueSites[I])
135 SiteCountArray[I] = 0;
136 else {
137 uint32_t Sz =
138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
139 offsetof(ValueProfRecord, SiteCountArray);
140 /* Only use alloca for this small byte array to avoid excessive
141 * stack growth. */
142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
143 memset(s: SiteCountArray[I], c: 0, n: Sz);
144 }
145 }
146
147 /* If NumValueKinds returned is 0, there is nothing to write, report
148 success and return. This should match the raw profile reader's behavior. */
149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
150 return 0;
151
152 /* First write the header structure. */
153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
154 VPHeader.NumValueKinds = NumValueKinds;
155 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&VPHeader,
156 Size: sizeof(ValueProfData)))
157 return -1;
158
159 /* Make sure nothing else needs to be written before value profile
160 * records. */
161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
162 (void *)(&VPHeader + 1))
163 return -1;
164
165 /* Write out the value profile record for each value kind
166 * one by one. */
167 for (I = 0; I <= IPVK_Last; I++) {
168 uint32_t J;
169 ValueProfRecord RecordHeader;
170 /* The size of the value prof record header without counting the
171 * site count array .*/
172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
173 uint32_t SiteCountArraySize;
174
175 if (!Data->NumValueSites[I])
176 continue;
177
178 /* Write out the record header. */
179 RecordHeader.Kind = I;
180 RecordHeader.NumValueSites = Data->NumValueSites[I];
181 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&RecordHeader,
182 Size: RecordHeaderSize))
183 return -1;
184
185 /* Write out the site value count array including padding space. */
186 SiteCountArraySize =
187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
188 RecordHeaderSize;
189 if (lprofBufferIOWrite(BufferIO, Data: SiteCountArray[I], Size: SiteCountArraySize))
190 return -1;
191
192 /* Write out the value profile data for each value site. */
193 for (J = 0; J < Data->NumValueSites[I]; J++) {
194 uint32_t NRead, NRemain;
195 ValueProfNode *NextStartNode = 0;
196 NRemain = VPDataReader->GetNumValueDataForSite(I, J);
197 if (!NRemain)
198 continue;
199 /* Read and write out value data in small chunks till it is done. */
200 do {
201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
202 NextStartNode =
203 VPDataReader->GetValueData(I, /* ValueKind */
204 J, /* Site */
205 &VPDataArray[0], NextStartNode, NRead);
206 if (lprofBufferIOWrite(BufferIO, Data: (const uint8_t *)&VPDataArray[0],
207 Size: NRead * sizeof(InstrProfValueData)))
208 return -1;
209 NRemain -= NRead;
210 } while (NRemain != 0);
211 }
212 }
213 /* All done report success. */
214 return 0;
215}
216
217static int writeValueProfData(ProfDataWriter *Writer,
218 VPDataReaderType *VPDataReader,
219 const __llvm_profile_data *DataBegin,
220 const __llvm_profile_data *DataEnd) {
221 ProfBufferIO *BufferIO;
222 const __llvm_profile_data *DI = 0;
223
224 if (!VPDataReader)
225 return 0;
226
227 BufferIO = lprofCreateBufferIO(FileWriter: Writer);
228
229 for (DI = DataBegin; DI < DataEnd; DI++) {
230 if (writeOneValueProfData(BufferIO, VPDataReader, Data: DI))
231 return -1;
232 }
233
234 if (lprofBufferIOFlush(BufferIO) != 0)
235 return -1;
236 lprofDeleteBufferIO(BufferIO);
237
238 return 0;
239}
240
241COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
242 VPDataReaderType *VPDataReader,
243 int SkipNameDataWrite) {
244 /* Match logic in __llvm_profile_write_buffer(). */
245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
247 const char *CountersBegin = __llvm_profile_begin_counters();
248 const char *CountersEnd = __llvm_profile_end_counters();
249 const char *BitmapBegin = __llvm_profile_begin_bitmap();
250 const char *BitmapEnd = __llvm_profile_end_bitmap();
251 const char *NamesBegin = __llvm_profile_begin_names();
252 const char *NamesEnd = __llvm_profile_end_names();
253 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
254 const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
255 const char *VNamesBegin = __llvm_profile_begin_vtabnames();
256 const char *VNamesEnd = __llvm_profile_end_vtabnames();
257 uint64_t Version = __llvm_profile_get_version();
258 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
259 CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
260 NamesBegin, NamesEnd, VTableBegin, VTableEnd,
261 VNamesBegin, VNamesEnd, SkipNameDataWrite, Version);
262}
263
264COMPILER_RT_VISIBILITY int lprofWriteDataImpl(
265 ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
266 const __llvm_profile_data *DataEnd, const char *CountersBegin,
267 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
268 VPDataReaderType *VPDataReader, const char *NamesBegin,
269 const char *NamesEnd, const VTableProfData *VTableBegin,
270 const VTableProfData *VTableEnd, const char *VNamesBegin,
271 const char *VNamesEnd, int SkipNameDataWrite, uint64_t Version) {
272 /* Calculate size of sections. */
273 const uint64_t DataSectionSize =
274 __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
275 const uint64_t NumData = __llvm_profile_get_num_data(Begin: DataBegin, End: DataEnd);
276 const uint64_t CountersSectionSize =
277 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
278 const uint64_t NumCounters =
279 __llvm_profile_get_num_counters(Begin: CountersBegin, End: CountersEnd);
280 const uint64_t NumBitmapBytes =
281 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
282 const uint64_t NamesSize = __llvm_profile_get_name_size(Begin: NamesBegin, End: NamesEnd);
283 const uint64_t NumVTables =
284 __llvm_profile_get_num_vtable(Begin: VTableBegin, End: VTableEnd);
285 const uint64_t VTableSectionSize =
286 __llvm_profile_get_vtable_section_size(Begin: VTableBegin, End: VTableEnd);
287 const uint64_t VNamesSize =
288 __llvm_profile_get_name_size(Begin: VNamesBegin, End: VNamesEnd);
289
290 /* Create the header. */
291 __llvm_profile_header Header;
292
293 /* Determine how much padding is needed before/after the counters and after
294 * the names. */
295 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
296 PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
297 PaddingBytesAfterVTable, PaddingBytesAfterVNames;
298 if (__llvm_profile_get_padding_sizes_for_counters(
299 DataSize: DataSectionSize, CountersSize: CountersSectionSize, NumBitmapBytes, NamesSize,
300 VTableSize: VTableSectionSize, VNameSize: VNamesSize, PaddingBytesBeforeCounters: &PaddingBytesBeforeCounters,
301 PaddingBytesAfterCounters: &PaddingBytesAfterCounters, PaddingBytesAfterBitmap: &PaddingBytesAfterBitmapBytes,
302 PaddingBytesAfterNames: &PaddingBytesAfterNames, PaddingBytesAfterVTable: &PaddingBytesAfterVTable,
303 PaddingBytesAfterVNames: &PaddingBytesAfterVNames) == -1)
304 return -1;
305
306 {
307/* Initialize header structure. */
308#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
309#include "profile/InstrProfData.inc"
310 }
311 Header.Version = Version;
312
313 /* On WIN64, label differences are truncated 32-bit values. Truncate
314 * CountersDelta to match. */
315#ifdef _WIN64
316 Header.CountersDelta = (uint32_t)Header.CountersDelta;
317 Header.BitmapDelta = (uint32_t)Header.BitmapDelta;
318#endif
319
320 /* The data and names sections are omitted in lightweight mode. */
321 if (NumData == 0 && NamesSize == 0) {
322 Header.CountersDelta = 0;
323 Header.NamesDelta = 0;
324 }
325
326 /* Write the profile header. */
327 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
328 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
329 return -1;
330
331 /* Write the binary id lengths and data. */
332 if (__llvm_write_binary_ids(Writer) == -1)
333 return -1;
334
335 /* Write the profile data. */
336 ProfDataIOVec IOVecData[] = {
337 {DataBegin, sizeof(uint8_t), DataSectionSize, 0},
338 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
339 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
340 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
341 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
342 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
343 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
344 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
345 {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
346 {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
347 {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0},
348 {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
349 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
350 return -1;
351
352 /* Value profiling is not yet supported in continuous mode and profile
353 * correlation mode. */
354 if (__llvm_profile_is_continuous_mode_enabled() ||
355 (NumData == 0 && NamesSize == 0))
356 return 0;
357
358 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
359}
360
361/*
362 * Write binary id length and then its data, because binary id does not
363 * have a fixed length.
364 */
365COMPILER_RT_VISIBILITY
366int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
367 const uint8_t *BinaryIdData,
368 uint64_t BinaryIdPadding) {
369 ProfDataIOVec BinaryIdIOVec[] = {
370 {&BinaryIdLen, sizeof(uint64_t), 1, 0},
371 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
372 {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
373 };
374 if (Writer->Write(Writer, BinaryIdIOVec,
375 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
376 return -1;
377
378 /* Successfully wrote binary id, report success. */
379 return 0;
380}
381