1/*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9// Note: This is linked into the Darwin kernel, and must remain compatible
10// with freestanding compilation. See `darwin_add_builtin_libraries`.
11
12#include "InstrProfiling.h"
13#include "InstrProfilingInternal.h"
14#include "InstrProfilingPort.h"
15
16/* When continuous mode is enabled (%c), this parameter is set to 1.
17 *
18 * This parameter is defined here in InstrProfilingBuffer.o, instead of in
19 * InstrProfilingFile.o, to sequester all libc-dependent code in
20 * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
21 * layering is violated. */
22static int ContinuouslySyncProfile = 0;
23
24/* The system page size. Only valid when non-zero. If 0, the page size is
25 * unavailable. */
26static unsigned PageSize = 0;
27
28COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
29 return ContinuouslySyncProfile && PageSize;
30}
31
32COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
33 ContinuouslySyncProfile = 1;
34}
35
36COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) {
37 ContinuouslySyncProfile = 0;
38}
39
40COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
41 PageSize = PS;
42}
43
44COMPILER_RT_VISIBILITY
45uint64_t __llvm_profile_get_size_for_buffer(void) {
46 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
47 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
48 const char *CountersBegin = __llvm_profile_begin_counters();
49 const char *CountersEnd = __llvm_profile_end_counters();
50 const char *BitmapBegin = __llvm_profile_begin_bitmap();
51 const char *BitmapEnd = __llvm_profile_end_bitmap();
52 const char *NamesBegin = __llvm_profile_begin_names();
53 const char *NamesEnd = __llvm_profile_end_names();
54 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
55 const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
56 const char *VNamesBegin = __llvm_profile_begin_vtabnames();
57 const char *VNamesEnd = __llvm_profile_end_vtabnames();
58
59 return __llvm_profile_get_size_for_buffer_internal(
60 DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,
61 NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
62}
63
64// NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open
65// interval [Begin, End). Namely, `End` is one-byte past the end of the array.
66COMPILER_RT_VISIBILITY
67uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
68 const __llvm_profile_data *End) {
69 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
70 return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
71 sizeof(__llvm_profile_data);
72}
73
74COMPILER_RT_VISIBILITY
75uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
76 const __llvm_profile_data *End) {
77 return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
78}
79
80// Counts the number of `VTableProfData` elements within the range of [Begin,
81// End). Caller should guarantee that End points to one byte past the inclusive
82// range.
83// FIXME: Add a compiler-rt test to make sure the number of vtables in the
84// raw profile is the same as the number of vtable elements in the instrumented
85// binary.
86COMPILER_RT_VISIBILITY
87uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
88 const VTableProfData *End) {
89 // Convert pointers to intptr_t to use integer arithmetic.
90 intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;
91 return (EndI - BeginI) / sizeof(VTableProfData);
92}
93
94COMPILER_RT_VISIBILITY
95uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
96 const VTableProfData *End) {
97 return (intptr_t)(End) - (intptr_t)(Begin);
98}
99
100COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
101 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
102 return sizeof(uint8_t);
103 return sizeof(uint64_t);
104}
105
106COMPILER_RT_VISIBILITY
107uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
108 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
109 return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
110 __llvm_profile_counter_entry_size();
111}
112
113COMPILER_RT_VISIBILITY
114uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
115 return __llvm_profile_get_num_counters(Begin, End) *
116 __llvm_profile_counter_entry_size();
117}
118
119COMPILER_RT_VISIBILITY
120uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
121 const char *End) {
122 return (End - Begin);
123}
124
125COMPILER_RT_VISIBILITY
126uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
127 return End - Begin;
128}
129
130/// Calculate the number of padding bytes needed to add to \p Offset in order
131/// for (\p Offset + Padding) to be page-aligned.
132static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
133 uint64_t OffsetModPage = Offset % PageSize;
134 if (OffsetModPage > 0)
135 return PageSize - OffsetModPage;
136 return 0;
137}
138
139static int needsCounterPadding(void) {
140#if defined(__APPLE__)
141 return __llvm_profile_is_continuous_mode_enabled();
142#else
143 return 0;
144#endif
145}
146
147COMPILER_RT_VISIBILITY
148int __llvm_profile_get_padding_sizes_for_counters(
149 uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
150 uint64_t NumUniformCounters, uint64_t NamesSize, uint64_t VTableSize,
151 uint64_t VNameSize, uint64_t *PaddingBytesBeforeCounters,
152 uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes,
153 uint64_t *PaddingBytesAfterUniformCounters,
154 uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
155 uint64_t *PaddingBytesAfterVName) {
156 // Counter padding is needed only if continuous mode is enabled.
157 if (!needsCounterPadding()) {
158 *PaddingBytesBeforeCounters = 0;
159 *PaddingBytesAfterCounters =
160 __llvm_profile_get_num_padding_bytes(SizeInBytes: CountersSize);
161 *PaddingBytesAfterBitmapBytes =
162 __llvm_profile_get_num_padding_bytes(SizeInBytes: NumBitmapBytes);
163 if (PaddingBytesAfterUniformCounters != NULL)
164 *PaddingBytesAfterUniformCounters = __llvm_profile_get_num_padding_bytes(
165 SizeInBytes: NumUniformCounters * sizeof(uint64_t));
166 *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(SizeInBytes: NamesSize);
167 if (PaddingBytesAfterVTable != NULL)
168 *PaddingBytesAfterVTable =
169 __llvm_profile_get_num_padding_bytes(SizeInBytes: VTableSize);
170 if (PaddingBytesAfterVName != NULL)
171 *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(SizeInBytes: VNameSize);
172 return 0;
173 }
174
175 // Value profiling not supported in continuous mode at profile-write time.
176 // Return -1 to alert the incompatibility.
177 if (VTableSize != 0 || VNameSize != 0)
178 return -1;
179
180 // In continuous mode, the file offsets for headers and for the start of
181 // counter sections need to be page-aligned.
182 *PaddingBytesBeforeCounters =
183 calculateBytesNeededToPageAlign(Offset: sizeof(__llvm_profile_header) + DataSize);
184 *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(Offset: CountersSize);
185 *PaddingBytesAfterBitmapBytes =
186 calculateBytesNeededToPageAlign(Offset: NumBitmapBytes);
187 if (PaddingBytesAfterUniformCounters != NULL)
188 *PaddingBytesAfterUniformCounters = 0;
189 *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(Offset: NamesSize);
190 // Set these two variables to zero to avoid uninitialized variables
191 // even if VTableSize and VNameSize are known to be zero.
192 if (PaddingBytesAfterVTable != NULL)
193 *PaddingBytesAfterVTable = 0;
194 if (PaddingBytesAfterVName != NULL)
195 *PaddingBytesAfterVName = 0;
196 return 0;
197}
198
199COMPILER_RT_VISIBILITY
200uint64_t __llvm_profile_get_size_for_buffer_internal(
201 const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
202 const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
203 const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd,
204 const VTableProfData *VTableBegin, const VTableProfData *VTableEnd,
205 const char *VNamesBegin, const char *VNamesEnd) {
206 /* Match logic in __llvm_profile_write_buffer(). */
207 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
208 uint64_t DataSize = __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
209 uint64_t CountersSize =
210 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
211 const uint64_t NumBitmapBytes =
212 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
213 const uint64_t VTableSize =
214 __llvm_profile_get_vtable_section_size(Begin: VTableBegin, End: VTableEnd);
215 const uint64_t VNameSize =
216 __llvm_profile_get_name_size(Begin: VNamesBegin, End: VNamesEnd);
217
218 /* Determine how much padding is needed before/after the counters and after
219 * the names. */
220 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
221 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
222 PaddingBytesAfterUniformCounters, PaddingBytesAfterVTable,
223 PaddingBytesAfterVNames;
224 __llvm_profile_get_padding_sizes_for_counters(
225 DataSize, CountersSize, NumBitmapBytes, NumUniformCounters: 0 /* NumUniformCounters */,
226 NamesSize, VTableSize: 0 /* VTableSize */, VNameSize: 0 /* VNameSize */,
227 PaddingBytesBeforeCounters: &PaddingBytesBeforeCounters, PaddingBytesAfterCounters: &PaddingBytesAfterCounters,
228 PaddingBytesAfterBitmapBytes: &PaddingBytesAfterBitmapBytes, PaddingBytesAfterUniformCounters: &PaddingBytesAfterUniformCounters,
229 PaddingBytesAfterNames: &PaddingBytesAfterNames, PaddingBytesAfterVTable: &PaddingBytesAfterVTable,
230 PaddingBytesAfterVName: &PaddingBytesAfterVNames);
231
232 return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
233 DataSize + PaddingBytesBeforeCounters + CountersSize +
234 PaddingBytesAfterCounters + NumBitmapBytes +
235 PaddingBytesAfterBitmapBytes + PaddingBytesAfterUniformCounters +
236 NamesSize + PaddingBytesAfterNames + VTableSize +
237 PaddingBytesAfterVTable + VNameSize + PaddingBytesAfterVNames;
238}
239
240COMPILER_RT_VISIBILITY
241void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {
242 BufferWriter->Write = lprofBufferWriter;
243 BufferWriter->WriterCtx = Buffer;
244}
245
246COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
247 ProfDataWriter BufferWriter;
248 initBufferWriter(BufferWriter: &BufferWriter, Buffer);
249 return lprofWriteData(Writer: &BufferWriter, VPDataReader: 0, SkipNameDataWrite: 0);
250}
251
252COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
253 char *Buffer, const __llvm_profile_data *DataBegin,
254 const __llvm_profile_data *DataEnd, const char *CountersBegin,
255 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
256 const char *NamesBegin, const char *NamesEnd) {
257 ProfDataWriter BufferWriter;
258 initBufferWriter(BufferWriter: &BufferWriter, Buffer);
259 // Set virtual table arguments to NULL since they are not supported yet.
260 return lprofWriteDataImpl(
261 Writer: &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,
262 BitmapBegin, BitmapEnd, /*UniformCountersBegin=*/NULL,
263 /*UniformCountersEnd=*/NULL, /*VPDataReader=*/0, NamesBegin, NamesEnd,
264 /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL,
265 /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0,
266 Version: __llvm_profile_get_version());
267}
268