1/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*
8|* This file defines the API needed for in-process merging of profile data
9|* stored in memory buffer.
10\*===---------------------------------------------------------------------===*/
11
12#include "InstrProfiling.h"
13#include "InstrProfilingInternal.h"
14
15#define INSTR_PROF_VALUE_PROF_DATA
16#include "profile/InstrProfData.inc"
17
18COMPILER_RT_VISIBILITY
19void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
20
21COMPILER_RT_VISIBILITY
22uint64_t lprofGetLoadModuleSignature(void) {
23 /* A very fast way to compute a module signature. */
24 uint64_t Version = __llvm_profile_get_version();
25 uint64_t NumCounters = __llvm_profile_get_num_counters(
26 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
27 uint64_t NumData = __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
28 End: __llvm_profile_end_data());
29 uint64_t NamesSize =
30 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
31 uint64_t NumVnodes =
32 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
33 const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
34
35 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
36 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
37 __llvm_profile_get_magic();
38}
39
40#ifdef __GNUC__
41#pragma GCC diagnostic push
42#pragma GCC diagnostic ignored "-Wcast-qual"
43#elif defined(__clang__)
44#pragma clang diagnostic push
45#pragma clang diagnostic ignored "-Wcast-qual"
46#endif
47
48/* Returns 1 if profile is not structurally compatible. */
49COMPILER_RT_VISIBILITY
50int __llvm_profile_check_compatibility(const char *ProfileData,
51 uint64_t ProfileSize) {
52 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
53 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
54 SrcDataStart =
55 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
56 Header->BinaryIdsSize);
57 SrcDataEnd = SrcDataStart + Header->NumData;
58
59 if (ProfileSize < sizeof(__llvm_profile_header))
60 return 1;
61
62 /* Check the header first. */
63 if (Header->Magic != __llvm_profile_get_magic() ||
64 Header->Version != __llvm_profile_get_version() ||
65 Header->NumData !=
66 __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
67 End: __llvm_profile_end_data()) ||
68 Header->NumCounters !=
69 __llvm_profile_get_num_counters(Begin: __llvm_profile_begin_counters(),
70 End: __llvm_profile_end_counters()) ||
71 Header->NumBitmapBytes !=
72 __llvm_profile_get_num_bitmap_bytes(Begin: __llvm_profile_begin_bitmap(),
73 End: __llvm_profile_end_bitmap()) ||
74 Header->NamesSize !=
75 __llvm_profile_get_name_size(Begin: __llvm_profile_begin_names(),
76 End: __llvm_profile_end_names()) ||
77 Header->ValueKindLast != IPVK_Last)
78 return 1;
79
80 if (ProfileSize <
81 sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
82 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
83 Header->NumCounters * __llvm_profile_counter_entry_size() +
84 Header->NumBitmapBytes)
85 return 1;
86
87 for (SrcData = SrcDataStart,
88 DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
89 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
90 if (SrcData->NameRef != DstData->NameRef ||
91 SrcData->FuncHash != DstData->FuncHash ||
92 SrcData->NumCounters != DstData->NumCounters ||
93 SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
94 return 1;
95 }
96
97 /* Matched! */
98 return 0;
99}
100
101static uintptr_t signextIfWin64(void *V) {
102#ifdef _WIN64
103 return (uintptr_t)(int32_t)(uintptr_t)V;
104#else
105 return (uintptr_t)V;
106#endif
107}
108
109// Skip names section, vtable profile data section and vtable names section
110// for runtime profile merge. To merge runtime addresses from multiple
111// profiles collected from the same instrumented binary, the binary should be
112// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
113// disabled). In this set-up these three sections remain unchanged.
114static uint64_t
115getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
116 const uint64_t VTableSectionSize =
117 Header->NumVTables * sizeof(VTableProfData);
118 const uint64_t PaddingBytesAfterVTableSection =
119 __llvm_profile_get_num_padding_bytes(SizeInBytes: VTableSectionSize);
120 const uint64_t VNamesSize = Header->VNamesSize;
121 const uint64_t PaddingBytesAfterVNamesSize =
122 __llvm_profile_get_num_padding_bytes(SizeInBytes: VNamesSize);
123 return Header->NamesSize +
124 __llvm_profile_get_num_padding_bytes(SizeInBytes: Header->NamesSize) +
125 VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
126 PaddingBytesAfterVNamesSize;
127}
128
129COMPILER_RT_VISIBILITY
130int __llvm_profile_merge_from_buffer(const char *ProfileData,
131 uint64_t ProfileSize) {
132 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
133 PROF_ERR("%s\n",
134 "Temporal profiles do not support profile merging at runtime. "
135 "Instead, merge raw profiles using the llvm-profdata tool.");
136 return 1;
137 }
138
139 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
140 uintptr_t CountersDelta = Header->CountersDelta;
141 uintptr_t BitmapDelta = Header->BitmapDelta;
142
143 __llvm_profile_data *SrcDataStart =
144 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
145 Header->BinaryIdsSize);
146 __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData;
147 uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd;
148 uintptr_t SrcCountersEnd =
149 SrcCountersStart +
150 Header->NumCounters * __llvm_profile_counter_entry_size();
151 uintptr_t SrcBitmapStart =
152 SrcCountersEnd +
153 __llvm_profile_get_num_padding_bytes(SizeInBytes: SrcCountersEnd - SrcCountersStart);
154 uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
155 uintptr_t SrcValueProfDataStart =
156 SrcNameStart + getDistanceFromCounterToValueProf(Header);
157 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
158 return 1;
159
160 // Merge counters by iterating the entire counter section when data section is
161 // empty due to correlation.
162 if (Header->NumData == 0) {
163 for (uintptr_t SrcCounter = SrcCountersStart,
164 DstCounter = (uintptr_t)__llvm_profile_begin_counters();
165 SrcCounter < SrcCountersEnd;) {
166 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
167 *(char *)DstCounter &= *(const char *)SrcCounter;
168 } else {
169 *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter;
170 }
171 SrcCounter += __llvm_profile_counter_entry_size();
172 DstCounter += __llvm_profile_counter_entry_size();
173 }
174 return 0;
175 }
176
177 __llvm_profile_data *SrcData, *DstData;
178 uintptr_t SrcValueProfData;
179 for (SrcData = SrcDataStart,
180 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
181 SrcValueProfData = SrcValueProfDataStart;
182 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
183 // For the in-memory destination, CounterPtr is the distance from the start
184 // address of the data to the start address of the counter. On WIN64,
185 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
186 // extend CounterPtr to get the original value.
187 uintptr_t DstCounters =
188 (uintptr_t)DstData + signextIfWin64(V: DstData->CounterPtr);
189 uintptr_t DstBitmap =
190 (uintptr_t)DstData + signextIfWin64(V: DstData->BitmapPtr);
191 unsigned NVK = 0;
192
193 // SrcData is a serialized representation of the memory image. We need to
194 // compute the in-buffer counter offset from the in-memory address distance.
195 // The initial CountersDelta is the in-memory address difference
196 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
197 // CountersDelta computes the offset into the in-buffer counter section.
198 //
199 // On WIN64, CountersDelta is truncated as well, so no need for signext.
200 uintptr_t SrcCounters =
201 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
202 // CountersDelta needs to be decreased as we advance to the next data
203 // record.
204 CountersDelta -= sizeof(*SrcData);
205 unsigned NC = SrcData->NumCounters;
206 if (NC == 0)
207 return 1;
208 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
209 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
210 return 1;
211 for (unsigned I = 0; I < NC; I++) {
212 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
213 // A value of zero signifies the function is covered.
214 ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I];
215 } else {
216 ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I];
217 }
218 }
219
220 uintptr_t SrcBitmap =
221 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
222 // BitmapDelta also needs to be decreased as we advance to the next data
223 // record.
224 BitmapDelta -= sizeof(*SrcData);
225 unsigned NB = SrcData->NumBitmapBytes;
226 // NumBitmapBytes may legitimately be 0. Just keep going.
227 if (NB != 0) {
228 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
229 return 1;
230 // Merge Src and Dst Bitmap bytes by simply ORing them together.
231 for (unsigned I = 0; I < NB; I++)
232 ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I];
233 }
234
235 /* Now merge value profile data. */
236 if (!VPMergeHook)
237 continue;
238
239 for (unsigned I = 0; I <= IPVK_Last; I++)
240 NVK += (SrcData->NumValueSites[I] != 0);
241
242 if (!NVK)
243 continue;
244
245 if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize)
246 return 1;
247 VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
248 SrcValueProfData =
249 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
250 }
251
252 return 0;
253}
254
255#ifdef __GNUC__
256#pragma GCC diagnostic pop
257#elif defined(__clang__)
258#pragma clang diagnostic pop
259#endif
260