1/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*
8|* This file defines the API needed for in-process merging of profile data
9|* stored in memory buffer.
10\*===---------------------------------------------------------------------===*/
11
12#include "InstrProfiling.h"
13#include "InstrProfilingInternal.h"
14#include "InstrProfilingUtil.h"
15
16#define INSTR_PROF_VALUE_PROF_DATA
17#include "profile/InstrProfData.inc"
18
19COMPILER_RT_VISIBILITY
20void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
21
22COMPILER_RT_VISIBILITY
23uint64_t lprofGetLoadModuleSignature(void) {
24 /* A very fast way to compute a module signature. */
25 uint64_t Version = __llvm_profile_get_version();
26 uint64_t NumCounters = __llvm_profile_get_num_counters(
27 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
28 uint64_t NumData = __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
29 End: __llvm_profile_end_data());
30 uint64_t NamesSize =
31 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
32 uint64_t NumVnodes =
33 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
34 const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
35
36 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
37 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
38 __llvm_profile_get_magic();
39}
40
41#ifdef __GNUC__
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wcast-qual"
44#elif defined(__clang__)
45#pragma clang diagnostic push
46#pragma clang diagnostic ignored "-Wcast-qual"
47#endif
48
49/* Returns 1 if profile is not structurally compatible. */
50COMPILER_RT_VISIBILITY
51int __llvm_profile_check_compatibility(const char *ProfileData,
52 uint64_t ProfileSize) {
53 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
54 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
55 SrcDataStart =
56 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
57 Header->BinaryIdsSize);
58 SrcDataEnd = SrcDataStart + Header->NumData;
59
60 if (ProfileSize < sizeof(__llvm_profile_header))
61 return 1;
62
63 /* Check the header first. */
64 if (Header->Magic != __llvm_profile_get_magic() ||
65 Header->Version != __llvm_profile_get_version() ||
66 Header->NumData !=
67 __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
68 End: __llvm_profile_end_data()) ||
69 Header->NumCounters !=
70 __llvm_profile_get_num_counters(Begin: __llvm_profile_begin_counters(),
71 End: __llvm_profile_end_counters()) ||
72 Header->NumBitmapBytes !=
73 __llvm_profile_get_num_bitmap_bytes(Begin: __llvm_profile_begin_bitmap(),
74 End: __llvm_profile_end_bitmap()) ||
75 Header->NamesSize !=
76 __llvm_profile_get_name_size(Begin: __llvm_profile_begin_names(),
77 End: __llvm_profile_end_names()) ||
78 Header->ValueKindLast != IPVK_Last)
79 return 1;
80
81 if (ProfileSize <
82 sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
83 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
84 Header->NumCounters * __llvm_profile_counter_entry_size() +
85 Header->NumBitmapBytes)
86 return 1;
87
88 for (SrcData = SrcDataStart,
89 DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
90 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
91 if (SrcData->NameRef != DstData->NameRef ||
92 SrcData->FuncHash != DstData->FuncHash ||
93 SrcData->NumCounters != DstData->NumCounters ||
94 SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
95 return 1;
96 }
97
98 /* Matched! */
99 return 0;
100}
101
102static uintptr_t signextIfWin64(void *V) {
103#ifdef _WIN64
104 return (uintptr_t)(int32_t)(uintptr_t)V;
105#else
106 return (uintptr_t)V;
107#endif
108}
109
110// Skip names section, vtable profile data section and vtable names section
111// for runtime profile merge. To merge runtime addresses from multiple
112// profiles collected from the same instrumented binary, the binary should be
113// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
114// disabled). In this set-up these three sections remain unchanged.
115static uint64_t
116getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
117 const uint64_t VTableSectionSize =
118 Header->NumVTables * sizeof(VTableProfData);
119 const uint64_t PaddingBytesAfterVTableSection =
120 __llvm_profile_get_num_padding_bytes(SizeInBytes: VTableSectionSize);
121 const uint64_t VNamesSize = Header->VNamesSize;
122 const uint64_t PaddingBytesAfterVNamesSize =
123 __llvm_profile_get_num_padding_bytes(SizeInBytes: VNamesSize);
124 return Header->NamesSize +
125 __llvm_profile_get_num_padding_bytes(SizeInBytes: Header->NamesSize) +
126 VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
127 PaddingBytesAfterVNamesSize;
128}
129
130COMPILER_RT_VISIBILITY
131int __llvm_profile_merge_from_buffer(const char *ProfileData,
132 uint64_t ProfileSize) {
133 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
134 PROF_ERR("%s\n",
135 "Temporal profiles do not support profile merging at runtime. "
136 "Instead, merge raw profiles using the llvm-profdata tool.");
137 return 1;
138 }
139
140 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
141 uintptr_t CountersDelta = Header->CountersDelta;
142 uintptr_t BitmapDelta = Header->BitmapDelta;
143
144 __llvm_profile_data *SrcDataStart =
145 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
146 Header->BinaryIdsSize);
147 __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData;
148 uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd;
149 uintptr_t SrcCountersEnd =
150 SrcCountersStart +
151 Header->NumCounters * __llvm_profile_counter_entry_size();
152 uintptr_t SrcBitmapStart =
153 SrcCountersEnd +
154 __llvm_profile_get_num_padding_bytes(SizeInBytes: SrcCountersEnd - SrcCountersStart);
155 uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
156 uintptr_t SrcValueProfDataStart =
157 SrcNameStart + getDistanceFromCounterToValueProf(Header);
158 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
159 return 1;
160
161 // Merge counters by iterating the entire counter section when data section is
162 // empty due to correlation.
163 if (Header->NumData == 0) {
164 for (uintptr_t SrcCounter = SrcCountersStart,
165 DstCounter = (uintptr_t)__llvm_profile_begin_counters();
166 SrcCounter < SrcCountersEnd;) {
167 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
168 *(char *)DstCounter &= *(const char *)SrcCounter;
169 } else {
170 *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter;
171 }
172 SrcCounter += __llvm_profile_counter_entry_size();
173 DstCounter += __llvm_profile_counter_entry_size();
174 }
175 return 0;
176 }
177
178 __llvm_profile_data *SrcData, *DstData;
179 uintptr_t SrcValueProfData;
180 for (SrcData = SrcDataStart,
181 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
182 SrcValueProfData = SrcValueProfDataStart;
183 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
184 // For the in-memory destination, CounterPtr is the distance from the start
185 // address of the data to the start address of the counter. On WIN64,
186 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
187 // extend CounterPtr to get the original value.
188 uintptr_t DstCounters =
189 (uintptr_t)DstData + signextIfWin64(V: DstData->CounterPtr);
190 uintptr_t DstBitmap =
191 (uintptr_t)DstData + signextIfWin64(V: DstData->BitmapPtr);
192 unsigned NVK = 0;
193
194 // SrcData is a serialized representation of the memory image. We need to
195 // compute the in-buffer counter offset from the in-memory address distance.
196 // The initial CountersDelta is the in-memory address difference
197 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
198 // CountersDelta computes the offset into the in-buffer counter section.
199 //
200 // On WIN64, CountersDelta is truncated as well, so no need for signext.
201 uintptr_t SrcCounters =
202 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
203 // CountersDelta needs to be decreased as we advance to the next data
204 // record.
205 CountersDelta -= sizeof(*SrcData);
206 unsigned NC = SrcData->NumCounters;
207 if (NC == 0)
208 return 1;
209 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
210 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
211 return 1;
212 for (unsigned I = 0; I < NC; I++) {
213 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
214 // A value of zero signifies the function is covered.
215 ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I];
216 } else {
217 ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I];
218 }
219 }
220
221 uintptr_t SrcBitmap =
222 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
223 // BitmapDelta also needs to be decreased as we advance to the next data
224 // record.
225 BitmapDelta -= sizeof(*SrcData);
226 unsigned NB = SrcData->NumBitmapBytes;
227 // NumBitmapBytes may legitimately be 0. Just keep going.
228 if (NB != 0) {
229 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
230 return 1;
231 // Merge Src and Dst Bitmap bytes by simply ORing them together.
232 for (unsigned I = 0; I < NB; I++)
233 ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I];
234 }
235
236 /* Now merge value profile data. */
237 if (!VPMergeHook)
238 continue;
239
240 for (unsigned I = 0; I <= IPVK_Last; I++)
241 NVK += (SrcData->NumValueSites[I] != 0);
242
243 if (!NVK)
244 continue;
245
246 if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize)
247 return 1;
248 VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
249 SrcValueProfData =
250 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
251 }
252
253 return 0;
254}
255
256#ifdef __GNUC__
257#pragma GCC diagnostic pop
258#elif defined(__clang__)
259#pragma clang diagnostic pop
260#endif
261