1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 * Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 * LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 * Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65// clang-format off:consider re-enabling clang-format if auto-formatted C macros
66// are readable (e.g., after `issue #82426` is fixed)
67/* INSTR_PROF_DATA start. */
68/* Definition of member fields of the per-function control structure. */
69#ifndef INSTR_PROF_DATA
70#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
71#else
72#define INSTR_PROF_DATA_DEFINED
73#endif
74INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
75 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
76 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
77INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
78 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
79 Inc->getHash()->getZExtValue()))
80INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
81INSTR_PROF_DATA(const IntPtrT, IntPtrTy, UniformCounterPtr, \
82 RelativeUniformCounterPtr)
83INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
84/* This is used to map function pointers for the indirect call targets to
85 * function name hashes during the conversion from raw to merged profile
86 * data.
87 */
88INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
89 FunctionAddr)
90INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
91 ValuesPtrExpr)
92INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
93 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
94INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
95 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
96INSTR_PROF_DATA(const uint16_t, llvm::Type::getInt16Ty(Ctx), \
97 OffloadDeviceWaveSize, \
98 ConstantInt::get(llvm::Type::getInt16Ty(Ctx), \
99 OffloadDeviceWaveSizeVal))
100INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
101 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
102#undef INSTR_PROF_DATA
103/* INSTR_PROF_DATA end. */
104
105/* For a virtual table object, record the name hash to associate profiled
106 * addresses with global variables, and record {starting address, size in bytes}
107 * to map the profiled virtual table (which usually have an offset from the
108 * starting address) back to a virtual table object. */
109#ifndef INSTR_PROF_VTABLE_DATA
110#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
111#else
112#define INSTR_PROF_VTABLE_DATA_DEFINED
113#endif
114INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \
115 VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
116 IndexedInstrProf::ComputeHash(PGOVTableName)))
117INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
118 VTablePointer, VTableAddr)
119INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
120 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
121 VTableSizeVal))
122#undef INSTR_PROF_VTABLE_DATA
123/* INSTR_PROF_VTABLE_DATA end. */
124
125/* This is an internal data structure used by value profiler. It
126 * is defined here to allow serialization code sharing by LLVM
127 * to be used in unit test.
128 *
129 * typedef struct ValueProfNode {
130 * // InstrProfValueData VData;
131 * uint64_t Value;
132 * uint64_t Count;
133 * struct ValueProfNode *Next;
134 * } ValueProfNode;
135 */
136/* INSTR_PROF_VALUE_NODE start. */
137#ifndef INSTR_PROF_VALUE_NODE
138#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
139#else
140#define INSTR_PROF_DATA_DEFINED
141#endif
142INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
143 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
144INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
145 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
146INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
147 ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))
148#undef INSTR_PROF_VALUE_NODE
149/* INSTR_PROF_VALUE_NODE end. */
150
151/* INSTR_PROF_GPU_SECT start. */
152/* Fields of the GPU profile section bounds structure, populated by the
153 * compiler runtime and read by the host to extract profiling data. */
154#ifndef INSTR_PROF_GPU_SECT
155#define INSTR_PROF_GPU_SECT(Type, LLVMType, Name, Initializer)
156#else
157#define INSTR_PROF_DATA_DEFINED
158#endif
159INSTR_PROF_GPU_SECT(const char *, llvm::PointerType::getUnqual(Ctx), \
160 NamesStart, \
161 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
162INSTR_PROF_GPU_SECT(const char *, llvm::PointerType::getUnqual(Ctx), \
163 NamesStop, \
164 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
165INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
166 CountersStart, \
167 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
168INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
169 CountersStop, \
170 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
171INSTR_PROF_GPU_SECT(const __llvm_profile_data *, llvm::PointerType::getUnqual( \
172 Ctx), DataStart, \
173 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
174INSTR_PROF_GPU_SECT(const __llvm_profile_data *, llvm::PointerType::getUnqual( \
175 Ctx), DataStop, \
176 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
177INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
178 UniformCountersStart, \
179 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
180INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
181 UniformCountersStop, \
182 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
183INSTR_PROF_GPU_SECT(uint64_t *, llvm::PointerType::getUnqual(Ctx), \
184 VersionVar, \
185 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
186#undef INSTR_PROF_GPU_SECT
187/* INSTR_PROF_GPU_SECT end. */
188
189/* INSTR_PROF_RAW_HEADER start */
190/* Definition of member fields of the raw profile header data structure. */
191/* Please update llvm/docs/InstrProfileFormat.md as appropriate when updating
192 raw profile format. */
193#ifndef INSTR_PROF_RAW_HEADER
194#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
195#else
196#define INSTR_PROF_DATA_DEFINED
197#endif
198INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
199INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
200INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
201INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
202INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
203INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
204INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
205INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
206INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
207INSTR_PROF_RAW_HEADER(uint64_t, NumUniformCounters, NumUniformCounters)
208INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterUniformCounters, PaddingBytesAfterUniformCounters)
209INSTR_PROF_RAW_HEADER(uint64_t, UniformCountersDelta,
210 UniformCountersBegin ? (uintptr_t)UniformCountersBegin -
211 (uintptr_t)DataBegin
212 : 0)
213INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
214INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
215 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
216INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
217 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
218INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
219INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
220INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
221INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
222#undef INSTR_PROF_RAW_HEADER
223/* INSTR_PROF_RAW_HEADER end */
224
225/* VALUE_PROF_FUNC_PARAM start */
226/* Definition of parameter types of the runtime API used to do value profiling
227 * for a given value site.
228 */
229#ifndef VALUE_PROF_FUNC_PARAM
230#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
231#define INSTR_PROF_COMMA
232#else
233#define INSTR_PROF_DATA_DEFINED
234#define INSTR_PROF_COMMA ,
235#endif
236VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
237 INSTR_PROF_COMMA
238VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
239VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
240#undef VALUE_PROF_FUNC_PARAM
241#undef INSTR_PROF_COMMA
242/* VALUE_PROF_FUNC_PARAM end */
243
244/* VALUE_PROF_KIND start */
245#ifndef VALUE_PROF_KIND
246#define VALUE_PROF_KIND(Enumerator, Value, Descr)
247#else
248#define INSTR_PROF_DATA_DEFINED
249#endif
250/* For indirect function call value profiling, the addresses of the target
251 * functions are profiled by the instrumented code. The target addresses are
252 * written in the raw profile data and converted to target function name's MD5
253 * hash by the profile reader during deserialization. Typically, this happens
254 * when the raw profile data is read during profile merging.
255 *
256 * For this remapping the ProfData is used. ProfData contains both the function
257 * name hash and the function address.
258 */
259VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
260/* For memory intrinsic functions size profiling. */
261VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
262/* For virtual table address profiling, the address point of the virtual table
263 * (i.e., the address contained in objects pointing to a virtual table) are
264 * profiled. Note this may not be the address of the per C++ class virtual table
265 * object (e.g., there might be an offset).
266 *
267 * The profiled addresses are stored in raw profile, together with the following
268 * two types of information.
269 * 1. The (starting and ending) addresses of per C++ class virtual table objects.
270 * 2. The (compressed) virtual table object names.
271 * RawInstrProfReader converts profiled virtual table addresses to virtual table
272 * objects' MD5 hash.
273 */
274VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")
275/* These two kinds must be the last to be
276 * declared. This is to make sure the string
277 * array created with the template can be
278 * indexed with the kind value.
279 */
280VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
281VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
282
283#undef VALUE_PROF_KIND
284/* VALUE_PROF_KIND end */
285
286#undef COVMAP_V2_OR_V3
287#ifdef COVMAP_V2
288#define COVMAP_V2_OR_V3
289#endif
290#ifdef COVMAP_V3
291#define COVMAP_V2_OR_V3
292#endif
293
294/* COVMAP_FUNC_RECORD start */
295/* Definition of member fields of the function record structure in coverage
296 * map.
297 */
298#ifndef COVMAP_FUNC_RECORD
299#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
300#else
301#define INSTR_PROF_DATA_DEFINED
302#endif
303#ifdef COVMAP_V1
304COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
305 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
306 llvm::PointerType::getUnqual(Ctx)))
307COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
308 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
309 NameValue.size()))
310#endif
311#ifdef COVMAP_V2_OR_V3
312COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
313 llvm::ConstantInt::get( \
314 llvm::Type::getInt64Ty(Ctx), NameHash))
315#endif
316COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
317 llvm::ConstantInt::get( \
318 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
319COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
320 llvm::ConstantInt::get( \
321 llvm::Type::getInt64Ty(Ctx), FuncHash))
322#ifdef COVMAP_V3
323COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
324 llvm::ConstantInt::get( \
325 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
326COVMAP_FUNC_RECORD(const char, \
327 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
328 CoverageMapping.size()), \
329 CoverageMapping,
330 llvm::ConstantDataArray::getRaw( \
331 CoverageMapping, CoverageMapping.size(), \
332 llvm::Type::getInt8Ty(Ctx)))
333#endif
334#undef COVMAP_FUNC_RECORD
335/* COVMAP_FUNC_RECORD end. */
336
337/* COVMAP_HEADER start */
338/* Definition of member fields of coverage map header.
339 */
340#ifndef COVMAP_HEADER
341#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
342#else
343#define INSTR_PROF_DATA_DEFINED
344#endif
345COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
346 llvm::ConstantInt::get(Int32Ty, NRecords))
347COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
348 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
349COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
350 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
351COVMAP_HEADER(uint32_t, Int32Ty, Version, \
352 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
353#undef COVMAP_HEADER
354/* COVMAP_HEADER end. */
355
356/* COVINIT_FUNC start */
357#ifndef COVINIT_FUNC
358#define COVINIT_FUNC(Type, LLVMType, Name, Initializer)
359#else
360#define INSTR_PROF_DATA_DEFINED
361#endif
362COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \
363 WriteoutF)
364COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \
365 ResetF)
366#undef COVINIT_FUNC
367/* COVINIT_FUNC end */
368
369#ifdef INSTR_PROF_SECT_ENTRY
370#define INSTR_PROF_DATA_DEFINED
371INSTR_PROF_SECT_ENTRY(IPSK_data, \
372 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
373 INSTR_PROF_DATA_COFF, "__DATA,")
374INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
375 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
376 INSTR_PROF_CNTS_COFF, "__DATA,")
377INSTR_PROF_SECT_ENTRY(IPSK_ucnts, \
378 INSTR_PROF_QUOTE(INSTR_PROF_UCNTS_COMMON), \
379 INSTR_PROF_UCNTS_COFF, "__DATA,")
380INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
381 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
382 INSTR_PROF_BITS_COFF, "__DATA,")
383INSTR_PROF_SECT_ENTRY(IPSK_name, \
384 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
385 INSTR_PROF_NAME_COFF, "__DATA,")
386INSTR_PROF_SECT_ENTRY(IPSK_vname, \
387 INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
388 INSTR_PROF_VNAME_COFF, "__DATA,")
389INSTR_PROF_SECT_ENTRY(IPSK_vals, \
390 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
391 INSTR_PROF_VALS_COFF, "__DATA,")
392INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
393 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
394 INSTR_PROF_VNODES_COFF, "__DATA,")
395INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
396 INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
397 INSTR_PROF_VTAB_COFF, "__DATA,")
398INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
399 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
400 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
401INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
402 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
403 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
404INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
405 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
406 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
407INSTR_PROF_SECT_ENTRY(IPSK_covname, \
408 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
409 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
410INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
411 INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \
412 INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")
413
414#undef INSTR_PROF_SECT_ENTRY
415#endif
416
417
418#ifdef INSTR_PROF_VALUE_PROF_DATA
419#define INSTR_PROF_DATA_DEFINED
420
421#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
422/*!
423 * This is the header of the data structure that defines the on-disk
424 * layout of the value profile data of a particular kind for one function.
425 */
426typedef struct ValueProfRecord {
427 /* The kind of the value profile record. */
428 uint32_t Kind;
429 /*
430 * The number of value profile sites. It is guaranteed to be non-zero;
431 * otherwise the record for this kind won't be emitted.
432 */
433 uint32_t NumValueSites;
434 /*
435 * The first element of the array that stores the number of profiled
436 * values for each value site. The size of the array is NumValueSites.
437 * Since NumValueSites is greater than zero, there is at least one
438 * element in the array.
439 */
440 uint8_t SiteCountArray[1];
441
442 /*
443 * The fake declaration is for documentation purpose only.
444 * Align the start of next field to be on 8 byte boundaries.
445 uint8_t Padding[X];
446 */
447
448 /* The array of value profile data. The size of the array is the sum
449 * of all elements in SiteCountArray[].
450 InstrProfValueData ValueData[];
451 */
452
453#ifdef __cplusplus
454 /*!
455 * Return the number of value sites.
456 */
457 uint32_t getNumValueSites() const { return NumValueSites; }
458 /*!
459 * Read data from this record and save it to Record.
460 */
461 LLVM_ABI void deserializeTo(InstrProfRecord &Record,
462 InstrProfSymtab *SymTab);
463 /*
464 * In-place byte swap:
465 * Do byte swap for this instance. \c Old is the original order before
466 * the swap, and \c New is the New byte order.
467 */
468 LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New);
469#endif
470} ValueProfRecord;
471
472/*!
473 * Per-function header/control data structure for value profiling
474 * data in indexed format.
475 */
476typedef struct ValueProfData {
477 /*
478 * Total size in bytes including this field. It must be a multiple
479 * of sizeof(uint64_t).
480 */
481 uint32_t TotalSize;
482 /*
483 *The number of value profile kinds that has value profile data.
484 * In this implementation, a value profile kind is considered to
485 * have profile data if the number of value profile sites for the
486 * kind is not zero. More aggressively, the implementation can
487 * choose to check the actual data value: if none of the value sites
488 * has any profiled values, the kind can be skipped.
489 */
490 uint32_t NumValueKinds;
491
492 /*
493 * Following are a sequence of variable length records. The prefix/header
494 * of each record is defined by ValueProfRecord type. The number of
495 * records is NumValueKinds.
496 * ValueProfRecord Record_1;
497 * ValueProfRecord Record_N;
498 */
499
500#if __cplusplus
501 /*!
502 * Return the total size in bytes of the on-disk value profile data
503 * given the data stored in Record.
504 */
505 LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record);
506 /*!
507 * Return a pointer to \c ValueProfData instance ready to be streamed.
508 */
509 LLVM_ABI static std::unique_ptr<ValueProfData>
510 serializeFrom(const InstrProfRecord &Record);
511 /*!
512 * Check the integrity of the record.
513 */
514 LLVM_ABI Error checkIntegrity();
515 /*!
516 * Return a pointer to \c ValueProfileData instance ready to be read.
517 * All data in the instance are properly byte swapped. The input
518 * data is assumed to be in little endian order.
519 */
520 LLVM_ABI static Expected<std::unique_ptr<ValueProfData>>
521 getValueProfData(const unsigned char *SrcBuffer,
522 const unsigned char *const SrcBufferEnd,
523 llvm::endianness SrcDataEndianness);
524 /*!
525 * Swap byte order from \c Endianness order to host byte order.
526 */
527 LLVM_ABI void swapBytesToHost(llvm::endianness Endianness);
528 /*!
529 * Swap byte order from host byte order to \c Endianness order.
530 */
531 LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness);
532 /*!
533 * Return the total size of \c ValueProfileData.
534 */
535 LLVM_ABI uint32_t getSize() const { return TotalSize; }
536 /*!
537 * Read data from this data and save it to \c Record.
538 */
539 LLVM_ABI void deserializeTo(InstrProfRecord &Record,
540 InstrProfSymtab *SymTab);
541 void operator delete(void *ptr) { ::operator delete(ptr); }
542#endif
543} ValueProfData;
544
545/*
546 * The closure is designed to abstract away two types of value profile data:
547 * - InstrProfRecord which is the primary data structure used to
548 * represent profile data in host tools (reader, writer, and profile-use)
549 * - value profile runtime data structure suitable to be used by C
550 * runtime library.
551 *
552 * Both sources of data need to serialize to disk/memory-buffer in common
553 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
554 * writer to share the same format and code with indexed profile writer.
555 *
556 * For documentation of the member methods below, refer to corresponding methods
557 * in class InstrProfRecord.
558 */
559typedef struct ValueProfRecordClosure {
560 const void *Record;
561 uint32_t (*GetNumValueKinds)(const void *Record);
562 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
563 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
564 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
565
566 /*
567 * After extracting the value profile data from the value profile record,
568 * this method is used to map the in-memory value to on-disk value. If
569 * the method is null, value will be written out untranslated.
570 */
571 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
572 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
573 uint32_t S);
574 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
575} ValueProfRecordClosure;
576
577INSTR_PROF_VISIBILITY ValueProfRecord *
578getFirstValueProfRecord(ValueProfData *VPD);
579INSTR_PROF_VISIBILITY ValueProfRecord *
580getValueProfRecordNext(ValueProfRecord *VPR);
581INSTR_PROF_VISIBILITY InstrProfValueData *
582getValueProfRecordValueData(ValueProfRecord *VPR);
583INSTR_PROF_VISIBILITY uint32_t
584getValueProfRecordHeaderSize(uint32_t NumValueSites);
585
586#undef INSTR_PROF_VALUE_PROF_DATA
587#endif /* INSTR_PROF_VALUE_PROF_DATA */
588
589
590#ifdef INSTR_PROF_COMMON_API_IMPL
591#define INSTR_PROF_DATA_DEFINED
592#ifdef __cplusplus
593#define INSTR_PROF_INLINE inline
594#define INSTR_PROF_NULLPTR nullptr
595#else
596#define INSTR_PROF_INLINE
597#define INSTR_PROF_NULLPTR NULL
598#endif
599
600#ifndef offsetof
601#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
602#endif
603
604// clang-format on
605
606/*!
607 * Return the \c ValueProfRecord header size including the
608 * padding bytes.
609 */
610INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
611getValueProfRecordHeaderSize(uint32_t NumValueSites) {
612 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
613 sizeof(uint8_t) * NumValueSites;
614 /* Round the size to multiple of 8 bytes. */
615 Size = (Size + 7) & ~7;
616 return Size;
617}
618
619/*!
620 * Return the total size of the value profile record including the
621 * header and the value data.
622 */
623INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
624getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
625 return getValueProfRecordHeaderSize(NumValueSites) +
626 sizeof(InstrProfValueData) * NumValueData;
627}
628
629/*!
630 * Return the pointer to the start of value data array.
631 */
632INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
633getValueProfRecordValueData(ValueProfRecord *This) {
634 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
635 This->NumValueSites));
636}
637
638/*!
639 * Return the total number of value data for \c This record.
640 */
641INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
642getValueProfRecordNumValueData(ValueProfRecord *This) {
643 uint32_t NumValueData = 0;
644 uint32_t I;
645 for (I = 0; I < This->NumValueSites; I++)
646 NumValueData += This->SiteCountArray[I];
647 return NumValueData;
648}
649
650/*!
651 * Use this method to advance to the next \c This \c ValueProfRecord.
652 */
653INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
654getValueProfRecordNext(ValueProfRecord *This) {
655 uint32_t NumValueData = getValueProfRecordNumValueData(This);
656 return (ValueProfRecord *)((char *)This +
657 getValueProfRecordSize(This->NumValueSites,
658 NumValueData));
659}
660
661/*!
662 * Return the first \c ValueProfRecord instance.
663 */
664INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
665getFirstValueProfRecord(ValueProfData *This) {
666 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
667}
668
669/* Closure based interfaces. */
670
671/*!
672 * Return the total size in bytes of the on-disk value profile data
673 * given the data stored in Record.
674 */
675INSTR_PROF_VISIBILITY uint32_t
676getValueProfDataSize(ValueProfRecordClosure *Closure) {
677 uint32_t Kind;
678 uint32_t TotalSize = sizeof(ValueProfData);
679 const void *Record = Closure->Record;
680
681 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
682 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
683 if (!NumValueSites)
684 continue;
685 TotalSize += getValueProfRecordSize(NumValueSites,
686 Closure->GetNumValueData(Record, Kind));
687 }
688 return TotalSize;
689}
690
691/*!
692 * Extract value profile data of a function for the profile kind \c ValueKind
693 * from the \c Closure and serialize the data into \c This record instance.
694 */
695INSTR_PROF_VISIBILITY void
696serializeValueProfRecordFrom(ValueProfRecord *This,
697 ValueProfRecordClosure *Closure,
698 uint32_t ValueKind, uint32_t NumValueSites) {
699 uint32_t S;
700 const void *Record = Closure->Record;
701 This->Kind = ValueKind;
702 This->NumValueSites = NumValueSites;
703 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
704
705 for (S = 0; S < NumValueSites; S++) {
706 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
707 This->SiteCountArray[S] = ND;
708 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
709 DstVD += ND;
710 }
711}
712
713/*!
714 * Extract value profile data of a function from the \c Closure
715 * and serialize the data into \c DstData if it is not NULL or heap
716 * memory allocated by the \c Closure's allocator method. If \c
717 * DstData is not null, the caller is expected to set the TotalSize
718 * in DstData.
719 */
720INSTR_PROF_VISIBILITY ValueProfData *
721serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
722 ValueProfData *DstData) {
723 uint32_t Kind;
724 uint32_t TotalSize =
725 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
726
727 ValueProfData *VPD =
728 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
729
730 VPD->TotalSize = TotalSize;
731 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
732 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
733 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
734 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
735 if (!NumValueSites)
736 continue;
737 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
738 VR = getValueProfRecordNext(VR);
739 }
740 return VPD;
741}
742
743#undef INSTR_PROF_COMMON_API_IMPL
744#endif /* INSTR_PROF_COMMON_API_IMPL */
745
746/*============================================================================*/
747
748// clang-format off:consider re-enabling clang-format if auto-formatted C macros
749// are readable (e.g., after `issue #82426` is fixed)
750#ifndef INSTR_PROF_DATA_DEFINED
751
752#ifndef INSTR_PROF_DATA_INC
753#define INSTR_PROF_DATA_INC
754
755/* Helper macros. */
756#define INSTR_PROF_SIMPLE_QUOTE(x) #x
757#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
758#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
759#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
760
761/* Magic number to detect file format and endianness.
762 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
763 * so that utilities, like strings, don't grab it as a string. 129 is also
764 * invalid UTF-8, and high enough to be interesting.
765 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
766 * for 32-bit platforms.
767 */
768#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
769 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
770 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
771#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
772 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
773 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
774
775/* Raw profile format version (start from 1). */
776#define INSTR_PROF_RAW_VERSION 11
777/* Indexed profile format version (start from 1). */
778#define INSTR_PROF_INDEX_VERSION 14
779/* Coverage mapping format version (start from 0). */
780#define INSTR_PROF_COVMAP_VERSION 6
781
782/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
783 * version for other variants of profile. We set the 8th most significant bit
784 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
785 * generated profile, and 0 if this is a Clang FE generated profile.
786 * 1 in bit 57 indicates there are context-sensitive records in the profile.
787 * The 54th bit indicates whether to always instrument loop entry blocks.
788 * The 58th bit indicates whether to always instrument function entry blocks.
789 * The 59th bit indicates whether to use debug info to correlate profiles.
790 * The 60th bit indicates single byte coverage instrumentation.
791 * The 61st bit indicates function entry instrumentation only.
792 * The 62nd bit indicates whether memory profile information is present.
793 * The 63rd bit indicates if this is a temporal profile.
794 */
795#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
796#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
797#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
798#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
799#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
800#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
801#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
802#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
803#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
804#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
805#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
806#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
807#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
808#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
809#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
810#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
811#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
812
813/* The variable that holds the name of the profile data
814 * specified via command line. */
815#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
816
817/* GPU profiling section bounds structure, populated by the compiler runtime
818 * and read by the host to extract profiling data. */
819#define INSTR_PROF_SECT_BOUNDS_TABLE __llvm_profile_sections
820
821/* section name strings common to all targets other
822 than WIN32 */
823#define INSTR_PROF_DATA_COMMON __llvm_prf_data
824#define INSTR_PROF_NAME_COMMON __llvm_prf_names
825#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns
826#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
827#define INSTR_PROF_UCNTS_COMMON __llvm_prf_ucnts
828#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
829#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
830#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
831#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
832#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
833#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
834#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
835#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
836#define INSTR_PROF_COVINIT_COMMON __llvm_covinit
837
838/* Windows section names. Because these section names contain dollar characters,
839 * they must be quoted.
840 */
841#define INSTR_PROF_DATA_COFF ".lprfd$M"
842#define INSTR_PROF_NAME_COFF ".lprfn$M"
843#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
844#define INSTR_PROF_CNTS_COFF ".lprfc$M"
845#define INSTR_PROF_UCNTS_COFF ".lprfuc$M"
846#define INSTR_PROF_BITS_COFF ".lprfb$M"
847#define INSTR_PROF_VALS_COFF ".lprfv$M"
848#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
849#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
850#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
851#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
852/* Since cov data and cov names sections are not allocated, we don't need to
853 * access them at runtime.
854 */
855#define INSTR_PROF_COVDATA_COFF ".lcovd"
856#define INSTR_PROF_COVNAME_COFF ".lcovn"
857
858// FIXME: Placeholder for Windows. Windows currently does not initialize
859// the GCOV functions in the runtime.
860#define INSTR_PROF_COVINIT_COFF ".lcovd$M"
861
862#ifdef _WIN32
863/* Runtime section names and name strings. */
864#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
865#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
866#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
867#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
868#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF
869#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF
870/* Array of pointers. Each pointer points to a list
871 * of value nodes associated with one value site.
872 */
873#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
874/* Value profile nodes section. */
875#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
876#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
877#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
878#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
879#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
880#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF
881#else
882/* Runtime section names and name strings. */
883#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
884#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
885#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
886#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
887#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)
888#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)
889/* Array of pointers. Each pointer points to a list
890 * of value nodes associated with one value site.
891 */
892#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
893/* Value profile nodes section. */
894#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
895#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
896#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
897#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
898#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
899#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)
900#endif
901
902/* Macros to define start/stop section symbol for a given
903 * section on Linux. For instance
904 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
905 * expand to __start___llvm_prof_data
906 */
907#define INSTR_PROF_SECT_START(Sect) \
908 INSTR_PROF_CONCAT(__start_,Sect)
909#define INSTR_PROF_SECT_STOP(Sect) \
910 INSTR_PROF_CONCAT(__stop_,Sect)
911
912/* Value Profiling API linkage name. */
913#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
914#define INSTR_PROF_VALUE_PROF_FUNC_STR \
915 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
916#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
917#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
918 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
919#define INSTR_PROF_INSTRUMENT_GPU_FUNC __llvm_profile_instrument_gpu
920#define INSTR_PROF_INSTRUMENT_GPU_FUNC_STR \
921 INSTR_PROF_QUOTE(INSTR_PROF_INSTRUMENT_GPU_FUNC)
922
923/* InstrProfile per-function control data alignment. */
924#define INSTR_PROF_DATA_ALIGNMENT 8
925
926/* The data structure that represents a tracked value by the
927 * value profiler.
928 */
929typedef struct InstrProfValueData {
930 /* Profiled value. */
931 uint64_t Value;
932 /* Number of times the value appears in the training run. */
933 uint64_t Count;
934} InstrProfValueData;
935
936#endif /* INSTR_PROF_DATA_INC */
937
938#else
939#undef INSTR_PROF_DATA_DEFINED
940#endif
941
942#undef COVMAP_V2_OR_V3
943
944#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
945
946#ifdef __cplusplus
947#define INSTR_PROF_INLINE inline
948#else
949#define INSTR_PROF_INLINE
950#endif
951
952/* The value range buckets (22 buckets) for the memop size value profiling looks
953 * like:
954 *
955 * [0, 0]
956 * [1, 1]
957 * [2, 2]
958 * [3, 3]
959 * [4, 4]
960 * [5, 5]
961 * [6, 6]
962 * [7, 7]
963 * [8, 8]
964 * [9, 15]
965 * [16, 16]
966 * [17, 31]
967 * [32, 32]
968 * [33, 63]
969 * [64, 64]
970 * [65, 127]
971 * [128, 128]
972 * [129, 255]
973 * [256, 256]
974 * [257, 511]
975 * [512, 512]
976 * [513, UINT64_MAX]
977 *
978 * Each range has a 'representative value' which is the lower end value of the
979 * range and used to store in the runtime profile data records and the VP
980 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
981 */
982#define INSTR_PROF_NUM_BUCKETS 22
983
984/*
985 * Clz and Popcount. This code was copied from
986 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
987 * llvm/include/llvm/Support/MathExtras.h.
988 */
989#if defined(_MSC_VER) && !defined(__clang__)
990
991#include <intrin.h>
992INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
993int InstProfClzll(unsigned long long X) {
994 unsigned long LeadZeroIdx = 0;
995#if !defined(_M_ARM64) && !defined(_M_X64)
996 // Scan the high 32 bits.
997 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
998 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
999 // from the MSB.
1000 // Scan the low 32 bits.
1001 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
1002 return (int)(63 - LeadZeroIdx);
1003#else
1004 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
1005#endif
1006 return 64;
1007}
1008INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
1009int InstProfPopcountll(unsigned long long X) {
1010 // This code originates from https://reviews.llvm.org/rG30626254510f.
1011 unsigned long long v = X;
1012 v = v - ((v >> 1) & 0x5555555555555555ULL);
1013 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
1014 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
1015 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
1016}
1017
1018#else
1019
1020INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
1021int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
1022INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
1023int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
1024
1025#endif /* defined(_MSC_VER) && !defined(__clang__) */
1026
1027// clang-format on
1028
1029/* Map an (observed) memop size value to the representative value of its range.
1030 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
1031INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
1032InstrProfGetRangeRepValue(uint64_t Value) {
1033 if (Value <= 8)
1034 // The first ranges are individually tracked. Use the value as is.
1035 return Value;
1036 else if (Value >= 513)
1037 // The last range is mapped to its lowest value.
1038 return 513;
1039 else if (InstProfPopcountll(Value) == 1)
1040 // If it's a power of two, use it as is.
1041 return Value;
1042 else
1043 // Otherwise, take to the previous power of two + 1.
1044 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
1045}
1046
1047/* Return true if the range that an (observed) memop size value belongs to has
1048 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
1049 * false, 64 -> true, 100 -> false, 513 -> false. */
1050INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
1051InstrProfIsSingleValRange(uint64_t Value) {
1052 if (Value <= 8)
1053 // The first ranges are individually tracked.
1054 return 1;
1055 else if (InstProfPopcountll(Value) == 1)
1056 // If it's a power of two, there's only one value.
1057 return 1;
1058 else
1059 // Otherwise, there's more than one value in the range.
1060 return 0;
1061}
1062
1063#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
1064