1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 * Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 * LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 * Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65// clang-format off:consider re-enabling clang-format if auto-formatted C macros
66// are readable (e.g., after `issue #82426` is fixed)
67/* INSTR_PROF_DATA start. */
68/* Definition of member fields of the per-function control structure. */
69#ifndef INSTR_PROF_DATA
70#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
71#else
72#define INSTR_PROF_DATA_DEFINED
73#endif
74INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
75 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
76 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
77INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
78 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
79 Inc->getHash()->getZExtValue()))
80INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
81INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
82/* This is used to map function pointers for the indirect call targets to
83 * function name hashes during the conversion from raw to merged profile
84 * data.
85 */
86INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
87 FunctionAddr)
88INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
89 ValuesPtrExpr)
90INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
91 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
92INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
93 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
94INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
95 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
96#undef INSTR_PROF_DATA
97/* INSTR_PROF_DATA end. */
98
99/* For a virtual table object, record the name hash to associate profiled
100 * addresses with global variables, and record {starting address, size in bytes}
101 * to map the profiled virtual table (which usually have an offset from the
102 * starting address) back to a virtual table object. */
103#ifndef INSTR_PROF_VTABLE_DATA
104#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
105#else
106#define INSTR_PROF_VTABLE_DATA_DEFINED
107#endif
108INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \
109 VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
110 IndexedInstrProf::ComputeHash(PGOVTableName)))
111INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
112 VTablePointer, VTableAddr)
113INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
114 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
115 VTableSizeVal))
116#undef INSTR_PROF_VTABLE_DATA
117/* INSTR_PROF_VTABLE_DATA end. */
118
119/* This is an internal data structure used by value profiler. It
120 * is defined here to allow serialization code sharing by LLVM
121 * to be used in unit test.
122 *
123 * typedef struct ValueProfNode {
124 * // InstrProfValueData VData;
125 * uint64_t Value;
126 * uint64_t Count;
127 * struct ValueProfNode *Next;
128 * } ValueProfNode;
129 */
130/* INSTR_PROF_VALUE_NODE start. */
131#ifndef INSTR_PROF_VALUE_NODE
132#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
133#else
134#define INSTR_PROF_DATA_DEFINED
135#endif
136INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
137 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
138INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
139 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
140INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
141 ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))
142#undef INSTR_PROF_VALUE_NODE
143/* INSTR_PROF_VALUE_NODE end. */
144
145/* INSTR_PROF_GPU_SECT start. */
146/* Fields of the GPU profile section bounds structure, populated by the
147 * compiler runtime and read by the host to extract profiling data. */
148#ifndef INSTR_PROF_GPU_SECT
149#define INSTR_PROF_GPU_SECT(Type, LLVMType, Name, Initializer)
150#else
151#define INSTR_PROF_DATA_DEFINED
152#endif
153INSTR_PROF_GPU_SECT(const char *, llvm::PointerType::getUnqual(Ctx), \
154 NamesStart, \
155 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
156INSTR_PROF_GPU_SECT(const char *, llvm::PointerType::getUnqual(Ctx), \
157 NamesStop, \
158 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
159INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
160 CountersStart, \
161 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
162INSTR_PROF_GPU_SECT(char *, llvm::PointerType::getUnqual(Ctx), \
163 CountersStop, \
164 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
165INSTR_PROF_GPU_SECT(const __llvm_profile_data *, llvm::PointerType::getUnqual( \
166 Ctx), DataStart, \
167 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
168INSTR_PROF_GPU_SECT(const __llvm_profile_data *, llvm::PointerType::getUnqual( \
169 Ctx), DataStop, \
170 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
171INSTR_PROF_GPU_SECT(uint64_t *, llvm::PointerType::getUnqual(Ctx), \
172 VersionVar, \
173 ConstantPointerNull::get(llvm::PointerType::getUnqual(Ctx)))
174#undef INSTR_PROF_GPU_SECT
175/* INSTR_PROF_GPU_SECT end. */
176
177/* INSTR_PROF_RAW_HEADER start */
178/* Definition of member fields of the raw profile header data structure. */
179/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
180 raw profile format. */
181#ifndef INSTR_PROF_RAW_HEADER
182#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
183#else
184#define INSTR_PROF_DATA_DEFINED
185#endif
186INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
187INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
188INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
189INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
190INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
191INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
192INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
193INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
194INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
195INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
196INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
197 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
198INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
199 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
200INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
201INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
202INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
203INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
204#undef INSTR_PROF_RAW_HEADER
205/* INSTR_PROF_RAW_HEADER end */
206
207/* VALUE_PROF_FUNC_PARAM start */
208/* Definition of parameter types of the runtime API used to do value profiling
209 * for a given value site.
210 */
211#ifndef VALUE_PROF_FUNC_PARAM
212#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
213#define INSTR_PROF_COMMA
214#else
215#define INSTR_PROF_DATA_DEFINED
216#define INSTR_PROF_COMMA ,
217#endif
218VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
219 INSTR_PROF_COMMA
220VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
221VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
222#undef VALUE_PROF_FUNC_PARAM
223#undef INSTR_PROF_COMMA
224/* VALUE_PROF_FUNC_PARAM end */
225
226/* VALUE_PROF_KIND start */
227#ifndef VALUE_PROF_KIND
228#define VALUE_PROF_KIND(Enumerator, Value, Descr)
229#else
230#define INSTR_PROF_DATA_DEFINED
231#endif
232/* For indirect function call value profiling, the addresses of the target
233 * functions are profiled by the instrumented code. The target addresses are
234 * written in the raw profile data and converted to target function name's MD5
235 * hash by the profile reader during deserialization. Typically, this happens
236 * when the raw profile data is read during profile merging.
237 *
238 * For this remapping the ProfData is used. ProfData contains both the function
239 * name hash and the function address.
240 */
241VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
242/* For memory intrinsic functions size profiling. */
243VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
244/* For virtual table address profiling, the address point of the virtual table
245 * (i.e., the address contained in objects pointing to a virtual table) are
246 * profiled. Note this may not be the address of the per C++ class virtual table
247 * object (e.g., there might be an offset).
248 *
249 * The profiled addresses are stored in raw profile, together with the following
250 * two types of information.
251 * 1. The (starting and ending) addresses of per C++ class virtual table objects.
252 * 2. The (compressed) virtual table object names.
253 * RawInstrProfReader converts profiled virtual table addresses to virtual table
254 * objects' MD5 hash.
255 */
256VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")
257/* These two kinds must be the last to be
258 * declared. This is to make sure the string
259 * array created with the template can be
260 * indexed with the kind value.
261 */
262VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
263VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
264
265#undef VALUE_PROF_KIND
266/* VALUE_PROF_KIND end */
267
268#undef COVMAP_V2_OR_V3
269#ifdef COVMAP_V2
270#define COVMAP_V2_OR_V3
271#endif
272#ifdef COVMAP_V3
273#define COVMAP_V2_OR_V3
274#endif
275
276/* COVMAP_FUNC_RECORD start */
277/* Definition of member fields of the function record structure in coverage
278 * map.
279 */
280#ifndef COVMAP_FUNC_RECORD
281#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
282#else
283#define INSTR_PROF_DATA_DEFINED
284#endif
285#ifdef COVMAP_V1
286COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
287 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
288 llvm::PointerType::getUnqual(Ctx)))
289COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
290 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
291 NameValue.size()))
292#endif
293#ifdef COVMAP_V2_OR_V3
294COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
295 llvm::ConstantInt::get( \
296 llvm::Type::getInt64Ty(Ctx), NameHash))
297#endif
298COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
299 llvm::ConstantInt::get( \
300 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
301COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
302 llvm::ConstantInt::get( \
303 llvm::Type::getInt64Ty(Ctx), FuncHash))
304#ifdef COVMAP_V3
305COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
306 llvm::ConstantInt::get( \
307 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
308COVMAP_FUNC_RECORD(const char, \
309 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
310 CoverageMapping.size()), \
311 CoverageMapping,
312 llvm::ConstantDataArray::getRaw( \
313 CoverageMapping, CoverageMapping.size(), \
314 llvm::Type::getInt8Ty(Ctx)))
315#endif
316#undef COVMAP_FUNC_RECORD
317/* COVMAP_FUNC_RECORD end. */
318
319/* COVMAP_HEADER start */
320/* Definition of member fields of coverage map header.
321 */
322#ifndef COVMAP_HEADER
323#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
324#else
325#define INSTR_PROF_DATA_DEFINED
326#endif
327COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
328 llvm::ConstantInt::get(Int32Ty, NRecords))
329COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
330 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
331COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
332 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
333COVMAP_HEADER(uint32_t, Int32Ty, Version, \
334 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
335#undef COVMAP_HEADER
336/* COVMAP_HEADER end. */
337
338/* COVINIT_FUNC start */
339#ifndef COVINIT_FUNC
340#define COVINIT_FUNC(Type, LLVMType, Name, Initializer)
341#else
342#define INSTR_PROF_DATA_DEFINED
343#endif
344COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \
345 WriteoutF)
346COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \
347 ResetF)
348#undef COVINIT_FUNC
349/* COVINIT_FUNC end */
350
351#ifdef INSTR_PROF_SECT_ENTRY
352#define INSTR_PROF_DATA_DEFINED
353INSTR_PROF_SECT_ENTRY(IPSK_data, \
354 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
355 INSTR_PROF_DATA_COFF, "__DATA,")
356INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
357 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
358 INSTR_PROF_CNTS_COFF, "__DATA,")
359INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
360 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
361 INSTR_PROF_BITS_COFF, "__DATA,")
362INSTR_PROF_SECT_ENTRY(IPSK_name, \
363 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
364 INSTR_PROF_NAME_COFF, "__DATA,")
365INSTR_PROF_SECT_ENTRY(IPSK_vname, \
366 INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
367 INSTR_PROF_VNAME_COFF, "__DATA,")
368INSTR_PROF_SECT_ENTRY(IPSK_vals, \
369 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
370 INSTR_PROF_VALS_COFF, "__DATA,")
371INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
372 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
373 INSTR_PROF_VNODES_COFF, "__DATA,")
374INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
375 INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
376 INSTR_PROF_VTAB_COFF, "__DATA,")
377INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
378 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
379 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
380INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
381 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
382 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
383INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
384 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
385 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
386INSTR_PROF_SECT_ENTRY(IPSK_covname, \
387 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
388 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
389INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
390 INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \
391 INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")
392
393#undef INSTR_PROF_SECT_ENTRY
394#endif
395
396
397#ifdef INSTR_PROF_VALUE_PROF_DATA
398#define INSTR_PROF_DATA_DEFINED
399
400#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
401/*!
402 * This is the header of the data structure that defines the on-disk
403 * layout of the value profile data of a particular kind for one function.
404 */
405typedef struct ValueProfRecord {
406 /* The kind of the value profile record. */
407 uint32_t Kind;
408 /*
409 * The number of value profile sites. It is guaranteed to be non-zero;
410 * otherwise the record for this kind won't be emitted.
411 */
412 uint32_t NumValueSites;
413 /*
414 * The first element of the array that stores the number of profiled
415 * values for each value site. The size of the array is NumValueSites.
416 * Since NumValueSites is greater than zero, there is at least one
417 * element in the array.
418 */
419 uint8_t SiteCountArray[1];
420
421 /*
422 * The fake declaration is for documentation purpose only.
423 * Align the start of next field to be on 8 byte boundaries.
424 uint8_t Padding[X];
425 */
426
427 /* The array of value profile data. The size of the array is the sum
428 * of all elements in SiteCountArray[].
429 InstrProfValueData ValueData[];
430 */
431
432#ifdef __cplusplus
433 /*!
434 * Return the number of value sites.
435 */
436 uint32_t getNumValueSites() const { return NumValueSites; }
437 /*!
438 * Read data from this record and save it to Record.
439 */
440 LLVM_ABI void deserializeTo(InstrProfRecord &Record,
441 InstrProfSymtab *SymTab);
442 /*
443 * In-place byte swap:
444 * Do byte swap for this instance. \c Old is the original order before
445 * the swap, and \c New is the New byte order.
446 */
447 LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New);
448#endif
449} ValueProfRecord;
450
451/*!
452 * Per-function header/control data structure for value profiling
453 * data in indexed format.
454 */
455typedef struct ValueProfData {
456 /*
457 * Total size in bytes including this field. It must be a multiple
458 * of sizeof(uint64_t).
459 */
460 uint32_t TotalSize;
461 /*
462 *The number of value profile kinds that has value profile data.
463 * In this implementation, a value profile kind is considered to
464 * have profile data if the number of value profile sites for the
465 * kind is not zero. More aggressively, the implementation can
466 * choose to check the actual data value: if none of the value sites
467 * has any profiled values, the kind can be skipped.
468 */
469 uint32_t NumValueKinds;
470
471 /*
472 * Following are a sequence of variable length records. The prefix/header
473 * of each record is defined by ValueProfRecord type. The number of
474 * records is NumValueKinds.
475 * ValueProfRecord Record_1;
476 * ValueProfRecord Record_N;
477 */
478
479#if __cplusplus
480 /*!
481 * Return the total size in bytes of the on-disk value profile data
482 * given the data stored in Record.
483 */
484 LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record);
485 /*!
486 * Return a pointer to \c ValueProfData instance ready to be streamed.
487 */
488 LLVM_ABI static std::unique_ptr<ValueProfData>
489 serializeFrom(const InstrProfRecord &Record);
490 /*!
491 * Check the integrity of the record.
492 */
493 LLVM_ABI Error checkIntegrity();
494 /*!
495 * Return a pointer to \c ValueProfileData instance ready to be read.
496 * All data in the instance are properly byte swapped. The input
497 * data is assumed to be in little endian order.
498 */
499 LLVM_ABI static Expected<std::unique_ptr<ValueProfData>>
500 getValueProfData(const unsigned char *SrcBuffer,
501 const unsigned char *const SrcBufferEnd,
502 llvm::endianness SrcDataEndianness);
503 /*!
504 * Swap byte order from \c Endianness order to host byte order.
505 */
506 LLVM_ABI void swapBytesToHost(llvm::endianness Endianness);
507 /*!
508 * Swap byte order from host byte order to \c Endianness order.
509 */
510 LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness);
511 /*!
512 * Return the total size of \c ValueProfileData.
513 */
514 LLVM_ABI uint32_t getSize() const { return TotalSize; }
515 /*!
516 * Read data from this data and save it to \c Record.
517 */
518 LLVM_ABI void deserializeTo(InstrProfRecord &Record,
519 InstrProfSymtab *SymTab);
520 void operator delete(void *ptr) { ::operator delete(ptr); }
521#endif
522} ValueProfData;
523
524/*
525 * The closure is designed to abstract away two types of value profile data:
526 * - InstrProfRecord which is the primary data structure used to
527 * represent profile data in host tools (reader, writer, and profile-use)
528 * - value profile runtime data structure suitable to be used by C
529 * runtime library.
530 *
531 * Both sources of data need to serialize to disk/memory-buffer in common
532 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
533 * writer to share the same format and code with indexed profile writer.
534 *
535 * For documentation of the member methods below, refer to corresponding methods
536 * in class InstrProfRecord.
537 */
538typedef struct ValueProfRecordClosure {
539 const void *Record;
540 uint32_t (*GetNumValueKinds)(const void *Record);
541 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
542 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
543 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
544
545 /*
546 * After extracting the value profile data from the value profile record,
547 * this method is used to map the in-memory value to on-disk value. If
548 * the method is null, value will be written out untranslated.
549 */
550 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
551 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
552 uint32_t S);
553 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
554} ValueProfRecordClosure;
555
556INSTR_PROF_VISIBILITY ValueProfRecord *
557getFirstValueProfRecord(ValueProfData *VPD);
558INSTR_PROF_VISIBILITY ValueProfRecord *
559getValueProfRecordNext(ValueProfRecord *VPR);
560INSTR_PROF_VISIBILITY InstrProfValueData *
561getValueProfRecordValueData(ValueProfRecord *VPR);
562INSTR_PROF_VISIBILITY uint32_t
563getValueProfRecordHeaderSize(uint32_t NumValueSites);
564
565#undef INSTR_PROF_VALUE_PROF_DATA
566#endif /* INSTR_PROF_VALUE_PROF_DATA */
567
568
569#ifdef INSTR_PROF_COMMON_API_IMPL
570#define INSTR_PROF_DATA_DEFINED
571#ifdef __cplusplus
572#define INSTR_PROF_INLINE inline
573#define INSTR_PROF_NULLPTR nullptr
574#else
575#define INSTR_PROF_INLINE
576#define INSTR_PROF_NULLPTR NULL
577#endif
578
579#ifndef offsetof
580#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
581#endif
582
583// clang-format on
584
585/*!
586 * Return the \c ValueProfRecord header size including the
587 * padding bytes.
588 */
589INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
590getValueProfRecordHeaderSize(uint32_t NumValueSites) {
591 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
592 sizeof(uint8_t) * NumValueSites;
593 /* Round the size to multiple of 8 bytes. */
594 Size = (Size + 7) & ~7;
595 return Size;
596}
597
598/*!
599 * Return the total size of the value profile record including the
600 * header and the value data.
601 */
602INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
603getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
604 return getValueProfRecordHeaderSize(NumValueSites) +
605 sizeof(InstrProfValueData) * NumValueData;
606}
607
608/*!
609 * Return the pointer to the start of value data array.
610 */
611INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
612getValueProfRecordValueData(ValueProfRecord *This) {
613 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
614 This->NumValueSites));
615}
616
617/*!
618 * Return the total number of value data for \c This record.
619 */
620INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
621getValueProfRecordNumValueData(ValueProfRecord *This) {
622 uint32_t NumValueData = 0;
623 uint32_t I;
624 for (I = 0; I < This->NumValueSites; I++)
625 NumValueData += This->SiteCountArray[I];
626 return NumValueData;
627}
628
629/*!
630 * Use this method to advance to the next \c This \c ValueProfRecord.
631 */
632INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
633getValueProfRecordNext(ValueProfRecord *This) {
634 uint32_t NumValueData = getValueProfRecordNumValueData(This);
635 return (ValueProfRecord *)((char *)This +
636 getValueProfRecordSize(This->NumValueSites,
637 NumValueData));
638}
639
640/*!
641 * Return the first \c ValueProfRecord instance.
642 */
643INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
644getFirstValueProfRecord(ValueProfData *This) {
645 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
646}
647
648/* Closure based interfaces. */
649
650/*!
651 * Return the total size in bytes of the on-disk value profile data
652 * given the data stored in Record.
653 */
654INSTR_PROF_VISIBILITY uint32_t
655getValueProfDataSize(ValueProfRecordClosure *Closure) {
656 uint32_t Kind;
657 uint32_t TotalSize = sizeof(ValueProfData);
658 const void *Record = Closure->Record;
659
660 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
661 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
662 if (!NumValueSites)
663 continue;
664 TotalSize += getValueProfRecordSize(NumValueSites,
665 Closure->GetNumValueData(Record, Kind));
666 }
667 return TotalSize;
668}
669
670/*!
671 * Extract value profile data of a function for the profile kind \c ValueKind
672 * from the \c Closure and serialize the data into \c This record instance.
673 */
674INSTR_PROF_VISIBILITY void
675serializeValueProfRecordFrom(ValueProfRecord *This,
676 ValueProfRecordClosure *Closure,
677 uint32_t ValueKind, uint32_t NumValueSites) {
678 uint32_t S;
679 const void *Record = Closure->Record;
680 This->Kind = ValueKind;
681 This->NumValueSites = NumValueSites;
682 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
683
684 for (S = 0; S < NumValueSites; S++) {
685 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
686 This->SiteCountArray[S] = ND;
687 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
688 DstVD += ND;
689 }
690}
691
692/*!
693 * Extract value profile data of a function from the \c Closure
694 * and serialize the data into \c DstData if it is not NULL or heap
695 * memory allocated by the \c Closure's allocator method. If \c
696 * DstData is not null, the caller is expected to set the TotalSize
697 * in DstData.
698 */
699INSTR_PROF_VISIBILITY ValueProfData *
700serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
701 ValueProfData *DstData) {
702 uint32_t Kind;
703 uint32_t TotalSize =
704 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
705
706 ValueProfData *VPD =
707 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
708
709 VPD->TotalSize = TotalSize;
710 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
711 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
712 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
713 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
714 if (!NumValueSites)
715 continue;
716 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
717 VR = getValueProfRecordNext(VR);
718 }
719 return VPD;
720}
721
722#undef INSTR_PROF_COMMON_API_IMPL
723#endif /* INSTR_PROF_COMMON_API_IMPL */
724
725/*============================================================================*/
726
727// clang-format off:consider re-enabling clang-format if auto-formatted C macros
728// are readable (e.g., after `issue #82426` is fixed)
729#ifndef INSTR_PROF_DATA_DEFINED
730
731#ifndef INSTR_PROF_DATA_INC
732#define INSTR_PROF_DATA_INC
733
734/* Helper macros. */
735#define INSTR_PROF_SIMPLE_QUOTE(x) #x
736#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
737#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
738#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
739
740/* Magic number to detect file format and endianness.
741 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
742 * so that utilities, like strings, don't grab it as a string. 129 is also
743 * invalid UTF-8, and high enough to be interesting.
744 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
745 * for 32-bit platforms.
746 */
747#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
748 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
749 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
750#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
751 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
752 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
753
754/* Raw profile format version (start from 1). */
755#define INSTR_PROF_RAW_VERSION 10
756/* Indexed profile format version (start from 1). */
757#define INSTR_PROF_INDEX_VERSION 13
758/* Coverage mapping format version (start from 0). */
759#define INSTR_PROF_COVMAP_VERSION 6
760
761/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
762 * version for other variants of profile. We set the 8th most significant bit
763 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
764 * generated profile, and 0 if this is a Clang FE generated profile.
765 * 1 in bit 57 indicates there are context-sensitive records in the profile.
766 * The 54th bit indicates whether to always instrument loop entry blocks.
767 * The 58th bit indicates whether to always instrument function entry blocks.
768 * The 59th bit indicates whether to use debug info to correlate profiles.
769 * The 60th bit indicates single byte coverage instrumentation.
770 * The 61st bit indicates function entry instrumentation only.
771 * The 62nd bit indicates whether memory profile information is present.
772 * The 63rd bit indicates if this is a temporal profile.
773 */
774#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
775#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
776#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
777#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
778#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
779#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
780#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
781#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
782#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
783#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
784#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
785#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
786#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
787#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
788#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
789#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
790#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
791
792/* The variable that holds the name of the profile data
793 * specified via command line. */
794#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
795
796/* GPU profiling section bounds structure, populated by the compiler runtime
797 * and read by the host to extract profiling data. */
798#define INSTR_PROF_SECT_BOUNDS_TABLE __llvm_profile_sections
799
800/* section name strings common to all targets other
801 than WIN32 */
802#define INSTR_PROF_DATA_COMMON __llvm_prf_data
803#define INSTR_PROF_NAME_COMMON __llvm_prf_names
804#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns
805#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
806#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
807#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
808#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
809#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
810#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
811#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
812#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
813#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
814#define INSTR_PROF_COVINIT_COMMON __llvm_covinit
815
816/* Windows section names. Because these section names contain dollar characters,
817 * they must be quoted.
818 */
819#define INSTR_PROF_DATA_COFF ".lprfd$M"
820#define INSTR_PROF_NAME_COFF ".lprfn$M"
821#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
822#define INSTR_PROF_CNTS_COFF ".lprfc$M"
823#define INSTR_PROF_BITS_COFF ".lprfb$M"
824#define INSTR_PROF_VALS_COFF ".lprfv$M"
825#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
826#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
827#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
828#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
829/* Since cov data and cov names sections are not allocated, we don't need to
830 * access them at runtime.
831 */
832#define INSTR_PROF_COVDATA_COFF ".lcovd"
833#define INSTR_PROF_COVNAME_COFF ".lcovn"
834
835// FIXME: Placeholder for Windows. Windows currently does not initialize
836// the GCOV functions in the runtime.
837#define INSTR_PROF_COVINIT_COFF ".lcovd$M"
838
839#ifdef _WIN32
840/* Runtime section names and name strings. */
841#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
842#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
843#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
844#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
845#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF
846#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF
847/* Array of pointers. Each pointer points to a list
848 * of value nodes associated with one value site.
849 */
850#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
851/* Value profile nodes section. */
852#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
853#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
854#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
855#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
856#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
857#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF
858#else
859/* Runtime section names and name strings. */
860#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
861#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
862#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
863#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
864#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)
865#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)
866/* Array of pointers. Each pointer points to a list
867 * of value nodes associated with one value site.
868 */
869#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
870/* Value profile nodes section. */
871#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
872#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
873#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
874#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
875#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
876#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)
877#endif
878
879/* Macros to define start/stop section symbol for a given
880 * section on Linux. For instance
881 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
882 * expand to __start___llvm_prof_data
883 */
884#define INSTR_PROF_SECT_START(Sect) \
885 INSTR_PROF_CONCAT(__start_,Sect)
886#define INSTR_PROF_SECT_STOP(Sect) \
887 INSTR_PROF_CONCAT(__stop_,Sect)
888
889/* Value Profiling API linkage name. */
890#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
891#define INSTR_PROF_VALUE_PROF_FUNC_STR \
892 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
893#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
894#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
895 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
896
897/* InstrProfile per-function control data alignment. */
898#define INSTR_PROF_DATA_ALIGNMENT 8
899
900/* The data structure that represents a tracked value by the
901 * value profiler.
902 */
903typedef struct InstrProfValueData {
904 /* Profiled value. */
905 uint64_t Value;
906 /* Number of times the value appears in the training run. */
907 uint64_t Count;
908} InstrProfValueData;
909
910#endif /* INSTR_PROF_DATA_INC */
911
912#else
913#undef INSTR_PROF_DATA_DEFINED
914#endif
915
916#undef COVMAP_V2_OR_V3
917
918#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
919
920#ifdef __cplusplus
921#define INSTR_PROF_INLINE inline
922#else
923#define INSTR_PROF_INLINE
924#endif
925
926/* The value range buckets (22 buckets) for the memop size value profiling looks
927 * like:
928 *
929 * [0, 0]
930 * [1, 1]
931 * [2, 2]
932 * [3, 3]
933 * [4, 4]
934 * [5, 5]
935 * [6, 6]
936 * [7, 7]
937 * [8, 8]
938 * [9, 15]
939 * [16, 16]
940 * [17, 31]
941 * [32, 32]
942 * [33, 63]
943 * [64, 64]
944 * [65, 127]
945 * [128, 128]
946 * [129, 255]
947 * [256, 256]
948 * [257, 511]
949 * [512, 512]
950 * [513, UINT64_MAX]
951 *
952 * Each range has a 'representative value' which is the lower end value of the
953 * range and used to store in the runtime profile data records and the VP
954 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
955 */
956#define INSTR_PROF_NUM_BUCKETS 22
957
958/*
959 * Clz and Popcount. This code was copied from
960 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
961 * llvm/include/llvm/Support/MathExtras.h.
962 */
963#if defined(_MSC_VER) && !defined(__clang__)
964
965#include <intrin.h>
966INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
967int InstProfClzll(unsigned long long X) {
968 unsigned long LeadZeroIdx = 0;
969#if !defined(_M_ARM64) && !defined(_M_X64)
970 // Scan the high 32 bits.
971 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
972 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
973 // from the MSB.
974 // Scan the low 32 bits.
975 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
976 return (int)(63 - LeadZeroIdx);
977#else
978 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
979#endif
980 return 64;
981}
982INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
983int InstProfPopcountll(unsigned long long X) {
984 // This code originates from https://reviews.llvm.org/rG30626254510f.
985 unsigned long long v = X;
986 v = v - ((v >> 1) & 0x5555555555555555ULL);
987 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
988 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
989 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
990}
991
992#else
993
994INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
995int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
996INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
997int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
998
999#endif /* defined(_MSC_VER) && !defined(__clang__) */
1000
1001// clang-format on
1002
1003/* Map an (observed) memop size value to the representative value of its range.
1004 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
1005INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
1006InstrProfGetRangeRepValue(uint64_t Value) {
1007 if (Value <= 8)
1008 // The first ranges are individually tracked. Use the value as is.
1009 return Value;
1010 else if (Value >= 513)
1011 // The last range is mapped to its lowest value.
1012 return 513;
1013 else if (InstProfPopcountll(Value) == 1)
1014 // If it's a power of two, use it as is.
1015 return Value;
1016 else
1017 // Otherwise, take to the previous power of two + 1.
1018 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
1019}
1020
1021/* Return true if the range that an (observed) memop size value belongs to has
1022 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
1023 * false, 64 -> true, 100 -> false, 513 -> false. */
1024INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
1025InstrProfIsSingleValRange(uint64_t Value) {
1026 if (Value <= 8)
1027 // The first ranges are individually tracked.
1028 return 1;
1029 else if (InstProfPopcountll(Value) == 1)
1030 // If it's a power of two, there's only one value.
1031 return 1;
1032 else
1033 // Otherwise, there's more than one value in the range.
1034 return 0;
1035}
1036
1037#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
1038