1/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9#if !defined(__Fuchsia__)
10
11#if defined(__linux__)
12// For fileno(), ftruncate(), getpagesize(), setenv()
13#define _DEFAULT_SOURCE
14#endif
15
16#include <assert.h>
17#include <errno.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#ifdef _MSC_VER
22/* For _alloca. */
23#include <malloc.h>
24#endif
25#if defined(_WIN32)
26#include "WindowsMMap.h"
27/* For _chsize_s */
28#include <io.h>
29#include <process.h>
30#else
31#include <sys/file.h>
32#include <sys/mman.h>
33#include <unistd.h>
34#if defined(__linux__)
35#include <sys/types.h>
36#endif
37#endif
38
39#include "InstrProfiling.h"
40#include "InstrProfilingInternal.h"
41#include "InstrProfilingPort.h"
42#include "InstrProfilingUtil.h"
43
44/* Weak so non-HIP programs do not force InstrProfilingPlatformROCm.o (and its
45 * transitive sanitizer_common / interception dependencies) into the host link
46 * out of libclang_rt.profile.a. HIP programs emit strong references to other
47 * ROCm-runtime symbols (e.g. __llvm_profile_offload_register_shadow_variable)
48 * that pull in the strong definition.
49 * No COMPILER_RT_VISIBILITY: a hidden weak-undefined symbol is non-preemptible
50 * and the address test at the call site would fold to true.
51 * Windows: __declspec(selectany) is data-only, and the ROCm interceptor path
52 * is not used there, so keep the original strong extern. */
53#if COMPILER_RT_BUILD_PROFILE_ROCM
54#if defined(_WIN32)
55extern int __llvm_profile_hip_collect_device_data(void);
56#else
57__attribute__((weak)) int __llvm_profile_hip_collect_device_data(void);
58#endif
59#endif
60
61/* From where is profile name specified.
62 * The order the enumerators define their
63 * precedence. Re-order them may lead to
64 * runtime behavior change. */
65typedef enum ProfileNameSpecifier {
66 PNS_unknown = 0,
67 PNS_default,
68 PNS_command_line,
69 PNS_environment,
70 PNS_runtime_api
71} ProfileNameSpecifier;
72
73static const char *getPNSStr(ProfileNameSpecifier PNS) {
74 switch (PNS) {
75 case PNS_default:
76 return "default setting";
77 case PNS_command_line:
78 return "command line";
79 case PNS_environment:
80 return "environment variable";
81 case PNS_runtime_api:
82 return "runtime API";
83 default:
84 return "Unknown";
85 }
86}
87
88#define MAX_PID_SIZE 16
89/* Data structure holding the result of parsed filename pattern. */
90typedef struct lprofFilename {
91 /* File name string possibly with %p or %h specifiers. */
92 const char *FilenamePat;
93 /* A flag indicating if FilenamePat's memory is allocated
94 * by runtime. */
95 unsigned OwnsFilenamePat;
96 const char *ProfilePathPrefix;
97 char PidChars[MAX_PID_SIZE];
98 char *TmpDir;
99 char Hostname[COMPILER_RT_MAX_HOSTLEN];
100 unsigned NumPids;
101 unsigned NumHosts;
102 unsigned NumBinaryIds;
103 /* When in-process merging is enabled, this parameter specifies
104 * the total number of profile data files shared by all the processes
105 * spawned from the same binary. By default the value is 1. If merging
106 * is not enabled, its value should be 0. This parameter is specified
107 * by the %[0-9]m specifier. For instance %2m enables merging using
108 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
109 * can only appear once at the end of the name pattern. */
110 unsigned MergePoolSize;
111 ProfileNameSpecifier PNS;
112} lprofFilename;
113
114static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, {0},
115 0, 0, 0, 0, PNS_unknown};
116
117static int ProfileMergeRequested = 0;
118static int getProfileFileSizeForMerging(FILE *ProfileFile,
119 uint64_t *ProfileFileSize);
120
121#if defined(__APPLE__)
122static const int ContinuousModeSupported = 1;
123static const int UseBiasVar = 0;
124static const char *FileOpenMode = "a+b";
125static void *BiasAddr = NULL;
126static void *BiasDefaultAddr = NULL;
127static void *BitmapBiasAddr = NULL;
128static void *BitmapBiasDefaultAddr = NULL;
129static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
130 /* Get the sizes of various profile data sections. Taken from
131 * __llvm_profile_get_size_for_buffer(). */
132 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
133 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
134 const char *CountersBegin = __llvm_profile_begin_counters();
135 const char *CountersEnd = __llvm_profile_end_counters();
136 const char *BitmapBegin = __llvm_profile_begin_bitmap();
137 const char *BitmapEnd = __llvm_profile_end_bitmap();
138 const char *NamesBegin = __llvm_profile_begin_names();
139 const char *NamesEnd = __llvm_profile_end_names();
140 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
141 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
142 uint64_t CountersSize =
143 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
144 uint64_t NumBitmapBytes =
145 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
146
147 /* Check that the counter, bitmap, and data sections in this image are
148 * page-aligned. */
149 unsigned PageSize = getpagesize();
150 if ((intptr_t)CountersBegin % PageSize != 0) {
151 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
152 CountersBegin, PageSize);
153 return 1;
154 }
155 if ((intptr_t)BitmapBegin % PageSize != 0) {
156 PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n",
157 BitmapBegin, PageSize);
158 return 1;
159 }
160 if ((intptr_t)DataBegin % PageSize != 0) {
161 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
162 DataBegin, PageSize);
163 return 1;
164 }
165
166 int Fileno = fileno(File);
167 /* Determine how much padding is needed before/after the counters and
168 * after the names. */
169 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
170 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
171 PaddingBytesAfterUniformCounters, PaddingBytesAfterVTable,
172 PaddingBytesAfterVNames;
173 __llvm_profile_get_padding_sizes_for_counters(
174 DataSize, CountersSize, NumBitmapBytes, 0 /* NumUniformCounters */,
175 NamesSize, /*VTableSize=*/0, /*VNameSize=*/0, &PaddingBytesBeforeCounters,
176 &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
177 &PaddingBytesAfterUniformCounters, &PaddingBytesAfterNames,
178 &PaddingBytesAfterVTable, &PaddingBytesAfterVNames);
179
180 uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
181 uint64_t FileOffsetToCounters = CurrentFileOffset +
182 sizeof(__llvm_profile_header) + DataSize +
183 PaddingBytesBeforeCounters;
184 void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
185 PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
186 Fileno, FileOffsetToCounters);
187 if (CounterMmap != CountersBegin) {
188 PROF_ERR(
189 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
190 " - CountersBegin: %p\n"
191 " - PageAlignedCountersLength: %" PRIu64 "\n"
192 " - Fileno: %d\n"
193 " - FileOffsetToCounters: %" PRIu64 "\n",
194 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
195 FileOffsetToCounters);
196 return 1;
197 }
198
199 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
200 * will fail with EINVAL. */
201 if (NumBitmapBytes == 0)
202 return 0;
203
204 uint64_t PageAlignedBitmapLength =
205 NumBitmapBytes + PaddingBytesAfterBitmapBytes;
206 uint64_t FileOffsetToBitmap =
207 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
208 void *BitmapMmap =
209 mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE,
210 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap);
211 if (BitmapMmap != BitmapBegin) {
212 PROF_ERR(
213 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
214 " - BitmapBegin: %p\n"
215 " - PageAlignedBitmapLength: %" PRIu64 "\n"
216 " - Fileno: %d\n"
217 " - FileOffsetToBitmap: %" PRIu64 "\n",
218 strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno,
219 FileOffsetToBitmap);
220 return 1;
221 }
222 return 0;
223}
224#elif defined(__ELF__) || defined(_WIN32) || defined(_AIX)
225
226#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
227 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
228COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
229#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
230 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
231COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
232
233/* This variable is a weak external reference which could be used to detect
234 * whether or not the compiler defined this symbol. */
235#if defined(_MSC_VER)
236COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
237COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
238#if defined(_M_IX86) || defined(__i386__)
239#define WIN_SYM_PREFIX "_"
240#else
241#define WIN_SYM_PREFIX
242#endif
243#pragma comment( \
244 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
245 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
246 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
247#pragma comment( \
248 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
249 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
250 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
251#else
252COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
253 __attribute__((weak, alias(INSTR_PROF_QUOTE(
254 INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
255COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
256 __attribute__((weak, alias(INSTR_PROF_QUOTE(
257 INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
258#endif
259static const int ContinuousModeSupported = 1;
260static const int UseBiasVar = 1;
261/* TODO: If there are two DSOs, the second DSO initialization will truncate the
262 * first profile file. */
263static const char *FileOpenMode = "w+b";
264/* This symbol is defined by the compiler when runtime counter relocation is
265 * used and runtime provides a weak alias so we can check if it's defined. */
266static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
267static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
268static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
269static void *BitmapBiasDefaultAddr =
270 &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
271static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
272 /* Get the sizes of various profile data sections. Taken from
273 * __llvm_profile_get_size_for_buffer(). */
274 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
275 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
276 const char *CountersBegin = __llvm_profile_begin_counters();
277 const char *CountersEnd = __llvm_profile_end_counters();
278 const char *BitmapBegin = __llvm_profile_begin_bitmap();
279 const char *BitmapEnd = __llvm_profile_end_bitmap();
280 uint64_t DataSize = __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
281 uint64_t CountersSize =
282 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
283 uint64_t NumBitmapBytes =
284 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
285 /* Get the file size. */
286 uint64_t FileSize = 0;
287 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &FileSize))
288 return 1;
289
290 int Fileno = fileno(stream: File);
291 uint64_t PaddingBytesAfterCounters =
292 __llvm_profile_get_num_padding_bytes(SizeInBytes: CountersSize);
293 uint64_t FileOffsetToCounters =
294 sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
295
296 /* Map the profile. */
297 char *Profile = (char *)mmap(NULL, len: FileSize, PROT_READ | PROT_WRITE,
298 MAP_SHARED, fd: Fileno, offset: 0);
299 if (Profile == MAP_FAILED) {
300 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
301 return 1;
302 }
303 /* Update the profile fields based on the current mapping. */
304 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
305 (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters;
306
307 /* Return the memory allocated for counters to OS. */
308 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)CountersBegin, End: (uintptr_t)CountersEnd);
309
310 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
311 * will fail with EINVAL. */
312 if (NumBitmapBytes == 0)
313 return 0;
314
315 /* Update profbm_bias. */
316 uint64_t FileOffsetToBitmap =
317 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
318 /* Update the profile fields based on the current mapping. */
319 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
320 (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
321
322 /* Return the memory allocated for counters to OS. */
323 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)BitmapBegin, End: (uintptr_t)BitmapEnd);
324 return 0;
325}
326#else
327static const int ContinuousModeSupported = 0;
328static const int UseBiasVar = 0;
329static const char *FileOpenMode = "a+b";
330static void *BiasAddr = NULL;
331static void *BiasDefaultAddr = NULL;
332static void *BitmapBiasAddr = NULL;
333static void *BitmapBiasDefaultAddr = NULL;
334static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
335 return 0;
336}
337#endif
338
339static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
340static void setProfileMergeRequested(int EnableMerge) {
341 ProfileMergeRequested = EnableMerge;
342}
343
344static FILE *ProfileFile = NULL;
345static FILE *getProfileFile(void) { return ProfileFile; }
346static void setProfileFile(FILE *File) { ProfileFile = File; }
347
348static int getCurFilenameLength(void);
349static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
350static unsigned doMerging(void) {
351 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
352}
353
354/* Return 1 if there is an error, otherwise return 0. */
355static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
356 uint32_t NumIOVecs) {
357 uint32_t I;
358 FILE *File = (FILE *)This->WriterCtx;
359 char Zeroes[sizeof(uint64_t)] = {0};
360 for (I = 0; I < NumIOVecs; I++) {
361 if (IOVecs[I].Data) {
362 if (fwrite(ptr: IOVecs[I].Data, size: IOVecs[I].ElmSize, n: IOVecs[I].NumElm, s: File) !=
363 IOVecs[I].NumElm)
364 return 1;
365 } else if (IOVecs[I].UseZeroPadding) {
366 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
367 while (BytesToWrite > 0) {
368 size_t PartialWriteLen =
369 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
370 if (fwrite(ptr: Zeroes, size: sizeof(uint8_t), n: PartialWriteLen, s: File) !=
371 PartialWriteLen) {
372 return 1;
373 }
374 BytesToWrite -= PartialWriteLen;
375 }
376 } else {
377 if (fseek(stream: File, off: IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
378 return 1;
379 }
380 }
381 return 0;
382}
383
384static void initFileWriter(ProfDataWriter *This, FILE *File) {
385 This->Write = fileWriter;
386 This->WriterCtx = File;
387}
388
389COMPILER_RT_VISIBILITY ProfBufferIO *
390lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
391 FreeHook = &free;
392 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: 1, size: BufferSz);
393 VPBufferSize = BufferSz;
394 ProfDataWriter *fileWriter =
395 (ProfDataWriter *)calloc(nmemb: 1, size: sizeof(ProfDataWriter));
396 initFileWriter(This: fileWriter, File);
397 ProfBufferIO *IO = lprofCreateBufferIO(FileWriter: fileWriter);
398 IO->OwnFileWriter = 1;
399 return IO;
400}
401
402static void setupIOBuffer(void) {
403 const char *BufferSzStr = 0;
404 BufferSzStr = getenv(name: "LLVM_VP_BUFFER_SIZE");
405 if (BufferSzStr && BufferSzStr[0]) {
406 VPBufferSize = atoi(nptr: BufferSzStr);
407 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: VPBufferSize, size: 1);
408 }
409}
410
411/* Get the size of the profile file. If there are any errors, print the
412 * message under the assumption that the profile is being read for merging
413 * purposes, and return -1. Otherwise return the file size in the inout param
414 * \p ProfileFileSize. */
415static int getProfileFileSizeForMerging(FILE *ProfileFile,
416 uint64_t *ProfileFileSize) {
417 if (fseek(stream: ProfileFile, off: 0L, SEEK_END) == -1) {
418 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
419 strerror(errno));
420 return -1;
421 }
422 *ProfileFileSize = ftell(stream: ProfileFile);
423
424 /* Restore file offset. */
425 if (fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
426 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
427 strerror(errno));
428 return -1;
429 }
430
431 if (*ProfileFileSize > 0 &&
432 *ProfileFileSize < sizeof(__llvm_profile_header)) {
433 PROF_WARN("Unable to merge profile data: %s\n",
434 "source profile file is too small.");
435 return -1;
436 }
437 return 0;
438}
439
440/* mmap() \p ProfileFile for profile merging purposes, assuming that an
441 * exclusive lock is held on the file and that \p ProfileFileSize is the
442 * length of the file. Return the mmap'd buffer in the inout variable
443 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
444 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
445static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
446 ManagedMemory *ProfileBuffer) {
447 lprofGetFileContentBuffer(F: ProfileFile, FileSize: ProfileFileSize, Buf: ProfileBuffer);
448
449 if (ProfileBuffer->Status == MS_INVALID) {
450 PROF_ERR("Unable to merge profile data: %s\n", "reading file failed");
451 return -1;
452 }
453
454 if (__llvm_profile_check_compatibility(Profile: ProfileBuffer->Addr,
455 Size: ProfileFileSize)) {
456 (void)lprofReleaseBuffer(FileBuffer: ProfileBuffer, Length: ProfileFileSize);
457 PROF_WARN("Unable to merge profile data: %s\n",
458 "source profile file is not compatible.");
459 return -1;
460 }
461 return 0;
462}
463
464/* Read profile data in \c ProfileFile and merge with in-memory
465 profile counters. Returns -1 if there is fatal error, otherwise
466 0 is returned. Returning 0 does not mean merge is actually
467 performed. If merge is actually done, *MergeDone is set to 1.
468*/
469static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
470 uint64_t ProfileFileSize;
471 ManagedMemory ProfileBuffer;
472
473 /* Get the size of the profile on disk. */
474 if (getProfileFileSizeForMerging(ProfileFile, ProfileFileSize: &ProfileFileSize) == -1)
475 return -1;
476
477 /* Nothing to merge. */
478 if (!ProfileFileSize)
479 return 0;
480
481 /* mmap() the profile and check that it is compatible with the data in
482 * the current image. */
483 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1)
484 return -1;
485
486 /* Now start merging */
487 if (__llvm_profile_merge_from_buffer(Profile: ProfileBuffer.Addr, Size: ProfileFileSize)) {
488 PROF_ERR("%s\n", "Invalid profile data to merge");
489 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
490 return -1;
491 }
492
493 // Truncate the file in case merging of value profile did not happen to
494 // prevent from leaving garbage data at the end of the profile file.
495 (void)COMPILER_RT_FTRUNCATE(ProfileFile,
496 __llvm_profile_get_size_for_buffer());
497
498 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
499 *MergeDone = 1;
500
501 return 0;
502}
503
504/* Create the directory holding the file, if needed. */
505static void createProfileDir(const char *Filename) {
506 size_t Length = strlen(s: Filename);
507 if (lprofFindFirstDirSeparator(Path: Filename)) {
508 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
509 strncpy(dest: Copy, src: Filename, n: Length + 1);
510 __llvm_profile_recursive_mkdir(Pathname: Copy);
511 }
512}
513
514/* Open the profile data for merging. It opens the file in r+b mode with
515 * file locking. If the file has content which is compatible with the
516 * current process, it also reads in the profile data in the file and merge
517 * it with in-memory counters. After the profile data is merged in memory,
518 * the original profile data is truncated and gets ready for the profile
519 * dumper. With profile merging enabled, each executable as well as any of
520 * its instrumented shared libraries dump profile data into their own data file.
521 */
522static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
523 FILE *ProfileFile = getProfileFile();
524 int rc;
525 // initializeProfileForContinuousMode will lock the profile, but if
526 // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed
527 // unlocked at this point.
528 if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) {
529 lprofLockFileHandle(F: ProfileFile);
530 }
531 if (!ProfileFile) {
532 createProfileDir(Filename: ProfileFileName);
533 ProfileFile = lprofOpenFileEx(Filename: ProfileFileName);
534 }
535 if (!ProfileFile)
536 return NULL;
537
538 rc = doProfileMerging(ProfileFile, MergeDone);
539 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
540 fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
541 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
542 strerror(errno));
543 fclose(stream: ProfileFile);
544 return NULL;
545 }
546 return ProfileFile;
547}
548
549static FILE *getFileObject(const char *OutputName) {
550 FILE *File;
551 File = getProfileFile();
552 if (File != NULL) {
553 return File;
554 }
555
556 return fopen(filename: OutputName, modes: "ab");
557}
558
559static void closeFileObject(FILE *OutputFile) {
560 if (OutputFile == getProfileFile()) {
561 fflush(stream: OutputFile);
562 if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) {
563 lprofUnlockFileHandle(F: OutputFile);
564 }
565 } else {
566 fclose(stream: OutputFile);
567 }
568}
569
570/* Write profile data to file \c OutputName. */
571static int writeFile(const char *OutputName) {
572 int RetVal;
573 FILE *OutputFile;
574
575 int MergeDone = 0;
576 VPMergeHook = &lprofMergeValueProfData;
577 if (doMerging())
578 OutputFile = openFileForMerging(ProfileFileName: OutputName, MergeDone: &MergeDone);
579 else
580 OutputFile = getFileObject(OutputName);
581
582 if (!OutputFile)
583 return -1;
584
585 FreeHook = &free;
586 setupIOBuffer();
587 ProfDataWriter fileWriter;
588 initFileWriter(This: &fileWriter, File: OutputFile);
589 RetVal = lprofWriteData(Writer: &fileWriter, VPDataReader: lprofGetVPDataReader(), SkipNameDataWrite: MergeDone);
590
591 closeFileObject(OutputFile);
592 return RetVal;
593}
594
595#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
596
597static void truncateCurrentFile(void) {
598 const char *Filename;
599 char *FilenameBuf;
600 FILE *File;
601 int Length;
602
603 Length = getCurFilenameLength();
604 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
605 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
606 if (!Filename)
607 return;
608
609 /* Only create the profile directory and truncate an existing profile once.
610 * In continuous mode, this is necessary, as the profile is written-to by the
611 * runtime initializer. */
612 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
613 if (initialized)
614 return;
615#if defined(_WIN32)
616 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
617#else
618 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, replace: 1);
619#endif
620
621 /* Create the profile dir (even if online merging is enabled), so that
622 * the profile file can be set up if continuous mode is enabled. */
623 createProfileDir(Filename);
624
625 /* By pass file truncation to allow online raw profile merging. */
626 if (lprofCurFilename.MergePoolSize)
627 return;
628
629 /* Truncate the file. Later we'll reopen and append. */
630 File = fopen(filename: Filename, modes: "w");
631 if (!File)
632 return;
633 fclose(stream: File);
634}
635
636/* Write a partial profile to \p Filename, which is required to be backed by
637 * the open file object \p File. */
638static int writeProfileWithFileObject(const char *Filename, FILE *File) {
639 setProfileFile(File);
640 int rc = writeFile(OutputName: Filename);
641 if (rc)
642 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
643 setProfileFile(NULL);
644 return rc;
645}
646
647static void initializeProfileForContinuousMode(void) {
648 if (!__llvm_profile_is_continuous_mode_enabled())
649 return;
650 if (!ContinuousModeSupported) {
651 PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
652 return;
653 }
654 if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
655 BitmapBiasAddr == BitmapBiasDefaultAddr) {
656 PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
657 "__llvm_profile_bitmap_bias is defined");
658 return;
659 }
660
661 /* Get the sizes of counter section. */
662 uint64_t CountersSize = __llvm_profile_get_counters_size(
663 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
664
665 int Length = getCurFilenameLength();
666 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
667 const char *Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
668 if (!Filename)
669 return;
670
671 FILE *File = NULL;
672 uint64_t CurrentFileOffset = 0;
673 if (doMerging()) {
674 /* We are merging profiles. Map the counter section as shared memory into
675 * the profile, i.e. into each participating process. An increment in one
676 * process should be visible to every other process with the same counter
677 * section mapped. */
678 File = lprofOpenFileEx(Filename);
679 if (!File)
680 return;
681
682 uint64_t ProfileFileSize = 0;
683 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
684 lprofUnlockFileHandle(F: File);
685 fclose(stream: File);
686 return;
687 }
688 if (ProfileFileSize == 0) {
689 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
690 * the file should stay open. */
691 if (writeProfileWithFileObject(Filename, File) != 0) {
692 lprofUnlockFileHandle(F: File);
693 fclose(stream: File);
694 return;
695 }
696 } else {
697 /* The merged profile has a non-zero length. Check that it is compatible
698 * with the data in this process. */
699 ManagedMemory ProfileBuffer;
700 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
701 lprofUnlockFileHandle(F: File);
702 fclose(stream: File);
703 return;
704 }
705 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
706 }
707 } else {
708 File = fopen(filename: Filename, modes: FileOpenMode);
709 if (!File)
710 return;
711 /* Check that the offset within the file is page-aligned. */
712 CurrentFileOffset = ftell(stream: File);
713 unsigned PageSize = getpagesize();
714 if (CurrentFileOffset % PageSize != 0) {
715 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
716 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
717 (uint64_t)CurrentFileOffset, PageSize);
718 fclose(stream: File);
719 return;
720 }
721 if (writeProfileWithFileObject(Filename, File) != 0) {
722 fclose(stream: File);
723 return;
724 }
725 }
726
727 /* mmap() the profile counters so long as there is at least one counter.
728 * If there aren't any counters, mmap() would fail with EINVAL. */
729 if (CountersSize > 0)
730 mmapForContinuousMode(CurrentFileOffset, File);
731
732 if (doMerging()) {
733 lprofUnlockFileHandle(F: File);
734 }
735 if (File != NULL) {
736 fclose(stream: File);
737 }
738}
739
740static const char *DefaultProfileName = "default.profraw";
741static void resetFilenameToDefault(void) {
742 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
743#ifdef __GNUC__
744#pragma GCC diagnostic push
745#pragma GCC diagnostic ignored "-Wcast-qual"
746#elif defined(__clang__)
747#pragma clang diagnostic push
748#pragma clang diagnostic ignored "-Wcast-qual"
749#endif
750 free(ptr: (void *)lprofCurFilename.FilenamePat);
751#ifdef __GNUC__
752#pragma GCC diagnostic pop
753#elif defined(__clang__)
754#pragma clang diagnostic pop
755#endif
756 }
757 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
758 lprofCurFilename.FilenamePat = DefaultProfileName;
759 lprofCurFilename.PNS = PNS_default;
760}
761
762static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
763 unsigned J = 0, Num = 0;
764 for (;; ++J) {
765 char C = FilenamePat[*I + J];
766 if (C == 'm') {
767 *I += J;
768 return Num ? Num : 1;
769 }
770 if (C < '0' || C > '9')
771 break;
772 Num = Num * 10 + C - '0';
773
774 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
775 * to be in-bound as the string is null terminated. */
776 }
777 return 0;
778}
779
780/* Assert that Idx does index past a string null terminator. Return the
781 * result of the check. */
782static int checkBounds(int Idx, int Strlen) {
783 assert(Idx <= Strlen && "Indexing past string null terminator");
784 return Idx <= Strlen;
785}
786
787/* Parses the pattern string \p FilenamePat and stores the result to
788 * lprofcurFilename structure. */
789static int parseFilenamePattern(const char *FilenamePat,
790 unsigned CopyFilenamePat) {
791 int NumPids = 0, NumHosts = 0, NumBinaryIds = 0, I;
792 char *PidChars = &lprofCurFilename.PidChars[0];
793 char *Hostname = &lprofCurFilename.Hostname[0];
794 int MergingEnabled = 0;
795 int FilenamePatLen = strlen(s: FilenamePat);
796
797#ifdef __GNUC__
798#pragma GCC diagnostic push
799#pragma GCC diagnostic ignored "-Wcast-qual"
800#elif defined(__clang__)
801#pragma clang diagnostic push
802#pragma clang diagnostic ignored "-Wcast-qual"
803#endif
804 /* Clean up cached prefix and filename. */
805 if (lprofCurFilename.ProfilePathPrefix)
806 free(ptr: (void *)lprofCurFilename.ProfilePathPrefix);
807
808 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
809 free(ptr: (void *)lprofCurFilename.FilenamePat);
810 }
811#ifdef __GNUC__
812#pragma GCC diagnostic pop
813#elif defined(__clang__)
814#pragma clang diagnostic pop
815#endif
816
817 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
818
819 if (!CopyFilenamePat)
820 lprofCurFilename.FilenamePat = FilenamePat;
821 else {
822 lprofCurFilename.FilenamePat = strdup(s: FilenamePat);
823 lprofCurFilename.OwnsFilenamePat = 1;
824 }
825 /* Check the filename for "%p", which indicates a pid-substitution. */
826 for (I = 0; checkBounds(Idx: I, Strlen: FilenamePatLen) && FilenamePat[I]; ++I) {
827 if (FilenamePat[I] == '%') {
828 ++I; /* Advance to the next character. */
829 if (!checkBounds(Idx: I, Strlen: FilenamePatLen))
830 break;
831 if (FilenamePat[I] == 'p') {
832 if (!NumPids++) {
833 if (snprintf(s: PidChars, MAX_PID_SIZE, format: "%ld", (long)getpid()) <= 0) {
834 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
835 "default name.",
836 FilenamePat);
837 return -1;
838 }
839 }
840 } else if (FilenamePat[I] == 'h') {
841 if (!NumHosts++)
842 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
843 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
844 "the default name.",
845 FilenamePat);
846 return -1;
847 }
848 } else if (FilenamePat[I] == 't') {
849 lprofCurFilename.TmpDir = getenv(name: "TMPDIR");
850 if (!lprofCurFilename.TmpDir) {
851 PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
852 "in %s. Using the default path.",
853 FilenamePat);
854 return -1;
855 }
856 } else if (FilenamePat[I] == 'b') {
857 if (!NumBinaryIds++) {
858 /* Check if binary ID does not exist or if its size is 0. */
859 if (__llvm_write_binary_ids(NULL) <= 0) {
860 PROF_WARN("Unable to get binary ID for filename pattern %s. Using "
861 "the default name.",
862 FilenamePat);
863 return -1;
864 }
865 }
866 } else if (FilenamePat[I] == 'c') {
867 if (__llvm_profile_is_continuous_mode_enabled()) {
868 PROF_WARN("%%c specifier can only be specified once in %s.\n",
869 FilenamePat);
870 __llvm_profile_disable_continuous_mode();
871 return -1;
872 }
873#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX)
874 __llvm_profile_set_page_size(PageSize: getpagesize());
875 __llvm_profile_enable_continuous_mode();
876#else
877 PROF_WARN("%s",
878 "Continuous mode is currently only supported for Mach-O,"
879 " ELF and COFF formats.");
880 return -1;
881#endif
882 } else {
883 unsigned MergePoolSize = getMergePoolSize(FilenamePat, I: &I);
884 if (!MergePoolSize)
885 continue;
886 if (MergingEnabled) {
887 PROF_WARN("%%m specifier can only be specified once in %s.\n",
888 FilenamePat);
889 return -1;
890 }
891 MergingEnabled = 1;
892 lprofCurFilename.MergePoolSize = MergePoolSize;
893 }
894 }
895 }
896
897 lprofCurFilename.NumPids = NumPids;
898 lprofCurFilename.NumHosts = NumHosts;
899 lprofCurFilename.NumBinaryIds = NumBinaryIds;
900 return 0;
901}
902
903static void parseAndSetFilename(const char *FilenamePat,
904 ProfileNameSpecifier PNS,
905 unsigned CopyFilenamePat) {
906
907 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
908 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
909
910 /* The old profile name specifier takes precedence over the old one. */
911 if (PNS < OldPNS)
912 return;
913
914 if (!FilenamePat)
915 FilenamePat = DefaultProfileName;
916
917 if (OldFilenamePat && !strcmp(s1: OldFilenamePat, s2: FilenamePat)) {
918 lprofCurFilename.PNS = PNS;
919 return;
920 }
921
922 /* When PNS >= OldPNS, the last one wins. */
923 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
924 resetFilenameToDefault();
925 lprofCurFilename.PNS = PNS;
926
927 if (!OldFilenamePat) {
928 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
929 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
930 lprofCurFilename.FilenamePat, getPNSStr(PNS));
931 } else {
932 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
933 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
934 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
935 getPNSStr(PNS));
936 }
937
938 truncateCurrentFile();
939 if (__llvm_profile_is_continuous_mode_enabled())
940 initializeProfileForContinuousMode();
941}
942
943/* Return buffer length that is required to store the current profile
944 * filename with PID and hostname substitutions. */
945/* The length to hold uint64_t followed by 3 digits pool id including '_' */
946#define SIGLEN 24
947/* The length to hold 160-bit hash in hexadecimal form */
948#define BINARY_ID_LEN 40
949static int getCurFilenameLength(void) {
950 int Len;
951 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
952 return 0;
953
954 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
955 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
956 lprofCurFilename.MergePoolSize))
957 return strlen(s: lprofCurFilename.FilenamePat);
958
959 Len = strlen(s: lprofCurFilename.FilenamePat) +
960 lprofCurFilename.NumPids * (strlen(s: lprofCurFilename.PidChars) - 2) +
961 lprofCurFilename.NumHosts * (strlen(s: lprofCurFilename.Hostname) - 2) +
962 lprofCurFilename.NumBinaryIds * BINARY_ID_LEN +
963 (lprofCurFilename.TmpDir ? (strlen(s: lprofCurFilename.TmpDir) - 1) : 0);
964 if (lprofCurFilename.MergePoolSize)
965 Len += SIGLEN;
966 return Len;
967}
968
969typedef struct lprofBinaryIdsBuffer {
970 char String[BINARY_ID_LEN + 1];
971 int Length;
972} lprofBinaryIdsBuffer;
973
974/* Reads binary ID length and then its data, writes it into lprofBinaryIdsBuffer
975 * in hexadecimal form. */
976static uint32_t binaryIdsStringWriter(ProfDataWriter *This,
977 ProfDataIOVec *IOVecs,
978 uint32_t NumIOVecs) {
979 if (NumIOVecs < 2 || IOVecs[0].ElmSize != sizeof(uint64_t))
980 return -1;
981 uint64_t BinaryIdLen = *(const uint64_t *)IOVecs[0].Data;
982 if (IOVecs[1].ElmSize != sizeof(uint8_t) || IOVecs[1].NumElm != BinaryIdLen)
983 return -1;
984 const uint8_t *BinaryIdData = (const uint8_t *)IOVecs[1].Data;
985 lprofBinaryIdsBuffer *Data = (lprofBinaryIdsBuffer *)This->WriterCtx;
986 for (uint64_t I = 0; I < BinaryIdLen; I++) {
987 Data->Length +=
988 snprintf(s: Data->String + Data->Length, BINARY_ID_LEN + 1 - Data->Length,
989 format: "%02hhx", BinaryIdData[I]);
990 }
991 return 0;
992}
993
994/* Return the pointer to the current profile file name (after substituting
995 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
996 * to store the resulting filename. If no substitution is needed, the
997 * current filename pattern string is directly returned, unless ForceUseBuf
998 * is enabled. */
999static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
1000 int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
1001 const char *FilenamePat = lprofCurFilename.FilenamePat;
1002
1003 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
1004 return 0;
1005
1006 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
1007 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
1008 lprofCurFilename.MergePoolSize ||
1009 __llvm_profile_is_continuous_mode_enabled())) {
1010 if (!ForceUseBuf)
1011 return lprofCurFilename.FilenamePat;
1012
1013 FilenamePatLength = strlen(s: lprofCurFilename.FilenamePat);
1014 memcpy(dest: FilenameBuf, src: lprofCurFilename.FilenamePat, n: FilenamePatLength);
1015 FilenameBuf[FilenamePatLength] = '\0';
1016 return FilenameBuf;
1017 }
1018
1019 PidLength = strlen(s: lprofCurFilename.PidChars);
1020 HostNameLength = strlen(s: lprofCurFilename.Hostname);
1021 TmpDirLength = lprofCurFilename.TmpDir ? strlen(s: lprofCurFilename.TmpDir) : 0;
1022 /* Construct the new filename. */
1023 for (I = 0, J = 0; FilenamePat[I]; ++I)
1024 if (FilenamePat[I] == '%') {
1025 if (FilenamePat[++I] == 'p') {
1026 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.PidChars, n: PidLength);
1027 J += PidLength;
1028 } else if (FilenamePat[I] == 'h') {
1029 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.Hostname, n: HostNameLength);
1030 J += HostNameLength;
1031 } else if (FilenamePat[I] == 't') {
1032 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.TmpDir, n: TmpDirLength);
1033 FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
1034 J += TmpDirLength + 1;
1035 } else if (FilenamePat[I] == 'b') {
1036 lprofBinaryIdsBuffer Data = {{0}, 0};
1037 ProfDataWriter Writer = {binaryIdsStringWriter, &Data};
1038 __llvm_write_binary_ids(Writer: &Writer);
1039 memcpy(dest: FilenameBuf + J, src: Data.String, n: Data.Length);
1040 J += Data.Length;
1041 } else {
1042 if (!getMergePoolSize(FilenamePat, I: &I))
1043 continue;
1044 char LoadModuleSignature[SIGLEN + 1];
1045 int S;
1046 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
1047 S = snprintf(s: LoadModuleSignature, SIGLEN + 1, format: "%" PRIu64 "_%d",
1048 lprofGetLoadModuleSignature(), ProfilePoolId);
1049 if (S == -1 || S > SIGLEN)
1050 S = SIGLEN;
1051 memcpy(dest: FilenameBuf + J, src: LoadModuleSignature, n: S);
1052 J += S;
1053 }
1054 /* Drop any unknown substitutions. */
1055 } else
1056 FilenameBuf[J++] = FilenamePat[I];
1057 FilenameBuf[J] = 0;
1058
1059 return FilenameBuf;
1060}
1061
1062/* Returns the pointer to the environment variable
1063 * string. Returns null if the env var is not set. */
1064static const char *getFilenamePatFromEnv(void) {
1065 const char *Filename = getenv(name: "LLVM_PROFILE_FILE");
1066 if (!Filename || !Filename[0])
1067 return 0;
1068 return Filename;
1069}
1070
1071COMPILER_RT_VISIBILITY
1072const char *__llvm_profile_get_path_prefix(void) {
1073 int Length;
1074 char *FilenameBuf, *Prefix;
1075 const char *Filename, *PrefixEnd;
1076
1077 if (lprofCurFilename.ProfilePathPrefix)
1078 return lprofCurFilename.ProfilePathPrefix;
1079
1080 Length = getCurFilenameLength();
1081 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1082 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1083 if (!Filename)
1084 return "\0";
1085
1086 PrefixEnd = lprofFindLastDirSeparator(Path: Filename);
1087 if (!PrefixEnd)
1088 return "\0";
1089
1090 Length = PrefixEnd - Filename + 1;
1091 Prefix = (char *)malloc(size: Length + 1);
1092 if (!Prefix) {
1093 PROF_ERR("Failed to %s\n", "allocate memory.");
1094 return "\0";
1095 }
1096 memcpy(dest: Prefix, src: Filename, n: Length);
1097 Prefix[Length] = '\0';
1098 lprofCurFilename.ProfilePathPrefix = Prefix;
1099 return Prefix;
1100}
1101
1102COMPILER_RT_VISIBILITY
1103const char *__llvm_profile_get_filename(void) {
1104 int Length;
1105 char *FilenameBuf;
1106 const char *Filename;
1107
1108 Length = getCurFilenameLength();
1109 FilenameBuf = (char *)malloc(size: Length + 1);
1110 if (!FilenameBuf) {
1111 PROF_ERR("Failed to %s\n", "allocate memory.");
1112 return "\0";
1113 }
1114 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 1);
1115 if (!Filename) {
1116 free(ptr: FilenameBuf);
1117 return "\0";
1118 }
1119
1120 return FilenameBuf;
1121}
1122
1123/* This API initializes the file handling, both user specified
1124 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
1125 * environment variable can override this default value.
1126 */
1127COMPILER_RT_VISIBILITY
1128void __llvm_profile_initialize_file(void) {
1129 const char *EnvFilenamePat;
1130 const char *SelectedPat = NULL;
1131 ProfileNameSpecifier PNS = PNS_unknown;
1132 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
1133
1134 EnvFilenamePat = getFilenamePatFromEnv();
1135 if (EnvFilenamePat) {
1136 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
1137 at the moment when __llvm_profile_write_file() gets executed. */
1138 parseAndSetFilename(FilenamePat: EnvFilenamePat, PNS: PNS_environment, CopyFilenamePat: 1);
1139 return;
1140 } else if (hasCommandLineOverrider) {
1141 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
1142 PNS = PNS_command_line;
1143 } else {
1144 SelectedPat = NULL;
1145 PNS = PNS_default;
1146 }
1147
1148 parseAndSetFilename(FilenamePat: SelectedPat, PNS, CopyFilenamePat: 0);
1149}
1150
1151/* This method is invoked by the runtime initialization hook
1152 * InstrProfilingRuntime.o if it is linked in.
1153 */
1154COMPILER_RT_VISIBILITY
1155void __llvm_profile_initialize(void) {
1156 __llvm_profile_initialize_file();
1157 if (!__llvm_profile_is_continuous_mode_enabled())
1158 __llvm_profile_register_write_file_atexit();
1159}
1160
1161/* This API is directly called by the user application code. It has the
1162 * highest precedence compared with LLVM_PROFILE_FILE environment variable
1163 * and command line option -fprofile-instr-generate=<profile_name>.
1164 */
1165COMPILER_RT_VISIBILITY
1166void __llvm_profile_set_filename(const char *FilenamePat) {
1167 if (__llvm_profile_is_continuous_mode_enabled())
1168 return;
1169 parseAndSetFilename(FilenamePat, PNS: PNS_runtime_api, CopyFilenamePat: 1);
1170}
1171
1172/* The public API for writing profile data into the file with name
1173 * set by previous calls to __llvm_profile_set_filename or
1174 * __llvm_profile_override_default_filename or
1175 * __llvm_profile_initialize_file. */
1176COMPILER_RT_VISIBILITY
1177int __llvm_profile_write_file(void) {
1178 int rc, Length;
1179 const char *Filename;
1180 char *FilenameBuf;
1181
1182 // Temporarily suspend getting SIGKILL when the parent exits.
1183 int PDeathSig = lprofSuspendSigKill();
1184
1185 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1186 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1187 if (PDeathSig == 1)
1188 lprofRestoreSigKill();
1189 return 0;
1190 }
1191
1192 Length = getCurFilenameLength();
1193 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1194 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1195
1196 /* Check the filename. */
1197 if (!Filename) {
1198 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1199 if (PDeathSig == 1)
1200 lprofRestoreSigKill();
1201 return -1;
1202 }
1203
1204 /* Check if there is llvm/runtime version mismatch. */
1205 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1206 PROF_ERR("Runtime and instrumentation version mismatch : "
1207 "expected %d, but get %d\n",
1208 INSTR_PROF_RAW_VERSION,
1209 (int)GET_VERSION(__llvm_profile_get_version()));
1210 if (PDeathSig == 1)
1211 lprofRestoreSigKill();
1212 return -1;
1213 }
1214
1215 /* Write profile data to the file. */
1216 rc = writeFile(OutputName: Filename);
1217 if (rc)
1218 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1219
1220 /* On non-Windows the declaration is weak: only invoked when
1221 * InstrProfilingPlatformROCm.o is in the link, which happens when the program
1222 * references other ROCm-runtime symbols (HIP-with-PGO). Warning on failure is
1223 * handled inside the callee. */
1224#if COMPILER_RT_BUILD_PROFILE_ROCM
1225#if defined(_WIN32)
1226 (void)__llvm_profile_hip_collect_device_data();
1227#else
1228 if (&__llvm_profile_hip_collect_device_data)
1229 (void)__llvm_profile_hip_collect_device_data();
1230#endif
1231#endif
1232
1233 // Restore SIGKILL.
1234 if (PDeathSig == 1)
1235 lprofRestoreSigKill();
1236
1237 return rc;
1238}
1239
1240COMPILER_RT_VISIBILITY
1241int __llvm_profile_dump(void) {
1242 if (!doMerging())
1243 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1244 " of previously dumped profile data : %s. Either use %%m "
1245 "in profile name or change profile name before dumping.\n",
1246 "online profile merging is not on");
1247 int rc = __llvm_profile_write_file();
1248 lprofSetProfileDumped(1);
1249 return rc;
1250}
1251
1252static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1253
1254COMPILER_RT_VISIBILITY
1255int __llvm_profile_register_write_file_atexit(void) {
1256 static int HasBeenRegistered = 0;
1257
1258 if (HasBeenRegistered)
1259 return 0;
1260
1261 lprofSetupValueProfiler();
1262
1263 HasBeenRegistered = 1;
1264 return lprofAtExit(writeFileWithoutReturn);
1265}
1266
1267COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
1268 int EnableMerge) {
1269 if (__llvm_profile_is_continuous_mode_enabled()) {
1270 if (!EnableMerge) {
1271 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
1272 "continuous sync mode when merging is disabled\n",
1273 fileno(File));
1274 return 1;
1275 }
1276 if (lprofLockFileHandle(F: File) != 0) {
1277 PROF_WARN("Data may be corrupted during profile merging : %s\n",
1278 "Fail to obtain file lock due to system limit.");
1279 }
1280 uint64_t ProfileFileSize = 0;
1281 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
1282 lprofUnlockFileHandle(F: File);
1283 return 1;
1284 }
1285 if (ProfileFileSize == 0) {
1286 FreeHook = &free;
1287 setupIOBuffer();
1288 ProfDataWriter fileWriter;
1289 initFileWriter(This: &fileWriter, File);
1290 if (lprofWriteData(Writer: &fileWriter, VPDataReader: 0, SkipNameDataWrite: 0)) {
1291 lprofUnlockFileHandle(F: File);
1292 PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
1293 strerror(errno));
1294 return 1;
1295 }
1296 fflush(stream: File);
1297 } else {
1298 /* The merged profile has a non-zero length. Check that it is compatible
1299 * with the data in this process. */
1300 ManagedMemory ProfileBuffer;
1301 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
1302 lprofUnlockFileHandle(F: File);
1303 return 1;
1304 }
1305 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
1306 }
1307 mmapForContinuousMode(CurrentFileOffset: 0, File);
1308 lprofUnlockFileHandle(F: File);
1309 } else {
1310 setProfileFile(File);
1311 setProfileMergeRequested(EnableMerge);
1312 }
1313 return 0;
1314}
1315
1316#ifndef __APPLE__
1317int __llvm_write_custom_profile(
1318 const char *Target, const __llvm_profile_data *DataBegin,
1319 const __llvm_profile_data *DataEnd, const char *CountersBegin,
1320 const char *CountersEnd, const char *UniformCountersBegin,
1321 const char *UniformCountersEnd, const char *NamesBegin,
1322 const char *NamesEnd, const uint64_t *VersionOverride) {
1323 int ReturnValue = 0, FilenameLength, TargetLength;
1324 char *FilenameBuf, *TargetFilename;
1325 const char *Filename;
1326
1327 /* Save old profile data */
1328 FILE *oldFile = getProfileFile();
1329
1330 // Temporarily suspend getting SIGKILL when the parent exits.
1331 int PDeathSig = lprofSuspendSigKill();
1332
1333 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1334 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1335 if (PDeathSig == 1)
1336 lprofRestoreSigKill();
1337 return 0;
1338 }
1339
1340 /* Check if there is llvm/runtime version mismatch. */
1341 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1342 PROF_ERR("Runtime and instrumentation version mismatch : "
1343 "expected %d, but get %d\n",
1344 INSTR_PROF_RAW_VERSION,
1345 (int)GET_VERSION(__llvm_profile_get_version()));
1346 if (PDeathSig == 1)
1347 lprofRestoreSigKill();
1348 return -1;
1349 }
1350
1351 /* Get current filename */
1352 FilenameLength = getCurFilenameLength();
1353 FilenameBuf = (char *)COMPILER_RT_ALLOCA(FilenameLength + 1);
1354 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1355
1356 /* Check the filename. */
1357 if (!Filename) {
1358 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1359 if (PDeathSig == 1)
1360 lprofRestoreSigKill();
1361 return -1;
1362 }
1363
1364 /* Allocate new space for our target-specific PGO filename */
1365 TargetLength = strlen(s: Target);
1366 TargetFilename =
1367 (char *)COMPILER_RT_ALLOCA(FilenameLength + TargetLength + 2);
1368
1369 /* Find file basename and path sizes */
1370 int32_t DirEnd = FilenameLength - 1;
1371 while (DirEnd >= 0 && !IS_DIR_SEPARATOR(Filename[DirEnd])) {
1372 DirEnd--;
1373 }
1374 uint32_t DirSize = DirEnd + 1, BaseSize = FilenameLength - DirSize;
1375
1376 /* Prepend "TARGET." to current filename */
1377 if (DirSize > 0) {
1378 memcpy(dest: TargetFilename, src: Filename, n: DirSize);
1379 }
1380 memcpy(dest: TargetFilename + DirSize, src: Target, n: TargetLength);
1381 TargetFilename[TargetLength + DirSize] = '.';
1382 memcpy(dest: TargetFilename + DirSize + 1 + TargetLength, src: Filename + DirSize,
1383 n: BaseSize);
1384 TargetFilename[FilenameLength + 1 + TargetLength] = 0;
1385
1386 /* Open and truncate target-specific PGO file */
1387 FILE *OutputFile = fopen(filename: TargetFilename, modes: "wb");
1388 setProfileFile(OutputFile);
1389
1390 if (!OutputFile) {
1391 PROF_ERR("Failed to open file : %s\n", TargetFilename);
1392 if (PDeathSig == 1)
1393 lprofRestoreSigKill();
1394 return -1;
1395 }
1396
1397 FreeHook = &free;
1398 setupIOBuffer();
1399
1400 /* Write custom data */
1401 ProfDataWriter fileWriter;
1402 initFileWriter(This: &fileWriter, File: OutputFile);
1403
1404 uint64_t Version = __llvm_profile_get_version();
1405 if (VersionOverride)
1406 Version = *VersionOverride;
1407
1408 /* Write custom data to the file */
1409 ReturnValue = lprofWriteDataImpl(
1410 Writer: &fileWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, NULL, NULL,
1411 UniformCountersBegin, UniformCountersEnd, VPDataReader: lprofGetVPDataReader(),
1412 NamesBegin, NamesEnd, NULL, NULL, NULL, NULL, SkipNameDataWrite: 0, Version);
1413 closeFileObject(OutputFile);
1414
1415 // Restore SIGKILL.
1416 if (PDeathSig == 1)
1417 lprofRestoreSigKill();
1418
1419 /* Restore old profiling file */
1420 setProfileFile(oldFile);
1421
1422 return ReturnValue;
1423}
1424#endif
1425
1426#endif
1427