1/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9#if !defined(__Fuchsia__)
10
11#include <assert.h>
12#include <errno.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <string.h>
16#ifdef _MSC_VER
17/* For _alloca. */
18#include <malloc.h>
19#endif
20#if defined(_WIN32)
21#include "WindowsMMap.h"
22/* For _chsize_s */
23#include <io.h>
24#include <process.h>
25#else
26#include <sys/file.h>
27#include <sys/mman.h>
28#include <unistd.h>
29#if defined(__linux__)
30#include <sys/types.h>
31#endif
32#endif
33
34#include "InstrProfiling.h"
35#include "InstrProfilingInternal.h"
36#include "InstrProfilingPort.h"
37#include "InstrProfilingUtil.h"
38
39/* From where is profile name specified.
40 * The order the enumerators define their
41 * precedence. Re-order them may lead to
42 * runtime behavior change. */
43typedef enum ProfileNameSpecifier {
44 PNS_unknown = 0,
45 PNS_default,
46 PNS_command_line,
47 PNS_environment,
48 PNS_runtime_api
49} ProfileNameSpecifier;
50
51static const char *getPNSStr(ProfileNameSpecifier PNS) {
52 switch (PNS) {
53 case PNS_default:
54 return "default setting";
55 case PNS_command_line:
56 return "command line";
57 case PNS_environment:
58 return "environment variable";
59 case PNS_runtime_api:
60 return "runtime API";
61 default:
62 return "Unknown";
63 }
64}
65
66#define MAX_PID_SIZE 16
67/* Data structure holding the result of parsed filename pattern. */
68typedef struct lprofFilename {
69 /* File name string possibly with %p or %h specifiers. */
70 const char *FilenamePat;
71 /* A flag indicating if FilenamePat's memory is allocated
72 * by runtime. */
73 unsigned OwnsFilenamePat;
74 const char *ProfilePathPrefix;
75 char PidChars[MAX_PID_SIZE];
76 char *TmpDir;
77 char Hostname[COMPILER_RT_MAX_HOSTLEN];
78 unsigned NumPids;
79 unsigned NumHosts;
80 unsigned NumBinaryIds;
81 /* When in-process merging is enabled, this parameter specifies
82 * the total number of profile data files shared by all the processes
83 * spawned from the same binary. By default the value is 1. If merging
84 * is not enabled, its value should be 0. This parameter is specified
85 * by the %[0-9]m specifier. For instance %2m enables merging using
86 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
87 * can only appear once at the end of the name pattern. */
88 unsigned MergePoolSize;
89 ProfileNameSpecifier PNS;
90} lprofFilename;
91
92static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, {0},
93 0, 0, 0, 0, PNS_unknown};
94
95static int ProfileMergeRequested = 0;
96static int getProfileFileSizeForMerging(FILE *ProfileFile,
97 uint64_t *ProfileFileSize);
98
99#if defined(__APPLE__)
100static const int ContinuousModeSupported = 1;
101static const int UseBiasVar = 0;
102static const char *FileOpenMode = "a+b";
103static void *BiasAddr = NULL;
104static void *BiasDefaultAddr = NULL;
105static void *BitmapBiasAddr = NULL;
106static void *BitmapBiasDefaultAddr = NULL;
107static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
108 /* Get the sizes of various profile data sections. Taken from
109 * __llvm_profile_get_size_for_buffer(). */
110 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
111 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
112 const char *CountersBegin = __llvm_profile_begin_counters();
113 const char *CountersEnd = __llvm_profile_end_counters();
114 const char *BitmapBegin = __llvm_profile_begin_bitmap();
115 const char *BitmapEnd = __llvm_profile_end_bitmap();
116 const char *NamesBegin = __llvm_profile_begin_names();
117 const char *NamesEnd = __llvm_profile_end_names();
118 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
119 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
120 uint64_t CountersSize =
121 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
122 uint64_t NumBitmapBytes =
123 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
124
125 /* Check that the counter, bitmap, and data sections in this image are
126 * page-aligned. */
127 unsigned PageSize = getpagesize();
128 if ((intptr_t)CountersBegin % PageSize != 0) {
129 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
130 CountersBegin, PageSize);
131 return 1;
132 }
133 if ((intptr_t)BitmapBegin % PageSize != 0) {
134 PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n",
135 BitmapBegin, PageSize);
136 return 1;
137 }
138 if ((intptr_t)DataBegin % PageSize != 0) {
139 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
140 DataBegin, PageSize);
141 return 1;
142 }
143
144 int Fileno = fileno(File);
145 /* Determine how much padding is needed before/after the counters and
146 * after the names. */
147 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
148 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
149 PaddingBytesAfterVTable, PaddingBytesAfterVNames;
150 __llvm_profile_get_padding_sizes_for_counters(
151 DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0,
152 /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
153 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames,
154 &PaddingBytesAfterVTable, &PaddingBytesAfterVNames);
155
156 uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
157 uint64_t FileOffsetToCounters = CurrentFileOffset +
158 sizeof(__llvm_profile_header) + DataSize +
159 PaddingBytesBeforeCounters;
160 void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
161 PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
162 Fileno, FileOffsetToCounters);
163 if (CounterMmap != CountersBegin) {
164 PROF_ERR(
165 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
166 " - CountersBegin: %p\n"
167 " - PageAlignedCountersLength: %" PRIu64 "\n"
168 " - Fileno: %d\n"
169 " - FileOffsetToCounters: %" PRIu64 "\n",
170 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
171 FileOffsetToCounters);
172 return 1;
173 }
174
175 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
176 * will fail with EINVAL. */
177 if (NumBitmapBytes == 0)
178 return 0;
179
180 uint64_t PageAlignedBitmapLength =
181 NumBitmapBytes + PaddingBytesAfterBitmapBytes;
182 uint64_t FileOffsetToBitmap =
183 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
184 void *BitmapMmap =
185 mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE,
186 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap);
187 if (BitmapMmap != BitmapBegin) {
188 PROF_ERR(
189 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
190 " - BitmapBegin: %p\n"
191 " - PageAlignedBitmapLength: %" PRIu64 "\n"
192 " - Fileno: %d\n"
193 " - FileOffsetToBitmap: %" PRIu64 "\n",
194 strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno,
195 FileOffsetToBitmap);
196 return 1;
197 }
198 return 0;
199}
200#elif defined(__ELF__) || defined(_WIN32) || defined(_AIX)
201
202#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
203 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
204COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
205#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
206 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
207COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
208
209/* This variable is a weak external reference which could be used to detect
210 * whether or not the compiler defined this symbol. */
211#if defined(_MSC_VER)
212COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
213COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
214#if defined(_M_IX86) || defined(__i386__)
215#define WIN_SYM_PREFIX "_"
216#else
217#define WIN_SYM_PREFIX
218#endif
219#pragma comment( \
220 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
221 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
222 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
223#pragma comment( \
224 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
225 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
226 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
227#else
228COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
229 __attribute__((weak, alias(INSTR_PROF_QUOTE(
230 INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
231COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
232 __attribute__((weak, alias(INSTR_PROF_QUOTE(
233 INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
234#endif
235static const int ContinuousModeSupported = 1;
236static const int UseBiasVar = 1;
237/* TODO: If there are two DSOs, the second DSO initialization will truncate the
238 * first profile file. */
239static const char *FileOpenMode = "w+b";
240/* This symbol is defined by the compiler when runtime counter relocation is
241 * used and runtime provides a weak alias so we can check if it's defined. */
242static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
243static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
244static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
245static void *BitmapBiasDefaultAddr =
246 &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
247static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
248 /* Get the sizes of various profile data sections. Taken from
249 * __llvm_profile_get_size_for_buffer(). */
250 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
251 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
252 const char *CountersBegin = __llvm_profile_begin_counters();
253 const char *CountersEnd = __llvm_profile_end_counters();
254 const char *BitmapBegin = __llvm_profile_begin_bitmap();
255 const char *BitmapEnd = __llvm_profile_end_bitmap();
256 uint64_t DataSize = __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
257 uint64_t CountersSize =
258 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
259 uint64_t NumBitmapBytes =
260 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
261 /* Get the file size. */
262 uint64_t FileSize = 0;
263 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &FileSize))
264 return 1;
265
266 int Fileno = fileno(stream: File);
267 uint64_t PaddingBytesAfterCounters =
268 __llvm_profile_get_num_padding_bytes(SizeInBytes: CountersSize);
269 uint64_t FileOffsetToCounters =
270 sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
271
272 /* Map the profile. */
273 char *Profile = (char *)mmap(NULL, len: FileSize, PROT_READ | PROT_WRITE,
274 MAP_SHARED, fd: Fileno, offset: 0);
275 if (Profile == MAP_FAILED) {
276 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
277 return 1;
278 }
279 /* Update the profile fields based on the current mapping. */
280 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
281 (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters;
282
283 /* Return the memory allocated for counters to OS. */
284 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)CountersBegin, End: (uintptr_t)CountersEnd);
285
286 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
287 * will fail with EINVAL. */
288 if (NumBitmapBytes == 0)
289 return 0;
290
291 /* Update profbm_bias. */
292 uint64_t FileOffsetToBitmap =
293 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
294 /* Update the profile fields based on the current mapping. */
295 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
296 (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
297
298 /* Return the memory allocated for counters to OS. */
299 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)BitmapBegin, End: (uintptr_t)BitmapEnd);
300 return 0;
301}
302#else
303static const int ContinuousModeSupported = 0;
304static const int UseBiasVar = 0;
305static const char *FileOpenMode = "a+b";
306static void *BiasAddr = NULL;
307static void *BiasDefaultAddr = NULL;
308static void *BitmapBiasAddr = NULL;
309static void *BitmapBiasDefaultAddr = NULL;
310static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
311 return 0;
312}
313#endif
314
315static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
316static void setProfileMergeRequested(int EnableMerge) {
317 ProfileMergeRequested = EnableMerge;
318}
319
320static FILE *ProfileFile = NULL;
321static FILE *getProfileFile(void) { return ProfileFile; }
322static void setProfileFile(FILE *File) { ProfileFile = File; }
323
324static int getCurFilenameLength(void);
325static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
326static unsigned doMerging(void) {
327 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
328}
329
330/* Return 1 if there is an error, otherwise return 0. */
331static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
332 uint32_t NumIOVecs) {
333 uint32_t I;
334 FILE *File = (FILE *)This->WriterCtx;
335 char Zeroes[sizeof(uint64_t)] = {0};
336 for (I = 0; I < NumIOVecs; I++) {
337 if (IOVecs[I].Data) {
338 if (fwrite(ptr: IOVecs[I].Data, size: IOVecs[I].ElmSize, n: IOVecs[I].NumElm, s: File) !=
339 IOVecs[I].NumElm)
340 return 1;
341 } else if (IOVecs[I].UseZeroPadding) {
342 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
343 while (BytesToWrite > 0) {
344 size_t PartialWriteLen =
345 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
346 if (fwrite(ptr: Zeroes, size: sizeof(uint8_t), n: PartialWriteLen, s: File) !=
347 PartialWriteLen) {
348 return 1;
349 }
350 BytesToWrite -= PartialWriteLen;
351 }
352 } else {
353 if (fseek(stream: File, off: IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
354 return 1;
355 }
356 }
357 return 0;
358}
359
360static void initFileWriter(ProfDataWriter *This, FILE *File) {
361 This->Write = fileWriter;
362 This->WriterCtx = File;
363}
364
365COMPILER_RT_VISIBILITY ProfBufferIO *
366lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
367 FreeHook = &free;
368 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: 1, size: BufferSz);
369 VPBufferSize = BufferSz;
370 ProfDataWriter *fileWriter =
371 (ProfDataWriter *)calloc(nmemb: 1, size: sizeof(ProfDataWriter));
372 initFileWriter(This: fileWriter, File);
373 ProfBufferIO *IO = lprofCreateBufferIO(FileWriter: fileWriter);
374 IO->OwnFileWriter = 1;
375 return IO;
376}
377
378static void setupIOBuffer(void) {
379 const char *BufferSzStr = 0;
380 BufferSzStr = getenv(name: "LLVM_VP_BUFFER_SIZE");
381 if (BufferSzStr && BufferSzStr[0]) {
382 VPBufferSize = atoi(nptr: BufferSzStr);
383 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: VPBufferSize, size: 1);
384 }
385}
386
387/* Get the size of the profile file. If there are any errors, print the
388 * message under the assumption that the profile is being read for merging
389 * purposes, and return -1. Otherwise return the file size in the inout param
390 * \p ProfileFileSize. */
391static int getProfileFileSizeForMerging(FILE *ProfileFile,
392 uint64_t *ProfileFileSize) {
393 if (fseek(stream: ProfileFile, off: 0L, SEEK_END) == -1) {
394 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
395 strerror(errno));
396 return -1;
397 }
398 *ProfileFileSize = ftell(stream: ProfileFile);
399
400 /* Restore file offset. */
401 if (fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
402 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
403 strerror(errno));
404 return -1;
405 }
406
407 if (*ProfileFileSize > 0 &&
408 *ProfileFileSize < sizeof(__llvm_profile_header)) {
409 PROF_WARN("Unable to merge profile data: %s\n",
410 "source profile file is too small.");
411 return -1;
412 }
413 return 0;
414}
415
416/* mmap() \p ProfileFile for profile merging purposes, assuming that an
417 * exclusive lock is held on the file and that \p ProfileFileSize is the
418 * length of the file. Return the mmap'd buffer in the inout variable
419 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
420 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
421static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
422 ManagedMemory *ProfileBuffer) {
423 lprofGetFileContentBuffer(F: ProfileFile, FileSize: ProfileFileSize, Buf: ProfileBuffer);
424
425 if (ProfileBuffer->Status == MS_INVALID) {
426 PROF_ERR("Unable to merge profile data: %s\n", "reading file failed");
427 return -1;
428 }
429
430 if (__llvm_profile_check_compatibility(Profile: ProfileBuffer->Addr,
431 Size: ProfileFileSize)) {
432 (void)lprofReleaseBuffer(FileBuffer: ProfileBuffer, Length: ProfileFileSize);
433 PROF_WARN("Unable to merge profile data: %s\n",
434 "source profile file is not compatible.");
435 return -1;
436 }
437 return 0;
438}
439
440/* Read profile data in \c ProfileFile and merge with in-memory
441 profile counters. Returns -1 if there is fatal error, otherwise
442 0 is returned. Returning 0 does not mean merge is actually
443 performed. If merge is actually done, *MergeDone is set to 1.
444*/
445static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
446 uint64_t ProfileFileSize;
447 ManagedMemory ProfileBuffer;
448
449 /* Get the size of the profile on disk. */
450 if (getProfileFileSizeForMerging(ProfileFile, ProfileFileSize: &ProfileFileSize) == -1)
451 return -1;
452
453 /* Nothing to merge. */
454 if (!ProfileFileSize)
455 return 0;
456
457 /* mmap() the profile and check that it is compatible with the data in
458 * the current image. */
459 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1)
460 return -1;
461
462 /* Now start merging */
463 if (__llvm_profile_merge_from_buffer(Profile: ProfileBuffer.Addr, Size: ProfileFileSize)) {
464 PROF_ERR("%s\n", "Invalid profile data to merge");
465 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
466 return -1;
467 }
468
469 // Truncate the file in case merging of value profile did not happen to
470 // prevent from leaving garbage data at the end of the profile file.
471 (void)COMPILER_RT_FTRUNCATE(ProfileFile,
472 __llvm_profile_get_size_for_buffer());
473
474 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
475 *MergeDone = 1;
476
477 return 0;
478}
479
480/* Create the directory holding the file, if needed. */
481static void createProfileDir(const char *Filename) {
482 size_t Length = strlen(s: Filename);
483 if (lprofFindFirstDirSeparator(Path: Filename)) {
484 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
485 strncpy(dest: Copy, src: Filename, n: Length + 1);
486 __llvm_profile_recursive_mkdir(Pathname: Copy);
487 }
488}
489
490/* Open the profile data for merging. It opens the file in r+b mode with
491 * file locking. If the file has content which is compatible with the
492 * current process, it also reads in the profile data in the file and merge
493 * it with in-memory counters. After the profile data is merged in memory,
494 * the original profile data is truncated and gets ready for the profile
495 * dumper. With profile merging enabled, each executable as well as any of
496 * its instrumented shared libraries dump profile data into their own data file.
497 */
498static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
499 FILE *ProfileFile = getProfileFile();
500 int rc;
501 // initializeProfileForContinuousMode will lock the profile, but if
502 // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed
503 // unlocked at this point.
504 if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) {
505 lprofLockFileHandle(F: ProfileFile);
506 }
507 if (!ProfileFile) {
508 createProfileDir(Filename: ProfileFileName);
509 ProfileFile = lprofOpenFileEx(Filename: ProfileFileName);
510 }
511 if (!ProfileFile)
512 return NULL;
513
514 rc = doProfileMerging(ProfileFile, MergeDone);
515 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
516 fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
517 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
518 strerror(errno));
519 fclose(stream: ProfileFile);
520 return NULL;
521 }
522 return ProfileFile;
523}
524
525static FILE *getFileObject(const char *OutputName) {
526 FILE *File;
527 File = getProfileFile();
528 if (File != NULL) {
529 return File;
530 }
531
532 return fopen(filename: OutputName, modes: "ab");
533}
534
535static void closeFileObject(FILE *OutputFile) {
536 if (OutputFile == getProfileFile()) {
537 fflush(stream: OutputFile);
538 if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) {
539 lprofUnlockFileHandle(F: OutputFile);
540 }
541 } else {
542 fclose(stream: OutputFile);
543 }
544}
545
546/* Write profile data to file \c OutputName. */
547static int writeFile(const char *OutputName) {
548 int RetVal;
549 FILE *OutputFile;
550
551 int MergeDone = 0;
552 VPMergeHook = &lprofMergeValueProfData;
553 if (doMerging())
554 OutputFile = openFileForMerging(ProfileFileName: OutputName, MergeDone: &MergeDone);
555 else
556 OutputFile = getFileObject(OutputName);
557
558 if (!OutputFile)
559 return -1;
560
561 FreeHook = &free;
562 setupIOBuffer();
563 ProfDataWriter fileWriter;
564 initFileWriter(This: &fileWriter, File: OutputFile);
565 RetVal = lprofWriteData(Writer: &fileWriter, VPDataReader: lprofGetVPDataReader(), SkipNameDataWrite: MergeDone);
566
567 closeFileObject(OutputFile);
568 return RetVal;
569}
570
571#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
572
573static void truncateCurrentFile(void) {
574 const char *Filename;
575 char *FilenameBuf;
576 FILE *File;
577 int Length;
578
579 Length = getCurFilenameLength();
580 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
581 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
582 if (!Filename)
583 return;
584
585 /* Only create the profile directory and truncate an existing profile once.
586 * In continuous mode, this is necessary, as the profile is written-to by the
587 * runtime initializer. */
588 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
589 if (initialized)
590 return;
591#if defined(_WIN32)
592 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
593#else
594 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, replace: 1);
595#endif
596
597 /* Create the profile dir (even if online merging is enabled), so that
598 * the profile file can be set up if continuous mode is enabled. */
599 createProfileDir(Filename);
600
601 /* By pass file truncation to allow online raw profile merging. */
602 if (lprofCurFilename.MergePoolSize)
603 return;
604
605 /* Truncate the file. Later we'll reopen and append. */
606 File = fopen(filename: Filename, modes: "w");
607 if (!File)
608 return;
609 fclose(stream: File);
610}
611
612/* Write a partial profile to \p Filename, which is required to be backed by
613 * the open file object \p File. */
614static int writeProfileWithFileObject(const char *Filename, FILE *File) {
615 setProfileFile(File);
616 int rc = writeFile(OutputName: Filename);
617 if (rc)
618 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
619 setProfileFile(NULL);
620 return rc;
621}
622
623static void initializeProfileForContinuousMode(void) {
624 if (!__llvm_profile_is_continuous_mode_enabled())
625 return;
626 if (!ContinuousModeSupported) {
627 PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
628 return;
629 }
630 if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
631 BitmapBiasAddr == BitmapBiasDefaultAddr) {
632 PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
633 "__llvm_profile_bitmap_bias is defined");
634 return;
635 }
636
637 /* Get the sizes of counter section. */
638 uint64_t CountersSize = __llvm_profile_get_counters_size(
639 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
640
641 int Length = getCurFilenameLength();
642 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
643 const char *Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
644 if (!Filename)
645 return;
646
647 FILE *File = NULL;
648 uint64_t CurrentFileOffset = 0;
649 if (doMerging()) {
650 /* We are merging profiles. Map the counter section as shared memory into
651 * the profile, i.e. into each participating process. An increment in one
652 * process should be visible to every other process with the same counter
653 * section mapped. */
654 File = lprofOpenFileEx(Filename);
655 if (!File)
656 return;
657
658 uint64_t ProfileFileSize = 0;
659 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
660 lprofUnlockFileHandle(F: File);
661 fclose(stream: File);
662 return;
663 }
664 if (ProfileFileSize == 0) {
665 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
666 * the file should stay open. */
667 if (writeProfileWithFileObject(Filename, File) != 0) {
668 lprofUnlockFileHandle(F: File);
669 fclose(stream: File);
670 return;
671 }
672 } else {
673 /* The merged profile has a non-zero length. Check that it is compatible
674 * with the data in this process. */
675 ManagedMemory ProfileBuffer;
676 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
677 lprofUnlockFileHandle(F: File);
678 fclose(stream: File);
679 return;
680 }
681 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
682 }
683 } else {
684 File = fopen(filename: Filename, modes: FileOpenMode);
685 if (!File)
686 return;
687 /* Check that the offset within the file is page-aligned. */
688 CurrentFileOffset = ftell(stream: File);
689 unsigned PageSize = getpagesize();
690 if (CurrentFileOffset % PageSize != 0) {
691 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
692 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
693 (uint64_t)CurrentFileOffset, PageSize);
694 fclose(stream: File);
695 return;
696 }
697 if (writeProfileWithFileObject(Filename, File) != 0) {
698 fclose(stream: File);
699 return;
700 }
701 }
702
703 /* mmap() the profile counters so long as there is at least one counter.
704 * If there aren't any counters, mmap() would fail with EINVAL. */
705 if (CountersSize > 0)
706 mmapForContinuousMode(CurrentFileOffset, File);
707
708 if (doMerging()) {
709 lprofUnlockFileHandle(F: File);
710 }
711 if (File != NULL) {
712 fclose(stream: File);
713 }
714}
715
716static const char *DefaultProfileName = "default.profraw";
717static void resetFilenameToDefault(void) {
718 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
719#ifdef __GNUC__
720#pragma GCC diagnostic push
721#pragma GCC diagnostic ignored "-Wcast-qual"
722#elif defined(__clang__)
723#pragma clang diagnostic push
724#pragma clang diagnostic ignored "-Wcast-qual"
725#endif
726 free(ptr: (void *)lprofCurFilename.FilenamePat);
727#ifdef __GNUC__
728#pragma GCC diagnostic pop
729#elif defined(__clang__)
730#pragma clang diagnostic pop
731#endif
732 }
733 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
734 lprofCurFilename.FilenamePat = DefaultProfileName;
735 lprofCurFilename.PNS = PNS_default;
736}
737
738static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
739 unsigned J = 0, Num = 0;
740 for (;; ++J) {
741 char C = FilenamePat[*I + J];
742 if (C == 'm') {
743 *I += J;
744 return Num ? Num : 1;
745 }
746 if (C < '0' || C > '9')
747 break;
748 Num = Num * 10 + C - '0';
749
750 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
751 * to be in-bound as the string is null terminated. */
752 }
753 return 0;
754}
755
756/* Assert that Idx does index past a string null terminator. Return the
757 * result of the check. */
758static int checkBounds(int Idx, int Strlen) {
759 assert(Idx <= Strlen && "Indexing past string null terminator");
760 return Idx <= Strlen;
761}
762
763/* Parses the pattern string \p FilenamePat and stores the result to
764 * lprofcurFilename structure. */
765static int parseFilenamePattern(const char *FilenamePat,
766 unsigned CopyFilenamePat) {
767 int NumPids = 0, NumHosts = 0, NumBinaryIds = 0, I;
768 char *PidChars = &lprofCurFilename.PidChars[0];
769 char *Hostname = &lprofCurFilename.Hostname[0];
770 int MergingEnabled = 0;
771 int FilenamePatLen = strlen(s: FilenamePat);
772
773#ifdef __GNUC__
774#pragma GCC diagnostic push
775#pragma GCC diagnostic ignored "-Wcast-qual"
776#elif defined(__clang__)
777#pragma clang diagnostic push
778#pragma clang diagnostic ignored "-Wcast-qual"
779#endif
780 /* Clean up cached prefix and filename. */
781 if (lprofCurFilename.ProfilePathPrefix)
782 free(ptr: (void *)lprofCurFilename.ProfilePathPrefix);
783
784 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
785 free(ptr: (void *)lprofCurFilename.FilenamePat);
786 }
787#ifdef __GNUC__
788#pragma GCC diagnostic pop
789#elif defined(__clang__)
790#pragma clang diagnostic pop
791#endif
792
793 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
794
795 if (!CopyFilenamePat)
796 lprofCurFilename.FilenamePat = FilenamePat;
797 else {
798 lprofCurFilename.FilenamePat = strdup(s: FilenamePat);
799 lprofCurFilename.OwnsFilenamePat = 1;
800 }
801 /* Check the filename for "%p", which indicates a pid-substitution. */
802 for (I = 0; checkBounds(Idx: I, Strlen: FilenamePatLen) && FilenamePat[I]; ++I) {
803 if (FilenamePat[I] == '%') {
804 ++I; /* Advance to the next character. */
805 if (!checkBounds(Idx: I, Strlen: FilenamePatLen))
806 break;
807 if (FilenamePat[I] == 'p') {
808 if (!NumPids++) {
809 if (snprintf(s: PidChars, MAX_PID_SIZE, format: "%ld", (long)getpid()) <= 0) {
810 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
811 "default name.",
812 FilenamePat);
813 return -1;
814 }
815 }
816 } else if (FilenamePat[I] == 'h') {
817 if (!NumHosts++)
818 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
819 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
820 "the default name.",
821 FilenamePat);
822 return -1;
823 }
824 } else if (FilenamePat[I] == 't') {
825 lprofCurFilename.TmpDir = getenv(name: "TMPDIR");
826 if (!lprofCurFilename.TmpDir) {
827 PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
828 "in %s. Using the default path.",
829 FilenamePat);
830 return -1;
831 }
832 } else if (FilenamePat[I] == 'b') {
833 if (!NumBinaryIds++) {
834 /* Check if binary ID does not exist or if its size is 0. */
835 if (__llvm_write_binary_ids(NULL) <= 0) {
836 PROF_WARN("Unable to get binary ID for filename pattern %s. Using "
837 "the default name.",
838 FilenamePat);
839 return -1;
840 }
841 }
842 } else if (FilenamePat[I] == 'c') {
843 if (__llvm_profile_is_continuous_mode_enabled()) {
844 PROF_WARN("%%c specifier can only be specified once in %s.\n",
845 FilenamePat);
846 __llvm_profile_disable_continuous_mode();
847 return -1;
848 }
849#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX)
850 __llvm_profile_set_page_size(PageSize: getpagesize());
851 __llvm_profile_enable_continuous_mode();
852#else
853 PROF_WARN("%s",
854 "Continuous mode is currently only supported for Mach-O,"
855 " ELF and COFF formats.");
856 return -1;
857#endif
858 } else {
859 unsigned MergePoolSize = getMergePoolSize(FilenamePat, I: &I);
860 if (!MergePoolSize)
861 continue;
862 if (MergingEnabled) {
863 PROF_WARN("%%m specifier can only be specified once in %s.\n",
864 FilenamePat);
865 return -1;
866 }
867 MergingEnabled = 1;
868 lprofCurFilename.MergePoolSize = MergePoolSize;
869 }
870 }
871 }
872
873 lprofCurFilename.NumPids = NumPids;
874 lprofCurFilename.NumHosts = NumHosts;
875 lprofCurFilename.NumBinaryIds = NumBinaryIds;
876 return 0;
877}
878
879static void parseAndSetFilename(const char *FilenamePat,
880 ProfileNameSpecifier PNS,
881 unsigned CopyFilenamePat) {
882
883 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
884 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
885
886 /* The old profile name specifier takes precedence over the old one. */
887 if (PNS < OldPNS)
888 return;
889
890 if (!FilenamePat)
891 FilenamePat = DefaultProfileName;
892
893 if (OldFilenamePat && !strcmp(s1: OldFilenamePat, s2: FilenamePat)) {
894 lprofCurFilename.PNS = PNS;
895 return;
896 }
897
898 /* When PNS >= OldPNS, the last one wins. */
899 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
900 resetFilenameToDefault();
901 lprofCurFilename.PNS = PNS;
902
903 if (!OldFilenamePat) {
904 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
905 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
906 lprofCurFilename.FilenamePat, getPNSStr(PNS));
907 } else {
908 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
909 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
910 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
911 getPNSStr(PNS));
912 }
913
914 truncateCurrentFile();
915 if (__llvm_profile_is_continuous_mode_enabled())
916 initializeProfileForContinuousMode();
917}
918
919/* Return buffer length that is required to store the current profile
920 * filename with PID and hostname substitutions. */
921/* The length to hold uint64_t followed by 3 digits pool id including '_' */
922#define SIGLEN 24
923/* The length to hold 160-bit hash in hexadecimal form */
924#define BINARY_ID_LEN 40
925static int getCurFilenameLength(void) {
926 int Len;
927 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
928 return 0;
929
930 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
931 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
932 lprofCurFilename.MergePoolSize))
933 return strlen(s: lprofCurFilename.FilenamePat);
934
935 Len = strlen(s: lprofCurFilename.FilenamePat) +
936 lprofCurFilename.NumPids * (strlen(s: lprofCurFilename.PidChars) - 2) +
937 lprofCurFilename.NumHosts * (strlen(s: lprofCurFilename.Hostname) - 2) +
938 lprofCurFilename.NumBinaryIds * BINARY_ID_LEN +
939 (lprofCurFilename.TmpDir ? (strlen(s: lprofCurFilename.TmpDir) - 1) : 0);
940 if (lprofCurFilename.MergePoolSize)
941 Len += SIGLEN;
942 return Len;
943}
944
945typedef struct lprofBinaryIdsBuffer {
946 char String[BINARY_ID_LEN + 1];
947 int Length;
948} lprofBinaryIdsBuffer;
949
950/* Reads binary ID length and then its data, writes it into lprofBinaryIdsBuffer
951 * in hexadecimal form. */
952static uint32_t binaryIdsStringWriter(ProfDataWriter *This,
953 ProfDataIOVec *IOVecs,
954 uint32_t NumIOVecs) {
955 if (NumIOVecs < 2 || IOVecs[0].ElmSize != sizeof(uint64_t))
956 return -1;
957 uint64_t BinaryIdLen = *(const uint64_t *)IOVecs[0].Data;
958 if (IOVecs[1].ElmSize != sizeof(uint8_t) || IOVecs[1].NumElm != BinaryIdLen)
959 return -1;
960 const uint8_t *BinaryIdData = (const uint8_t *)IOVecs[1].Data;
961 lprofBinaryIdsBuffer *Data = (lprofBinaryIdsBuffer *)This->WriterCtx;
962 for (uint64_t I = 0; I < BinaryIdLen; I++) {
963 Data->Length +=
964 snprintf(s: Data->String + Data->Length, BINARY_ID_LEN + 1 - Data->Length,
965 format: "%02hhx", BinaryIdData[I]);
966 }
967 return 0;
968}
969
970/* Return the pointer to the current profile file name (after substituting
971 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
972 * to store the resulting filename. If no substitution is needed, the
973 * current filename pattern string is directly returned, unless ForceUseBuf
974 * is enabled. */
975static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
976 int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
977 const char *FilenamePat = lprofCurFilename.FilenamePat;
978
979 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
980 return 0;
981
982 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
983 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
984 lprofCurFilename.MergePoolSize ||
985 __llvm_profile_is_continuous_mode_enabled())) {
986 if (!ForceUseBuf)
987 return lprofCurFilename.FilenamePat;
988
989 FilenamePatLength = strlen(s: lprofCurFilename.FilenamePat);
990 memcpy(dest: FilenameBuf, src: lprofCurFilename.FilenamePat, n: FilenamePatLength);
991 FilenameBuf[FilenamePatLength] = '\0';
992 return FilenameBuf;
993 }
994
995 PidLength = strlen(s: lprofCurFilename.PidChars);
996 HostNameLength = strlen(s: lprofCurFilename.Hostname);
997 TmpDirLength = lprofCurFilename.TmpDir ? strlen(s: lprofCurFilename.TmpDir) : 0;
998 /* Construct the new filename. */
999 for (I = 0, J = 0; FilenamePat[I]; ++I)
1000 if (FilenamePat[I] == '%') {
1001 if (FilenamePat[++I] == 'p') {
1002 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.PidChars, n: PidLength);
1003 J += PidLength;
1004 } else if (FilenamePat[I] == 'h') {
1005 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.Hostname, n: HostNameLength);
1006 J += HostNameLength;
1007 } else if (FilenamePat[I] == 't') {
1008 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.TmpDir, n: TmpDirLength);
1009 FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
1010 J += TmpDirLength + 1;
1011 } else if (FilenamePat[I] == 'b') {
1012 lprofBinaryIdsBuffer Data = {{0}, 0};
1013 ProfDataWriter Writer = {binaryIdsStringWriter, &Data};
1014 __llvm_write_binary_ids(Writer: &Writer);
1015 memcpy(dest: FilenameBuf + J, src: Data.String, n: Data.Length);
1016 J += Data.Length;
1017 } else {
1018 if (!getMergePoolSize(FilenamePat, I: &I))
1019 continue;
1020 char LoadModuleSignature[SIGLEN + 1];
1021 int S;
1022 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
1023 S = snprintf(s: LoadModuleSignature, SIGLEN + 1, format: "%" PRIu64 "_%d",
1024 lprofGetLoadModuleSignature(), ProfilePoolId);
1025 if (S == -1 || S > SIGLEN)
1026 S = SIGLEN;
1027 memcpy(dest: FilenameBuf + J, src: LoadModuleSignature, n: S);
1028 J += S;
1029 }
1030 /* Drop any unknown substitutions. */
1031 } else
1032 FilenameBuf[J++] = FilenamePat[I];
1033 FilenameBuf[J] = 0;
1034
1035 return FilenameBuf;
1036}
1037
1038/* Returns the pointer to the environment variable
1039 * string. Returns null if the env var is not set. */
1040static const char *getFilenamePatFromEnv(void) {
1041 const char *Filename = getenv(name: "LLVM_PROFILE_FILE");
1042 if (!Filename || !Filename[0])
1043 return 0;
1044 return Filename;
1045}
1046
1047COMPILER_RT_VISIBILITY
1048const char *__llvm_profile_get_path_prefix(void) {
1049 int Length;
1050 char *FilenameBuf, *Prefix;
1051 const char *Filename, *PrefixEnd;
1052
1053 if (lprofCurFilename.ProfilePathPrefix)
1054 return lprofCurFilename.ProfilePathPrefix;
1055
1056 Length = getCurFilenameLength();
1057 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1058 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1059 if (!Filename)
1060 return "\0";
1061
1062 PrefixEnd = lprofFindLastDirSeparator(Path: Filename);
1063 if (!PrefixEnd)
1064 return "\0";
1065
1066 Length = PrefixEnd - Filename + 1;
1067 Prefix = (char *)malloc(size: Length + 1);
1068 if (!Prefix) {
1069 PROF_ERR("Failed to %s\n", "allocate memory.");
1070 return "\0";
1071 }
1072 memcpy(dest: Prefix, src: Filename, n: Length);
1073 Prefix[Length] = '\0';
1074 lprofCurFilename.ProfilePathPrefix = Prefix;
1075 return Prefix;
1076}
1077
1078COMPILER_RT_VISIBILITY
1079const char *__llvm_profile_get_filename(void) {
1080 int Length;
1081 char *FilenameBuf;
1082 const char *Filename;
1083
1084 Length = getCurFilenameLength();
1085 FilenameBuf = (char *)malloc(size: Length + 1);
1086 if (!FilenameBuf) {
1087 PROF_ERR("Failed to %s\n", "allocate memory.");
1088 return "\0";
1089 }
1090 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 1);
1091 if (!Filename) {
1092 free(ptr: FilenameBuf);
1093 return "\0";
1094 }
1095
1096 return FilenameBuf;
1097}
1098
1099/* This API initializes the file handling, both user specified
1100 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
1101 * environment variable can override this default value.
1102 */
1103COMPILER_RT_VISIBILITY
1104void __llvm_profile_initialize_file(void) {
1105 const char *EnvFilenamePat;
1106 const char *SelectedPat = NULL;
1107 ProfileNameSpecifier PNS = PNS_unknown;
1108 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
1109
1110 EnvFilenamePat = getFilenamePatFromEnv();
1111 if (EnvFilenamePat) {
1112 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
1113 at the moment when __llvm_profile_write_file() gets executed. */
1114 parseAndSetFilename(FilenamePat: EnvFilenamePat, PNS: PNS_environment, CopyFilenamePat: 1);
1115 return;
1116 } else if (hasCommandLineOverrider) {
1117 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
1118 PNS = PNS_command_line;
1119 } else {
1120 SelectedPat = NULL;
1121 PNS = PNS_default;
1122 }
1123
1124 parseAndSetFilename(FilenamePat: SelectedPat, PNS, CopyFilenamePat: 0);
1125}
1126
1127/* This method is invoked by the runtime initialization hook
1128 * InstrProfilingRuntime.o if it is linked in.
1129 */
1130COMPILER_RT_VISIBILITY
1131void __llvm_profile_initialize(void) {
1132 __llvm_profile_initialize_file();
1133 if (!__llvm_profile_is_continuous_mode_enabled())
1134 __llvm_profile_register_write_file_atexit();
1135}
1136
1137/* This API is directly called by the user application code. It has the
1138 * highest precedence compared with LLVM_PROFILE_FILE environment variable
1139 * and command line option -fprofile-instr-generate=<profile_name>.
1140 */
1141COMPILER_RT_VISIBILITY
1142void __llvm_profile_set_filename(const char *FilenamePat) {
1143 if (__llvm_profile_is_continuous_mode_enabled())
1144 return;
1145 parseAndSetFilename(FilenamePat, PNS: PNS_runtime_api, CopyFilenamePat: 1);
1146}
1147
1148/* The public API for writing profile data into the file with name
1149 * set by previous calls to __llvm_profile_set_filename or
1150 * __llvm_profile_override_default_filename or
1151 * __llvm_profile_initialize_file. */
1152COMPILER_RT_VISIBILITY
1153int __llvm_profile_write_file(void) {
1154 int rc, Length;
1155 const char *Filename;
1156 char *FilenameBuf;
1157
1158 // Temporarily suspend getting SIGKILL when the parent exits.
1159 int PDeathSig = lprofSuspendSigKill();
1160
1161 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1162 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1163 if (PDeathSig == 1)
1164 lprofRestoreSigKill();
1165 return 0;
1166 }
1167
1168 Length = getCurFilenameLength();
1169 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1170 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1171
1172 /* Check the filename. */
1173 if (!Filename) {
1174 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1175 if (PDeathSig == 1)
1176 lprofRestoreSigKill();
1177 return -1;
1178 }
1179
1180 /* Check if there is llvm/runtime version mismatch. */
1181 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1182 PROF_ERR("Runtime and instrumentation version mismatch : "
1183 "expected %d, but get %d\n",
1184 INSTR_PROF_RAW_VERSION,
1185 (int)GET_VERSION(__llvm_profile_get_version()));
1186 if (PDeathSig == 1)
1187 lprofRestoreSigKill();
1188 return -1;
1189 }
1190
1191 /* Write profile data to the file. */
1192 rc = writeFile(OutputName: Filename);
1193 if (rc)
1194 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1195
1196 // Restore SIGKILL.
1197 if (PDeathSig == 1)
1198 lprofRestoreSigKill();
1199
1200 return rc;
1201}
1202
1203COMPILER_RT_VISIBILITY
1204int __llvm_profile_dump(void) {
1205 if (!doMerging())
1206 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1207 " of previously dumped profile data : %s. Either use %%m "
1208 "in profile name or change profile name before dumping.\n",
1209 "online profile merging is not on");
1210 int rc = __llvm_profile_write_file();
1211 lprofSetProfileDumped(1);
1212 return rc;
1213}
1214
1215static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1216
1217COMPILER_RT_VISIBILITY
1218int __llvm_profile_register_write_file_atexit(void) {
1219 static int HasBeenRegistered = 0;
1220
1221 if (HasBeenRegistered)
1222 return 0;
1223
1224 lprofSetupValueProfiler();
1225
1226 HasBeenRegistered = 1;
1227 return lprofAtExit(writeFileWithoutReturn);
1228}
1229
1230COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
1231 int EnableMerge) {
1232 if (__llvm_profile_is_continuous_mode_enabled()) {
1233 if (!EnableMerge) {
1234 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
1235 "continuous sync mode when merging is disabled\n",
1236 fileno(File));
1237 return 1;
1238 }
1239 if (lprofLockFileHandle(F: File) != 0) {
1240 PROF_WARN("Data may be corrupted during profile merging : %s\n",
1241 "Fail to obtain file lock due to system limit.");
1242 }
1243 uint64_t ProfileFileSize = 0;
1244 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
1245 lprofUnlockFileHandle(F: File);
1246 return 1;
1247 }
1248 if (ProfileFileSize == 0) {
1249 FreeHook = &free;
1250 setupIOBuffer();
1251 ProfDataWriter fileWriter;
1252 initFileWriter(This: &fileWriter, File);
1253 if (lprofWriteData(Writer: &fileWriter, VPDataReader: 0, SkipNameDataWrite: 0)) {
1254 lprofUnlockFileHandle(F: File);
1255 PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
1256 strerror(errno));
1257 return 1;
1258 }
1259 fflush(stream: File);
1260 } else {
1261 /* The merged profile has a non-zero length. Check that it is compatible
1262 * with the data in this process. */
1263 ManagedMemory ProfileBuffer;
1264 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
1265 lprofUnlockFileHandle(F: File);
1266 return 1;
1267 }
1268 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
1269 }
1270 mmapForContinuousMode(CurrentFileOffset: 0, File);
1271 lprofUnlockFileHandle(F: File);
1272 } else {
1273 setProfileFile(File);
1274 setProfileMergeRequested(EnableMerge);
1275 }
1276 return 0;
1277}
1278
1279#ifndef __APPLE__
1280int __llvm_write_custom_profile(const char *Target,
1281 const __llvm_profile_data *DataBegin,
1282 const __llvm_profile_data *DataEnd,
1283 const char *CountersBegin,
1284 const char *CountersEnd, const char *NamesBegin,
1285 const char *NamesEnd,
1286 const uint64_t *VersionOverride) {
1287 int ReturnValue = 0, FilenameLength, TargetLength;
1288 char *FilenameBuf, *TargetFilename;
1289 const char *Filename;
1290
1291 /* Save old profile data */
1292 FILE *oldFile = getProfileFile();
1293
1294 // Temporarily suspend getting SIGKILL when the parent exits.
1295 int PDeathSig = lprofSuspendSigKill();
1296
1297 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1298 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1299 if (PDeathSig == 1)
1300 lprofRestoreSigKill();
1301 return 0;
1302 }
1303
1304 /* Check if there is llvm/runtime version mismatch. */
1305 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1306 PROF_ERR("Runtime and instrumentation version mismatch : "
1307 "expected %d, but get %d\n",
1308 INSTR_PROF_RAW_VERSION,
1309 (int)GET_VERSION(__llvm_profile_get_version()));
1310 if (PDeathSig == 1)
1311 lprofRestoreSigKill();
1312 return -1;
1313 }
1314
1315 /* Get current filename */
1316 FilenameLength = getCurFilenameLength();
1317 FilenameBuf = (char *)COMPILER_RT_ALLOCA(FilenameLength + 1);
1318 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1319
1320 /* Check the filename. */
1321 if (!Filename) {
1322 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1323 if (PDeathSig == 1)
1324 lprofRestoreSigKill();
1325 return -1;
1326 }
1327
1328 /* Allocate new space for our target-specific PGO filename */
1329 TargetLength = strlen(s: Target);
1330 TargetFilename =
1331 (char *)COMPILER_RT_ALLOCA(FilenameLength + TargetLength + 2);
1332
1333 /* Find file basename and path sizes */
1334 int32_t DirEnd = FilenameLength - 1;
1335 while (DirEnd >= 0 && !IS_DIR_SEPARATOR(Filename[DirEnd])) {
1336 DirEnd--;
1337 }
1338 uint32_t DirSize = DirEnd + 1, BaseSize = FilenameLength - DirSize;
1339
1340 /* Prepend "TARGET." to current filename */
1341 if (DirSize > 0) {
1342 memcpy(dest: TargetFilename, src: Filename, n: DirSize);
1343 }
1344 memcpy(dest: TargetFilename + DirSize, src: Target, n: TargetLength);
1345 TargetFilename[TargetLength + DirSize] = '.';
1346 memcpy(dest: TargetFilename + DirSize + 1 + TargetLength, src: Filename + DirSize,
1347 n: BaseSize);
1348 TargetFilename[FilenameLength + 1 + TargetLength] = 0;
1349
1350 /* Open and truncate target-specific PGO file */
1351 FILE *OutputFile = fopen(filename: TargetFilename, modes: "w");
1352 setProfileFile(OutputFile);
1353
1354 if (!OutputFile) {
1355 PROF_ERR("Failed to open file : %s\n", TargetFilename);
1356 if (PDeathSig == 1)
1357 lprofRestoreSigKill();
1358 return -1;
1359 }
1360
1361 FreeHook = &free;
1362 setupIOBuffer();
1363
1364 /* Write custom data */
1365 ProfDataWriter fileWriter;
1366 initFileWriter(This: &fileWriter, File: OutputFile);
1367
1368 uint64_t Version = __llvm_profile_get_version();
1369 if (VersionOverride)
1370 Version = *VersionOverride;
1371
1372 /* Write custom data to the file */
1373 ReturnValue =
1374 lprofWriteDataImpl(Writer: &fileWriter, DataBegin, DataEnd, CountersBegin,
1375 CountersEnd, NULL, NULL, VPDataReader: lprofGetVPDataReader(), NULL,
1376 NULL, NULL, NULL, VNamesBegin: NamesBegin, VNamesEnd: NamesEnd, SkipNameDataWrite: 0, Version);
1377 closeFileObject(OutputFile);
1378
1379 // Restore SIGKILL.
1380 if (PDeathSig == 1)
1381 lprofRestoreSigKill();
1382
1383 /* Restore old profiling file */
1384 setProfileFile(oldFile);
1385
1386 return ReturnValue;
1387}
1388#endif
1389
1390#endif
1391