1/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9#if !defined(__Fuchsia__)
10
11#if defined(__linux__)
12// For fileno(), ftruncate(), getpagesize(), setenv()
13#define _DEFAULT_SOURCE
14#endif
15
16#include <assert.h>
17#include <errno.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#ifdef _MSC_VER
22/* For _alloca. */
23#include <malloc.h>
24#endif
25#if defined(_WIN32)
26#include "WindowsMMap.h"
27/* For _chsize_s */
28#include <io.h>
29#include <process.h>
30#else
31#include <sys/file.h>
32#include <sys/mman.h>
33#include <unistd.h>
34#if defined(__linux__)
35#include <sys/types.h>
36#endif
37#endif
38
39#include "InstrProfiling.h"
40#include "InstrProfilingInternal.h"
41#include "InstrProfilingPort.h"
42#include "InstrProfilingUtil.h"
43
44/* From where is profile name specified.
45 * The order the enumerators define their
46 * precedence. Re-order them may lead to
47 * runtime behavior change. */
48typedef enum ProfileNameSpecifier {
49 PNS_unknown = 0,
50 PNS_default,
51 PNS_command_line,
52 PNS_environment,
53 PNS_runtime_api
54} ProfileNameSpecifier;
55
56static const char *getPNSStr(ProfileNameSpecifier PNS) {
57 switch (PNS) {
58 case PNS_default:
59 return "default setting";
60 case PNS_command_line:
61 return "command line";
62 case PNS_environment:
63 return "environment variable";
64 case PNS_runtime_api:
65 return "runtime API";
66 default:
67 return "Unknown";
68 }
69}
70
71#define MAX_PID_SIZE 16
72/* Data structure holding the result of parsed filename pattern. */
73typedef struct lprofFilename {
74 /* File name string possibly with %p or %h specifiers. */
75 const char *FilenamePat;
76 /* A flag indicating if FilenamePat's memory is allocated
77 * by runtime. */
78 unsigned OwnsFilenamePat;
79 const char *ProfilePathPrefix;
80 char PidChars[MAX_PID_SIZE];
81 char *TmpDir;
82 char Hostname[COMPILER_RT_MAX_HOSTLEN];
83 unsigned NumPids;
84 unsigned NumHosts;
85 unsigned NumBinaryIds;
86 /* When in-process merging is enabled, this parameter specifies
87 * the total number of profile data files shared by all the processes
88 * spawned from the same binary. By default the value is 1. If merging
89 * is not enabled, its value should be 0. This parameter is specified
90 * by the %[0-9]m specifier. For instance %2m enables merging using
91 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
92 * can only appear once at the end of the name pattern. */
93 unsigned MergePoolSize;
94 ProfileNameSpecifier PNS;
95} lprofFilename;
96
97static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, {0},
98 0, 0, 0, 0, PNS_unknown};
99
100static int ProfileMergeRequested = 0;
101static int getProfileFileSizeForMerging(FILE *ProfileFile,
102 uint64_t *ProfileFileSize);
103
104#if defined(__APPLE__)
105static const int ContinuousModeSupported = 1;
106static const int UseBiasVar = 0;
107static const char *FileOpenMode = "a+b";
108static void *BiasAddr = NULL;
109static void *BiasDefaultAddr = NULL;
110static void *BitmapBiasAddr = NULL;
111static void *BitmapBiasDefaultAddr = NULL;
112static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
113 /* Get the sizes of various profile data sections. Taken from
114 * __llvm_profile_get_size_for_buffer(). */
115 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
116 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
117 const char *CountersBegin = __llvm_profile_begin_counters();
118 const char *CountersEnd = __llvm_profile_end_counters();
119 const char *BitmapBegin = __llvm_profile_begin_bitmap();
120 const char *BitmapEnd = __llvm_profile_end_bitmap();
121 const char *NamesBegin = __llvm_profile_begin_names();
122 const char *NamesEnd = __llvm_profile_end_names();
123 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
124 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
125 uint64_t CountersSize =
126 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
127 uint64_t NumBitmapBytes =
128 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
129
130 /* Check that the counter, bitmap, and data sections in this image are
131 * page-aligned. */
132 unsigned PageSize = getpagesize();
133 if ((intptr_t)CountersBegin % PageSize != 0) {
134 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
135 CountersBegin, PageSize);
136 return 1;
137 }
138 if ((intptr_t)BitmapBegin % PageSize != 0) {
139 PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n",
140 BitmapBegin, PageSize);
141 return 1;
142 }
143 if ((intptr_t)DataBegin % PageSize != 0) {
144 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
145 DataBegin, PageSize);
146 return 1;
147 }
148
149 int Fileno = fileno(File);
150 /* Determine how much padding is needed before/after the counters and
151 * after the names. */
152 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
153 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
154 PaddingBytesAfterVTable, PaddingBytesAfterVNames;
155 __llvm_profile_get_padding_sizes_for_counters(
156 DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0,
157 /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
158 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames,
159 &PaddingBytesAfterVTable, &PaddingBytesAfterVNames);
160
161 uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
162 uint64_t FileOffsetToCounters = CurrentFileOffset +
163 sizeof(__llvm_profile_header) + DataSize +
164 PaddingBytesBeforeCounters;
165 void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
166 PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
167 Fileno, FileOffsetToCounters);
168 if (CounterMmap != CountersBegin) {
169 PROF_ERR(
170 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
171 " - CountersBegin: %p\n"
172 " - PageAlignedCountersLength: %" PRIu64 "\n"
173 " - Fileno: %d\n"
174 " - FileOffsetToCounters: %" PRIu64 "\n",
175 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
176 FileOffsetToCounters);
177 return 1;
178 }
179
180 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
181 * will fail with EINVAL. */
182 if (NumBitmapBytes == 0)
183 return 0;
184
185 uint64_t PageAlignedBitmapLength =
186 NumBitmapBytes + PaddingBytesAfterBitmapBytes;
187 uint64_t FileOffsetToBitmap =
188 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
189 void *BitmapMmap =
190 mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE,
191 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap);
192 if (BitmapMmap != BitmapBegin) {
193 PROF_ERR(
194 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
195 " - BitmapBegin: %p\n"
196 " - PageAlignedBitmapLength: %" PRIu64 "\n"
197 " - Fileno: %d\n"
198 " - FileOffsetToBitmap: %" PRIu64 "\n",
199 strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno,
200 FileOffsetToBitmap);
201 return 1;
202 }
203 return 0;
204}
205#elif defined(__ELF__) || defined(_WIN32) || defined(_AIX)
206
207#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
208 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
209COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
210#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
211 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
212COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
213
214/* This variable is a weak external reference which could be used to detect
215 * whether or not the compiler defined this symbol. */
216#if defined(_MSC_VER)
217COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
218COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
219#if defined(_M_IX86) || defined(__i386__)
220#define WIN_SYM_PREFIX "_"
221#else
222#define WIN_SYM_PREFIX
223#endif
224#pragma comment( \
225 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
226 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
227 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
228#pragma comment( \
229 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
230 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
231 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
232#else
233COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
234 __attribute__((weak, alias(INSTR_PROF_QUOTE(
235 INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
236COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
237 __attribute__((weak, alias(INSTR_PROF_QUOTE(
238 INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
239#endif
240static const int ContinuousModeSupported = 1;
241static const int UseBiasVar = 1;
242/* TODO: If there are two DSOs, the second DSO initialization will truncate the
243 * first profile file. */
244static const char *FileOpenMode = "w+b";
245/* This symbol is defined by the compiler when runtime counter relocation is
246 * used and runtime provides a weak alias so we can check if it's defined. */
247static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
248static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
249static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
250static void *BitmapBiasDefaultAddr =
251 &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
252static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
253 /* Get the sizes of various profile data sections. Taken from
254 * __llvm_profile_get_size_for_buffer(). */
255 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
256 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
257 const char *CountersBegin = __llvm_profile_begin_counters();
258 const char *CountersEnd = __llvm_profile_end_counters();
259 const char *BitmapBegin = __llvm_profile_begin_bitmap();
260 const char *BitmapEnd = __llvm_profile_end_bitmap();
261 uint64_t DataSize = __llvm_profile_get_data_size(Begin: DataBegin, End: DataEnd);
262 uint64_t CountersSize =
263 __llvm_profile_get_counters_size(Begin: CountersBegin, End: CountersEnd);
264 uint64_t NumBitmapBytes =
265 __llvm_profile_get_num_bitmap_bytes(Begin: BitmapBegin, End: BitmapEnd);
266 /* Get the file size. */
267 uint64_t FileSize = 0;
268 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &FileSize))
269 return 1;
270
271 int Fileno = fileno(stream: File);
272 uint64_t PaddingBytesAfterCounters =
273 __llvm_profile_get_num_padding_bytes(SizeInBytes: CountersSize);
274 uint64_t FileOffsetToCounters =
275 sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
276
277 /* Map the profile. */
278 char *Profile = (char *)mmap(NULL, len: FileSize, PROT_READ | PROT_WRITE,
279 MAP_SHARED, fd: Fileno, offset: 0);
280 if (Profile == MAP_FAILED) {
281 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
282 return 1;
283 }
284 /* Update the profile fields based on the current mapping. */
285 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
286 (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters;
287
288 /* Return the memory allocated for counters to OS. */
289 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)CountersBegin, End: (uintptr_t)CountersEnd);
290
291 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
292 * will fail with EINVAL. */
293 if (NumBitmapBytes == 0)
294 return 0;
295
296 /* Update profbm_bias. */
297 uint64_t FileOffsetToBitmap =
298 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
299 /* Update the profile fields based on the current mapping. */
300 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
301 (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
302
303 /* Return the memory allocated for counters to OS. */
304 lprofReleaseMemoryPagesToOS(Begin: (uintptr_t)BitmapBegin, End: (uintptr_t)BitmapEnd);
305 return 0;
306}
307#else
308static const int ContinuousModeSupported = 0;
309static const int UseBiasVar = 0;
310static const char *FileOpenMode = "a+b";
311static void *BiasAddr = NULL;
312static void *BiasDefaultAddr = NULL;
313static void *BitmapBiasAddr = NULL;
314static void *BitmapBiasDefaultAddr = NULL;
315static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
316 return 0;
317}
318#endif
319
320static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
321static void setProfileMergeRequested(int EnableMerge) {
322 ProfileMergeRequested = EnableMerge;
323}
324
325static FILE *ProfileFile = NULL;
326static FILE *getProfileFile(void) { return ProfileFile; }
327static void setProfileFile(FILE *File) { ProfileFile = File; }
328
329static int getCurFilenameLength(void);
330static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
331static unsigned doMerging(void) {
332 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
333}
334
335/* Return 1 if there is an error, otherwise return 0. */
336static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
337 uint32_t NumIOVecs) {
338 uint32_t I;
339 FILE *File = (FILE *)This->WriterCtx;
340 char Zeroes[sizeof(uint64_t)] = {0};
341 for (I = 0; I < NumIOVecs; I++) {
342 if (IOVecs[I].Data) {
343 if (fwrite(ptr: IOVecs[I].Data, size: IOVecs[I].ElmSize, n: IOVecs[I].NumElm, s: File) !=
344 IOVecs[I].NumElm)
345 return 1;
346 } else if (IOVecs[I].UseZeroPadding) {
347 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
348 while (BytesToWrite > 0) {
349 size_t PartialWriteLen =
350 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
351 if (fwrite(ptr: Zeroes, size: sizeof(uint8_t), n: PartialWriteLen, s: File) !=
352 PartialWriteLen) {
353 return 1;
354 }
355 BytesToWrite -= PartialWriteLen;
356 }
357 } else {
358 if (fseek(stream: File, off: IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
359 return 1;
360 }
361 }
362 return 0;
363}
364
365static void initFileWriter(ProfDataWriter *This, FILE *File) {
366 This->Write = fileWriter;
367 This->WriterCtx = File;
368}
369
370COMPILER_RT_VISIBILITY ProfBufferIO *
371lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
372 FreeHook = &free;
373 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: 1, size: BufferSz);
374 VPBufferSize = BufferSz;
375 ProfDataWriter *fileWriter =
376 (ProfDataWriter *)calloc(nmemb: 1, size: sizeof(ProfDataWriter));
377 initFileWriter(This: fileWriter, File);
378 ProfBufferIO *IO = lprofCreateBufferIO(FileWriter: fileWriter);
379 IO->OwnFileWriter = 1;
380 return IO;
381}
382
383static void setupIOBuffer(void) {
384 const char *BufferSzStr = 0;
385 BufferSzStr = getenv(name: "LLVM_VP_BUFFER_SIZE");
386 if (BufferSzStr && BufferSzStr[0]) {
387 VPBufferSize = atoi(nptr: BufferSzStr);
388 DynamicBufferIOBuffer = (uint8_t *)calloc(nmemb: VPBufferSize, size: 1);
389 }
390}
391
392/* Get the size of the profile file. If there are any errors, print the
393 * message under the assumption that the profile is being read for merging
394 * purposes, and return -1. Otherwise return the file size in the inout param
395 * \p ProfileFileSize. */
396static int getProfileFileSizeForMerging(FILE *ProfileFile,
397 uint64_t *ProfileFileSize) {
398 if (fseek(stream: ProfileFile, off: 0L, SEEK_END) == -1) {
399 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
400 strerror(errno));
401 return -1;
402 }
403 *ProfileFileSize = ftell(stream: ProfileFile);
404
405 /* Restore file offset. */
406 if (fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
407 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
408 strerror(errno));
409 return -1;
410 }
411
412 if (*ProfileFileSize > 0 &&
413 *ProfileFileSize < sizeof(__llvm_profile_header)) {
414 PROF_WARN("Unable to merge profile data: %s\n",
415 "source profile file is too small.");
416 return -1;
417 }
418 return 0;
419}
420
421/* mmap() \p ProfileFile for profile merging purposes, assuming that an
422 * exclusive lock is held on the file and that \p ProfileFileSize is the
423 * length of the file. Return the mmap'd buffer in the inout variable
424 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
425 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
426static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
427 ManagedMemory *ProfileBuffer) {
428 lprofGetFileContentBuffer(F: ProfileFile, FileSize: ProfileFileSize, Buf: ProfileBuffer);
429
430 if (ProfileBuffer->Status == MS_INVALID) {
431 PROF_ERR("Unable to merge profile data: %s\n", "reading file failed");
432 return -1;
433 }
434
435 if (__llvm_profile_check_compatibility(Profile: ProfileBuffer->Addr,
436 Size: ProfileFileSize)) {
437 (void)lprofReleaseBuffer(FileBuffer: ProfileBuffer, Length: ProfileFileSize);
438 PROF_WARN("Unable to merge profile data: %s\n",
439 "source profile file is not compatible.");
440 return -1;
441 }
442 return 0;
443}
444
445/* Read profile data in \c ProfileFile and merge with in-memory
446 profile counters. Returns -1 if there is fatal error, otherwise
447 0 is returned. Returning 0 does not mean merge is actually
448 performed. If merge is actually done, *MergeDone is set to 1.
449*/
450static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
451 uint64_t ProfileFileSize;
452 ManagedMemory ProfileBuffer;
453
454 /* Get the size of the profile on disk. */
455 if (getProfileFileSizeForMerging(ProfileFile, ProfileFileSize: &ProfileFileSize) == -1)
456 return -1;
457
458 /* Nothing to merge. */
459 if (!ProfileFileSize)
460 return 0;
461
462 /* mmap() the profile and check that it is compatible with the data in
463 * the current image. */
464 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1)
465 return -1;
466
467 /* Now start merging */
468 if (__llvm_profile_merge_from_buffer(Profile: ProfileBuffer.Addr, Size: ProfileFileSize)) {
469 PROF_ERR("%s\n", "Invalid profile data to merge");
470 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
471 return -1;
472 }
473
474 // Truncate the file in case merging of value profile did not happen to
475 // prevent from leaving garbage data at the end of the profile file.
476 (void)COMPILER_RT_FTRUNCATE(ProfileFile,
477 __llvm_profile_get_size_for_buffer());
478
479 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
480 *MergeDone = 1;
481
482 return 0;
483}
484
485/* Create the directory holding the file, if needed. */
486static void createProfileDir(const char *Filename) {
487 size_t Length = strlen(s: Filename);
488 if (lprofFindFirstDirSeparator(Path: Filename)) {
489 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
490 strncpy(dest: Copy, src: Filename, n: Length + 1);
491 __llvm_profile_recursive_mkdir(Pathname: Copy);
492 }
493}
494
495/* Open the profile data for merging. It opens the file in r+b mode with
496 * file locking. If the file has content which is compatible with the
497 * current process, it also reads in the profile data in the file and merge
498 * it with in-memory counters. After the profile data is merged in memory,
499 * the original profile data is truncated and gets ready for the profile
500 * dumper. With profile merging enabled, each executable as well as any of
501 * its instrumented shared libraries dump profile data into their own data file.
502 */
503static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
504 FILE *ProfileFile = getProfileFile();
505 int rc;
506 // initializeProfileForContinuousMode will lock the profile, but if
507 // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed
508 // unlocked at this point.
509 if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) {
510 lprofLockFileHandle(F: ProfileFile);
511 }
512 if (!ProfileFile) {
513 createProfileDir(Filename: ProfileFileName);
514 ProfileFile = lprofOpenFileEx(Filename: ProfileFileName);
515 }
516 if (!ProfileFile)
517 return NULL;
518
519 rc = doProfileMerging(ProfileFile, MergeDone);
520 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
521 fseek(stream: ProfileFile, off: 0L, SEEK_SET) == -1) {
522 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
523 strerror(errno));
524 fclose(stream: ProfileFile);
525 return NULL;
526 }
527 return ProfileFile;
528}
529
530static FILE *getFileObject(const char *OutputName) {
531 FILE *File;
532 File = getProfileFile();
533 if (File != NULL) {
534 return File;
535 }
536
537 return fopen(filename: OutputName, modes: "ab");
538}
539
540static void closeFileObject(FILE *OutputFile) {
541 if (OutputFile == getProfileFile()) {
542 fflush(stream: OutputFile);
543 if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) {
544 lprofUnlockFileHandle(F: OutputFile);
545 }
546 } else {
547 fclose(stream: OutputFile);
548 }
549}
550
551/* Write profile data to file \c OutputName. */
552static int writeFile(const char *OutputName) {
553 int RetVal;
554 FILE *OutputFile;
555
556 int MergeDone = 0;
557 VPMergeHook = &lprofMergeValueProfData;
558 if (doMerging())
559 OutputFile = openFileForMerging(ProfileFileName: OutputName, MergeDone: &MergeDone);
560 else
561 OutputFile = getFileObject(OutputName);
562
563 if (!OutputFile)
564 return -1;
565
566 FreeHook = &free;
567 setupIOBuffer();
568 ProfDataWriter fileWriter;
569 initFileWriter(This: &fileWriter, File: OutputFile);
570 RetVal = lprofWriteData(Writer: &fileWriter, VPDataReader: lprofGetVPDataReader(), SkipNameDataWrite: MergeDone);
571
572 closeFileObject(OutputFile);
573 return RetVal;
574}
575
576#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
577
578static void truncateCurrentFile(void) {
579 const char *Filename;
580 char *FilenameBuf;
581 FILE *File;
582 int Length;
583
584 Length = getCurFilenameLength();
585 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
586 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
587 if (!Filename)
588 return;
589
590 /* Only create the profile directory and truncate an existing profile once.
591 * In continuous mode, this is necessary, as the profile is written-to by the
592 * runtime initializer. */
593 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
594 if (initialized)
595 return;
596#if defined(_WIN32)
597 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
598#else
599 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, replace: 1);
600#endif
601
602 /* Create the profile dir (even if online merging is enabled), so that
603 * the profile file can be set up if continuous mode is enabled. */
604 createProfileDir(Filename);
605
606 /* By pass file truncation to allow online raw profile merging. */
607 if (lprofCurFilename.MergePoolSize)
608 return;
609
610 /* Truncate the file. Later we'll reopen and append. */
611 File = fopen(filename: Filename, modes: "w");
612 if (!File)
613 return;
614 fclose(stream: File);
615}
616
617/* Write a partial profile to \p Filename, which is required to be backed by
618 * the open file object \p File. */
619static int writeProfileWithFileObject(const char *Filename, FILE *File) {
620 setProfileFile(File);
621 int rc = writeFile(OutputName: Filename);
622 if (rc)
623 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
624 setProfileFile(NULL);
625 return rc;
626}
627
628static void initializeProfileForContinuousMode(void) {
629 if (!__llvm_profile_is_continuous_mode_enabled())
630 return;
631 if (!ContinuousModeSupported) {
632 PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
633 return;
634 }
635 if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
636 BitmapBiasAddr == BitmapBiasDefaultAddr) {
637 PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
638 "__llvm_profile_bitmap_bias is defined");
639 return;
640 }
641
642 /* Get the sizes of counter section. */
643 uint64_t CountersSize = __llvm_profile_get_counters_size(
644 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
645
646 int Length = getCurFilenameLength();
647 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
648 const char *Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
649 if (!Filename)
650 return;
651
652 FILE *File = NULL;
653 uint64_t CurrentFileOffset = 0;
654 if (doMerging()) {
655 /* We are merging profiles. Map the counter section as shared memory into
656 * the profile, i.e. into each participating process. An increment in one
657 * process should be visible to every other process with the same counter
658 * section mapped. */
659 File = lprofOpenFileEx(Filename);
660 if (!File)
661 return;
662
663 uint64_t ProfileFileSize = 0;
664 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
665 lprofUnlockFileHandle(F: File);
666 fclose(stream: File);
667 return;
668 }
669 if (ProfileFileSize == 0) {
670 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
671 * the file should stay open. */
672 if (writeProfileWithFileObject(Filename, File) != 0) {
673 lprofUnlockFileHandle(F: File);
674 fclose(stream: File);
675 return;
676 }
677 } else {
678 /* The merged profile has a non-zero length. Check that it is compatible
679 * with the data in this process. */
680 ManagedMemory ProfileBuffer;
681 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
682 lprofUnlockFileHandle(F: File);
683 fclose(stream: File);
684 return;
685 }
686 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
687 }
688 } else {
689 File = fopen(filename: Filename, modes: FileOpenMode);
690 if (!File)
691 return;
692 /* Check that the offset within the file is page-aligned. */
693 CurrentFileOffset = ftell(stream: File);
694 unsigned PageSize = getpagesize();
695 if (CurrentFileOffset % PageSize != 0) {
696 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
697 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
698 (uint64_t)CurrentFileOffset, PageSize);
699 fclose(stream: File);
700 return;
701 }
702 if (writeProfileWithFileObject(Filename, File) != 0) {
703 fclose(stream: File);
704 return;
705 }
706 }
707
708 /* mmap() the profile counters so long as there is at least one counter.
709 * If there aren't any counters, mmap() would fail with EINVAL. */
710 if (CountersSize > 0)
711 mmapForContinuousMode(CurrentFileOffset, File);
712
713 if (doMerging()) {
714 lprofUnlockFileHandle(F: File);
715 }
716 if (File != NULL) {
717 fclose(stream: File);
718 }
719}
720
721static const char *DefaultProfileName = "default.profraw";
722static void resetFilenameToDefault(void) {
723 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
724#ifdef __GNUC__
725#pragma GCC diagnostic push
726#pragma GCC diagnostic ignored "-Wcast-qual"
727#elif defined(__clang__)
728#pragma clang diagnostic push
729#pragma clang diagnostic ignored "-Wcast-qual"
730#endif
731 free(ptr: (void *)lprofCurFilename.FilenamePat);
732#ifdef __GNUC__
733#pragma GCC diagnostic pop
734#elif defined(__clang__)
735#pragma clang diagnostic pop
736#endif
737 }
738 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
739 lprofCurFilename.FilenamePat = DefaultProfileName;
740 lprofCurFilename.PNS = PNS_default;
741}
742
743static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
744 unsigned J = 0, Num = 0;
745 for (;; ++J) {
746 char C = FilenamePat[*I + J];
747 if (C == 'm') {
748 *I += J;
749 return Num ? Num : 1;
750 }
751 if (C < '0' || C > '9')
752 break;
753 Num = Num * 10 + C - '0';
754
755 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
756 * to be in-bound as the string is null terminated. */
757 }
758 return 0;
759}
760
761/* Assert that Idx does index past a string null terminator. Return the
762 * result of the check. */
763static int checkBounds(int Idx, int Strlen) {
764 assert(Idx <= Strlen && "Indexing past string null terminator");
765 return Idx <= Strlen;
766}
767
768/* Parses the pattern string \p FilenamePat and stores the result to
769 * lprofcurFilename structure. */
770static int parseFilenamePattern(const char *FilenamePat,
771 unsigned CopyFilenamePat) {
772 int NumPids = 0, NumHosts = 0, NumBinaryIds = 0, I;
773 char *PidChars = &lprofCurFilename.PidChars[0];
774 char *Hostname = &lprofCurFilename.Hostname[0];
775 int MergingEnabled = 0;
776 int FilenamePatLen = strlen(s: FilenamePat);
777
778#ifdef __GNUC__
779#pragma GCC diagnostic push
780#pragma GCC diagnostic ignored "-Wcast-qual"
781#elif defined(__clang__)
782#pragma clang diagnostic push
783#pragma clang diagnostic ignored "-Wcast-qual"
784#endif
785 /* Clean up cached prefix and filename. */
786 if (lprofCurFilename.ProfilePathPrefix)
787 free(ptr: (void *)lprofCurFilename.ProfilePathPrefix);
788
789 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
790 free(ptr: (void *)lprofCurFilename.FilenamePat);
791 }
792#ifdef __GNUC__
793#pragma GCC diagnostic pop
794#elif defined(__clang__)
795#pragma clang diagnostic pop
796#endif
797
798 memset(s: &lprofCurFilename, c: 0, n: sizeof(lprofCurFilename));
799
800 if (!CopyFilenamePat)
801 lprofCurFilename.FilenamePat = FilenamePat;
802 else {
803 lprofCurFilename.FilenamePat = strdup(s: FilenamePat);
804 lprofCurFilename.OwnsFilenamePat = 1;
805 }
806 /* Check the filename for "%p", which indicates a pid-substitution. */
807 for (I = 0; checkBounds(Idx: I, Strlen: FilenamePatLen) && FilenamePat[I]; ++I) {
808 if (FilenamePat[I] == '%') {
809 ++I; /* Advance to the next character. */
810 if (!checkBounds(Idx: I, Strlen: FilenamePatLen))
811 break;
812 if (FilenamePat[I] == 'p') {
813 if (!NumPids++) {
814 if (snprintf(s: PidChars, MAX_PID_SIZE, format: "%ld", (long)getpid()) <= 0) {
815 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
816 "default name.",
817 FilenamePat);
818 return -1;
819 }
820 }
821 } else if (FilenamePat[I] == 'h') {
822 if (!NumHosts++)
823 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
824 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
825 "the default name.",
826 FilenamePat);
827 return -1;
828 }
829 } else if (FilenamePat[I] == 't') {
830 lprofCurFilename.TmpDir = getenv(name: "TMPDIR");
831 if (!lprofCurFilename.TmpDir) {
832 PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
833 "in %s. Using the default path.",
834 FilenamePat);
835 return -1;
836 }
837 } else if (FilenamePat[I] == 'b') {
838 if (!NumBinaryIds++) {
839 /* Check if binary ID does not exist or if its size is 0. */
840 if (__llvm_write_binary_ids(NULL) <= 0) {
841 PROF_WARN("Unable to get binary ID for filename pattern %s. Using "
842 "the default name.",
843 FilenamePat);
844 return -1;
845 }
846 }
847 } else if (FilenamePat[I] == 'c') {
848 if (__llvm_profile_is_continuous_mode_enabled()) {
849 PROF_WARN("%%c specifier can only be specified once in %s.\n",
850 FilenamePat);
851 __llvm_profile_disable_continuous_mode();
852 return -1;
853 }
854#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX)
855 __llvm_profile_set_page_size(PageSize: getpagesize());
856 __llvm_profile_enable_continuous_mode();
857#else
858 PROF_WARN("%s",
859 "Continuous mode is currently only supported for Mach-O,"
860 " ELF and COFF formats.");
861 return -1;
862#endif
863 } else {
864 unsigned MergePoolSize = getMergePoolSize(FilenamePat, I: &I);
865 if (!MergePoolSize)
866 continue;
867 if (MergingEnabled) {
868 PROF_WARN("%%m specifier can only be specified once in %s.\n",
869 FilenamePat);
870 return -1;
871 }
872 MergingEnabled = 1;
873 lprofCurFilename.MergePoolSize = MergePoolSize;
874 }
875 }
876 }
877
878 lprofCurFilename.NumPids = NumPids;
879 lprofCurFilename.NumHosts = NumHosts;
880 lprofCurFilename.NumBinaryIds = NumBinaryIds;
881 return 0;
882}
883
884static void parseAndSetFilename(const char *FilenamePat,
885 ProfileNameSpecifier PNS,
886 unsigned CopyFilenamePat) {
887
888 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
889 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
890
891 /* The old profile name specifier takes precedence over the old one. */
892 if (PNS < OldPNS)
893 return;
894
895 if (!FilenamePat)
896 FilenamePat = DefaultProfileName;
897
898 if (OldFilenamePat && !strcmp(s1: OldFilenamePat, s2: FilenamePat)) {
899 lprofCurFilename.PNS = PNS;
900 return;
901 }
902
903 /* When PNS >= OldPNS, the last one wins. */
904 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
905 resetFilenameToDefault();
906 lprofCurFilename.PNS = PNS;
907
908 if (!OldFilenamePat) {
909 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
910 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
911 lprofCurFilename.FilenamePat, getPNSStr(PNS));
912 } else {
913 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
914 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
915 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
916 getPNSStr(PNS));
917 }
918
919 truncateCurrentFile();
920 if (__llvm_profile_is_continuous_mode_enabled())
921 initializeProfileForContinuousMode();
922}
923
924/* Return buffer length that is required to store the current profile
925 * filename with PID and hostname substitutions. */
926/* The length to hold uint64_t followed by 3 digits pool id including '_' */
927#define SIGLEN 24
928/* The length to hold 160-bit hash in hexadecimal form */
929#define BINARY_ID_LEN 40
930static int getCurFilenameLength(void) {
931 int Len;
932 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
933 return 0;
934
935 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
936 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
937 lprofCurFilename.MergePoolSize))
938 return strlen(s: lprofCurFilename.FilenamePat);
939
940 Len = strlen(s: lprofCurFilename.FilenamePat) +
941 lprofCurFilename.NumPids * (strlen(s: lprofCurFilename.PidChars) - 2) +
942 lprofCurFilename.NumHosts * (strlen(s: lprofCurFilename.Hostname) - 2) +
943 lprofCurFilename.NumBinaryIds * BINARY_ID_LEN +
944 (lprofCurFilename.TmpDir ? (strlen(s: lprofCurFilename.TmpDir) - 1) : 0);
945 if (lprofCurFilename.MergePoolSize)
946 Len += SIGLEN;
947 return Len;
948}
949
950typedef struct lprofBinaryIdsBuffer {
951 char String[BINARY_ID_LEN + 1];
952 int Length;
953} lprofBinaryIdsBuffer;
954
955/* Reads binary ID length and then its data, writes it into lprofBinaryIdsBuffer
956 * in hexadecimal form. */
957static uint32_t binaryIdsStringWriter(ProfDataWriter *This,
958 ProfDataIOVec *IOVecs,
959 uint32_t NumIOVecs) {
960 if (NumIOVecs < 2 || IOVecs[0].ElmSize != sizeof(uint64_t))
961 return -1;
962 uint64_t BinaryIdLen = *(const uint64_t *)IOVecs[0].Data;
963 if (IOVecs[1].ElmSize != sizeof(uint8_t) || IOVecs[1].NumElm != BinaryIdLen)
964 return -1;
965 const uint8_t *BinaryIdData = (const uint8_t *)IOVecs[1].Data;
966 lprofBinaryIdsBuffer *Data = (lprofBinaryIdsBuffer *)This->WriterCtx;
967 for (uint64_t I = 0; I < BinaryIdLen; I++) {
968 Data->Length +=
969 snprintf(s: Data->String + Data->Length, BINARY_ID_LEN + 1 - Data->Length,
970 format: "%02hhx", BinaryIdData[I]);
971 }
972 return 0;
973}
974
975/* Return the pointer to the current profile file name (after substituting
976 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
977 * to store the resulting filename. If no substitution is needed, the
978 * current filename pattern string is directly returned, unless ForceUseBuf
979 * is enabled. */
980static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
981 int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
982 const char *FilenamePat = lprofCurFilename.FilenamePat;
983
984 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
985 return 0;
986
987 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
988 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
989 lprofCurFilename.MergePoolSize ||
990 __llvm_profile_is_continuous_mode_enabled())) {
991 if (!ForceUseBuf)
992 return lprofCurFilename.FilenamePat;
993
994 FilenamePatLength = strlen(s: lprofCurFilename.FilenamePat);
995 memcpy(dest: FilenameBuf, src: lprofCurFilename.FilenamePat, n: FilenamePatLength);
996 FilenameBuf[FilenamePatLength] = '\0';
997 return FilenameBuf;
998 }
999
1000 PidLength = strlen(s: lprofCurFilename.PidChars);
1001 HostNameLength = strlen(s: lprofCurFilename.Hostname);
1002 TmpDirLength = lprofCurFilename.TmpDir ? strlen(s: lprofCurFilename.TmpDir) : 0;
1003 /* Construct the new filename. */
1004 for (I = 0, J = 0; FilenamePat[I]; ++I)
1005 if (FilenamePat[I] == '%') {
1006 if (FilenamePat[++I] == 'p') {
1007 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.PidChars, n: PidLength);
1008 J += PidLength;
1009 } else if (FilenamePat[I] == 'h') {
1010 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.Hostname, n: HostNameLength);
1011 J += HostNameLength;
1012 } else if (FilenamePat[I] == 't') {
1013 memcpy(dest: FilenameBuf + J, src: lprofCurFilename.TmpDir, n: TmpDirLength);
1014 FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
1015 J += TmpDirLength + 1;
1016 } else if (FilenamePat[I] == 'b') {
1017 lprofBinaryIdsBuffer Data = {{0}, 0};
1018 ProfDataWriter Writer = {binaryIdsStringWriter, &Data};
1019 __llvm_write_binary_ids(Writer: &Writer);
1020 memcpy(dest: FilenameBuf + J, src: Data.String, n: Data.Length);
1021 J += Data.Length;
1022 } else {
1023 if (!getMergePoolSize(FilenamePat, I: &I))
1024 continue;
1025 char LoadModuleSignature[SIGLEN + 1];
1026 int S;
1027 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
1028 S = snprintf(s: LoadModuleSignature, SIGLEN + 1, format: "%" PRIu64 "_%d",
1029 lprofGetLoadModuleSignature(), ProfilePoolId);
1030 if (S == -1 || S > SIGLEN)
1031 S = SIGLEN;
1032 memcpy(dest: FilenameBuf + J, src: LoadModuleSignature, n: S);
1033 J += S;
1034 }
1035 /* Drop any unknown substitutions. */
1036 } else
1037 FilenameBuf[J++] = FilenamePat[I];
1038 FilenameBuf[J] = 0;
1039
1040 return FilenameBuf;
1041}
1042
1043/* Returns the pointer to the environment variable
1044 * string. Returns null if the env var is not set. */
1045static const char *getFilenamePatFromEnv(void) {
1046 const char *Filename = getenv(name: "LLVM_PROFILE_FILE");
1047 if (!Filename || !Filename[0])
1048 return 0;
1049 return Filename;
1050}
1051
1052COMPILER_RT_VISIBILITY
1053const char *__llvm_profile_get_path_prefix(void) {
1054 int Length;
1055 char *FilenameBuf, *Prefix;
1056 const char *Filename, *PrefixEnd;
1057
1058 if (lprofCurFilename.ProfilePathPrefix)
1059 return lprofCurFilename.ProfilePathPrefix;
1060
1061 Length = getCurFilenameLength();
1062 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1063 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1064 if (!Filename)
1065 return "\0";
1066
1067 PrefixEnd = lprofFindLastDirSeparator(Path: Filename);
1068 if (!PrefixEnd)
1069 return "\0";
1070
1071 Length = PrefixEnd - Filename + 1;
1072 Prefix = (char *)malloc(size: Length + 1);
1073 if (!Prefix) {
1074 PROF_ERR("Failed to %s\n", "allocate memory.");
1075 return "\0";
1076 }
1077 memcpy(dest: Prefix, src: Filename, n: Length);
1078 Prefix[Length] = '\0';
1079 lprofCurFilename.ProfilePathPrefix = Prefix;
1080 return Prefix;
1081}
1082
1083COMPILER_RT_VISIBILITY
1084const char *__llvm_profile_get_filename(void) {
1085 int Length;
1086 char *FilenameBuf;
1087 const char *Filename;
1088
1089 Length = getCurFilenameLength();
1090 FilenameBuf = (char *)malloc(size: Length + 1);
1091 if (!FilenameBuf) {
1092 PROF_ERR("Failed to %s\n", "allocate memory.");
1093 return "\0";
1094 }
1095 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 1);
1096 if (!Filename) {
1097 free(ptr: FilenameBuf);
1098 return "\0";
1099 }
1100
1101 return FilenameBuf;
1102}
1103
1104/* This API initializes the file handling, both user specified
1105 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
1106 * environment variable can override this default value.
1107 */
1108COMPILER_RT_VISIBILITY
1109void __llvm_profile_initialize_file(void) {
1110 const char *EnvFilenamePat;
1111 const char *SelectedPat = NULL;
1112 ProfileNameSpecifier PNS = PNS_unknown;
1113 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
1114
1115 EnvFilenamePat = getFilenamePatFromEnv();
1116 if (EnvFilenamePat) {
1117 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
1118 at the moment when __llvm_profile_write_file() gets executed. */
1119 parseAndSetFilename(FilenamePat: EnvFilenamePat, PNS: PNS_environment, CopyFilenamePat: 1);
1120 return;
1121 } else if (hasCommandLineOverrider) {
1122 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
1123 PNS = PNS_command_line;
1124 } else {
1125 SelectedPat = NULL;
1126 PNS = PNS_default;
1127 }
1128
1129 parseAndSetFilename(FilenamePat: SelectedPat, PNS, CopyFilenamePat: 0);
1130}
1131
1132/* This method is invoked by the runtime initialization hook
1133 * InstrProfilingRuntime.o if it is linked in.
1134 */
1135COMPILER_RT_VISIBILITY
1136void __llvm_profile_initialize(void) {
1137 __llvm_profile_initialize_file();
1138 if (!__llvm_profile_is_continuous_mode_enabled())
1139 __llvm_profile_register_write_file_atexit();
1140}
1141
1142/* This API is directly called by the user application code. It has the
1143 * highest precedence compared with LLVM_PROFILE_FILE environment variable
1144 * and command line option -fprofile-instr-generate=<profile_name>.
1145 */
1146COMPILER_RT_VISIBILITY
1147void __llvm_profile_set_filename(const char *FilenamePat) {
1148 if (__llvm_profile_is_continuous_mode_enabled())
1149 return;
1150 parseAndSetFilename(FilenamePat, PNS: PNS_runtime_api, CopyFilenamePat: 1);
1151}
1152
1153/* The public API for writing profile data into the file with name
1154 * set by previous calls to __llvm_profile_set_filename or
1155 * __llvm_profile_override_default_filename or
1156 * __llvm_profile_initialize_file. */
1157COMPILER_RT_VISIBILITY
1158int __llvm_profile_write_file(void) {
1159 int rc, Length;
1160 const char *Filename;
1161 char *FilenameBuf;
1162
1163 // Temporarily suspend getting SIGKILL when the parent exits.
1164 int PDeathSig = lprofSuspendSigKill();
1165
1166 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1167 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1168 if (PDeathSig == 1)
1169 lprofRestoreSigKill();
1170 return 0;
1171 }
1172
1173 Length = getCurFilenameLength();
1174 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1175 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1176
1177 /* Check the filename. */
1178 if (!Filename) {
1179 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1180 if (PDeathSig == 1)
1181 lprofRestoreSigKill();
1182 return -1;
1183 }
1184
1185 /* Check if there is llvm/runtime version mismatch. */
1186 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1187 PROF_ERR("Runtime and instrumentation version mismatch : "
1188 "expected %d, but get %d\n",
1189 INSTR_PROF_RAW_VERSION,
1190 (int)GET_VERSION(__llvm_profile_get_version()));
1191 if (PDeathSig == 1)
1192 lprofRestoreSigKill();
1193 return -1;
1194 }
1195
1196 /* Write profile data to the file. */
1197 rc = writeFile(OutputName: Filename);
1198 if (rc)
1199 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1200
1201 // Restore SIGKILL.
1202 if (PDeathSig == 1)
1203 lprofRestoreSigKill();
1204
1205 return rc;
1206}
1207
1208COMPILER_RT_VISIBILITY
1209int __llvm_profile_dump(void) {
1210 if (!doMerging())
1211 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1212 " of previously dumped profile data : %s. Either use %%m "
1213 "in profile name or change profile name before dumping.\n",
1214 "online profile merging is not on");
1215 int rc = __llvm_profile_write_file();
1216 lprofSetProfileDumped(1);
1217 return rc;
1218}
1219
1220static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1221
1222COMPILER_RT_VISIBILITY
1223int __llvm_profile_register_write_file_atexit(void) {
1224 static int HasBeenRegistered = 0;
1225
1226 if (HasBeenRegistered)
1227 return 0;
1228
1229 lprofSetupValueProfiler();
1230
1231 HasBeenRegistered = 1;
1232 return lprofAtExit(writeFileWithoutReturn);
1233}
1234
1235COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
1236 int EnableMerge) {
1237 if (__llvm_profile_is_continuous_mode_enabled()) {
1238 if (!EnableMerge) {
1239 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
1240 "continuous sync mode when merging is disabled\n",
1241 fileno(File));
1242 return 1;
1243 }
1244 if (lprofLockFileHandle(F: File) != 0) {
1245 PROF_WARN("Data may be corrupted during profile merging : %s\n",
1246 "Fail to obtain file lock due to system limit.");
1247 }
1248 uint64_t ProfileFileSize = 0;
1249 if (getProfileFileSizeForMerging(ProfileFile: File, ProfileFileSize: &ProfileFileSize) == -1) {
1250 lprofUnlockFileHandle(F: File);
1251 return 1;
1252 }
1253 if (ProfileFileSize == 0) {
1254 FreeHook = &free;
1255 setupIOBuffer();
1256 ProfDataWriter fileWriter;
1257 initFileWriter(This: &fileWriter, File);
1258 if (lprofWriteData(Writer: &fileWriter, VPDataReader: 0, SkipNameDataWrite: 0)) {
1259 lprofUnlockFileHandle(F: File);
1260 PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
1261 strerror(errno));
1262 return 1;
1263 }
1264 fflush(stream: File);
1265 } else {
1266 /* The merged profile has a non-zero length. Check that it is compatible
1267 * with the data in this process. */
1268 ManagedMemory ProfileBuffer;
1269 if (mmapProfileForMerging(ProfileFile: File, ProfileFileSize, ProfileBuffer: &ProfileBuffer) == -1) {
1270 lprofUnlockFileHandle(F: File);
1271 return 1;
1272 }
1273 (void)lprofReleaseBuffer(FileBuffer: &ProfileBuffer, Length: ProfileFileSize);
1274 }
1275 mmapForContinuousMode(CurrentFileOffset: 0, File);
1276 lprofUnlockFileHandle(F: File);
1277 } else {
1278 setProfileFile(File);
1279 setProfileMergeRequested(EnableMerge);
1280 }
1281 return 0;
1282}
1283
1284#ifndef __APPLE__
1285int __llvm_write_custom_profile(const char *Target,
1286 const __llvm_profile_data *DataBegin,
1287 const __llvm_profile_data *DataEnd,
1288 const char *CountersBegin,
1289 const char *CountersEnd, const char *NamesBegin,
1290 const char *NamesEnd,
1291 const uint64_t *VersionOverride) {
1292 int ReturnValue = 0, FilenameLength, TargetLength;
1293 char *FilenameBuf, *TargetFilename;
1294 const char *Filename;
1295
1296 /* Save old profile data */
1297 FILE *oldFile = getProfileFile();
1298
1299 // Temporarily suspend getting SIGKILL when the parent exits.
1300 int PDeathSig = lprofSuspendSigKill();
1301
1302 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1303 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1304 if (PDeathSig == 1)
1305 lprofRestoreSigKill();
1306 return 0;
1307 }
1308
1309 /* Check if there is llvm/runtime version mismatch. */
1310 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1311 PROF_ERR("Runtime and instrumentation version mismatch : "
1312 "expected %d, but get %d\n",
1313 INSTR_PROF_RAW_VERSION,
1314 (int)GET_VERSION(__llvm_profile_get_version()));
1315 if (PDeathSig == 1)
1316 lprofRestoreSigKill();
1317 return -1;
1318 }
1319
1320 /* Get current filename */
1321 FilenameLength = getCurFilenameLength();
1322 FilenameBuf = (char *)COMPILER_RT_ALLOCA(FilenameLength + 1);
1323 Filename = getCurFilename(FilenameBuf, ForceUseBuf: 0);
1324
1325 /* Check the filename. */
1326 if (!Filename) {
1327 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1328 if (PDeathSig == 1)
1329 lprofRestoreSigKill();
1330 return -1;
1331 }
1332
1333 /* Allocate new space for our target-specific PGO filename */
1334 TargetLength = strlen(s: Target);
1335 TargetFilename =
1336 (char *)COMPILER_RT_ALLOCA(FilenameLength + TargetLength + 2);
1337
1338 /* Find file basename and path sizes */
1339 int32_t DirEnd = FilenameLength - 1;
1340 while (DirEnd >= 0 && !IS_DIR_SEPARATOR(Filename[DirEnd])) {
1341 DirEnd--;
1342 }
1343 uint32_t DirSize = DirEnd + 1, BaseSize = FilenameLength - DirSize;
1344
1345 /* Prepend "TARGET." to current filename */
1346 if (DirSize > 0) {
1347 memcpy(dest: TargetFilename, src: Filename, n: DirSize);
1348 }
1349 memcpy(dest: TargetFilename + DirSize, src: Target, n: TargetLength);
1350 TargetFilename[TargetLength + DirSize] = '.';
1351 memcpy(dest: TargetFilename + DirSize + 1 + TargetLength, src: Filename + DirSize,
1352 n: BaseSize);
1353 TargetFilename[FilenameLength + 1 + TargetLength] = 0;
1354
1355 /* Open and truncate target-specific PGO file */
1356 FILE *OutputFile = fopen(filename: TargetFilename, modes: "w");
1357 setProfileFile(OutputFile);
1358
1359 if (!OutputFile) {
1360 PROF_ERR("Failed to open file : %s\n", TargetFilename);
1361 if (PDeathSig == 1)
1362 lprofRestoreSigKill();
1363 return -1;
1364 }
1365
1366 FreeHook = &free;
1367 setupIOBuffer();
1368
1369 /* Write custom data */
1370 ProfDataWriter fileWriter;
1371 initFileWriter(This: &fileWriter, File: OutputFile);
1372
1373 uint64_t Version = __llvm_profile_get_version();
1374 if (VersionOverride)
1375 Version = *VersionOverride;
1376
1377 /* Write custom data to the file */
1378 ReturnValue =
1379 lprofWriteDataImpl(Writer: &fileWriter, DataBegin, DataEnd, CountersBegin,
1380 CountersEnd, NULL, NULL, VPDataReader: lprofGetVPDataReader(), NULL,
1381 NULL, NULL, NULL, VNamesBegin: NamesBegin, VNamesEnd: NamesEnd, SkipNameDataWrite: 0, Version);
1382 closeFileObject(OutputFile);
1383
1384 // Restore SIGKILL.
1385 if (PDeathSig == 1)
1386 lprofRestoreSigKill();
1387
1388 /* Restore old profiling file */
1389 setProfileFile(oldFile);
1390
1391 return ReturnValue;
1392}
1393#endif
1394
1395#endif
1396