1/*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9#ifdef _WIN32
10#include <direct.h>
11#include <process.h>
12#include <windows.h>
13#include "WindowsMMap.h"
14#else
15#if defined(__linux__)
16// For fdopen(), fileno(), getpagesize(), madvise()
17#define _DEFAULT_SOURCE
18#endif
19
20#include <errno.h>
21#include <fcntl.h>
22#include <sys/file.h>
23#include <sys/mman.h>
24#include <sys/stat.h>
25#include <sys/types.h>
26#include <unistd.h>
27#endif
28
29#ifdef _AIX
30#include <sys/statfs.h>
31// <sys/vmount.h> depends on `uint` to be a typedef from <sys/types.h> to
32// `uint_t`; however, <sys/types.h> does not always declare `uint`. We provide
33// the typedef prior to including <sys/vmount.h> to work around this issue.
34typedef uint_t uint;
35#include <sys/vmount.h>
36#endif
37
38#ifdef COMPILER_RT_HAS_UNAME
39#include <sys/utsname.h>
40#endif
41
42#include <stdlib.h>
43#include <string.h>
44
45#if defined(__linux__)
46#include <signal.h>
47#include <sys/prctl.h>
48#endif
49
50#if defined(__Fuchsia__)
51#include <zircon/process.h>
52#include <zircon/syscalls.h>
53#endif
54
55#if defined(__FreeBSD__)
56#include <signal.h>
57#include <sys/procctl.h>
58#endif
59
60#include "InstrProfiling.h"
61#include "InstrProfilingUtil.h"
62
63COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755;
64
65COMPILER_RT_VISIBILITY
66void __llvm_profile_recursive_mkdir(char *path) {
67 int i;
68 int start = 1;
69
70#if defined(__ANDROID__) && defined(__ANDROID_API__) && \
71 defined(__ANDROID_API_FUTURE__) && \
72 __ANDROID_API__ == __ANDROID_API_FUTURE__
73 // Avoid spammy selinux denial messages in Android by not attempting to
74 // create directories in GCOV_PREFIX. These denials occur when creating (or
75 // even attempting to stat()) top-level directories like "/data".
76 //
77 // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir().
78 const char *gcov_prefix = getenv("GCOV_PREFIX");
79 if (gcov_prefix != NULL) {
80 const int gcov_prefix_len = strlen(gcov_prefix);
81 if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0)
82 start = gcov_prefix_len;
83 }
84#endif
85
86 for (i = start; path[i] != '\0'; ++i) {
87 char save = path[i];
88 if (!IS_DIR_SEPARATOR(path[i]))
89 continue;
90 path[i] = '\0';
91#ifdef _WIN32
92 _mkdir(path);
93#else
94 /* Some of these will fail, ignore it. */
95 mkdir(path: path, mode: __llvm_profile_get_dir_mode());
96#endif
97 path[i] = save;
98 }
99}
100
101COMPILER_RT_VISIBILITY
102void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; }
103
104COMPILER_RT_VISIBILITY
105unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; }
106
107#if COMPILER_RT_HAS_ATOMICS != 1
108COMPILER_RT_VISIBILITY
109uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
110 void *R = *Ptr;
111 if (R == OldV) {
112 *Ptr = NewV;
113 return 1;
114 }
115 return 0;
116}
117COMPILER_RT_VISIBILITY
118void *lprofPtrFetchAdd(void **Mem, long ByteIncr) {
119 void *Old = *Mem;
120 *((char **)Mem) += ByteIncr;
121 return Old;
122}
123
124#endif
125
126#ifdef _WIN32
127COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
128 WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN];
129 DWORD BufferSize = sizeof(Buffer);
130 BOOL Result =
131 GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize);
132 if (!Result)
133 return -1;
134 if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0)
135 return -1;
136 return 0;
137}
138#elif defined(COMPILER_RT_HAS_UNAME)
139COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
140 struct utsname N;
141 int R = uname(name: &N);
142 if (R >= 0) {
143 strncpy(dest: Name, src: N.nodename, n: Len);
144 return 0;
145 }
146 return R;
147}
148#endif
149
150COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {
151#ifdef COMPILER_RT_HAS_FCNTL_LCK
152 struct flock s_flock;
153
154 s_flock.l_whence = SEEK_SET;
155 s_flock.l_start = 0;
156 s_flock.l_len = 0; /* Until EOF. */
157 s_flock.l_pid = getpid();
158 s_flock.l_type = F_WRLCK;
159
160 while (fcntl(fd: fd, F_SETLKW, &s_flock) == -1) {
161 if (errno != EINTR) {
162 if (errno == ENOLCK) {
163 return -1;
164 }
165 break;
166 }
167 }
168 return 0;
169#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
170 // Windows doesn't have flock but WindowsMMap.h provides a shim
171 flock(fd, LOCK_EX);
172 return 0;
173#else
174 return 0;
175#endif
176}
177
178COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {
179#ifdef COMPILER_RT_HAS_FCNTL_LCK
180 struct flock s_flock;
181
182 s_flock.l_whence = SEEK_SET;
183 s_flock.l_start = 0;
184 s_flock.l_len = 0; /* Until EOF. */
185 s_flock.l_pid = getpid();
186 s_flock.l_type = F_UNLCK;
187
188 while (fcntl(fd: fd, F_SETLKW, &s_flock) == -1) {
189 if (errno != EINTR) {
190 if (errno == ENOLCK) {
191 return -1;
192 }
193 break;
194 }
195 }
196 return 0;
197#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
198 // Windows doesn't have flock but WindowsMMap.h provides a shim
199 flock(fd, LOCK_UN);
200 return 0;
201#else
202 return 0;
203#endif
204}
205
206COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) {
207 int fd;
208#if defined(_WIN32)
209 fd = _fileno(F);
210#else
211 fd = fileno(stream: F);
212#endif
213 return lprofLockFd(fd);
214}
215
216COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) {
217 int fd;
218#if defined(_WIN32)
219 fd = _fileno(F);
220#else
221 fd = fileno(stream: F);
222#endif
223 return lprofUnlockFd(fd);
224}
225
226COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
227 FILE *f;
228 int fd;
229#ifdef COMPILER_RT_HAS_FCNTL_LCK
230 fd = open(file: ProfileName, O_RDWR | O_CREAT, 0666);
231 if (fd < 0)
232 return NULL;
233
234 if (lprofLockFd(fd) != 0)
235 PROF_WARN("Data may be corrupted during profile merging : %s\n",
236 "Fail to obtain file lock due to system limit.");
237
238 f = fdopen(fd: fd, modes: "r+b");
239#elif defined(_WIN32)
240 // FIXME: Use the wide variants to handle Unicode filenames.
241 HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE,
242 FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
243 FILE_ATTRIBUTE_NORMAL, 0);
244 if (h == INVALID_HANDLE_VALUE)
245 return NULL;
246
247 fd = _open_osfhandle((intptr_t)h, 0);
248 if (fd == -1) {
249 CloseHandle(h);
250 return NULL;
251 }
252
253 if (lprofLockFd(fd) != 0)
254 PROF_WARN("Data may be corrupted during profile merging : %s\n",
255 "Fail to obtain file lock due to system limit.");
256
257 f = _fdopen(fd, "r+b");
258 if (f == 0) {
259 CloseHandle(h);
260 return NULL;
261 }
262#else
263 /* Worst case no locking applied. */
264 PROF_WARN("Concurrent file access is not supported : %s\n",
265 "lack file locking");
266 fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
267 if (fd < 0)
268 return NULL;
269 f = fdopen(fd, "r+b");
270#endif
271
272 return f;
273}
274
275#if defined(_AIX)
276// Return 1 (true) if the file descriptor Fd represents a file that is on a
277// local filesystem, otherwise return 0.
278static int isLocalFilesystem(int Fd) {
279 struct statfs Vfs;
280 if (fstatfs(Fd, &Vfs) != 0) {
281 PROF_ERR("%s: fstatfs(%d) failed: %s\n", __func__, Fd, strerror(errno));
282 return 0;
283 }
284
285 int Ret;
286 size_t BufSize = 2048u;
287 char *Buf;
288 int Tries = 3;
289 while (Tries--) {
290 Buf = malloc(BufSize);
291 // mntctl returns -1 if `Buf` is `NULL`.
292 Ret = mntctl(MCTL_QUERY, BufSize, Buf);
293 if (Ret != 0)
294 break;
295 BufSize = *(unsigned int *)Buf;
296 free(Buf);
297 }
298
299 if (Ret != -1) {
300 // Look for the correct vmount entry.
301 char *CurObjPtr = Buf;
302 while (Ret--) {
303 struct vmount *Vp = (struct vmount *)CurObjPtr;
304 _Static_assert(sizeof(Vfs.f_fsid) == sizeof(Vp->vmt_fsid),
305 "fsid length mismatch");
306 if (memcmp(&Vfs.f_fsid, &Vp->vmt_fsid, sizeof Vfs.f_fsid) == 0) {
307 int Answer = (Vp->vmt_flags & MNT_REMOTE) == 0;
308 free(Buf);
309 return Answer;
310 }
311 CurObjPtr += Vp->vmt_length;
312 }
313 }
314
315 free(Buf);
316 // There was an error in mntctl or vmount entry not found; "remote" is the
317 // conservative answer.
318 return 0;
319}
320#endif
321
322static int isMmapSafe(int Fd) {
323 if (getenv(name: "LLVM_PROFILE_NO_MMAP")) // For testing purposes.
324 return 0;
325#ifdef _AIX
326 return isLocalFilesystem(Fd);
327#else
328 return 1;
329#endif
330}
331
332COMPILER_RT_VISIBILITY void lprofGetFileContentBuffer(FILE *F, uint64_t Length,
333 ManagedMemory *Buf) {
334 Buf->Status = MS_INVALID;
335 if (isMmapSafe(Fd: fileno(stream: F))) {
336 Buf->Addr =
337 mmap(NULL, len: Length, PROT_READ, MAP_SHARED | MAP_FILE, fd: fileno(stream: F), offset: 0);
338 if (Buf->Addr == MAP_FAILED)
339 PROF_ERR("%s: mmap failed: %s\n", __func__, strerror(errno))
340 else
341 Buf->Status = MS_MMAP;
342 return;
343 }
344
345 if (getenv(name: "LLVM_PROFILE_VERBOSE"))
346 PROF_NOTE("%s\n", "could not use mmap; using fread instead");
347
348 void *Buffer = malloc(size: Length);
349 if (!Buffer) {
350 PROF_ERR("%s: malloc failed: %s\n", __func__, strerror(errno));
351 return;
352 }
353 if (ftell(stream: F) != 0) {
354 PROF_ERR("%s: expecting ftell to return zero\n", __func__);
355 free(ptr: Buffer);
356 return;
357 }
358
359 // Read the entire file into memory.
360 size_t BytesRead = fread(ptr: Buffer, size: 1, n: Length, stream: F);
361 if (BytesRead != (size_t)Length) {
362 PROF_ERR("%s: fread failed%s\n", __func__,
363 feof(F) ? ": end of file reached" : "");
364 free(ptr: Buffer);
365 return;
366 }
367
368 // Reading was successful, record the result in the Buf parameter.
369 Buf->Addr = Buffer;
370 Buf->Status = MS_MALLOC;
371}
372
373COMPILER_RT_VISIBILITY
374void lprofReleaseBuffer(ManagedMemory *Buf, size_t Length) {
375 switch (Buf->Status) {
376 case MS_MALLOC:
377 free(ptr: Buf->Addr);
378 break;
379 case MS_MMAP:
380 (void)munmap(addr: Buf->Addr, len: Length);
381 break;
382 default:
383 PROF_ERR("%s: Buffer has invalid state: %d\n", __func__, Buf->Status);
384 break;
385 }
386 Buf->Addr = NULL;
387 Buf->Status = MS_INVALID;
388}
389
390COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip,
391 size_t *PrefixLen) {
392 const char *Prefix = getenv(name: "GCOV_PREFIX");
393 const char *PrefixStripStr = getenv(name: "GCOV_PREFIX_STRIP");
394
395 *PrefixLen = 0;
396 *PrefixStrip = 0;
397 if (Prefix == NULL || Prefix[0] == '\0')
398 return NULL;
399
400 if (PrefixStripStr) {
401 *PrefixStrip = atoi(nptr: PrefixStripStr);
402
403 /* Negative GCOV_PREFIX_STRIP values are ignored */
404 if (*PrefixStrip < 0)
405 *PrefixStrip = 0;
406 } else {
407 *PrefixStrip = 0;
408 }
409 *PrefixLen = strlen(s: Prefix);
410
411 return Prefix;
412}
413
414COMPILER_RT_VISIBILITY void
415lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix,
416 size_t PrefixLen, int PrefixStrip) {
417
418 const char *Ptr;
419 int Level;
420 const char *StrippedPathStr = PathStr;
421
422 for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) {
423 if (*Ptr == '\0')
424 break;
425
426 if (!IS_DIR_SEPARATOR(*Ptr))
427 continue;
428
429 StrippedPathStr = Ptr;
430 ++Level;
431 }
432
433 memcpy(dest: Dest, src: Prefix, n: PrefixLen);
434
435 if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1]))
436 Dest[PrefixLen++] = DIR_SEPARATOR;
437
438 memcpy(dest: Dest + PrefixLen, src: StrippedPathStr, n: strlen(s: StrippedPathStr) + 1);
439}
440
441COMPILER_RT_VISIBILITY const char *
442lprofFindFirstDirSeparator(const char *Path) {
443 const char *Sep = strchr(s: Path, DIR_SEPARATOR);
444#if defined(DIR_SEPARATOR_2)
445 const char *Sep2 = strchr(Path, DIR_SEPARATOR_2);
446 if (Sep2 && (!Sep || Sep2 < Sep))
447 Sep = Sep2;
448#endif
449 return Sep;
450}
451
452COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) {
453 const char *Sep = strrchr(s: Path, DIR_SEPARATOR);
454#if defined(DIR_SEPARATOR_2)
455 const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2);
456 if (Sep2 && (!Sep || Sep2 > Sep))
457 Sep = Sep2;
458#endif
459 return Sep;
460}
461
462COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) {
463#if defined(__linux__)
464 int PDeachSig = 0;
465 /* Temporarily suspend getting SIGKILL upon exit of the parent process. */
466 if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL)
467 prctl(PR_SET_PDEATHSIG, 0);
468 return (PDeachSig == SIGKILL);
469#elif defined(__FreeBSD__)
470 int PDeachSig = 0, PDisableSig = 0;
471 if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 &&
472 PDeachSig == SIGKILL)
473 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig);
474 return (PDeachSig == SIGKILL);
475#else
476 return 0;
477#endif
478}
479
480COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {
481#if defined(__linux__)
482 prctl(PR_SET_PDEATHSIG, SIGKILL);
483#elif defined(__FreeBSD__)
484 int PEnableSig = SIGKILL;
485 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig);
486#endif
487}
488
489COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
490 uintptr_t End) {
491#if defined(__ve__) || defined(__wasi__)
492 // VE and WASI doesn't support madvise.
493 return 0;
494#else
495 size_t PageSize = getpagesize();
496 uintptr_t BeginAligned = lprofRoundUpTo(x: (uintptr_t)Begin, boundary: PageSize);
497 uintptr_t EndAligned = lprofRoundDownTo(x: (uintptr_t)End, boundary: PageSize);
498 if (BeginAligned < EndAligned) {
499#if defined(__Fuchsia__)
500 return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT,
501 (zx_vaddr_t)BeginAligned,
502 EndAligned - BeginAligned, NULL, 0);
503#else
504 return madvise(addr: (void *)BeginAligned, len: EndAligned - BeginAligned,
505 MADV_DONTNEED);
506#endif
507 }
508 return 0;
509#endif
510}
511
512#ifdef _AIX
513typedef struct fn_node {
514 AtExit_Fn_ptr func;
515 struct fn_node *next;
516} fn_node;
517typedef struct {
518 fn_node *top;
519} fn_stack;
520
521static void fn_stack_push(fn_stack *, AtExit_Fn_ptr);
522static AtExit_Fn_ptr fn_stack_pop(fn_stack *);
523/* return 1 if stack is empty, 0 otherwise */
524static int fn_stack_is_empty(fn_stack *);
525
526static fn_stack AtExit_stack = {0};
527#define ATEXIT_STACK (&AtExit_stack)
528
529/* On AIX, atexit() functions registered by a shared library do not get called
530 * when the library is dlclose'd, causing a crash when they are eventually
531 * called at main program exit. However, a destructor does get called. So we
532 * collect all atexit functions registered by profile-rt and at program
533 * termination time (normal exit, shared library unload, or dlclose) we walk
534 * the list and execute any function that is still sitting in the atexit system
535 * queue.
536 */
537__attribute__((__destructor__)) static void cleanup() {
538 while (!fn_stack_is_empty(ATEXIT_STACK)) {
539 AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK);
540 if (func && unatexit(func) == 0)
541 func();
542 }
543}
544
545static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) {
546 fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node));
547 n->func = func;
548
549 while (1) {
550 old_top = st->top;
551 n->next = old_top;
552 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n))
553 return;
554 }
555}
556static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) {
557 fn_node *old_top, *new_top;
558 while (1) {
559 old_top = st->top;
560 if (old_top == 0)
561 return 0;
562 new_top = old_top->next;
563 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) {
564 AtExit_Fn_ptr func = old_top->func;
565 free(old_top);
566 return func;
567 }
568 }
569}
570
571static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; }
572#endif
573
574COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) {
575#ifdef _AIX
576 fn_stack_push(ATEXIT_STACK, func);
577#endif
578 return atexit(func: func);
579}
580