1/*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*|
8|*
9|* This file implements the call back routines for the gcov profiling
10|* instrumentation pass. Link against this library when running code through
11|* the -insert-gcov-profiling LLVM pass.
12|*
13|* We emit files in a corrupt version of GCOV's "gcda" file format. These files
14|* are only close enough that LCOV will happily parse them. Anything that lcov
15|* ignores is missing.
16|*
17|* TODO: gcov is multi-process safe by having each exit open the existing file
18|* and append to it. We'd like to achieve that and be thread-safe too.
19|*
20\*===----------------------------------------------------------------------===*/
21
22#if !defined(__Fuchsia__)
23
24#include <errno.h>
25#include <fcntl.h>
26#include <stdint.h>
27#include <stdio.h>
28#include <stdlib.h>
29#include <string.h>
30
31#if defined(_WIN32)
32#define WIN32_LEAN_AND_MEAN
33#include <windows.h>
34#include "WindowsMMap.h"
35#else
36#include <sys/file.h>
37#include <sys/mman.h>
38#include <sys/types.h>
39#include <unistd.h>
40#endif
41
42#include "InstrProfiling.h"
43#include "InstrProfilingUtil.h"
44
45/* #define DEBUG_GCDAPROFILING */
46
47enum {
48 GCOV_DATA_MAGIC = 0x67636461, // "gcda"
49
50 GCOV_TAG_FUNCTION = 0x01000000,
51 GCOV_TAG_COUNTER_ARCS = 0x01a10000,
52 // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9.
53 GCOV_TAG_OBJECT_SUMMARY = 0xa1000000,
54 GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000,
55};
56
57/*
58 * --- GCOV file format I/O primitives ---
59 */
60
61/*
62 * The current file name we're outputting. Used primarily for error logging.
63 */
64static char *filename = NULL;
65
66/*
67 * The current file we're outputting.
68 */
69static FILE *output_file = NULL;
70
71/*
72 * Buffer that we write things into.
73 */
74#define WRITE_BUFFER_SIZE (128 * 1024)
75static unsigned char *write_buffer = NULL;
76static uint64_t cur_buffer_size = 0;
77static uint64_t cur_pos = 0;
78static uint64_t file_size = 0;
79static int new_file = 0;
80static int gcov_version;
81#if defined(_WIN32)
82static HANDLE mmap_handle = NULL;
83#endif
84static int fd = -1;
85
86typedef void (*fn_ptr)(void);
87
88typedef void* dynamic_object_id;
89// The address of this variable identifies a given dynamic object.
90static dynamic_object_id current_id;
91#define CURRENT_ID (&current_id)
92
93struct fn_node {
94 dynamic_object_id id;
95 fn_ptr fn;
96 struct fn_node* next;
97};
98
99struct fn_list {
100 struct fn_node *head, *tail;
101};
102
103/*
104 * A list of functions to write out the data, shared between all dynamic objects.
105 */
106struct fn_list writeout_fn_list;
107
108/*
109 * A list of reset functions, shared between all dynamic objects.
110 */
111struct fn_list reset_fn_list;
112
113static void fn_list_insert(struct fn_list* list, fn_ptr fn) {
114 struct fn_node* new_node = malloc(size: sizeof(struct fn_node));
115 new_node->fn = fn;
116 new_node->next = NULL;
117 new_node->id = CURRENT_ID;
118
119 if (!list->head) {
120 list->head = list->tail = new_node;
121 } else {
122 list->tail->next = new_node;
123 list->tail = new_node;
124 }
125}
126
127static void fn_list_remove(struct fn_list* list) {
128 struct fn_node* curr = list->head;
129 struct fn_node* prev = NULL;
130 struct fn_node* next = NULL;
131
132 while (curr) {
133 next = curr->next;
134
135 if (curr->id == CURRENT_ID) {
136 if (curr == list->head) {
137 list->head = next;
138 }
139
140 if (curr == list->tail) {
141 list->tail = prev;
142 }
143
144 if (prev) {
145 prev->next = next;
146 }
147
148 free(ptr: curr);
149 } else {
150 prev = curr;
151 }
152
153 curr = next;
154 }
155}
156
157static void resize_write_buffer(uint64_t size) {
158 if (!new_file) return;
159 size += cur_pos;
160 if (size <= cur_buffer_size) return;
161 size = (size - 1) / WRITE_BUFFER_SIZE + 1;
162 size *= WRITE_BUFFER_SIZE;
163 write_buffer = realloc(ptr: write_buffer, size: size);
164 cur_buffer_size = size;
165}
166
167static void write_bytes(const char *s, size_t len) {
168 resize_write_buffer(size: len);
169 memcpy(dest: &write_buffer[cur_pos], src: s, n: len);
170 cur_pos += len;
171}
172
173static void write_32bit_value(uint32_t i) {
174 write_bytes(s: (char*)&i, len: 4);
175}
176
177static void write_64bit_value(uint64_t i) {
178 // GCOV uses a lo-/hi-word format even on big-endian systems.
179 // See also GCOVBuffer::readInt64 in LLVM.
180 uint32_t lo = (uint32_t) i;
181 uint32_t hi = (uint32_t) (i >> 32);
182 write_32bit_value(i: lo);
183 write_32bit_value(i: hi);
184}
185
186static uint32_t read_32bit_value(void) {
187 uint32_t val;
188
189 if (new_file)
190 return (uint32_t)-1;
191
192 val = *(uint32_t*)&write_buffer[cur_pos];
193 cur_pos += 4;
194 return val;
195}
196
197static uint64_t read_64bit_value(void) {
198 // GCOV uses a lo-/hi-word format even on big-endian systems.
199 // See also GCOVBuffer::readInt64 in LLVM.
200 uint32_t lo = read_32bit_value();
201 uint32_t hi = read_32bit_value();
202 return ((uint64_t)hi << 32) | ((uint64_t)lo);
203}
204
205static char *mangle_filename(const char *orig_filename) {
206 char *new_filename;
207 size_t prefix_len;
208 int prefix_strip;
209 const char *prefix = lprofGetPathPrefix(PrefixStrip: &prefix_strip, PrefixLen: &prefix_len);
210
211 if (prefix == NULL)
212 return strdup(s: orig_filename);
213
214 new_filename = malloc(size: prefix_len + 1 + strlen(s: orig_filename) + 1);
215 lprofApplyPathPrefix(Dest: new_filename, PathStr: orig_filename, Prefix: prefix, PrefixLen: prefix_len,
216 PrefixStrip: prefix_strip);
217
218 return new_filename;
219}
220
221static int map_file(void) {
222 fseek(stream: output_file, off: 0L, SEEK_END);
223 file_size = ftell(stream: output_file);
224
225 /* A size of 0 means the file has been created just now (possibly by another
226 * process in lock-after-open race condition). No need to mmap. */
227 if (file_size == 0)
228 return -1;
229
230#if defined(_WIN32)
231 HANDLE mmap_fd;
232 if (fd == -1)
233 mmap_fd = INVALID_HANDLE_VALUE;
234 else
235 mmap_fd = (HANDLE)_get_osfhandle(fd);
236
237 mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL);
238 if (mmap_handle == NULL) {
239 fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n",
240 filename, GetLastError());
241 return -1;
242 }
243
244 write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size);
245 if (write_buffer == NULL) {
246 fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename,
247 GetLastError());
248 CloseHandle(mmap_handle);
249 return -1;
250 }
251#else
252 write_buffer = mmap(addr: 0, len: file_size, PROT_READ | PROT_WRITE,
253 MAP_FILE | MAP_SHARED, fd: fd, offset: 0);
254 if (write_buffer == (void *)-1) {
255 int errnum = errno;
256 fprintf(stderr, format: "profiling: %s: cannot map: %s\n", filename,
257 strerror(errnum: errnum));
258 return -1;
259 }
260#endif
261
262 return 0;
263}
264
265static void unmap_file(void) {
266#if defined(_WIN32)
267 if (!UnmapViewOfFile(write_buffer)) {
268 fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
269 GetLastError());
270 }
271
272 if (!CloseHandle(mmap_handle)) {
273 fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n",
274 filename, GetLastError());
275 }
276
277 mmap_handle = NULL;
278#else
279 if (munmap(addr: write_buffer, len: file_size) == -1) {
280 int errnum = errno;
281 fprintf(stderr, format: "profiling: %s: cannot munmap: %s\n", filename,
282 strerror(errnum: errnum));
283 }
284#endif
285
286 write_buffer = NULL;
287 file_size = 0;
288}
289
290/*
291 * --- LLVM line counter API ---
292 */
293
294/* A file in this case is a translation unit. Each .o file built with line
295 * profiling enabled will emit to a different file. Only one file may be
296 * started at a time.
297 */
298COMPILER_RT_VISIBILITY
299void llvm_gcda_start_file(const char *orig_filename, uint32_t version,
300 uint32_t checksum) {
301 const char *mode = "r+b";
302 filename = mangle_filename(orig_filename);
303
304 /* Try just opening the file. */
305 fd = open(file: filename, O_RDWR | O_BINARY);
306
307 if (fd == -1) {
308 /* Try creating the file. */
309 fd = open(file: filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
310 if (fd != -1) {
311 mode = "w+b";
312 } else {
313 /* Try creating the directories first then opening the file. */
314 __llvm_profile_recursive_mkdir(Pathname: filename);
315 fd = open(file: filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
316 if (fd != -1) {
317 mode = "w+b";
318 } else {
319 /* Another process may have created the file just now.
320 * Try opening it without O_CREAT and O_EXCL. */
321 fd = open(file: filename, O_RDWR | O_BINARY);
322 if (fd == -1) {
323 /* Bah! It's hopeless. */
324 int errnum = errno;
325 fprintf(stderr, format: "profiling: %s: cannot open: %s\n", filename,
326 strerror(errnum: errnum));
327 return;
328 }
329 }
330 }
331 }
332
333 /* Try to flock the file to serialize concurrent processes writing out to the
334 * same GCDA. This can fail if the filesystem doesn't support it, but in that
335 * case we'll just carry on with the old racy behaviour and hope for the best.
336 */
337 lprofLockFd(fd);
338 output_file = fdopen(fd: fd, modes: mode);
339
340 /* Initialize the write buffer. */
341 new_file = 0;
342 write_buffer = NULL;
343 cur_buffer_size = 0;
344 cur_pos = 0;
345
346 if (map_file() == -1) {
347 /* The file has been created just now (file_size == 0) or mmap failed
348 * unexpectedly. In the latter case, try to recover by clobbering. */
349 new_file = 1;
350 write_buffer = NULL;
351 resize_write_buffer(WRITE_BUFFER_SIZE);
352 memset(s: write_buffer, c: 0, WRITE_BUFFER_SIZE);
353 }
354
355 /* gcda file, version, stamp checksum. */
356 {
357 uint8_t c3 = version >> 24;
358 uint8_t c2 = (version >> 16) & 255;
359 uint8_t c1 = (version >> 8) & 255;
360 gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
361 : (c3 - '0') * 10 + c1 - '0';
362 }
363 write_32bit_value(i: GCOV_DATA_MAGIC);
364 write_32bit_value(i: version);
365 write_32bit_value(i: checksum);
366
367#ifdef DEBUG_GCDAPROFILING
368 fprintf(stderr, "llvmgcda: [%s]\n", orig_filename);
369#endif
370}
371
372COMPILER_RT_VISIBILITY
373void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
374 uint32_t cfg_checksum) {
375 uint32_t len = 2;
376 int use_extra_checksum = gcov_version >= 47;
377
378 if (use_extra_checksum)
379 len++;
380#ifdef DEBUG_GCDAPROFILING
381 fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident);
382#endif
383 if (!output_file) return;
384
385 /* function tag */
386 write_32bit_value(i: GCOV_TAG_FUNCTION);
387 write_32bit_value(i: len);
388 write_32bit_value(i: ident);
389 write_32bit_value(i: func_checksum);
390 if (use_extra_checksum)
391 write_32bit_value(i: cfg_checksum);
392}
393
394COMPILER_RT_VISIBILITY
395void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
396 uint32_t i;
397 uint64_t *old_ctrs = NULL;
398 uint32_t val = 0;
399 uint64_t save_cur_pos = cur_pos;
400
401 if (!output_file) return;
402
403 val = read_32bit_value();
404
405 if (val != (uint32_t)-1) {
406 /* There are counters present in the file. Merge them. */
407 if (val != GCOV_TAG_COUNTER_ARCS) {
408 fprintf(stderr, format: "profiling: %s: cannot merge previous GCDA file: "
409 "corrupt arc tag (0x%08x)\n",
410 filename, val);
411 return;
412 }
413
414 val = read_32bit_value();
415 if (val == (uint32_t)-1 || val / 2 != num_counters) {
416 fprintf(stderr, format: "profiling: %s: cannot merge previous GCDA file: "
417 "mismatched number of counters (%d)\n",
418 filename, val);
419 return;
420 }
421
422 old_ctrs = malloc(size: sizeof(uint64_t) * num_counters);
423 for (i = 0; i < num_counters; ++i)
424 old_ctrs[i] = read_64bit_value();
425 }
426
427 cur_pos = save_cur_pos;
428
429 /* Counter #1 (arcs) tag */
430 write_32bit_value(i: GCOV_TAG_COUNTER_ARCS);
431 write_32bit_value(i: num_counters * 2);
432 for (i = 0; i < num_counters; ++i) {
433 counters[i] += (old_ctrs ? old_ctrs[i] : 0);
434 write_64bit_value(i: counters[i]);
435 }
436
437 free(ptr: old_ctrs);
438
439#ifdef DEBUG_GCDAPROFILING
440 fprintf(stderr, "llvmgcda: %u arcs\n", num_counters);
441 for (i = 0; i < num_counters; ++i)
442 fprintf(stderr, "llvmgcda: %llu\n", (unsigned long long)counters[i]);
443#endif
444}
445
446COMPILER_RT_VISIBILITY
447void llvm_gcda_summary_info(void) {
448 uint32_t runs = 1;
449 static uint32_t run_counted = 0; // We only want to increase the run count once.
450 uint32_t val = 0;
451 uint64_t save_cur_pos = cur_pos;
452
453 if (!output_file) return;
454
455 val = read_32bit_value();
456
457 if (val != (uint32_t)-1) {
458 /* There are counters present in the file. Merge them. */
459 uint32_t gcov_tag =
460 gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY;
461 if (val != gcov_tag) {
462 fprintf(stderr,
463 format: "profiling: %s: cannot merge previous run count: "
464 "corrupt object tag (0x%08x)\n",
465 filename, val);
466 return;
467 }
468
469 val = read_32bit_value(); /* length */
470 uint32_t prev_runs;
471 if (gcov_version < 90) {
472 read_32bit_value();
473 read_32bit_value();
474 prev_runs = read_32bit_value();
475 } else {
476 prev_runs = read_32bit_value();
477 read_32bit_value();
478 }
479 for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i)
480 read_32bit_value();
481 /* Add previous run count to new counter, if not already counted before. */
482 runs = run_counted ? prev_runs : prev_runs + 1;
483 }
484
485 cur_pos = save_cur_pos;
486
487 if (gcov_version >= 90) {
488 write_32bit_value(i: GCOV_TAG_OBJECT_SUMMARY);
489 write_32bit_value(i: 2);
490 write_32bit_value(i: runs);
491 write_32bit_value(i: 0); // sum_max
492 } else {
493 // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set
494 // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which
495 // can make gcov read "Runs:".
496 write_32bit_value(i: GCOV_TAG_PROGRAM_SUMMARY);
497 write_32bit_value(i: 3);
498 write_32bit_value(i: 0);
499 write_32bit_value(i: 0);
500 write_32bit_value(i: runs);
501 }
502
503 run_counted = 1;
504
505#ifdef DEBUG_GCDAPROFILING
506 fprintf(stderr, "llvmgcda: %u runs\n", runs);
507#endif
508}
509
510COMPILER_RT_VISIBILITY
511void llvm_gcda_end_file(void) {
512 /* Write out EOF record. */
513 if (output_file) {
514 write_bytes(s: "\0\0\0\0\0\0\0\0", len: 8);
515
516 if (new_file) {
517 fwrite(ptr: write_buffer, size: cur_pos, n: 1, s: output_file);
518 free(ptr: write_buffer);
519 } else {
520 unmap_file();
521 }
522
523 fflush(stream: output_file);
524 lprofUnlockFd(fd);
525 fclose(stream: output_file);
526 output_file = NULL;
527 write_buffer = NULL;
528 }
529 free(ptr: filename);
530
531#ifdef DEBUG_GCDAPROFILING
532 fprintf(stderr, "llvmgcda: -----\n");
533#endif
534}
535
536COMPILER_RT_VISIBILITY
537void llvm_register_writeout_function(fn_ptr fn) {
538 fn_list_insert(list: &writeout_fn_list, fn);
539}
540
541COMPILER_RT_VISIBILITY
542void llvm_writeout_files(void) {
543 struct fn_node *curr = writeout_fn_list.head;
544
545 while (curr) {
546 if (curr->id == CURRENT_ID) {
547 curr->fn();
548 }
549 curr = curr->next;
550 }
551}
552
553#ifndef _WIN32
554// __attribute__((destructor)) and destructors whose priorities are greater than
555// 100 run before this function and can thus be tracked. The priority is
556// compatible with GCC 7 onwards.
557#if __GNUC__ >= 9
558#pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
559#endif
560__attribute__((destructor(100)))
561#endif
562static void llvm_writeout_and_clear(void) {
563 llvm_writeout_files();
564 fn_list_remove(list: &writeout_fn_list);
565}
566
567COMPILER_RT_VISIBILITY
568void llvm_register_reset_function(fn_ptr fn) {
569 fn_list_insert(list: &reset_fn_list, fn);
570}
571
572COMPILER_RT_VISIBILITY
573void llvm_delete_reset_function_list(void) { fn_list_remove(list: &reset_fn_list); }
574
575COMPILER_RT_VISIBILITY
576void llvm_reset_counters(void) {
577 struct fn_node *curr = reset_fn_list.head;
578
579 while (curr) {
580 if (curr->id == CURRENT_ID) {
581 curr->fn();
582 }
583 curr = curr->next;
584 }
585}
586
587#if !defined(_WIN32) && !defined(__wasm__)
588COMPILER_RT_VISIBILITY
589pid_t __gcov_fork() {
590 pid_t parent_pid = getpid();
591 pid_t pid = fork();
592
593 if (pid == 0) {
594 pid_t child_pid = getpid();
595 if (child_pid != parent_pid) {
596 // The pid changed so we've a fork (one could have its own fork function)
597 // Just reset the counters for this child process
598 // threads.
599 llvm_reset_counters();
600 }
601 }
602 return pid;
603}
604#endif
605
606COMPILER_RT_VISIBILITY
607void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) {
608 static int atexit_ran = 0;
609
610 if (wfn)
611 llvm_register_writeout_function(fn: wfn);
612
613 if (rfn)
614 llvm_register_reset_function(fn: rfn);
615
616 if (atexit_ran == 0) {
617 atexit_ran = 1;
618
619 /* Make sure we write out the data and delete the data structures. */
620 lprofAtExit(llvm_delete_reset_function_list);
621#ifdef _WIN32
622 lprofAtExit(llvm_writeout_and_clear);
623#endif
624 }
625}
626
627#if defined(_AIX)
628COMPILER_RT_VISIBILITY __attribute__((constructor)) void
629__llvm_profile_gcov_initialize() {
630 const __llvm_gcov_init_func_struct *InitFuncStart =
631 __llvm_profile_begin_covinit();
632 const __llvm_gcov_init_func_struct *InitFuncEnd =
633 __llvm_profile_end_covinit();
634
635 for (const __llvm_gcov_init_func_struct *Ptr = InitFuncStart;
636 Ptr != InitFuncEnd; ++Ptr) {
637 fn_ptr wfn = (fn_ptr)Ptr->WriteoutFunction;
638 fn_ptr rfn = (fn_ptr)Ptr->ResetFunction;
639 if (!(wfn && rfn))
640 continue;
641 llvm_gcov_init(wfn, rfn);
642 }
643}
644#endif
645
646void __gcov_dump(void) {
647 for (struct fn_node *f = writeout_fn_list.head; f; f = f->next)
648 f->fn();
649}
650
651void __gcov_reset(void) {
652 for (struct fn_node *f = reset_fn_list.head; f; f = f->next)
653 f->fn();
654}
655
656#endif
657