1//===-- xray_init.cpp -------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of XRay, a dynamic runtime instrumentation system.
10//
11// XRay initialisation logic.
12//===----------------------------------------------------------------------===//
13
14#include <fcntl.h>
15#include <strings.h>
16#include <unistd.h>
17
18#include "sanitizer_common/sanitizer_common.h"
19#include "xray/xray_interface.h"
20#include "xray_allocator.h"
21#include "xray_defs.h"
22#include "xray_flags.h"
23#include "xray_interface_internal.h"
24
25extern "C" {
26void __xray_init();
27extern const XRaySledEntry __start_xray_instr_map[] __attribute__((weak));
28extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak));
29extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak));
30extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak));
31
32#if SANITIZER_APPLE
33// HACK: This is a temporary workaround to make XRay build on
34// Darwin, but it will probably not work at runtime.
35const XRaySledEntry __start_xray_instr_map[] = {};
36extern const XRaySledEntry __stop_xray_instr_map[] = {};
37extern const XRayFunctionSledIndex __start_xray_fn_idx[] = {};
38extern const XRayFunctionSledIndex __stop_xray_fn_idx[] = {};
39#endif
40}
41
42using namespace __xray;
43
44// When set to 'true' this means the XRay runtime has been initialised. We use
45// the weak symbols defined above (__start_xray_inst_map and
46// __stop_xray_instr_map) to initialise the instrumentation map that XRay uses
47// for runtime patching/unpatching of instrumentation points.
48atomic_uint8_t XRayInitialized{.val_dont_use: 0};
49
50// This should always be updated before XRayInitialized is updated.
51SpinMutex XRayInstrMapMutex;
52
53// Contains maps for the main executable as well as DSOs.
54XRaySledMap *XRayInstrMaps;
55
56// Number of binary objects registered.
57atomic_uint32_t XRayNumObjects{.val_dont_use: 0};
58
59// Global flag to determine whether the flags have been initialized.
60atomic_uint8_t XRayFlagsInitialized{.val_dont_use: 0};
61
62// A mutex to allow only one thread to initialize the XRay data structures.
63SpinMutex XRayInitMutex;
64
65// Registers XRay sleds and trampolines coming from the main executable or one
66// of the linked DSOs.
67// Returns the object ID if registration is successful, -1 otherwise.
68int32_t
69__xray_register_sleds(const XRaySledEntry *SledsBegin,
70 const XRaySledEntry *SledsEnd,
71 const XRayFunctionSledIndex *FnIndexBegin,
72 const XRayFunctionSledIndex *FnIndexEnd, bool FromDSO,
73 XRayTrampolines Trampolines) XRAY_NEVER_INSTRUMENT {
74 if (!SledsBegin || !SledsEnd) {
75 Report(format: "Invalid XRay sleds.\n");
76 return -1;
77 }
78 XRaySledMap SledMap;
79 SledMap.FromDSO = FromDSO;
80 SledMap.Loaded = true;
81 SledMap.Trampolines = Trampolines;
82 SledMap.Sleds = SledsBegin;
83 SledMap.Entries = SledsEnd - SledsBegin;
84 if (FnIndexBegin != nullptr) {
85 SledMap.SledsIndex = FnIndexBegin;
86 SledMap.Functions = FnIndexEnd - FnIndexBegin;
87 } else {
88 size_t CountFunctions = 0;
89 uint64_t LastFnAddr = 0;
90
91 for (std::size_t I = 0; I < SledMap.Entries; I++) {
92 const auto &Sled = SledMap.Sleds[I];
93 const auto Function = Sled.function();
94 if (Function != LastFnAddr) {
95 CountFunctions++;
96 LastFnAddr = Function;
97 }
98 }
99 SledMap.SledsIndex = nullptr;
100 SledMap.Functions = CountFunctions;
101 }
102 if (SledMap.Functions >= XRayMaxFunctions) {
103 Report(format: "Too many functions! Maximum is %ld\n", XRayMaxFunctions);
104 return -1;
105 }
106
107 if (Verbosity())
108 Report(format: "Registering %d new functions!\n", SledMap.Functions);
109
110 {
111 SpinMutexLock Guard(&XRayInstrMapMutex);
112 auto Idx = atomic_fetch_add(a: &XRayNumObjects, v: 1, mo: memory_order_acq_rel);
113 if (Idx >= XRayMaxObjects) {
114 Report(format: "Too many objects registered! Maximum is %ld\n", XRayMaxObjects);
115 return -1;
116 }
117 XRayInstrMaps[Idx] = std::move(t&: SledMap);
118 return Idx;
119 }
120}
121
122// __xray_init() will do the actual loading of the current process' memory map
123// and then proceed to look for the .xray_instr_map section/segment.
124void __xray_init() XRAY_NEVER_INSTRUMENT {
125 SpinMutexLock Guard(&XRayInitMutex);
126 // Short-circuit if we've already initialized XRay before.
127 if (atomic_load(a: &XRayInitialized, mo: memory_order_acquire))
128 return;
129
130 // XRAY is not compatible with PaX MPROTECT
131 CheckMPROTECT();
132
133 if (!atomic_load(a: &XRayFlagsInitialized, mo: memory_order_acquire)) {
134 initializeFlags();
135 atomic_store(a: &XRayFlagsInitialized, v: true, mo: memory_order_release);
136 }
137
138 if (__start_xray_instr_map == nullptr) {
139 if (Verbosity())
140 Report(format: "XRay instrumentation map missing. Not initializing XRay.\n");
141 return;
142 }
143
144 atomic_store(a: &XRayNumObjects, v: 0, mo: memory_order_release);
145
146 // Pre-allocation takes up approx. 5kB for XRayMaxObjects=64.
147 XRayInstrMaps = allocateBuffer<XRaySledMap>(S: XRayMaxObjects);
148
149 int MainBinaryId =
150 __xray_register_sleds(SledsBegin: __start_xray_instr_map, SledsEnd: __stop_xray_instr_map,
151 FnIndexBegin: __start_xray_fn_idx, FnIndexEnd: __stop_xray_fn_idx, FromDSO: false, Trampolines: {});
152
153 // The executable should always get ID 0.
154 if (MainBinaryId != 0) {
155 Report(format: "Registering XRay sleds failed.\n");
156 return;
157 }
158
159 atomic_store(a: &XRayInitialized, v: true, mo: memory_order_release);
160
161#ifndef XRAY_NO_PREINIT
162 if (flags()->patch_premain)
163 __xray_patch();
164#endif
165}
166
167// Registers XRay sleds and trampolines of an instrumented DSO.
168// Returns the object ID if registration is successful, -1 otherwise.
169//
170// Default visibility is hidden, so we have to explicitly make it visible to
171// DSO.
172SANITIZER_INTERFACE_ATTRIBUTE int32_t __xray_register_dso(
173 const XRaySledEntry *SledsBegin, const XRaySledEntry *SledsEnd,
174 const XRayFunctionSledIndex *FnIndexBegin,
175 const XRayFunctionSledIndex *FnIndexEnd,
176 XRayTrampolines Trampolines) XRAY_NEVER_INSTRUMENT {
177 // Make sure XRay has been initialized in the main executable.
178 __xray_init();
179
180 if (__xray_num_objects() == 0) {
181 if (Verbosity())
182 Report(format: "No XRay instrumentation map in main executable. Not initializing "
183 "XRay for DSO.\n");
184 return -1;
185 }
186
187 // Register sleds in global map.
188 int ObjId = __xray_register_sleds(SledsBegin, SledsEnd, FnIndexBegin,
189 FnIndexEnd, FromDSO: true, Trampolines);
190
191#ifndef XRAY_NO_PREINIT
192 if (ObjId >= 0 && flags()->patch_premain)
193 __xray_patch_object(ObjId);
194#endif
195
196 return ObjId;
197}
198
199// Deregisters a DSO from the main XRay runtime.
200// Called from the DSO-local runtime when the library is unloaded (e.g. if
201// dlclose is called).
202// Returns true if the object ID is valid and the DSO was successfully
203// deregistered.
204SANITIZER_INTERFACE_ATTRIBUTE bool
205__xray_deregister_dso(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
206
207 if (!atomic_load(a: &XRayInitialized, mo: memory_order_acquire)) {
208 if (Verbosity())
209 Report(format: "XRay has not been initialized. Cannot deregister DSO.\n");
210 return false;
211 }
212
213 if (ObjId <= 0 || static_cast<uint32_t>(ObjId) >= __xray_num_objects()) {
214 if (Verbosity())
215 Report(format: "Can't deregister object with ID %d: ID is invalid.\n", ObjId);
216 return false;
217 }
218
219 {
220 SpinMutexLock Guard(&XRayInstrMapMutex);
221 auto &Entry = XRayInstrMaps[ObjId];
222 if (!Entry.FromDSO) {
223 if (Verbosity())
224 Report(format: "Can't deregister object with ID %d: object does not correspond "
225 "to a shared library.\n",
226 ObjId);
227 return false;
228 }
229 if (!Entry.Loaded) {
230 if (Verbosity())
231 Report(format: "Can't deregister object with ID %d: object is not loaded.\n",
232 ObjId);
233 return true;
234 }
235 // Mark DSO as unloaded. No need to unpatch.
236 Entry.Loaded = false;
237 }
238
239 if (Verbosity())
240 Report(format: "Deregistered object with ID %d.\n", ObjId);
241
242 return true;
243}
244
245// FIXME: Make check-xray tests work on FreeBSD without
246// SANITIZER_CAN_USE_PREINIT_ARRAY.
247// See sanitizer_internal_defs.h where the macro is defined.
248// Calling unresolved PLT functions in .preinit_array can lead to deadlock on
249// FreeBSD but here it seems benign.
250#if !defined(XRAY_NO_PREINIT) && \
251 (SANITIZER_CAN_USE_PREINIT_ARRAY || SANITIZER_FREEBSD)
252// Only add the preinit array initialization if the sanitizers can.
253__attribute__((section(".preinit_array"),
254 used)) void (*__local_xray_preinit)(void) = __xray_init;
255#else
256// If we cannot use the .preinit_array section, we should instead use dynamic
257// initialisation.
258__attribute__ ((constructor (0)))
259static void __local_xray_dyninit() {
260 __xray_init();
261}
262#endif
263