1//===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of XRay, a dynamic runtime instrumentation system.
10//
11// Implementation of the API functions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "xray_interface_internal.h"
16
17#include <cinttypes>
18#include <cstdio>
19#include <errno.h>
20#include <limits>
21#include <string.h>
22#include <sys/mman.h>
23
24#if SANITIZER_FUCHSIA
25#include <zircon/process.h>
26#include <zircon/sanitizer.h>
27#include <zircon/status.h>
28#include <zircon/syscalls.h>
29#endif
30
31#include "sanitizer_common/sanitizer_addrhashmap.h"
32#include "sanitizer_common/sanitizer_common.h"
33
34#include "xray_defs.h"
35#include "xray_flags.h"
36
37extern __sanitizer::SpinMutex XRayInstrMapMutex;
38extern __sanitizer::atomic_uint8_t XRayInitialized;
39extern __xray::XRaySledMap *XRayInstrMaps;
40extern __sanitizer::atomic_uint32_t XRayNumObjects;
41
42namespace __xray {
43
44#if defined(__x86_64__)
45static const int16_t cSledLength = 12;
46#elif defined(__aarch64__)
47static const int16_t cSledLength = 32;
48#elif defined(__arm__)
49static const int16_t cSledLength = 28;
50#elif SANITIZER_LOONGARCH64
51static const int16_t cSledLength = 48;
52#elif SANITIZER_MIPS32
53static const int16_t cSledLength = 48;
54#elif SANITIZER_MIPS64
55static const int16_t cSledLength = 64;
56#elif defined(__powerpc64__)
57static const int16_t cSledLength = 8;
58#elif defined(__hexagon__)
59static const int16_t cSledLength = 20;
60#elif defined(__riscv) && (__riscv_xlen == 64)
61static const int16_t cSledLength = 68;
62#elif defined(__riscv) && (__riscv_xlen == 32)
63static const int16_t cSledLength = 52;
64#elif defined(__s390x__)
65static const int16_t cSledLength = 18;
66#else
67#error "Unsupported CPU Architecture"
68#endif /* CPU architecture */
69
70// This is the function to call when we encounter the entry or exit sleds.
71atomic_uintptr_t XRayPatchedFunction SANITIZER_INTERFACE_ATTRIBUTE{.val_dont_use: 0};
72
73// This is the function to call from the arg1-enabled sleds/trampolines.
74atomic_uintptr_t XRayArgLogger SANITIZER_INTERFACE_ATTRIBUTE{.val_dont_use: 0};
75
76// This is the function to call when we encounter a custom event log call.
77atomic_uintptr_t XRayPatchedCustomEvent SANITIZER_INTERFACE_ATTRIBUTE{.val_dont_use: 0};
78
79// This is the function to call when we encounter a typed event log call.
80atomic_uintptr_t XRayPatchedTypedEvent SANITIZER_INTERFACE_ATTRIBUTE{.val_dont_use: 0};
81
82// This is the global status to determine whether we are currently
83// patching/unpatching.
84atomic_uint8_t XRayPatching{.val_dont_use: 0};
85
86struct TypeDescription {
87 uint32_t type_id;
88 std::size_t description_string_length;
89};
90
91using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
92// An address map from immutable descriptors to type ids.
93TypeDescriptorMapType TypeDescriptorAddressMap{};
94
95atomic_uint32_t TypeEventDescriptorCounter{.val_dont_use: 0};
96
97// MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
98// undo any successful mprotect(...) changes. This is used to make a page
99// writeable and executable, and upon destruction if it was successful in
100// doing so returns the page into a read-only and executable page.
101//
102// This is only used specifically for runtime-patching of the XRay
103// instrumentation points. This assumes that the executable pages are
104// originally read-and-execute only.
105class MProtectHelper {
106 void *PageAlignedAddr;
107 std::size_t MProtectLen;
108 bool MustCleanup;
109
110public:
111 explicit MProtectHelper(void *PageAlignedAddr,
112 std::size_t MProtectLen,
113 std::size_t PageSize) XRAY_NEVER_INSTRUMENT
114 : PageAlignedAddr(PageAlignedAddr),
115 MProtectLen(MProtectLen),
116 MustCleanup(false) {
117#if SANITIZER_FUCHSIA
118 MProtectLen = RoundUpTo(MProtectLen, PageSize);
119#endif
120 }
121
122 int MakeWriteable() XRAY_NEVER_INSTRUMENT {
123#if SANITIZER_FUCHSIA
124 auto R = __sanitizer_change_code_protection(
125 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
126 if (R != ZX_OK) {
127 Report("XRay: cannot change code protection: %s\n",
128 _zx_status_get_string(R));
129 return -1;
130 }
131 MustCleanup = true;
132 return 0;
133#else
134 auto R = mprotect(addr: PageAlignedAddr, len: MProtectLen,
135 PROT_READ | PROT_WRITE | PROT_EXEC);
136 if (R != -1)
137 MustCleanup = true;
138 return R;
139#endif
140 }
141
142 ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
143 if (MustCleanup) {
144#if SANITIZER_FUCHSIA
145 auto R = __sanitizer_change_code_protection(
146 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
147 if (R != ZX_OK) {
148 Report("XRay: cannot change code protection: %s\n",
149 _zx_status_get_string(R));
150 }
151#else
152 mprotect(addr: PageAlignedAddr, len: MProtectLen, PROT_READ | PROT_EXEC);
153#endif
154 }
155 }
156};
157
158namespace {
159
160bool isObjectLoaded(int32_t ObjId) {
161 SpinMutexLock Guard(&XRayInstrMapMutex);
162 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
163 atomic_load(a: &XRayNumObjects, mo: memory_order_acquire)) {
164 return false;
165 }
166 return XRayInstrMaps[ObjId].Loaded;
167}
168
169bool patchSled(const XRaySledEntry &Sled, bool Enable, int32_t FuncId,
170 const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
171 bool Success = false;
172 switch (Sled.Kind) {
173 case XRayEntryType::ENTRY:
174 Success = patchFunctionEntry(Enable, FuncId, Sled, Trampolines,
175 /*LogArgs=*/false);
176 break;
177 case XRayEntryType::EXIT:
178 Success = patchFunctionExit(Enable, FuncId, Sled, Trampolines);
179 break;
180 case XRayEntryType::TAIL:
181 Success = patchFunctionTailExit(Enable, FuncId, Sled, Trampolines);
182 break;
183 case XRayEntryType::LOG_ARGS_ENTRY:
184 Success = patchFunctionEntry(Enable, FuncId, Sled, Trampolines,
185 /*LogArgs=*/true);
186 break;
187 case XRayEntryType::CUSTOM_EVENT:
188 Success = patchCustomEvent(Enable, FuncId, Sled);
189 break;
190 case XRayEntryType::TYPED_EVENT:
191 Success = patchTypedEvent(Enable, FuncId, Sled);
192 break;
193 default:
194 Report(format: "Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address,
195 int(Sled.Kind));
196 return false;
197 }
198 return Success;
199}
200
201const XRayFunctionSledIndex
202findFunctionSleds(int32_t FuncId,
203 const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
204 int32_t CurFn = 0;
205 uint64_t LastFnAddr = 0;
206 XRayFunctionSledIndex Index = {.Begin: nullptr, .Size: 0};
207
208 for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
209 const auto &Sled = InstrMap.Sleds[I];
210 const auto Function = Sled.function();
211 if (Function != LastFnAddr) {
212 CurFn++;
213 LastFnAddr = Function;
214 }
215
216 if (CurFn == FuncId) {
217 if (Index.Begin == nullptr)
218 Index.Begin = &Sled;
219 Index.Size = &Sled - Index.Begin + 1;
220 }
221 }
222
223 return Index;
224}
225
226XRayPatchingStatus patchFunction(int32_t FuncId, int32_t ObjId,
227 bool Enable) XRAY_NEVER_INSTRUMENT {
228 if (!atomic_load(a: &XRayInitialized, mo: memory_order_acquire))
229 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
230
231 uint8_t NotPatching = false;
232 if (!atomic_compare_exchange_strong(
233 a: &XRayPatching, cmp: &NotPatching, xchg: true, mo: memory_order_acq_rel))
234 return XRayPatchingStatus::ONGOING; // Already patching.
235
236 // Next, we look for the function index.
237 XRaySledMap InstrMap;
238 {
239 SpinMutexLock Guard(&XRayInstrMapMutex);
240 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
241 atomic_load(a: &XRayNumObjects, mo: memory_order_acquire)) {
242 Report(format: "Unable to patch function: invalid sled map index: %d", ObjId);
243 return XRayPatchingStatus::FAILED;
244 }
245 InstrMap = XRayInstrMaps[ObjId];
246 }
247
248 // If we don't have an index, we can't patch individual functions.
249 if (InstrMap.Functions == 0)
250 return XRayPatchingStatus::NOT_INITIALIZED;
251
252 // Check if the corresponding DSO has been unloaded.
253 if (!InstrMap.Loaded) {
254 Report(format: "Invalid function id provided: %d\n", FuncId);
255 return XRayPatchingStatus::NOT_INITIALIZED;
256 }
257
258 // FuncId must be a positive number, less than the number of functions
259 // instrumented.
260 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
261 Report(format: "Invalid function id provided: %d\n", FuncId);
262 return XRayPatchingStatus::FAILED;
263 }
264
265 auto PackedId = __xray::MakePackedId(FnId: FuncId, ObjId);
266
267 // Now we patch ths sleds for this specific function.
268 XRayFunctionSledIndex SledRange;
269 if (InstrMap.SledsIndex) {
270 SledRange = {.Begin: InstrMap.SledsIndex[FuncId - 1].fromPCRelative(),
271 .Size: InstrMap.SledsIndex[FuncId - 1].Size};
272 } else {
273 SledRange = findFunctionSleds(FuncId, InstrMap);
274 }
275
276 auto *f = SledRange.Begin;
277 bool SucceedOnce = false;
278 for (size_t i = 0; i != SledRange.Size; ++i)
279 SucceedOnce |= patchSled(Sled: f[i], Enable, FuncId: PackedId, Trampolines: InstrMap.Trampolines);
280
281 atomic_store(a: &XRayPatching, v: false, mo: memory_order_release);
282
283 if (!SucceedOnce) {
284 Report(format: "Failed patching any sled for function '%d'.", FuncId);
285 return XRayPatchingStatus::FAILED;
286 }
287
288 return XRayPatchingStatus::SUCCESS;
289}
290
291// controlPatching implements the common internals of the patching/unpatching
292// implementation. |Enable| defines whether we're enabling or disabling the
293// runtime XRay instrumentation.
294// This function should only be called after ensuring that XRay is initialized
295// and no other thread is currently patching.
296XRayPatchingStatus controlPatchingObjectUnchecked(bool Enable, int32_t ObjId) {
297 XRaySledMap InstrMap;
298 {
299 SpinMutexLock Guard(&XRayInstrMapMutex);
300 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
301 atomic_load(a: &XRayNumObjects, mo: memory_order_acquire)) {
302 Report(format: "Unable to patch functions: invalid sled map index: %d\n", ObjId);
303 return XRayPatchingStatus::FAILED;
304 }
305 InstrMap = XRayInstrMaps[ObjId];
306 }
307 if (InstrMap.Entries == 0)
308 return XRayPatchingStatus::NOT_INITIALIZED;
309
310 if (Verbosity())
311 Report(format: "Patching object %d with %d functions.\n", ObjId,
312 (int)InstrMap.Entries);
313
314 // Check if the corresponding DSO has been unloaded.
315 if (!InstrMap.Loaded) {
316 Report(format: "Object is not loaded at index: %d\n", ObjId);
317 return XRayPatchingStatus::FAILED;
318 }
319
320 uint32_t FuncId = 1;
321 uint64_t CurFun = 0;
322
323 // First we want to find the bounds for which we have instrumentation points,
324 // and try to get as few calls to mprotect(...) as possible. We're assuming
325 // that all the sleds for the instrumentation map are contiguous as a single
326 // set of pages. When we do support dynamic shared object instrumentation,
327 // we'll need to do this for each set of page load offsets per DSO loaded. For
328 // now we're assuming we can mprotect the whole section of text between the
329 // minimum sled address and the maximum sled address (+ the largest sled
330 // size).
331 auto *MinSled = &InstrMap.Sleds[0];
332 auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
333 for (std::size_t I = 0; I < InstrMap.Entries; I++) {
334 const auto &Sled = InstrMap.Sleds[I];
335 if (Sled.address() < MinSled->address())
336 MinSled = &Sled;
337 if (Sled.address() > MaxSled->address())
338 MaxSled = &Sled;
339 }
340
341 const size_t PageSize = flags()->xray_page_size_override > 0
342 ? flags()->xray_page_size_override
343 : GetPageSizeCached();
344 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
345 Report(format: "System page size is not a power of two: %zu\n", PageSize);
346 return XRayPatchingStatus::FAILED;
347 }
348
349 void *PageAlignedAddr =
350 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
351 size_t MProtectLen =
352 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
353 cSledLength;
354 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
355 if (Protector.MakeWriteable() == -1) {
356 Report(format: "Failed mprotect: %d\n", errno);
357 return XRayPatchingStatus::FAILED;
358 }
359
360 for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
361 auto &Sled = InstrMap.Sleds[I];
362 auto F = Sled.function();
363 if (CurFun == 0)
364 CurFun = F;
365 if (F != CurFun) {
366 ++FuncId;
367 CurFun = F;
368 }
369 auto PackedId = __xray::MakePackedId(FnId: FuncId, ObjId);
370 patchSled(Sled, Enable, FuncId: PackedId, Trampolines: InstrMap.Trampolines);
371 }
372 atomic_store(a: &XRayPatching, v: false, mo: memory_order_release);
373 return XRayPatchingStatus::SUCCESS;
374}
375
376// Controls patching for all registered objects.
377// Returns: SUCCESS, if patching succeeds for all objects.
378// NOT_INITIALIZED, if one or more objects returned NOT_INITIALIZED
379// but none failed.
380// FAILED, if patching of one or more objects failed.
381XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
382 if (!atomic_load(a: &XRayInitialized, mo: memory_order_acquire))
383 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
384
385 uint8_t NotPatching = false;
386 if (!atomic_compare_exchange_strong(a: &XRayPatching, cmp: &NotPatching, xchg: true,
387 mo: memory_order_acq_rel))
388 return XRayPatchingStatus::ONGOING; // Already patching.
389
390 auto XRayPatchingStatusResetter = at_scope_exit(
391 fn: [] { atomic_store(a: &XRayPatching, v: false, mo: memory_order_release); });
392
393 unsigned NumObjects = __xray_num_objects();
394
395 XRayPatchingStatus CombinedStatus{NOT_INITIALIZED};
396 for (unsigned I = 0; I < NumObjects; ++I) {
397 if (!isObjectLoaded(ObjId: I))
398 continue;
399 auto LastStatus = controlPatchingObjectUnchecked(Enable, ObjId: I);
400 switch (LastStatus) {
401 case SUCCESS:
402 if (CombinedStatus == NOT_INITIALIZED)
403 CombinedStatus = SUCCESS;
404 break;
405 case FAILED:
406 // Report failure, but try to patch the remaining objects
407 CombinedStatus = FAILED;
408 break;
409 case NOT_INITIALIZED:
410 // XRay has been initialized but there are no sleds available for this
411 // object. Try to patch remaining objects.
412 if (CombinedStatus != FAILED)
413 CombinedStatus = NOT_INITIALIZED;
414 break;
415 case ONGOING:
416 UNREACHABLE("Status ONGOING should not appear at this point");
417 }
418 }
419 return CombinedStatus;
420}
421
422// Controls patching for one object.
423XRayPatchingStatus controlPatching(bool Enable,
424 int32_t ObjId) XRAY_NEVER_INSTRUMENT {
425
426 if (!atomic_load(a: &XRayInitialized, mo: memory_order_acquire))
427 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
428
429 uint8_t NotPatching = false;
430 if (!atomic_compare_exchange_strong(a: &XRayPatching, cmp: &NotPatching, xchg: true,
431 mo: memory_order_acq_rel))
432 return XRayPatchingStatus::ONGOING; // Already patching.
433
434 auto XRayPatchingStatusResetter = at_scope_exit(
435 fn: [] { atomic_store(a: &XRayPatching, v: false, mo: memory_order_release); });
436
437 return controlPatchingObjectUnchecked(Enable, ObjId);
438}
439
440XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, int32_t ObjId,
441 bool Enable) XRAY_NEVER_INSTRUMENT {
442 XRaySledMap InstrMap;
443 {
444 SpinMutexLock Guard(&XRayInstrMapMutex);
445 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
446 atomic_load(a: &XRayNumObjects, mo: memory_order_acquire)) {
447 Report(format: "Unable to patch function: invalid sled map index: %d\n", ObjId);
448 return XRayPatchingStatus::FAILED;
449 }
450 InstrMap = XRayInstrMaps[ObjId];
451 }
452
453 // Check if the corresponding DSO has been unloaded.
454 if (!InstrMap.Loaded) {
455 Report(format: "Object is not loaded at index: %d\n", ObjId);
456 return XRayPatchingStatus::FAILED;
457 }
458
459 // FuncId must be a positive number, less than the number of functions
460 // instrumented.
461 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
462 Report(format: "Invalid function id provided: %d\n", FuncId);
463 return XRayPatchingStatus::FAILED;
464 }
465
466 const size_t PageSize = flags()->xray_page_size_override > 0
467 ? flags()->xray_page_size_override
468 : GetPageSizeCached();
469 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
470 Report(format: "Provided page size is not a power of two: %zu\n", PageSize);
471 return XRayPatchingStatus::FAILED;
472 }
473
474 // Here we compute the minimum sled and maximum sled associated with a
475 // particular function ID.
476 XRayFunctionSledIndex SledRange;
477 if (InstrMap.SledsIndex) {
478 SledRange = {.Begin: InstrMap.SledsIndex[FuncId - 1].fromPCRelative(),
479 .Size: InstrMap.SledsIndex[FuncId - 1].Size};
480 } else {
481 SledRange = findFunctionSleds(FuncId, InstrMap);
482 }
483 auto *f = SledRange.Begin;
484 auto *e = SledRange.Begin + SledRange.Size;
485 auto *MinSled = f;
486 auto *MaxSled = e - 1;
487 while (f != e) {
488 if (f->address() < MinSled->address())
489 MinSled = f;
490 if (f->address() > MaxSled->address())
491 MaxSled = f;
492 ++f;
493 }
494
495 void *PageAlignedAddr =
496 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
497 size_t MProtectLen =
498 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
499 cSledLength;
500 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
501 if (Protector.MakeWriteable() == -1) {
502 Report(format: "Failed mprotect: %d\n", errno);
503 return XRayPatchingStatus::FAILED;
504 }
505 return patchFunction(FuncId, ObjId, Enable);
506}
507
508} // namespace
509
510} // namespace __xray
511
512using namespace __xray;
513
514// The following functions are declared `extern "C" {...}` in the header, hence
515// they're defined in the global namespace.
516
517int __xray_set_handler(void (*entry)(int32_t,
518 XRayEntryType)) XRAY_NEVER_INSTRUMENT {
519 if (atomic_load(a: &XRayInitialized, mo: memory_order_acquire)) {
520
521 atomic_store(a: &__xray::XRayPatchedFunction,
522 v: reinterpret_cast<uintptr_t>(entry), mo: memory_order_release);
523 return 1;
524 }
525 return 0;
526}
527
528int __xray_set_customevent_handler(void (*entry)(void *, size_t))
529 XRAY_NEVER_INSTRUMENT {
530 if (atomic_load(a: &XRayInitialized, mo: memory_order_acquire)) {
531 atomic_store(a: &__xray::XRayPatchedCustomEvent,
532 v: reinterpret_cast<uintptr_t>(entry), mo: memory_order_release);
533 return 1;
534 }
535 return 0;
536}
537
538int __xray_set_typedevent_handler(void (*entry)(size_t, const void *,
539 size_t)) XRAY_NEVER_INSTRUMENT {
540 if (atomic_load(a: &XRayInitialized, mo: memory_order_acquire)) {
541 atomic_store(a: &__xray::XRayPatchedTypedEvent,
542 v: reinterpret_cast<uintptr_t>(entry), mo: memory_order_release);
543 return 1;
544 }
545 return 0;
546}
547
548int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
549 return __xray_set_handler(entry: nullptr);
550}
551
552int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
553 return __xray_set_customevent_handler(entry: nullptr);
554}
555
556int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
557 return __xray_set_typedevent_handler(entry: nullptr);
558}
559
560uint16_t __xray_register_event_type(
561 const char *const event_type) XRAY_NEVER_INSTRUMENT {
562 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
563 if (h.created()) {
564 h->type_id = atomic_fetch_add(
565 a: &TypeEventDescriptorCounter, v: 1, mo: memory_order_acq_rel);
566 h->description_string_length = strnlen(string: event_type, maxlen: 1024);
567 }
568 return h->type_id;
569}
570
571XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
572 return controlPatching(Enable: true);
573}
574
575XRayPatchingStatus __xray_patch_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
576 return controlPatching(Enable: true, ObjId);
577}
578
579XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
580 return controlPatching(Enable: false);
581}
582
583XRayPatchingStatus __xray_unpatch_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
584 return controlPatching(Enable: false, ObjId);
585}
586
587XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
588 auto Ids = __xray::UnpackId(PackedId: FuncId);
589 auto ObjId = Ids.first;
590 auto FnId = Ids.second;
591 return mprotectAndPatchFunction(FuncId: FnId, ObjId, Enable: true);
592}
593
594XRayPatchingStatus
595__xray_patch_function_in_object(int32_t FuncId,
596 int32_t ObjId) XRAY_NEVER_INSTRUMENT {
597 return mprotectAndPatchFunction(FuncId, ObjId, Enable: true);
598}
599
600XRayPatchingStatus
601__xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
602 auto Ids = __xray::UnpackId(PackedId: FuncId);
603 auto ObjId = Ids.first;
604 auto FnId = Ids.second;
605 return mprotectAndPatchFunction(FuncId: FnId, ObjId, Enable: false);
606}
607
608XRayPatchingStatus
609__xray_unpatch_function_in_object(int32_t FuncId,
610 int32_t ObjId) XRAY_NEVER_INSTRUMENT {
611 return mprotectAndPatchFunction(FuncId, ObjId, Enable: false);
612}
613
614int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
615 if (!atomic_load(a: &XRayInitialized, mo: memory_order_acquire))
616 return 0;
617
618 // A relaxed write might not be visible even if the current thread gets
619 // scheduled on a different CPU/NUMA node. We need to wait for everyone to
620 // have this handler installed for consistency of collected data across CPUs.
621 atomic_store(a: &XRayArgLogger, v: reinterpret_cast<uint64_t>(entry),
622 mo: memory_order_release);
623 return 1;
624}
625
626int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(entry: nullptr); }
627
628uintptr_t
629__xray_function_address(int32_t CombinedFuncId) XRAY_NEVER_INSTRUMENT {
630 auto Ids = __xray::UnpackId(PackedId: CombinedFuncId);
631 return __xray_function_address_in_object(FuncId: Ids.second, ObjId: Ids.first);
632}
633
634uintptr_t __xray_function_address_in_object(int32_t FuncId, int32_t ObjId)
635 XRAY_NEVER_INSTRUMENT {
636 XRaySledMap InstrMap;
637 {
638 SpinMutexLock Guard(&XRayInstrMapMutex);
639 auto count = atomic_load(a: &XRayNumObjects, mo: memory_order_acquire);
640 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >= count) {
641 Report(format: "Unable to determine function address: invalid sled map index %d "
642 "(size is %d)\n",
643 ObjId, (int)count);
644 return 0;
645 }
646 InstrMap = XRayInstrMaps[ObjId];
647 }
648
649 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
650 return 0;
651 const XRaySledEntry *Sled =
652 InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1].fromPCRelative()
653 : findFunctionSleds(FuncId, InstrMap).Begin;
654 return Sled->function()
655// On PPC, function entries are always aligned to 16 bytes. The beginning of a
656// sled might be a local entry, which is always +8 based on the global entry.
657// Always return the global entry.
658#ifdef __PPC__
659 & ~0xf
660#endif
661 ;
662}
663
664size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
665 return __xray_max_function_id_in_object(ObjId: 0);
666}
667
668size_t __xray_max_function_id_in_object(int32_t ObjId) XRAY_NEVER_INSTRUMENT {
669 SpinMutexLock Guard(&XRayInstrMapMutex);
670 if (ObjId < 0 || static_cast<uint32_t>(ObjId) >=
671 atomic_load(a: &XRayNumObjects, mo: memory_order_acquire))
672 return 0;
673 return XRayInstrMaps[ObjId].Functions;
674}
675
676size_t __xray_num_objects() XRAY_NEVER_INSTRUMENT {
677 SpinMutexLock Guard(&XRayInstrMapMutex);
678 return atomic_load(a: &XRayNumObjects, mo: memory_order_acquire);
679}
680
681int32_t __xray_unpack_function_id(int32_t PackedId) {
682 return __xray::UnpackId(PackedId).second;
683}
684
685int32_t __xray_unpack_object_id(int32_t PackedId) {
686 return __xray::UnpackId(PackedId).first;
687}
688
689int32_t __xray_pack_id(int32_t FuncId, int32_t ObjId) {
690 return __xray::MakePackedId(FnId: FuncId, ObjId);
691}
692