1//===- InstrProfilingPlatformROCm.cpp - Profile data ROCm platform -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9extern "C" {
10#include "InstrProfiling.h"
11#include "InstrProfilingPort.h"
12#include "InstrProfilingUtil.h"
13}
14
15#include "interception/interception.h"
16// C library headers (not <cstdio> etc.): clang_rt.profile is built with
17// -nostdinc++ and avoids the C++ standard library (see profile/CMakeLists.txt).
18#include <stddef.h>
19#include <stdio.h>
20#include <stdlib.h>
21#include <string.h>
22#ifdef _WIN32
23#include <wchar.h>
24#endif
25
26#ifdef _WIN32
27#define WIN32_LEAN_AND_MEAN
28#include <windows.h>
29// windows.h needs to be included before tlhelp32.h.
30#include <tlhelp32.h>
31#else
32#include <dlfcn.h>
33#include <pthread.h>
34#endif
35
36#include "InstrProfilingPlatformROCmInternal.h"
37
38// shortcut to shared helper names
39using namespace __prof_rocm;
40
41/* Serialize one-time HIP loader resolution and DynamicModules mutations.
42 * Inline to avoid a sanitizer_common dependency. */
43#ifdef _WIN32
44static INIT_ONCE HipLoadedOnce = INIT_ONCE_STATIC_INIT;
45static CRITICAL_SECTION DynamicModulesLock;
46static INIT_ONCE DynamicModulesLockInit = INIT_ONCE_STATIC_INIT;
47static BOOL CALLBACK initDynamicModulesLockCb(PINIT_ONCE, PVOID, PVOID *) {
48 InitializeCriticalSection(&DynamicModulesLock);
49 return TRUE;
50}
51static void lockDynamicModules(void) {
52 InitOnceExecuteOnce(&DynamicModulesLockInit, initDynamicModulesLockCb, NULL,
53 NULL);
54 EnterCriticalSection(&DynamicModulesLock);
55}
56static void unlockDynamicModules(void) {
57 LeaveCriticalSection(&DynamicModulesLock);
58}
59#else
60static pthread_once_t HipLoadedOnce = PTHREAD_ONCE_INIT;
61static pthread_mutex_t DynamicModulesLock = PTHREAD_MUTEX_INITIALIZER;
62static void lockDynamicModules(void) {
63 pthread_mutex_lock(mutex: &DynamicModulesLock);
64}
65static void unlockDynamicModules(void) {
66 pthread_mutex_unlock(mutex: &DynamicModulesLock);
67}
68#endif
69
70int __prof_rocm::isVerboseMode() {
71 static int IsVerbose = -1;
72 if (IsVerbose == -1)
73 IsVerbose = getenv(name: "LLVM_PROFILE_VERBOSE") != nullptr;
74 return IsVerbose;
75}
76
77/* -------------------------------------------------------------------------- */
78/* Dynamic loading of HIP runtime symbols */
79/* -------------------------------------------------------------------------- */
80
81typedef int (*hipGetSymbolAddressTy)(void **, const void *);
82typedef int (*hipGetSymbolSizeTy)(size_t *, const void *);
83typedef int (*hipMemcpyTy)(void *, const void *, size_t, int);
84typedef int (*hipModuleGetGlobalTy)(void **, size_t *, void *, const char *);
85typedef int (*hipGetDeviceCountTy)(int *);
86typedef int (*hipGetDeviceTy)(int *);
87typedef int (*hipSetDeviceTy)(int);
88#if defined(__linux__) && !defined(_WIN32)
89typedef void *HipStream;
90typedef int (*hipStreamGetDeviceTy)(HipStream, int *);
91#endif
92
93/* Minimal hipDeviceProp_t (HIP 6.x R0600): only gcnArchName at offset 1160
94 * is read. Padded to 4096 to tolerate ABI growth. */
95typedef struct {
96 char padding[1160];
97 char gcnArchName[256];
98 char tail_padding[2680];
99} HipDevicePropMinimal;
100typedef int (*hipGetDevicePropertiesTy)(HipDevicePropMinimal *, int);
101
102static hipGetSymbolAddressTy pHipGetSymbolAddress = nullptr;
103static hipGetSymbolSizeTy pHipGetSymbolSize = nullptr;
104static hipMemcpyTy pHipMemcpy = nullptr;
105static hipModuleGetGlobalTy pHipModuleGetGlobal = nullptr;
106static hipGetDeviceCountTy pHipGetDeviceCount = nullptr;
107static hipGetDeviceTy pHipGetDevice = nullptr;
108static hipSetDeviceTy pHipSetDevice = nullptr;
109#if defined(__linux__) && !defined(_WIN32)
110static hipStreamGetDeviceTy pHipStreamGetDevice = nullptr;
111#endif
112static hipGetDevicePropertiesTy pHipGetDeviceProperties = nullptr;
113
114static int NumDevices = 0;
115/* 256 matches hipDeviceProp_t::gcnArchName, the source field width. */
116static char (*DeviceArchNames)[256] = nullptr;
117#if defined(__linux__) && !defined(_WIN32)
118static unsigned char *UsedDevices = nullptr;
119static int AnyDeviceUsed = 0;
120#endif
121
122#ifdef _WIN32
123static wchar_t toLowerAsciiW(wchar_t C) {
124 return C >= L'A' && C <= L'Z' ? C - L'A' + L'a' : C;
125}
126
127static int wcsEqualNoCase(const wchar_t *A, const wchar_t *B) {
128 while (*A && *B) {
129 if (toLowerAsciiW(*A) != toLowerAsciiW(*B))
130 return 0;
131 ++A;
132 ++B;
133 }
134 return *A == *B;
135}
136
137static int wcsStartsWithNoCase(const wchar_t *S, const wchar_t *Prefix) {
138 while (*Prefix) {
139 if (toLowerAsciiW(*S) != toLowerAsciiW(*Prefix))
140 return 0;
141 ++S;
142 ++Prefix;
143 }
144 return 1;
145}
146
147static int wcsEndsWithNoCase(const wchar_t *S, const wchar_t *Suffix) {
148 size_t SLen = wcslen(S);
149 size_t SuffixLen = wcslen(Suffix);
150 return SLen >= SuffixLen && wcsEqualNoCase(S + SLen - SuffixLen, Suffix);
151}
152
153static int isHipRuntimeModuleName(const wchar_t *Name) {
154 return wcsEqualNoCase(Name, L"amdhip64.dll") ||
155 (wcsStartsWithNoCase(Name, L"amdhip64_") &&
156 wcsEndsWithNoCase(Name, L".dll"));
157}
158
159static void *findLoadedHipRuntime(void) {
160 HMODULE Handle = GetModuleHandleW(L"amdhip64.dll");
161 if (Handle)
162 return (void *)Handle;
163
164 HANDLE Snapshot = CreateToolhelp32Snapshot(
165 TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, GetCurrentProcessId());
166 if (Snapshot == INVALID_HANDLE_VALUE)
167 return nullptr;
168
169 MODULEENTRY32W Entry;
170 Entry.dwSize = sizeof(Entry);
171 if (Module32FirstW(Snapshot, &Entry)) {
172 do {
173 if (isHipRuntimeModuleName(Entry.szModule)) {
174 Handle = Entry.hModule;
175 break;
176 }
177 } while (Module32NextW(Snapshot, &Entry));
178 }
179
180 CloseHandle(Snapshot);
181 return (void *)Handle;
182}
183#endif
184
185/* -------------------------------------------------------------------------- */
186/* Device-to-host copies */
187/* Keep HIP-only to avoid an HSA dependency. */
188/* -------------------------------------------------------------------------- */
189
190static void doEnsureHipLoaded(void) {
191 if (!__interception::DynamicLoaderAvailable()) {
192 if (isVerboseMode())
193 PROF_NOTE("%s", "Dynamic library loading not available - "
194 "HIP profiling disabled\n");
195 return;
196 }
197
198#ifdef _WIN32
199 /* Use the app's loaded HIP runtime to avoid binding another ROCm version. */
200 void *Handle = findLoadedHipRuntime();
201#else
202 const char *HipLibName = "libamdhip64.so";
203 void *Handle = __interception::OpenLibrary(name: HipLibName);
204#endif
205 if (!Handle)
206 return;
207
208 pHipGetSymbolAddress = (hipGetSymbolAddressTy)__interception::LookupSymbol(
209 handle: Handle, symbol: "hipGetSymbolAddress");
210 pHipGetSymbolSize = (hipGetSymbolSizeTy)__interception::LookupSymbol(
211 handle: Handle, symbol: "hipGetSymbolSize");
212 pHipMemcpy = (hipMemcpyTy)__interception::LookupSymbol(handle: Handle, symbol: "hipMemcpy");
213 pHipModuleGetGlobal = (hipModuleGetGlobalTy)__interception::LookupSymbol(
214 handle: Handle, symbol: "hipModuleGetGlobal");
215 pHipGetDeviceCount = (hipGetDeviceCountTy)__interception::LookupSymbol(
216 handle: Handle, symbol: "hipGetDeviceCount");
217 pHipGetDevice =
218 (hipGetDeviceTy)__interception::LookupSymbol(handle: Handle, symbol: "hipGetDevice");
219 pHipSetDevice =
220 (hipSetDeviceTy)__interception::LookupSymbol(handle: Handle, symbol: "hipSetDevice");
221#if defined(__linux__) && !defined(_WIN32)
222 pHipStreamGetDevice = (hipStreamGetDeviceTy)__interception::LookupSymbol(
223 handle: Handle, symbol: "hipStreamGetDevice");
224#endif
225 pHipGetDeviceProperties =
226 (hipGetDevicePropertiesTy)__interception::LookupSymbol(
227 handle: Handle, symbol: "hipGetDevicePropertiesR0600");
228 if (!pHipGetDeviceProperties)
229 pHipGetDeviceProperties =
230 (hipGetDevicePropertiesTy)__interception::LookupSymbol(
231 handle: Handle, symbol: "hipGetDeviceProperties");
232
233 if (pHipGetDeviceCount && pHipGetDeviceProperties) {
234 int Count = 0;
235 if (pHipGetDeviceCount(&Count) == 0 && Count > 0) {
236 DeviceArchNames = (char (*)[256])calloc(nmemb: Count, size: sizeof(*DeviceArchNames));
237 if (!DeviceArchNames) {
238 PROF_ERR("%s\n", "failed to allocate device arch name table");
239 return;
240 }
241#if defined(__linux__) && !defined(_WIN32)
242 UsedDevices = (unsigned char *)calloc(nmemb: Count, size: sizeof(*UsedDevices));
243 if (!UsedDevices && isVerboseMode())
244 PROF_NOTE("%s\n", "Device-use tracking disabled");
245#endif
246 HipDevicePropMinimal Prop;
247 for (int i = 0; i < Count; ++i) {
248 __builtin_memset(&Prop, 0, sizeof(Prop));
249 if (pHipGetDeviceProperties(&Prop, i) == 0) {
250 strncpy(dest: DeviceArchNames[i], src: Prop.gcnArchName,
251 n: sizeof(DeviceArchNames[i]) - 1);
252 DeviceArchNames[i][sizeof(DeviceArchNames[i]) - 1] = '\0';
253 if (isVerboseMode())
254 PROF_NOTE("Device %d arch: %s\n", i, DeviceArchNames[i]);
255 }
256 }
257 NumDevices = Count;
258 }
259 }
260}
261
262#ifdef _WIN32
263static BOOL CALLBACK ensureHipLoadedCb(PINIT_ONCE, PVOID, PVOID *) {
264 doEnsureHipLoaded();
265 return TRUE;
266}
267#endif
268
269void __prof_rocm::ensureHipLoaded(void) {
270#ifdef _WIN32
271 InitOnceExecuteOnce(&HipLoadedOnce, ensureHipLoadedCb, NULL, NULL);
272#else
273 pthread_once(once_control: &HipLoadedOnce, init_routine: doEnsureHipLoaded);
274#endif
275}
276
277// Accessor for the HSA drain: true once the loaded HIP runtime exposes
278// hipMemcpy. Kept here so pHipMemcpy stays file-private to this TU.
279int __prof_rocm::hipMemcpyAvailable() { return pHipMemcpy != nullptr; }
280
281/* -------------------------------------------------------------------------- */
282/* Public wrappers that forward to the loaded HIP symbols */
283/* -------------------------------------------------------------------------- */
284
285static int hipGetSymbolAddress(void **devPtr, const void *symbol) {
286 ensureHipLoaded();
287 return pHipGetSymbolAddress ? pHipGetSymbolAddress(devPtr, symbol) : -1;
288}
289
290static int hipGetSymbolSize(size_t *size, const void *symbol) {
291 ensureHipLoaded();
292 return pHipGetSymbolSize ? pHipGetSymbolSize(size, symbol) : -1;
293}
294
295static int hipMemcpy(void *dest, const void *src, size_t len,
296 int kind /*2=DToH*/) {
297 ensureHipLoaded();
298 return pHipMemcpy ? pHipMemcpy(dest, src, len, kind) : -1;
299}
300
301/* Device section symbols must be registered with CLR first; otherwise
302 * hipMemcpy may take a CPU path and crash. */
303int __prof_rocm::memcpyDeviceToHost(void *Dst, const void *Src, size_t Size) {
304 return hipMemcpy(dest: Dst, src: Src, len: Size, kind: 2 /* DToH */);
305}
306
307[[maybe_unused]]
308static int hipModuleGetGlobal(void **DevPtr, size_t *Bytes, void *Module,
309 const char *Name) {
310 ensureHipLoaded();
311 return pHipModuleGetGlobal ? pHipModuleGetGlobal(DevPtr, Bytes, Module, Name)
312 : -1;
313}
314
315static int hipGetDevice(int *DeviceId) {
316 ensureHipLoaded();
317 return pHipGetDevice ? pHipGetDevice(DeviceId) : -1;
318}
319
320static int hipSetDevice(int DeviceId) {
321 ensureHipLoaded();
322 return pHipSetDevice ? pHipSetDevice(DeviceId) : -1;
323}
324
325#if defined(__linux__) && !defined(_WIN32)
326static int hipStreamGetDevice(HipStream Stream, int *DeviceId) {
327 ensureHipLoaded();
328 return pHipStreamGetDevice ? pHipStreamGetDevice(Stream, DeviceId) : -1;
329}
330
331static void markDeviceUsed(int DeviceId) {
332 if (DeviceId < 0 || DeviceId >= NumDevices || !UsedDevices)
333 return;
334 __atomic_store_n(&UsedDevices[DeviceId], 1, __ATOMIC_RELAXED);
335 __atomic_store_n(&AnyDeviceUsed, 1, __ATOMIC_RELEASE);
336}
337
338static void markCurrentDeviceUsed(void) {
339 int DeviceId = -1;
340 if (hipGetDevice(DeviceId: &DeviceId) == 0)
341 markDeviceUsed(DeviceId);
342}
343
344static void markLaunchStreamDeviceUsed(HipStream Stream) {
345 int DeviceId = -1;
346 if (Stream && hipStreamGetDevice(Stream, DeviceId: &DeviceId) == 0) {
347 markDeviceUsed(DeviceId);
348 return;
349 }
350 markCurrentDeviceUsed();
351}
352
353static int shouldCollectDevice(int DeviceId) {
354 if (UsedDevices && __atomic_load_n(&AnyDeviceUsed, __ATOMIC_ACQUIRE) &&
355 !__atomic_load_n(&UsedDevices[DeviceId], __ATOMIC_RELAXED))
356 return 0;
357 return 1;
358}
359#else
360static int shouldCollectDevice(int) { return 1; }
361#endif
362
363static const char *getDeviceArchName(int DeviceId) {
364 if (DeviceId < 0 || DeviceId >= NumDevices || !DeviceArchNames[DeviceId][0])
365 return "amdgpu";
366 return DeviceArchNames[DeviceId];
367}
368
369/* -------------------------------------------------------------------------- */
370/* Dynamic module tracking */
371/* -------------------------------------------------------------------------- */
372
373/* Per-TU profile entry inside a dynamic module.
374 * A single dynamic module may contain multiple TUs (e.g. -fgpu-rdc). */
375typedef struct {
376 void *DeviceVar; /* device address of __llvm_profile_sections_<CUID> */
377 int Processed; /* 0 = not yet collected, 1 = data already copied */
378} OffloadDynamicTUInfo;
379
380/* One entry per hipModuleLoad call. */
381typedef struct {
382 void *ModulePtr; /* hipModule_t handle */
383 OffloadDynamicTUInfo *TUs; /* array of per-TU entries */
384 int NumTUs;
385 int CapTUs;
386} OffloadDynamicModuleInfo;
387
388static OffloadDynamicModuleInfo *DynamicModules = nullptr;
389static int NumDynamicModules = 0;
390static int CapDynamicModules = 0;
391
392/* -------------------------------------------------------------------------- */
393/* ELF symbol enumeration (manual parse: compiler-rt cannot link LLVM Support)
394 */
395/* -------------------------------------------------------------------------- */
396
397#if __has_include(<elf.h>)
398#include <elf.h>
399
400/* Callback invoked for every matching symbol name found in the ELF image.
401 * Return 0 to continue iteration, non-zero to stop. */
402typedef int (*SymbolCallback)(const char *Name, void *UserData);
403
404/* If Image is a clang offload bundle, return a pointer to the first embedded
405 * ELF. Returns Image if not a bundle, nullptr if a bundle holds no ELF. */
406static const void *unwrapOffloadBundle(const void *Image) {
407 static const char BundleMagic[] = "__CLANG_OFFLOAD_BUNDLE__";
408 if (memcmp(s1: Image, s2: BundleMagic, n: sizeof(BundleMagic) - 1) != 0)
409 return Image; /* Not a bundle, return as-is. */
410
411 const char *Buf = (const char *)Image;
412 uint64_t NumEntries;
413 __builtin_memcpy(&NumEntries, Buf + sizeof(BundleMagic) - 1,
414 sizeof(uint64_t));
415
416 /* Walk the entry table (starts at offset 32). */
417 const char *Cursor = Buf + 32;
418 for (uint64_t I = 0; I < NumEntries; ++I) {
419 uint64_t EntryOffset, EntrySize, IDSize;
420 __builtin_memcpy(&EntryOffset, Cursor, sizeof(EntryOffset));
421 Cursor += sizeof(EntryOffset);
422 __builtin_memcpy(&EntrySize, Cursor, sizeof(EntrySize));
423 Cursor += sizeof(EntrySize);
424 __builtin_memcpy(&IDSize, Cursor, sizeof(IDSize));
425 Cursor += sizeof(IDSize);
426 Cursor += IDSize; /* skip entry ID */
427
428 if (EntrySize >= sizeof(Elf64_Ehdr)) {
429 const Elf64_Ehdr *E = (const Elf64_Ehdr *)(Buf + EntryOffset);
430 if (E->e_ident[EI_MAG0] == ELFMAG0 && E->e_ident[EI_MAG1] == ELFMAG1 &&
431 E->e_ident[EI_MAG2] == ELFMAG2 && E->e_ident[EI_MAG3] == ELFMAG3) {
432 return (const void *)(Buf + EntryOffset);
433 }
434 }
435 }
436
437 PROF_WARN("%s", "offload bundle contains no valid ELF entries\n");
438 return nullptr;
439}
440
441/* Invoke CB for every global symbol in Image (an AMDGPU ELF or offload bundle)
442 * whose name starts with PREFIX. Image may be null. */
443static void enumerateElfSymbols(const void *Image, const char *Prefix,
444 SymbolCallback CB, void *UserData) {
445 if (!Image)
446 return;
447
448 Image = unwrapOffloadBundle(Image);
449 if (!Image)
450 return;
451
452 const Elf64_Ehdr *Ehdr = (const Elf64_Ehdr *)Image;
453 if (Ehdr->e_ident[EI_MAG0] != ELFMAG0 || Ehdr->e_ident[EI_MAG1] != ELFMAG1 ||
454 Ehdr->e_ident[EI_MAG2] != ELFMAG2 || Ehdr->e_ident[EI_MAG3] != ELFMAG3) {
455 if (isVerboseMode())
456 PROF_NOTE("%s", "Image is not a valid ELF, skipping enumeration\n");
457 return;
458 }
459
460 size_t PrefixLen = strlen(s: Prefix);
461 const char *Base = (const char *)Image;
462 const Elf64_Shdr *Shdrs = (const Elf64_Shdr *)(Base + Ehdr->e_shoff);
463
464 for (int i = 0; i < Ehdr->e_shnum; ++i) {
465 if (Shdrs[i].sh_type != SHT_SYMTAB)
466 continue;
467
468 const Elf64_Sym *Syms = (const Elf64_Sym *)(Base + Shdrs[i].sh_offset);
469 int NumSyms = Shdrs[i].sh_size / sizeof(Elf64_Sym);
470 /* String table is the section referenced by sh_link. */
471 const char *StrTab = Base + Shdrs[Shdrs[i].sh_link].sh_offset;
472
473 for (int j = 0; j < NumSyms; ++j) {
474 if (Syms[j].st_name == 0)
475 continue;
476 const char *Name = StrTab + Syms[j].st_name;
477 if (strncmp(s1: Name, s2: Prefix, n: PrefixLen) == 0) {
478 if (CB(Name, UserData))
479 return;
480 }
481 }
482 }
483}
484
485/* State passed through the enumeration callback. */
486typedef struct {
487 void *Module; /* hipModule_t */
488 OffloadDynamicModuleInfo *ModInfo;
489} EnumState;
490
491/* Register one __llvm_profile_sections_<CUID> symbol on the module entry.
492 * hipModuleGetGlobal also registers the device address with CLR so hipMemcpy
493 * can copy from it later. */
494static int registerPrfSymbol(const char *Name, void *UserData) {
495 EnumState *S = (EnumState *)UserData;
496 OffloadDynamicModuleInfo *MI = S->ModInfo;
497
498 /* The symbol is the per-TU sections struct itself, not a pointer
499 * indirection, so this address is the hipMemcpy source. */
500 void *DeviceVar = nullptr;
501 size_t Bytes = 0;
502 if (hipModuleGetGlobal(DevPtr: &DeviceVar, Bytes: &Bytes, Module: S->Module, Name) != 0) {
503 PROF_WARN("failed to get symbol %s for module %p\n", Name, S->Module);
504 return 0; /* continue */
505 }
506
507 if (growArray(Arr: (void **)&MI->TUs, Cap: &MI->CapTUs, MinCount: MI->NumTUs + 1, InitCap: 4,
508 ElemSize: sizeof(*MI->TUs))) {
509 PROF_ERR("%s\n", "failed to grow TU array");
510 return 0;
511 }
512 OffloadDynamicTUInfo *TU = &MI->TUs[MI->NumTUs++];
513 TU->DeviceVar = DeviceVar;
514 TU->Processed = 0;
515
516 (void)Name;
517 return 0; /* continue enumeration */
518}
519
520#endif /* __has_include(<elf.h>) */
521
522/* -------------------------------------------------------------------------- */
523/* Registration / un-registration helpers */
524/* -------------------------------------------------------------------------- */
525
526extern "C" void
527__llvm_profile_offload_register_dynamic_module(int ModuleLoadRc, void **Ptr,
528 const void *Image) {
529 if (ModuleLoadRc)
530 return;
531
532 lockDynamicModules();
533
534 if (isVerboseMode())
535 PROF_NOTE("Registering loaded module %d: rc=%d, module=%p, image=%p\n",
536 NumDynamicModules, ModuleLoadRc, *Ptr, Image);
537
538 if (growArray(Arr: (void **)&DynamicModules, Cap: &CapDynamicModules,
539 MinCount: NumDynamicModules + 1, InitCap: 64, ElemSize: sizeof(*DynamicModules))) {
540 unlockDynamicModules();
541 return;
542 }
543
544 OffloadDynamicModuleInfo *MI = &DynamicModules[NumDynamicModules++];
545 MI->ModulePtr = *Ptr;
546 MI->TUs = nullptr;
547 MI->NumTUs = 0;
548 MI->CapTUs = 0;
549
550 /* Dynamic-module profiling needs ELF parsing for symbol enumeration. */
551#if __has_include(<elf.h>)
552 EnumState State = {.Module: *Ptr, .ModInfo: MI};
553 enumerateElfSymbols(Image, Prefix: "__llvm_profile_sections_", CB: registerPrfSymbol,
554 UserData: &State);
555#else
556 (void)Image;
557 if (isVerboseMode())
558 PROF_NOTE("%s",
559 "Dynamic module profiling not supported on this platform\n");
560#endif
561
562 if (MI->NumTUs == 0) {
563 PROF_WARN("no __llvm_profile_sections_* symbols found in module %p\n",
564 *Ptr);
565 } else if (isVerboseMode()) {
566 PROF_NOTE("Module %p: registered %d TU(s)\n", *Ptr, MI->NumTUs);
567 }
568
569 unlockDynamicModules();
570}
571
572extern "C" void __llvm_profile_offload_unregister_dynamic_module(void *Ptr) {
573 lockDynamicModules();
574 for (int i = 0; i < NumDynamicModules; ++i) {
575 OffloadDynamicModuleInfo *MI = &DynamicModules[i];
576
577 /* HIP recycles hipModule_t addresses; drained slots are cleared so a
578 * recycled handle finds the new slot, not the dead one. */
579 if (MI->ModulePtr != Ptr)
580 continue;
581
582 if (isVerboseMode())
583 PROF_NOTE("Unregistering module %p (%d TUs)\n", MI->ModulePtr,
584 MI->NumTUs);
585
586 static int NextTUIndex = 0;
587 for (int t = 0; t < MI->NumTUs; ++t) {
588 OffloadDynamicTUInfo *TU = &MI->TUs[t];
589 if (TU->Processed) {
590 if (isVerboseMode())
591 PROF_NOTE("Module %p TU %d already processed, skipping\n", Ptr, t);
592 continue;
593 }
594 int TUIndex = __atomic_fetch_add(&NextTUIndex, 1, __ATOMIC_RELAXED);
595 if (TU->DeviceVar) {
596 int CurDev = 0;
597 hipGetDevice(DeviceId: &CurDev);
598 const char *ArchName = getDeviceArchName(DeviceId: CurDev);
599 /* Encode TUIndex in Target so each drain writes a distinct profraw;
600 * otherwise back-to-back drains overwrite the same file. */
601 char TargetWithTU[64];
602 snprintf(s: TargetWithTU, maxlen: sizeof(TargetWithTU), format: "%s.%d", ArchName,
603 TUIndex);
604 if (processDeviceOffloadPrf(DeviceOffloadPrf: TU->DeviceVar, Target: TargetWithTU, Sections: nullptr) == 0)
605 TU->Processed = 1;
606 else
607 PROF_WARN("failed to process profile data for module %p TU %d\n", Ptr,
608 t);
609 }
610 }
611 MI->ModulePtr = nullptr;
612 unlockDynamicModules();
613 return;
614 }
615
616 if (isVerboseMode())
617 PROF_WARN("unregister called for unknown module %p\n", Ptr);
618 unlockDynamicModules();
619}
620
621static void **OffloadShadowVariables = nullptr;
622static int NumShadowVariables = 0;
623static int CapShadowVariables = 0;
624static bool CurrentShadowVariableRegistered = false;
625
626struct OffloadSectionShadow {
627 void *Data;
628 void *Counters;
629 void *UniformCounters;
630 void *Names;
631};
632
633struct OffloadSectionShadowGroup {
634 void *VersionShadow;
635 OffloadSectionShadow *Shadows;
636 int NumShadows;
637 int CapShadows;
638 int NumSections;
639};
640
641static OffloadSectionShadowGroup *OffloadSectionShadowGroups = nullptr;
642static int CapSectionShadowGroups = 0;
643
644static int ensureSectionShadowGroupCapacity(void) {
645 return growArray(Arr: (void **)&OffloadSectionShadowGroups,
646 Cap: &CapSectionShadowGroups, MinCount: CapShadowVariables,
647 InitCap: CapShadowVariables, ElemSize: sizeof(*OffloadSectionShadowGroups));
648}
649
650static int ensureSectionShadowCapacity(OffloadSectionShadowGroup *Group,
651 int MinCapacity) {
652 return growArray(Arr: (void **)&Group->Shadows, Cap: &Group->CapShadows, MinCount: MinCapacity, InitCap: 4,
653 ElemSize: sizeof(*Group->Shadows));
654}
655
656extern "C" void __llvm_profile_offload_register_shadow_variable(void *ptr) {
657 // Dependent callbacks must not use the previous group if this attempt fails.
658 CurrentShadowVariableRegistered = false;
659 if (growArray(Arr: (void **)&OffloadShadowVariables, Cap: &CapShadowVariables,
660 MinCount: NumShadowVariables + 1, InitCap: 64, ElemSize: sizeof(*OffloadShadowVariables)))
661 return;
662 if (ensureSectionShadowGroupCapacity())
663 return;
664 int Index = NumShadowVariables++;
665 OffloadShadowVariables[Index] = ptr;
666 __builtin_memset(&OffloadSectionShadowGroups[Index], 0,
667 sizeof(OffloadSectionShadowGroups[Index]));
668 CurrentShadowVariableRegistered = true;
669}
670
671extern "C" void
672__llvm_profile_offload_register_section_shadow_variable(void *ptr) {
673 if (!CurrentShadowVariableRegistered)
674 return;
675
676 /* Match CGCUDANV.cpp: data, counters, uniform counters, then names for each
677 * kernel. */
678 OffloadSectionShadowGroup *Group =
679 &OffloadSectionShadowGroups[NumShadowVariables - 1];
680 int ShadowIndex = Group->NumSections / 4;
681 if (ensureSectionShadowCapacity(Group, MinCapacity: ShadowIndex + 1))
682 return;
683 if (ShadowIndex >= Group->NumShadows)
684 Group->NumShadows = ShadowIndex + 1;
685
686 OffloadSectionShadow *Shadow = &Group->Shadows[ShadowIndex];
687 switch (Group->NumSections % 4) {
688 case 0:
689 Shadow->Data = ptr;
690 break;
691 case 1:
692 Shadow->Counters = ptr;
693 break;
694 case 2:
695 Shadow->UniformCounters = ptr;
696 break;
697 case 3:
698 Shadow->Names = ptr;
699 break;
700 }
701 ++Group->NumSections;
702}
703
704extern "C" void
705__llvm_profile_offload_register_version_shadow_variable(void *ptr) {
706 if (!CurrentShadowVariableRegistered)
707 return;
708 OffloadSectionShadowGroups[NumShadowVariables - 1].VersionShadow = ptr;
709}
710
711namespace {
712
713struct ProfileSectionCopy {
714 const char *Name;
715 const void *DevBegin;
716 size_t Size;
717 const void *&CachedDevBegin;
718 char *&CachedHost;
719 size_t &CachedSize;
720 UniqueFree Owner;
721 char *HostBegin = nullptr;
722 bool Reused = false;
723
724 ProfileSectionCopy(const char *Name, const void *DevBegin, size_t Size,
725 const void *&CachedDevBegin, char *&CachedHost,
726 size_t &CachedSize)
727 : Name(Name), DevBegin(DevBegin), Size(Size),
728 CachedDevBegin(CachedDevBegin), CachedHost(CachedHost),
729 CachedSize(CachedSize) {}
730
731 ProfileSectionCopy(const ProfileSectionCopy &) = delete;
732 ProfileSectionCopy &operator=(const ProfileSectionCopy &) = delete;
733
734 int prepare() {
735 if (Size == 0)
736 return 0;
737 if (DevBegin == CachedDevBegin && Size == CachedSize) {
738 HostBegin = CachedHost;
739 Reused = true;
740 if (isVerboseMode())
741 PROF_NOTE("Reusing cached %s section (%zu bytes)\n", Name, Size);
742 } else {
743 HostBegin = static_cast<char *>(malloc(size: Size));
744 Owner.reset(P: HostBegin);
745 }
746 return HostBegin ? 0 : -1;
747 }
748
749 int copy() {
750 if (Size == 0 || Reused)
751 return 0;
752 return memcpyDeviceToHost(Dst: HostBegin, Src: DevBegin, Size);
753 }
754
755 void commitCache() {
756 if (Reused || Size == 0)
757 return;
758 CachedDevBegin = DevBegin;
759 CachedHost = HostBegin;
760 CachedSize = Size;
761 Owner.release();
762 }
763};
764
765} // namespace
766
767static int getRegisteredSectionBounds(void *Shadow, void **DevicePtr,
768 size_t *Size) {
769 *DevicePtr = nullptr;
770 *Size = 0;
771 int AddrRc = hipGetSymbolAddress(devPtr: DevicePtr, symbol: Shadow);
772 int SizeRc = hipGetSymbolSize(size: Size, symbol: Shadow);
773 return AddrRc == 0 && SizeRc == 0 && *DevicePtr && *Size > 0 ? 0 : -1;
774}
775
776struct RegisteredSectionRange {
777 const void *Data;
778 const void *Counters;
779 const void *UniformCounters;
780 const void *Names;
781 size_t DataSize;
782 size_t CountersSize;
783 size_t UniformCountersSize;
784 size_t NamesSize;
785 size_t DataOffset;
786 size_t CountersOffset;
787 size_t UniformCountersOffset;
788 size_t NamesOffset;
789};
790
791static int
792hasCompleteSectionShadows(const OffloadSectionShadowGroup *Sections) {
793 if (!Sections || Sections->NumShadows == 0 || Sections->NumSections % 4 != 0)
794 return 0;
795 for (int I = 0; I < Sections->NumShadows; ++I) {
796 if (!Sections->Shadows[I].Data || !Sections->Shadows[I].Counters ||
797 !Sections->Shadows[I].UniformCounters || !Sections->Shadows[I].Names)
798 return 0;
799 }
800 return 1;
801}
802
803int __prof_rocm::processDeviceOffloadPrf(
804 void *DeviceOffloadPrf, const char *Target,
805 const OffloadSectionShadowGroup *Sections) {
806 __llvm_profile_gpu_sections HostSections;
807
808 if (hipMemcpy(dest: &HostSections, src: DeviceOffloadPrf, len: sizeof(HostSections),
809 kind: 2 /*DToH*/) != 0) {
810 PROF_ERR("%s\n", "failed to copy offload prf structure from device");
811 return -1;
812 }
813
814 const void *DevCntsBegin = HostSections.CountersStart;
815 const void *DevDataBegin = HostSections.DataStart;
816 const void *DevNamesBegin = HostSections.NamesStart;
817 const void *DevUniformCntsBegin = HostSections.UniformCountersStart;
818 const void *DevCntsEnd = HostSections.CountersStop;
819 const void *DevDataEnd = HostSections.DataStop;
820 const void *DevNamesEnd = HostSections.NamesStop;
821 const void *DevUniformCntsEnd = HostSections.UniformCountersStop;
822
823 size_t CountersSize = (const char *)DevCntsEnd - (const char *)DevCntsBegin;
824 size_t DataSize = (const char *)DevDataEnd - (const char *)DevDataBegin;
825 size_t NamesSize = (const char *)DevNamesEnd - (const char *)DevNamesBegin;
826 size_t UniformCountersSize =
827 (const char *)DevUniformCntsEnd - (const char *)DevUniformCntsBegin;
828
829 int UseRegisteredSections = hasCompleteSectionShadows(Sections);
830 RegisteredSectionRange *RegisteredRanges = nullptr;
831 int NumRegisteredRanges = 0;
832
833 if (isVerboseMode())
834 PROF_NOTE("Section pointers: Cnts=[%p,%p]=%zu Data=[%p,%p]=%zu "
835 "Names=[%p,%p]=%zu UCnts=[%p,%p]=%zu\n",
836 DevCntsBegin, DevCntsEnd, CountersSize, DevDataBegin, DevDataEnd,
837 DataSize, DevNamesBegin, DevNamesEnd, NamesSize,
838 DevUniformCntsBegin, DevUniformCntsEnd, UniformCountersSize);
839
840 if (CountersSize == 0 || DataSize == 0)
841 return 0;
842
843 // The host and device may use different instrumentation layouts.
844#ifdef _WIN32
845 // Use the host handle to make the version address known to HIP.
846 void *VersionVar = nullptr;
847 if (!Sections || !Sections->VersionShadow ||
848 hipGetSymbolAddress(&VersionVar, Sections->VersionShadow) != 0 ||
849 !VersionVar) {
850 PROF_ERR("%s\n", "failed to get profile version symbol address");
851 return -1;
852 }
853#else
854 const void *VersionVar = HostSections.VersionVar;
855#endif
856 uint64_t Version;
857 if (!VersionVar ||
858 hipMemcpy(dest: &Version, src: VersionVar, len: sizeof(Version), kind: 2 /*DToH*/) != 0) {
859 PROF_ERR("%s\n", "failed to copy profile version from device");
860 return -1;
861 }
862 if (GET_VERSION(Version) != INSTR_PROF_RAW_VERSION) {
863 PROF_ERR("Device runtime and instrumentation version mismatch: "
864 "expected %d, but get %d\n",
865 INSTR_PROF_RAW_VERSION, (int)GET_VERSION(Version));
866 return -1;
867 }
868
869 int ret = -1;
870
871 /* Sections using linker-defined __start_/__stop_ bounds are shared across
872 TU structs in RDC mode. Deduplicate by caching the last copied range. */
873 static const void *CachedDevNamesBegin = nullptr;
874 static char *CachedHostNames = nullptr;
875 static size_t CachedNamesSize = 0;
876
877 static const void *CachedDevCntsBegin = nullptr;
878 static char *CachedHostCnts = nullptr;
879 static size_t CachedCntsSize = 0;
880
881 static const void *CachedDevDataBegin = nullptr;
882 static char *CachedHostData = nullptr;
883 static size_t CachedDataSize = 0;
884
885 static const void *CachedDevUCntsBegin = nullptr;
886 static char *CachedHostUCnts = nullptr;
887 static size_t CachedUCntsSize = 0;
888
889 ProfileSectionCopy Cnts("counters", DevCntsBegin, CountersSize,
890 CachedDevCntsBegin, CachedHostCnts, CachedCntsSize);
891 ProfileSectionCopy Data("data", DevDataBegin, DataSize, CachedDevDataBegin,
892 CachedHostData, CachedDataSize);
893 ProfileSectionCopy Names("names", DevNamesBegin, NamesSize,
894 CachedDevNamesBegin, CachedHostNames,
895 CachedNamesSize);
896 ProfileSectionCopy UCnts("ucnts", DevUniformCntsBegin, UniformCountersSize,
897 CachedDevUCntsBegin, CachedHostUCnts,
898 CachedUCntsSize);
899
900 UniqueFree RegisteredRangeOwner;
901
902 if (UseRegisteredSections) {
903 NumRegisteredRanges = Sections->NumShadows;
904 RegisteredRangeOwner.reset(
905 P: malloc(size: NumRegisteredRanges * sizeof(RegisteredSectionRange)));
906 RegisteredRanges = (RegisteredSectionRange *)RegisteredRangeOwner.get();
907 if (!RegisteredRanges) {
908 PROF_ERR("%s\n", "failed to allocate registered section table");
909 return -1;
910 }
911 __builtin_memset(RegisteredRanges, 0,
912 NumRegisteredRanges * sizeof(*RegisteredRanges));
913
914 size_t RegisteredDataSize = 0;
915 size_t RegisteredCountersSize = 0;
916 size_t RegisteredUniformCountersSize = 0;
917 size_t RegisteredNamesSize = 0;
918 for (int I = 0; I < NumRegisteredRanges; ++I) {
919 void *Data = nullptr;
920 void *Counters = nullptr;
921 void *UniformCounters = nullptr;
922 void *Names = nullptr;
923 size_t ThisDataSize = 0;
924 size_t ThisCountersSize = 0;
925 size_t ThisUniformCountersSize = 0;
926 size_t ThisNamesSize = 0;
927 OffloadSectionShadow *Shadow = &Sections->Shadows[I];
928 if (getRegisteredSectionBounds(Shadow: Shadow->Data, DevicePtr: &Data, Size: &ThisDataSize) != 0 ||
929 getRegisteredSectionBounds(Shadow: Shadow->Counters, DevicePtr: &Counters,
930 Size: &ThisCountersSize) != 0 ||
931 getRegisteredSectionBounds(Shadow: Shadow->UniformCounters, DevicePtr: &UniformCounters,
932 Size: &ThisUniformCountersSize) != 0 ||
933 getRegisteredSectionBounds(Shadow: Shadow->Names, DevicePtr: &Names, Size: &ThisNamesSize) !=
934 0) {
935 PROF_ERR("%s\n", "failed to get registered section bounds");
936 return -1;
937 }
938
939 RegisteredRanges[I].Data = Data;
940 RegisteredRanges[I].Counters = Counters;
941 RegisteredRanges[I].UniformCounters = UniformCounters;
942 RegisteredRanges[I].Names = Names;
943 RegisteredRanges[I].DataSize = ThisDataSize;
944 RegisteredRanges[I].CountersSize = ThisCountersSize;
945 RegisteredRanges[I].UniformCountersSize = ThisUniformCountersSize;
946 RegisteredRanges[I].NamesSize = ThisNamesSize;
947 RegisteredRanges[I].DataOffset = RegisteredDataSize;
948 RegisteredRanges[I].CountersOffset = RegisteredCountersSize;
949 RegisteredRanges[I].UniformCountersOffset = RegisteredUniformCountersSize;
950 RegisteredDataSize += ThisDataSize;
951 RegisteredCountersSize += ThisCountersSize;
952 RegisteredUniformCountersSize += ThisUniformCountersSize;
953
954 int ReuseNames = 0;
955 for (int J = 0; J < I; ++J) {
956 if (RegisteredRanges[J].Names == Names &&
957 RegisteredRanges[J].NamesSize == ThisNamesSize) {
958 RegisteredRanges[I].NamesOffset = RegisteredRanges[J].NamesOffset;
959 ReuseNames = 1;
960 break;
961 }
962 }
963 if (!ReuseNames) {
964 RegisteredRanges[I].NamesOffset = RegisteredNamesSize;
965 RegisteredNamesSize += ThisNamesSize;
966 }
967 }
968
969 DataSize = RegisteredDataSize;
970 CountersSize = RegisteredCountersSize;
971 UniformCountersSize = RegisteredUniformCountersSize;
972 NamesSize = RegisteredNamesSize;
973 Data.HostBegin = DataSize ? (char *)malloc(size: DataSize) : nullptr;
974 Cnts.HostBegin = CountersSize ? (char *)malloc(size: CountersSize) : nullptr;
975 UCnts.HostBegin =
976 UniformCountersSize ? (char *)malloc(size: UniformCountersSize) : nullptr;
977 Names.HostBegin = NamesSize ? (char *)malloc(size: NamesSize) : nullptr;
978 Data.Owner.reset(P: Data.HostBegin);
979 Cnts.Owner.reset(P: Cnts.HostBegin);
980 UCnts.Owner.reset(P: UCnts.HostBegin);
981 Names.Owner.reset(P: Names.HostBegin);
982 if ((DataSize > 0 && !Data.HostBegin) ||
983 (CountersSize > 0 && !Cnts.HostBegin) ||
984 (UniformCountersSize > 0 && !UCnts.HostBegin) ||
985 (NamesSize > 0 && !Names.HostBegin)) {
986 PROF_ERR("%s\n", "failed to allocate host memory for device sections");
987 return -1;
988 }
989
990 for (int I = 0; I < NumRegisteredRanges; ++I) {
991 RegisteredSectionRange *R = &RegisteredRanges[I];
992 if (memcpyDeviceToHost(Dst: Data.HostBegin + R->DataOffset, Src: R->Data,
993 Size: R->DataSize) != 0 ||
994 memcpyDeviceToHost(Dst: Cnts.HostBegin + R->CountersOffset, Src: R->Counters,
995 Size: R->CountersSize) != 0 ||
996 memcpyDeviceToHost(Dst: UCnts.HostBegin + R->UniformCountersOffset,
997 Src: R->UniformCounters, Size: R->UniformCountersSize) != 0) {
998 PROF_ERR("%s\n", "failed to copy profile sections from device");
999 return -1;
1000 }
1001
1002 int CopyNames = 1;
1003 for (int J = 0; J < I; ++J) {
1004 if (RegisteredRanges[J].Names == R->Names &&
1005 RegisteredRanges[J].NamesSize == R->NamesSize) {
1006 CopyNames = 0;
1007 break;
1008 }
1009 }
1010 if (CopyNames && R->NamesSize > 0 &&
1011 memcpyDeviceToHost(Dst: Names.HostBegin + R->NamesOffset, Src: R->Names,
1012 Size: R->NamesSize) != 0) {
1013 PROF_ERR("%s\n", "failed to copy profile sections from device");
1014 return -1;
1015 }
1016 }
1017 } else {
1018 if (Cnts.prepare() != 0 || Data.prepare() != 0 || Names.prepare() != 0 ||
1019 UCnts.prepare() != 0) {
1020 PROF_ERR("%s\n", "failed to allocate host memory for device sections");
1021 return -1;
1022 }
1023
1024 if (Data.copy() != 0 || Cnts.copy() != 0 || Names.copy() != 0 ||
1025 UCnts.copy() != 0) {
1026 PROF_ERR("%s\n", "failed to copy profile sections from device");
1027 return -1;
1028 }
1029
1030 /* Cache buffers so RDC-mode multi-shadow drains can reuse them.
1031 * release() prevents the scope guards from freeing what the cache owns. */
1032 Cnts.commitCache();
1033 Data.commitCache();
1034 Names.commitCache();
1035 UCnts.commitCache();
1036 }
1037
1038 if (isVerboseMode())
1039 PROF_NOTE("Copied device sections: Counters=%zu, Data=%zu, Names=%zu, "
1040 "UniformCounters=%zu\n",
1041 CountersSize, DataSize, NamesSize, UniformCountersSize);
1042
1043 // Arrange buffer as [Data][Padding][Counters][Names] to match the layout
1044 // expected by lprofWriteDataImpl (CountersDelta = CountersBegin - DataBegin).
1045 const uint64_t NumData = DataSize / sizeof(__llvm_profile_data);
1046 const uint64_t NumBitmapBytes = 0;
1047 const uint64_t NumUniformCounters = UniformCountersSize / sizeof(uint64_t);
1048 const uint64_t VTableSectionSize = 0;
1049 const uint64_t VNamesSize = 0;
1050 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
1051 PaddingBytesAfterBitmapBytes, PaddingBytesAfterUniformCounters,
1052 PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
1053
1054 if (__llvm_profile_get_padding_sizes_for_counters(
1055 DataSize, CountersSize, NumBitmapBytes, NumUniformCounters, NamesSize,
1056 VTableSize: VTableSectionSize, VNameSize: VNamesSize, PaddingBytesBeforeCounters: &PaddingBytesBeforeCounters,
1057 PaddingBytesAfterCounters: &PaddingBytesAfterCounters, PaddingBytesAfterBitmap: &PaddingBytesAfterBitmapBytes,
1058 PaddingBytesAfterUniformCounters: &PaddingBytesAfterUniformCounters, PaddingBytesAfterNames: &PaddingBytesAfterNames,
1059 PaddingBytesAfterVTable: &PaddingBytesAfterVTable, PaddingBytesAfterVNames: &PaddingBytesAfterVNames) != 0) {
1060 PROF_ERR("%s\n", "failed to get padding sizes");
1061 return -1;
1062 }
1063
1064 size_t ContiguousBufferSize =
1065 DataSize + PaddingBytesBeforeCounters + CountersSize + NamesSize;
1066 UniqueFree ContiguousBuf(malloc(size: ContiguousBufferSize));
1067 if (!ContiguousBuf.get()) {
1068 PROF_ERR("%s\n", "failed to allocate contiguous buffer");
1069 return -1;
1070 }
1071 char *ContiguousBuffer = ContiguousBuf.get();
1072 __builtin_memset(ContiguousBuffer, 0, ContiguousBufferSize);
1073
1074 char *BufDataBegin = ContiguousBuffer;
1075 char *BufCountersBegin =
1076 ContiguousBuffer + DataSize + PaddingBytesBeforeCounters;
1077 char *BufNamesBegin = BufCountersBegin + CountersSize;
1078
1079 __builtin_memcpy(BufDataBegin, Data.HostBegin, DataSize);
1080 __builtin_memcpy(BufCountersBegin, Cnts.HostBegin, CountersSize);
1081 __builtin_memcpy(BufNamesBegin, Names.HostBegin, NamesSize);
1082
1083 // CounterPtr and UniformCounterPtr are device-relative offsets; relocate
1084 // them for the file layout where the Data section precedes the Counters and
1085 // UniformCounters sections. Uniform counters are copied in linker (section)
1086 // order and located via their relative pointer, exactly like the regular
1087 // counters: llvm-profdata reads them through UniformCounterPtr (decrementing
1088 // UniformCountersDelta per record, just like CountersDelta) and does not
1089 // assume data-record order, so no reordering is needed.
1090 ptrdiff_t UCFileOffset = DataSize + PaddingBytesBeforeCounters +
1091 CountersSize + PaddingBytesAfterCounters +
1092 NumBitmapBytes + PaddingBytesAfterBitmapBytes;
1093 __llvm_profile_data *RelocatedData = (__llvm_profile_data *)BufDataBegin;
1094 for (uint64_t i = 0; i < NumData; ++i) {
1095 size_t DataRecordOffset = i * sizeof(__llvm_profile_data);
1096 const char *RangeDevDataBegin = (const char *)DevDataBegin;
1097 const char *RangeDevCountersBegin = (const char *)DevCntsBegin;
1098 const char *RangeDevUCntsBegin = (const char *)DevUniformCntsBegin;
1099 size_t RangeCountersOffset = 0;
1100 size_t RangeUCntsOffset = 0;
1101 if (UseRegisteredSections) {
1102 int FoundRange = 0;
1103 for (int R = 0; R < NumRegisteredRanges; ++R) {
1104 RegisteredSectionRange *Range = &RegisteredRanges[R];
1105 if (DataRecordOffset < Range->DataOffset ||
1106 DataRecordOffset >= Range->DataOffset + Range->DataSize)
1107 continue;
1108 RangeDevDataBegin = (const char *)Range->Data;
1109 RangeDevCountersBegin = (const char *)Range->Counters;
1110 RangeDevUCntsBegin = (const char *)Range->UniformCounters;
1111 RangeCountersOffset = Range->CountersOffset;
1112 RangeUCntsOffset = Range->UniformCountersOffset;
1113 DataRecordOffset -= Range->DataOffset;
1114 FoundRange = 1;
1115 break;
1116 }
1117 if (!FoundRange) {
1118 PROF_ERR("%s\n", "failed to locate profile data record range");
1119 return -1;
1120 }
1121 }
1122 const char *DeviceDataStructAddr = RangeDevDataBegin + DataRecordOffset;
1123 if (RelocatedData[i].CounterPtr) {
1124 const char *DeviceCountersAddr =
1125 DeviceDataStructAddr + (ptrdiff_t)RelocatedData[i].CounterPtr;
1126 ptrdiff_t OffsetIntoCountersSection =
1127 DeviceCountersAddr - RangeDevCountersBegin;
1128 ptrdiff_t NewRelativeOffset =
1129 DataSize + PaddingBytesBeforeCounters + RangeCountersOffset +
1130 OffsetIntoCountersSection - (i * sizeof(__llvm_profile_data));
1131 __builtin_memcpy((char *)RelocatedData + i * sizeof(__llvm_profile_data) +
1132 offsetof(__llvm_profile_data, CounterPtr),
1133 &NewRelativeOffset, sizeof(NewRelativeOffset));
1134 }
1135 if (UCnts.HostBegin && RelocatedData[i].UniformCounterPtr) {
1136 const char *DeviceUCAddr =
1137 DeviceDataStructAddr + (ptrdiff_t)RelocatedData[i].UniformCounterPtr;
1138 ptrdiff_t OffsetIntoUCSection = DeviceUCAddr - RangeDevUCntsBegin;
1139 ptrdiff_t NewUCRelativeOffset = UCFileOffset + RangeUCntsOffset +
1140 OffsetIntoUCSection -
1141 (i * sizeof(__llvm_profile_data));
1142 __builtin_memcpy((char *)RelocatedData + i * sizeof(__llvm_profile_data) +
1143 offsetof(__llvm_profile_data, UniformCounterPtr),
1144 &NewUCRelativeOffset, sizeof(NewUCRelativeOffset));
1145 } else {
1146 __builtin_memset((char *)RelocatedData + i * sizeof(__llvm_profile_data) +
1147 offsetof(__llvm_profile_data, UniformCounterPtr),
1148 0, sizeof(RelocatedData[i].UniformCounterPtr));
1149 }
1150 __builtin_memset((char *)RelocatedData + i * sizeof(__llvm_profile_data) +
1151 offsetof(__llvm_profile_data, BitmapPtr),
1152 0,
1153 sizeof(RelocatedData[i].BitmapPtr) +
1154 sizeof(RelocatedData[i].FunctionPointer) +
1155 sizeof(RelocatedData[i].Values));
1156 }
1157
1158 ret = __llvm_write_custom_profile(
1159 Target, DataBegin: (__llvm_profile_data *)BufDataBegin,
1160 DataEnd: (__llvm_profile_data *)(BufDataBegin + DataSize), CountersBegin: BufCountersBegin,
1161 CountersEnd: BufCountersBegin + CountersSize, UniformCountersBegin: UCnts.HostBegin,
1162 UniformCountersEnd: UCnts.HostBegin ? UCnts.HostBegin + UniformCountersSize : nullptr,
1163 NamesBegin: BufNamesBegin, NamesEnd: BufNamesBegin + NamesSize, VersionOverride: &Version);
1164
1165 if (ret != 0) {
1166 PROF_ERR("%s\n", "failed to write device profile using shared API");
1167 } else {
1168#if defined(__linux__) && !defined(_WIN32)
1169 // Dedup against the supplemental HSA pass: this section is now drained, so
1170 // the HSA walk must not drain the same device code object again.
1171 profRecordDrainedBounds(Data: DevDataBegin, Counters: DevCntsBegin, Names: DevNamesBegin);
1172#endif
1173 if (isVerboseMode())
1174 PROF_NOTE("%s\n", "Successfully wrote device profile using shared API");
1175 }
1176
1177 return ret;
1178}
1179
1180static int processShadowVariable(int Index, const char *Target) {
1181 void *ShadowVar = OffloadShadowVariables[Index];
1182 void *DeviceSections = nullptr;
1183 if (hipGetSymbolAddress(devPtr: &DeviceSections, symbol: ShadowVar) != 0) {
1184 PROF_WARN("failed to get symbol address for shadow variable %p\n",
1185 ShadowVar);
1186 return -1;
1187 }
1188 /* DeviceSections points at the per-TU sections struct itself. */
1189 const OffloadSectionShadowGroup *Sections = nullptr;
1190 if (Index < CapSectionShadowGroups)
1191 Sections = &OffloadSectionShadowGroups[Index];
1192 if (!hasCompleteSectionShadows(Sections))
1193 return 0;
1194 return processDeviceOffloadPrf(DeviceOffloadPrf: DeviceSections, Target, Sections);
1195}
1196
1197static int isHipAvailable(void) {
1198 ensureHipLoaded();
1199 return pHipMemcpy != nullptr && pHipGetSymbolAddress != nullptr;
1200}
1201
1202/* -------------------------------------------------------------------------- */
1203/* Collect device-side profile data */
1204/* -------------------------------------------------------------------------- */
1205
1206/* Host-shadow drain: static-linked kernels on Windows, or if HSA loader
1207 * introspection is unavailable on Linux, plus intercepted dynamic modules.
1208 * The Linux HSA pass normally drains resident static images without loading
1209 * images that were only registered by the host. */
1210static int collectHostShadowData(void) {
1211 int Ret = 0;
1212
1213 /* On Linux, use host shadows only if HSA cannot enumerate resident images.
1214 * Looking up every shadow otherwise loads unused static images. */
1215#if defined(__linux__) && !defined(_WIN32)
1216 const bool CollectStaticShadows =
1217 NumShadowVariables != 0 && !hsaRuntimeAvailable();
1218#else
1219 constexpr bool CollectStaticShadows = true;
1220#endif
1221 if (CollectStaticShadows && NumShadowVariables > 0) {
1222 int OrigDevice = -1;
1223 hipGetDevice(DeviceId: &OrigDevice);
1224
1225 for (int Dev = 0; Dev < NumDevices; ++Dev) {
1226 if (!shouldCollectDevice(DeviceId: Dev)) {
1227 if (isVerboseMode())
1228 PROF_NOTE("Skipping unused device %d\n", Dev);
1229 continue;
1230 }
1231#if defined(__linux__) && !defined(_WIN32)
1232 /* When no kernel launch was tracked at all, shouldCollectDevice() falls
1233 * back to collect-all, which can fault/hang reading a non-resident
1234 * device's sections on a multi-GPU host. On Linux the supplemental HSA
1235 * drain covers those cases safely. */
1236 if (!__atomic_load_n(&AnyDeviceUsed, __ATOMIC_ACQUIRE)) {
1237 if (isVerboseMode())
1238 PROF_NOTE("No tracked launch; deferring device %d to HSA drain\n",
1239 Dev);
1240 continue;
1241 }
1242#endif
1243 if (hipSetDevice(DeviceId: Dev) != 0) {
1244 if (isVerboseMode())
1245 PROF_NOTE("Failed to set device %d, skipping\n", Dev);
1246 continue;
1247 }
1248 const char *ArchName = getDeviceArchName(DeviceId: Dev);
1249 if (isVerboseMode())
1250 PROF_NOTE("Collecting static profile data from device %d (%s)\n", Dev,
1251 ArchName);
1252 for (int i = 0; i < NumShadowVariables; ++i) {
1253 /* Stable name per shadow so a repeated drain (explicit collect plus the
1254 * atexit drain) overwrites its own profraw rather than emitting a
1255 * second one: bare arch for a single TU, arch.<i> for RDC multi-TU. */
1256 const char *Target = ArchName;
1257 char TargetWithIdx[64];
1258 if (NumShadowVariables > 1) {
1259 snprintf(s: TargetWithIdx, maxlen: sizeof(TargetWithIdx), format: "%s.%d", ArchName, i);
1260 Target = TargetWithIdx;
1261 }
1262 if (processShadowVariable(Index: i, Target) != 0)
1263 Ret = -1;
1264 }
1265 }
1266
1267 if (OrigDevice >= 0)
1268 hipSetDevice(DeviceId: OrigDevice);
1269 }
1270
1271 /* Warn about unprocessed TUs; skip cleared slots (already drained). */
1272 lockDynamicModules();
1273 for (int i = 0; i < NumDynamicModules; ++i) {
1274 OffloadDynamicModuleInfo *MI = &DynamicModules[i];
1275 if (!MI->ModulePtr)
1276 continue;
1277 for (int t = 0; t < MI->NumTUs; ++t) {
1278 if (!MI->TUs[t].Processed) {
1279 PROF_WARN("dynamic module %p TU %d was not processed before exit\n",
1280 MI->ModulePtr, t);
1281 Ret = -1;
1282 }
1283 }
1284 }
1285 unlockDynamicModules();
1286
1287 return Ret;
1288}
1289
1290extern "C" int __llvm_profile_hip_collect_device_data(void) {
1291 int Ret = 0;
1292
1293 if ((NumShadowVariables != 0 || NumDynamicModules != 0) && isHipAvailable() &&
1294 collectHostShadowData() != 0)
1295 Ret = -1;
1296
1297#if defined(__linux__) && !defined(_WIN32)
1298 /* Drain resident device images through HSA. */
1299 if (drainDevicesViaHsa() != 0)
1300 Ret = -1;
1301#endif
1302
1303 if (Ret != 0)
1304 PROF_WARN("%s\n", "failed to collect device profile data");
1305 return Ret;
1306}
1307
1308/* Linux HIP interceptors. */
1309
1310#if defined(__linux__) && !defined(_WIN32)
1311
1312typedef struct {
1313 unsigned int x;
1314 unsigned int y;
1315 unsigned int z;
1316} HipDim3;
1317
1318typedef struct {
1319 void *Func;
1320 HipDim3 GridDim;
1321 HipDim3 BlockDim;
1322 void **Args;
1323 size_t SharedMem;
1324 HipStream Stream;
1325} HipLaunchParams;
1326
1327typedef struct {
1328 HipDim3 GridDim;
1329 HipDim3 BlockDim;
1330 size_t DynamicSmemBytes;
1331 HipStream Stream;
1332 void *Attrs;
1333 unsigned NumAttrs;
1334} HipLaunchConfig;
1335
1336typedef void *HipFunction;
1337typedef void *HipEvent;
1338typedef void *HipGraphExec;
1339
1340static int recordHipLaunchResult(int Rc, HipStream Stream) {
1341 if (Rc == 0)
1342 markLaunchStreamDeviceUsed(Stream);
1343 return Rc;
1344}
1345
1346static int recordHipMultiDeviceLaunchResult(int Rc,
1347 HipLaunchParams *LaunchParams,
1348 int NumLaunches) {
1349 if (Rc != 0 || !LaunchParams || NumLaunches <= 0)
1350 return Rc;
1351 for (int I = 0; I < NumLaunches; ++I)
1352 markLaunchStreamDeviceUsed(Stream: LaunchParams[I].Stream);
1353 return Rc;
1354}
1355
1356// interceptors must have external linkage
1357// NOLINTBEGIN(misc-use-internal-linkage)
1358INTERCEPTOR(int, hipLaunchKernel, const void *Function, HipDim3 GridDim,
1359 HipDim3 BlockDim, void **Args, size_t SharedMemBytes,
1360 HipStream Stream) {
1361 return recordHipLaunchResult(REAL(hipLaunchKernel)(Function, GridDim,
1362 BlockDim, Args,
1363 SharedMemBytes, Stream),
1364 Stream);
1365}
1366
1367INTERCEPTOR(int, hipLaunchKernel_spt, const void *Function, HipDim3 GridDim,
1368 HipDim3 BlockDim, void **Args, size_t SharedMemBytes,
1369 HipStream Stream) {
1370 return recordHipLaunchResult(
1371 REAL(hipLaunchKernel_spt)(Function, GridDim, BlockDim, Args,
1372 SharedMemBytes, Stream),
1373 Stream);
1374}
1375
1376INTERCEPTOR(int, hipExtLaunchKernel, const void *Function, HipDim3 GridDim,
1377 HipDim3 BlockDim, void **Args, size_t SharedMemBytes,
1378 HipStream Stream, HipEvent StartEvent, HipEvent StopEvent,
1379 int Flags) {
1380 return recordHipLaunchResult(
1381 REAL(hipExtLaunchKernel)(Function, GridDim, BlockDim, Args,
1382 SharedMemBytes, Stream, StartEvent, StopEvent,
1383 Flags),
1384 Stream);
1385}
1386
1387INTERCEPTOR(int, hipLaunchKernelExC, const HipLaunchConfig *Config,
1388 const void *Function, void **Args) {
1389 int Rc = REAL(hipLaunchKernelExC)(Config, Function, Args);
1390 return recordHipLaunchResult(Rc, Stream: Config ? Config->Stream : nullptr);
1391}
1392
1393INTERCEPTOR(int, hipLaunchCooperativeKernel, const void *Function,
1394 HipDim3 GridDim, HipDim3 BlockDim, void **KernelParams,
1395 unsigned SharedMemBytes, HipStream Stream) {
1396 return recordHipLaunchResult(
1397 REAL(hipLaunchCooperativeKernel)(Function, GridDim, BlockDim,
1398 KernelParams, SharedMemBytes, Stream),
1399 Stream);
1400}
1401
1402INTERCEPTOR(int, hipLaunchCooperativeKernel_spt, const void *Function,
1403 HipDim3 GridDim, HipDim3 BlockDim, void **KernelParams,
1404 unsigned SharedMemBytes, HipStream Stream) {
1405 return recordHipLaunchResult(
1406 REAL(hipLaunchCooperativeKernel_spt)(
1407 Function, GridDim, BlockDim, KernelParams, SharedMemBytes, Stream),
1408 Stream);
1409}
1410
1411INTERCEPTOR(int, hipLaunchCooperativeKernelMultiDevice,
1412 HipLaunchParams *LaunchParams, int NumDevices, unsigned Flags) {
1413 return recordHipMultiDeviceLaunchResult(
1414 REAL(hipLaunchCooperativeKernelMultiDevice)(LaunchParams, NumDevices,
1415 Flags),
1416 LaunchParams, NumLaunches: NumDevices);
1417}
1418
1419INTERCEPTOR(int, hipExtLaunchMultiKernelMultiDevice,
1420 HipLaunchParams *LaunchParams, int NumDevices, unsigned Flags) {
1421 return recordHipMultiDeviceLaunchResult(
1422 REAL(hipExtLaunchMultiKernelMultiDevice)(LaunchParams, NumDevices, Flags),
1423 LaunchParams, NumLaunches: NumDevices);
1424}
1425
1426INTERCEPTOR(int, hipModuleLaunchKernel, HipFunction Function, unsigned GridDimX,
1427 unsigned GridDimY, unsigned GridDimZ, unsigned BlockDimX,
1428 unsigned BlockDimY, unsigned BlockDimZ, unsigned SharedMemBytes,
1429 HipStream Stream, void **KernelParams, void **Extra) {
1430 return recordHipLaunchResult(
1431 REAL(hipModuleLaunchKernel)(Function, GridDimX, GridDimY, GridDimZ,
1432 BlockDimX, BlockDimY, BlockDimZ,
1433 SharedMemBytes, Stream, KernelParams, Extra),
1434 Stream);
1435}
1436
1437INTERCEPTOR(int, hipExtModuleLaunchKernel, HipFunction Function,
1438 unsigned GridDimX, unsigned GridDimY, unsigned GridDimZ,
1439 unsigned BlockDimX, unsigned BlockDimY, unsigned BlockDimZ,
1440 size_t SharedMemBytes, HipStream Stream, void **KernelParams,
1441 void **Extra, HipEvent StartEvent, HipEvent StopEvent,
1442 unsigned Flags) {
1443 return recordHipLaunchResult(
1444 REAL(hipExtModuleLaunchKernel)(Function, GridDimX, GridDimY, GridDimZ,
1445 BlockDimX, BlockDimY, BlockDimZ,
1446 SharedMemBytes, Stream, KernelParams,
1447 Extra, StartEvent, StopEvent, Flags),
1448 Stream);
1449}
1450
1451INTERCEPTOR(int, hipGraphLaunch, HipGraphExec GraphExec, HipStream Stream) {
1452 return recordHipLaunchResult(REAL(hipGraphLaunch)(GraphExec, Stream), Stream);
1453}
1454
1455INTERCEPTOR(int, hipGraphLaunch_spt, HipGraphExec GraphExec, HipStream Stream) {
1456 return recordHipLaunchResult(REAL(hipGraphLaunch_spt)(GraphExec, Stream),
1457 Stream);
1458}
1459
1460static void registerDynamicModuleFromFile(int Rc, void **Module,
1461 const char *Filename) {
1462 if (Rc)
1463 return;
1464
1465 FILE *File = fopen(filename: Filename, modes: "rb");
1466 if (!File) {
1467 PROF_WARN("failed to open dynamic module file %s\n", Filename);
1468 return;
1469 }
1470
1471 if (fseek(stream: File, off: 0, SEEK_END) != 0) {
1472 PROF_WARN("failed to seek dynamic module file %s\n", Filename);
1473 fclose(stream: File);
1474 return;
1475 }
1476 long FileSize = ftell(stream: File);
1477 if (FileSize <= 0 || fseek(stream: File, off: 0, SEEK_SET) != 0) {
1478 PROF_WARN("failed to get size of dynamic module file %s\n", Filename);
1479 fclose(stream: File);
1480 return;
1481 }
1482
1483 ManagedMemory Image;
1484 lprofGetFileContentBuffer(F: File, FileSize: (uint64_t)FileSize, Buf: &Image);
1485 fclose(stream: File);
1486 if (Image.Status == MS_INVALID) {
1487 PROF_WARN("failed to read dynamic module file %s\n", Filename);
1488 return;
1489 }
1490
1491 __llvm_profile_offload_register_dynamic_module(ModuleLoadRc: Rc, Ptr: Module, Image: Image.Addr);
1492 lprofReleaseBuffer(FileBuffer: &Image, Length: (size_t)FileSize);
1493}
1494
1495INTERCEPTOR(int, hipModuleLoad, void **module, const char *fname) {
1496 int rc = REAL(hipModuleLoad)(module, fname);
1497 registerDynamicModuleFromFile(Rc: rc, Module: module, Filename: fname);
1498 return rc;
1499}
1500
1501INTERCEPTOR(int, hipModuleLoadData, void **module, const void *image) {
1502 int rc = REAL(hipModuleLoadData)(module, image);
1503 __llvm_profile_offload_register_dynamic_module(ModuleLoadRc: rc, Ptr: module, Image: image);
1504 return rc;
1505}
1506
1507INTERCEPTOR(int, hipModuleLoadDataEx, void **module, const void *image,
1508 unsigned numOptions, void **options, void **optionValues) {
1509 int rc = REAL(hipModuleLoadDataEx)(module, image, numOptions, options,
1510 optionValues);
1511 __llvm_profile_offload_register_dynamic_module(ModuleLoadRc: rc, Ptr: module, Image: image);
1512 return rc;
1513}
1514
1515INTERCEPTOR(int, hipModuleUnload, void *module) {
1516 /* Drain counters before the module is destroyed; device addresses
1517 * captured at register time are invalid after unload. */
1518 __llvm_profile_offload_unregister_dynamic_module(Ptr: module);
1519 return REAL(hipModuleUnload)(module);
1520}
1521// NOLINTEND(misc-use-internal-linkage)
1522
1523__attribute__((constructor)) static void installHipInterceptors() {
1524 /* Avoid interception unless the HIP runtime is already loaded. */
1525 int HasModuleLoad = dlsym(RTLD_DEFAULT, name: "hipModuleLoad") != nullptr;
1526 int InstalledLaunch = 0;
1527#define TRY_INTERCEPT_LAUNCH(Name) \
1528 do { \
1529 if (dlsym(RTLD_DEFAULT, #Name)) \
1530 InstalledLaunch |= INTERCEPT_FUNCTION(Name); \
1531 } while (0)
1532 TRY_INTERCEPT_LAUNCH(hipLaunchKernel);
1533 TRY_INTERCEPT_LAUNCH(hipLaunchKernel_spt);
1534 TRY_INTERCEPT_LAUNCH(hipExtLaunchKernel);
1535 TRY_INTERCEPT_LAUNCH(hipLaunchKernelExC);
1536 TRY_INTERCEPT_LAUNCH(hipLaunchCooperativeKernel);
1537 TRY_INTERCEPT_LAUNCH(hipLaunchCooperativeKernel_spt);
1538 TRY_INTERCEPT_LAUNCH(hipLaunchCooperativeKernelMultiDevice);
1539 TRY_INTERCEPT_LAUNCH(hipExtLaunchMultiKernelMultiDevice);
1540 TRY_INTERCEPT_LAUNCH(hipModuleLaunchKernel);
1541 TRY_INTERCEPT_LAUNCH(hipExtModuleLaunchKernel);
1542 TRY_INTERCEPT_LAUNCH(hipGraphLaunch);
1543 TRY_INTERCEPT_LAUNCH(hipGraphLaunch_spt);
1544#undef TRY_INTERCEPT_LAUNCH
1545 int InstalledAny = InstalledLaunch;
1546 if (HasModuleLoad) {
1547 HasModuleLoad = INTERCEPT_FUNCTION(hipModuleLoad);
1548 InstalledAny |= HasModuleLoad;
1549 }
1550 if (!InstalledAny)
1551 return;
1552 if (isVerboseMode())
1553 PROF_NOTE("%s", "Installing HIP interceptors\n");
1554 if (HasModuleLoad) {
1555 INTERCEPT_FUNCTION(hipModuleLoadData);
1556 INTERCEPT_FUNCTION(hipModuleLoadDataEx);
1557 INTERCEPT_FUNCTION(hipModuleUnload);
1558 }
1559}
1560
1561#endif /* __linux__ */
1562