1//===- InstrProfilingPlatformROCmHSA.cpp - ROCm HSA device drain ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// HSA resident-image drain (Linux only).
10//
11// This pass walks loaded executables on each GPU agent and drains their
12// __llvm_profile_sections tables. It covers static HIP images and device-linked
13// programs without host shadows, while avoiding lookups that would load unused
14// static images. It shares processDeviceOffloadPrf() and section-bounds dedup
15// with the dynamic-module drain.
16//
17//===----------------------------------------------------------------------===//
18
19#if defined(__linux__)
20
21extern "C" {
22#include "InstrProfiling.h"
23#include "InstrProfilingPort.h"
24}
25
26#include "InstrProfilingPlatformROCmInternal.h"
27#include "interception/interception.h"
28#include "sanitizer_common/sanitizer_offload_hsa.h"
29// C (not C++) headers: clang_rt.profile is built -nostdinc++.
30#include <stddef.h>
31#include <stdint.h>
32#include <stdio.h>
33#include <stdlib.h>
34#include <string.h>
35
36using namespace __prof_rocm;
37
38/* Status-check shorthands, in the spirit of the thin HIP wrappers in
39 * InstrProfilingPlatformROCm.cpp: every HSA entry point returns
40 * hsa_status_t. hsaOkOrBreak() also accepts INFO_BREAK, which the
41 * iterate_* callbacks use to stop early and is not an error. */
42static inline bool hsaOk(hsa_status_t St) { return St == HSA_STATUS_SUCCESS; }
43static inline bool hsaOkOrBreak(hsa_status_t St) {
44 return St == HSA_STATUS_SUCCESS || St == HSA_STATUS_INFO_BREAK;
45}
46
47/* 0 = not attempted, 1 = ready, -1 = unavailable. Release publishes the HSA
48 * pointers to a thread that observes ready. */
49static int HsaRuntimeState = 0;
50
51typedef decltype(&hsa_iterate_agents) hsa_iterate_agents_ty;
52typedef decltype(&hsa_agent_get_info) hsa_agent_get_info_ty;
53typedef decltype(&hsa_executable_iterate_agent_symbols)
54 hsa_executable_iterate_agent_symbols_ty;
55typedef decltype(&hsa_executable_symbol_get_info)
56 hsa_executable_symbol_get_info_ty;
57typedef decltype(LoaderApi::QuerySegmentDescriptors) hsa_query_segments_ty;
58
59static hsa_iterate_agents_ty pHsaIterateAgents = nullptr;
60static hsa_agent_get_info_ty pHsaAgentGetInfo = nullptr;
61static hsa_executable_iterate_agent_symbols_ty pHsaExecIterAgentSyms = nullptr;
62static hsa_executable_symbol_get_info_ty pHsaSymGetInfo = nullptr;
63static hsa_query_segments_ty pQuerySegDescs = nullptr;
64
65static int setHsaRuntimeState(int S) {
66 __atomic_store_n(&HsaRuntimeState, S, __ATOMIC_RELEASE);
67 return S > 0 ? 0 : -1;
68}
69
70/* Dlopen libhsa and call its hsa_init, then keep the loader slot this drain
71 * calls. Runs on the first drain, never from a constructor -- see the
72 * fork-safety note at end of file. */
73static int bindHsaRuntime(void) {
74 int State = __atomic_load_n(&HsaRuntimeState, __ATOMIC_ACQUIRE);
75 if (State)
76 return State > 0 ? 0 : -1;
77
78 if (!__interception::DynamicLoaderAvailable()) {
79 if (isVerboseMode())
80 PROF_NOTE("%s", "Dynamic library loading not available - "
81 "HSA device profiling disabled\n");
82 return setHsaRuntimeState(-1);
83 }
84
85 void *Hsa = __interception::OpenLibrary(SANITIZER_HSA_LIBRARY ".so");
86 if (!Hsa)
87 Hsa = __interception::OpenLibrary(SANITIZER_HSA_LIBRARY ".so.1");
88 if (!Hsa) {
89 if (isVerboseMode())
90 PROF_NOTE("%s", "libhsa-runtime64.so not loadable - "
91 "HSA device profiling disabled\n");
92 return setHsaRuntimeState(-1);
93 }
94
95 auto *pHsaInit =
96 (decltype(&hsa_init))__interception::LookupSymbol(handle: Hsa, symbol: "hsa_init");
97 auto *pGetExtTable = (decltype(&hsa_system_get_major_extension_table))
98 __interception::LookupSymbol(handle: Hsa, symbol: "hsa_system_get_major_extension_table");
99 pHsaIterateAgents = (hsa_iterate_agents_ty)__interception::LookupSymbol(
100 handle: Hsa, symbol: "hsa_iterate_agents");
101 pHsaAgentGetInfo = (hsa_agent_get_info_ty)__interception::LookupSymbol(
102 handle: Hsa, symbol: "hsa_agent_get_info");
103 pHsaExecIterAgentSyms =
104 (hsa_executable_iterate_agent_symbols_ty)__interception::LookupSymbol(
105 handle: Hsa, symbol: "hsa_executable_iterate_agent_symbols");
106 pHsaSymGetInfo =
107 (hsa_executable_symbol_get_info_ty)__interception::LookupSymbol(
108 handle: Hsa, symbol: "hsa_executable_symbol_get_info");
109 if (!pHsaInit || !pGetExtTable || !pHsaIterateAgents || !pHsaAgentGetInfo ||
110 !pHsaExecIterAgentSyms || !pHsaSymGetInfo) {
111 PROF_WARN("%s",
112 "required HSA symbols missing - HSA device profiling disabled\n");
113 return setHsaRuntimeState(-1);
114 }
115
116 hsa_status_t St = pHsaInit();
117 if (!hsaOkOrBreak(St)) {
118 if (isVerboseMode())
119 PROF_NOTE("hsa_init failed (0x%x) - HSA device profiling disabled\n", St);
120 return setHsaRuntimeState(-1);
121 }
122
123 // Full 1.04 table. A shorter runtime may succeed and leave the tail null.
124 LoaderApi LoaderTable = {};
125 St = pGetExtTable(HSA_EXTENSION_AMD_LOADER, 1, sizeof(LoaderTable),
126 &LoaderTable);
127 if (!hsaOk(St) || !LoaderTable.QuerySegmentDescriptors) {
128 PROF_WARN("AMD loader extension unavailable (0x%x) - "
129 "HSA device profiling disabled\n",
130 St);
131 return setHsaRuntimeState(-1);
132 }
133 pQuerySegDescs = LoaderTable.QuerySegmentDescriptors;
134
135 /* The device-to-host copies go through the shared HIP loader. */
136 ensureHipLoaded();
137 if (!hipMemcpyAvailable()) {
138 PROF_WARN("%s", "hipMemcpy unavailable - HSA device profiling disabled\n");
139 return setHsaRuntimeState(-1);
140 }
141
142 if (isVerboseMode())
143 PROF_NOTE("%s", "HSA + HIP runtime resolved for device profiling\n");
144 return setHsaRuntimeState(1);
145}
146
147int __prof_rocm::hsaRuntimeAvailable(void) { return bindHsaRuntime() == 0; }
148
149/* The canonical device bounds-table symbol from InstrProfilingPlatformGPU.c. */
150static const char ProfileSectionsSymbol[] = "__llvm_profile_sections";
151
152/* Dedup of drained section-bounds tuples, shared with the host-shadow path
153 * (processDeviceOffloadPrf records here on every successful drain) so each
154 * unique counter set is drained exactly once across both paths.
155 */
156static ProfBoundsSet SeenBounds;
157
158/* Has this bounds tuple already been drained? Pure check, no state mutation. */
159static int profBoundsAlreadyDrained(const void *D, const void *C,
160 const void *N) {
161 return SeenBounds.contains(D, C, N);
162}
163
164/* Record a drained bounds tuple. Idempotent; call only after a successful drain
165 * so a failed attempt stays retryable. */
166void __prof_rocm::profRecordDrainedBounds(const void *D, const void *C,
167 const void *N) {
168 SeenBounds.record(D, C, N);
169}
170
171#define PROF_MAX_GPU_AGENTS 64
172
173/* Buffer size for HSA agent names and symbol names we read back; both the
174 * device arch string and the __llvm_profile_sections symbol are far shorter. */
175#define PROF_HSA_NAME_MAX 64
176
177namespace {
178struct GpuAgent {
179 hsa_agent_t agent;
180 char arch[PROF_HSA_NAME_MAX];
181};
182
183struct WalkState {
184 GpuAgent agents[PROF_MAX_GPU_AGENTS];
185 int num_agents;
186 int total_found;
187 int total_drained;
188};
189
190/* Per (agent, executable) symbol-iteration state. */
191struct SymbolState {
192 const char *arch;
193 int found;
194 int drained;
195};
196} // namespace
197
198/* HSA per-symbol callback: when it finds a __llvm_profile_sections variable,
199 * drain it via processDeviceOffloadPrf() unless the host-shadow path (or an
200 * earlier agent) already handled the same bounds. */
201static hsa_status_t onSymbol(hsa_executable_t, hsa_agent_t,
202 hsa_executable_symbol_t Sym, void *Data) {
203 SymbolState *S = (SymbolState *)Data;
204
205 hsa_symbol_kind_t Kind;
206 if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &Kind)) ||
207 Kind != HSA_SYMBOL_KIND_VARIABLE)
208 return HSA_STATUS_SUCCESS;
209
210 uint32_t NameLen = 0;
211 if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH,
212 &NameLen)) ||
213 NameLen != sizeof(ProfileSectionsSymbol) - 1)
214 return HSA_STATUS_SUCCESS;
215
216 char NameBuf[PROF_HSA_NAME_MAX];
217 if (NameLen + 1 > sizeof(NameBuf))
218 return HSA_STATUS_SUCCESS;
219 if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_NAME, NameBuf)))
220 return HSA_STATUS_SUCCESS;
221 NameBuf[NameLen] = '\0';
222
223 if (strcmp(s1: NameBuf, s2: ProfileSectionsSymbol) != 0)
224 return HSA_STATUS_SUCCESS;
225
226 uint64_t Addr = 0;
227 if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS,
228 &Addr)) ||
229 Addr == 0) {
230 if (isVerboseMode())
231 PROF_NOTE("%s", "failed to read __llvm_profile_sections address\n");
232 return HSA_STATUS_SUCCESS;
233 }
234
235 S->found++;
236
237 // Read the bounds table first to dedup (and detect empty sections) before
238 // the full copy/relocate done by processDeviceOffloadPrf.
239 __llvm_profile_gpu_sections Sec;
240 if (memcpyDeviceToHost(Dst: &Sec, Src: (void *)(uintptr_t)Addr, Size: sizeof(Sec)) != 0) {
241 PROF_WARN("%s", "failed to copy device bounds table\n");
242 return HSA_STATUS_SUCCESS;
243 }
244 if (profBoundsAlreadyDrained(D: Sec.DataStart, C: Sec.CountersStart,
245 N: Sec.NamesStart)) {
246 if (isVerboseMode())
247 PROF_NOTE("%s", "device bounds already drained, skipping\n");
248 return HSA_STATUS_SUCCESS;
249 }
250
251 size_t DataBytes = (const char *)Sec.DataStop - (const char *)Sec.DataStart;
252 size_t CntsBytes =
253 (const char *)Sec.CountersStop - (const char *)Sec.CountersStart;
254 if (DataBytes == 0 || CntsBytes == 0) {
255 // Empty code object: nothing to write. Mark seen so we don't revisit it.
256 profRecordDrainedBounds(D: Sec.DataStart, C: Sec.CountersStart, N: Sec.NamesStart);
257 return HSA_STATUS_SUCCESS;
258 }
259
260 // Name HSA-drained objects in their own ".hsaN" suffix space so they never
261 // collide with the host-shadow path's "arch"/"arch.<i>" filenames. The drain
262 // latch (HsaDrainCompleted) already prevents re-draining an object, so a
263 // plain per-drain counter is enough for uniqueness.
264 static int DrainIndex = 0;
265 char Target[96];
266 snprintf(s: Target, maxlen: sizeof(Target), format: "%s.hsa%d", S->arch, DrainIndex);
267
268 // Record the bounds (and advance the index) only on a successful write so a
269 // transient error stays retryable on a later agent or collect call.
270 if (processDeviceOffloadPrf(DeviceOffloadPrf: (void *)(uintptr_t)Addr, Target, Sections: nullptr) == 0) {
271 S->drained++;
272 DrainIndex++;
273 profRecordDrainedBounds(D: Sec.DataStart, C: Sec.CountersStart, N: Sec.NamesStart);
274 }
275
276 return HSA_STATUS_SUCCESS;
277}
278
279static hsa_status_t collectAgent(hsa_agent_t Agent, void *Data) {
280 hsa_device_type_t DevType;
281 if (!hsaOk(St: pHsaAgentGetInfo(Agent, HSA_AGENT_INFO_DEVICE, &DevType)) ||
282 DevType != HSA_DEVICE_TYPE_GPU)
283 return HSA_STATUS_SUCCESS;
284
285 WalkState *W = (WalkState *)Data;
286 if (W->num_agents >= PROF_MAX_GPU_AGENTS)
287 return HSA_STATUS_SUCCESS;
288
289 GpuAgent &GA = W->agents[W->num_agents++];
290 GA.agent = Agent;
291 char Name[PROF_HSA_NAME_MAX];
292 __builtin_memset(Name, 0, sizeof(Name));
293 pHsaAgentGetInfo(Agent, HSA_AGENT_INFO_NAME, Name);
294 size_t N = strnlen(string: Name, maxlen: sizeof(GA.arch) - 1);
295 __builtin_memcpy(GA.arch, Name, N);
296 GA.arch[N] = '\0';
297 if (!GA.arch[0])
298 strncpy(dest: GA.arch, src: "amdgpu", n: sizeof(GA.arch) - 1);
299
300 if (isVerboseMode())
301 PROF_NOTE("GPU agent %d: %s\n", W->num_agents - 1, GA.arch);
302 return HSA_STATUS_SUCCESS;
303}
304
305/* Reentrancy guard and "drained at least once" latch (both acquire/release). */
306static int HsaDrainInProgress = 0;
307static int HsaDrainCompleted = 0;
308
309int __prof_rocm::drainDevicesViaHsa(void) {
310 if (__atomic_load_n(&HsaDrainCompleted, __ATOMIC_ACQUIRE))
311 return 0;
312
313 int Expected = 0;
314 if (!__atomic_compare_exchange_n(&HsaDrainInProgress, &Expected, 1,
315 /*weak=*/0, __ATOMIC_ACQ_REL,
316 __ATOMIC_ACQUIRE))
317 return 0;
318
319 struct InProgressGuard {
320 ~InProgressGuard() {
321 __atomic_store_n(&HsaDrainInProgress, 0, __ATOMIC_RELEASE);
322 }
323 } _Guard;
324
325 if (bindHsaRuntime() != 0)
326 return 0; /* Runtime unavailable: stay retryable. */
327
328 WalkState W;
329 __builtin_memset(&W, 0, sizeof(W));
330 hsa_status_t St = pHsaIterateAgents(collectAgent, &W);
331 if (!hsaOkOrBreak(St)) {
332 PROF_WARN("hsa_iterate_agents failed (0x%x)\n", St);
333 return -1;
334 }
335 if (W.num_agents == 0) {
336 if (isVerboseMode())
337 PROF_NOTE("%s", "no GPU agents present; nothing to drain (will retry)\n");
338 return 0;
339 }
340
341 /* query_segment_descriptors ships in every loader-extension version, is more
342 * permissive than iterate_executables on ROCm, and yields the loaded
343 * (agent, executable) pairs directly. */
344 size_t NumSegs = 0;
345 St = pQuerySegDescs(nullptr, &NumSegs);
346 if (!hsaOk(St)) {
347 PROF_WARN("query_segment_descriptors(count) failed (0x%x)\n", St);
348 return -1;
349 }
350 if (NumSegs == 0) {
351 if (isVerboseMode())
352 PROF_NOTE("%s", "no loaded segments; nothing to drain (will retry)\n");
353 return 0;
354 }
355
356 hsa_ven_amd_loader_segment_descriptor_t *Segs =
357 (hsa_ven_amd_loader_segment_descriptor_t *)calloc(nmemb: NumSegs, size: sizeof(*Segs));
358 if (!Segs) {
359 PROF_ERR("%s\n", "failed to allocate segment descriptor array");
360 return -1;
361 }
362 UniqueFree SegsOwner(Segs);
363
364 St = pQuerySegDescs(Segs, &NumSegs);
365 if (!hsaOk(St)) {
366 PROF_WARN("query_segment_descriptors(fetch) failed (0x%x)\n", St);
367 return -1;
368 }
369
370 if (isVerboseMode())
371 PROF_NOTE("query_segment_descriptors: %zu segments\n", NumSegs);
372
373 // Walk each unique (agent, executable) pair once.
374 struct SeenPair {
375 uint64_t agent;
376 uint64_t exec;
377 };
378 enum { kSeenPairsInitCap = 64 };
379 SeenPair *Seen = nullptr;
380 int NumPairs = 0;
381 int CapPairs = 0;
382 int IterFailures = 0;
383
384 for (size_t i = 0; i < NumSegs; ++i) {
385 if (Segs[i].executable.handle == 0 || Segs[i].agent.handle == 0)
386 continue;
387
388 bool AlreadySeen = false;
389 for (int j = 0; j < NumPairs; ++j)
390 if (Seen[j].agent == Segs[i].agent.handle &&
391 Seen[j].exec == Segs[i].executable.handle) {
392 AlreadySeen = true;
393 break;
394 }
395 if (AlreadySeen)
396 continue;
397 if (growArray(Arr: (void **)&Seen, Cap: &CapPairs, MinCount: NumPairs + 1, InitCap: kSeenPairsInitCap,
398 ElemSize: sizeof(*Seen)) == 0) {
399 Seen[NumPairs].agent = Segs[i].agent.handle;
400 Seen[NumPairs].exec = Segs[i].executable.handle;
401 NumPairs++;
402 }
403
404 const char *Arch = nullptr;
405 for (int k = 0; k < W.num_agents; ++k)
406 if (W.agents[k].agent.handle == Segs[i].agent.handle) {
407 Arch = W.agents[k].arch;
408 break;
409 }
410 if (!Arch)
411 continue; /* not a GPU agent we collected */
412
413 SymbolState S;
414 __builtin_memset(&S, 0, sizeof(S));
415 S.arch = Arch;
416 if (isVerboseMode())
417 PROF_NOTE("walking executable 0x%llx on %s\n",
418 (unsigned long long)Segs[i].executable.handle, Arch);
419 hsa_status_t IterSt =
420 pHsaExecIterAgentSyms(Segs[i].executable, Segs[i].agent, onSymbol, &S);
421 if (!hsaOkOrBreak(St: IterSt)) {
422 PROF_WARN("hsa_executable_iterate_agent_symbols on executable 0x%llx "
423 "failed (0x%x)\n",
424 (unsigned long long)Segs[i].executable.handle, IterSt);
425 IterFailures++;
426 }
427 W.total_found += S.found;
428 W.total_drained += S.drained;
429 }
430
431 if (isVerboseMode())
432 PROF_NOTE("HSA walk complete: agents=%d pairs=%d found=%d drained=%d "
433 "iter-failures=%d\n",
434 W.num_agents, NumPairs, W.total_found, W.total_drained,
435 IterFailures);
436
437 free(ptr: Seen);
438
439 /* Latch only when we actually drained data. A "found nothing new" walk is
440 * deliberately not latched: an early collect can precede any kernel launch,
441 * and latching it would suppress the real exit-time drain. No-op walks are
442 * cheap to repeat. */
443 if (W.total_drained > 0)
444 __atomic_store_n(&HsaDrainCompleted, 1, __ATOMIC_RELEASE);
445 return (IterFailures > 0) ? -1 : 0;
446}
447
448/* Fork-safety: deliberately no library constructor calling hsa_init(). */
449
450#endif /* defined(__linux__) */
451