| 1 | //===- InstrProfilingPlatformROCmHSA.cpp - ROCm HSA device drain ---------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // HSA resident-image drain (Linux only). |
| 10 | // |
| 11 | // This pass walks loaded executables on each GPU agent and drains their |
| 12 | // __llvm_profile_sections tables. It covers static HIP images and device-linked |
| 13 | // programs without host shadows, while avoiding lookups that would load unused |
| 14 | // static images. It shares processDeviceOffloadPrf() and section-bounds dedup |
| 15 | // with the dynamic-module drain. |
| 16 | // |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | |
| 19 | #if defined(__linux__) |
| 20 | |
| 21 | extern "C" { |
| 22 | #include "InstrProfiling.h" |
| 23 | #include "InstrProfilingPort.h" |
| 24 | } |
| 25 | |
| 26 | #include "InstrProfilingPlatformROCmInternal.h" |
| 27 | #include "interception/interception.h" |
| 28 | #include "sanitizer_common/sanitizer_offload_hsa.h" |
| 29 | // C (not C++) headers: clang_rt.profile is built -nostdinc++. |
| 30 | #include <stddef.h> |
| 31 | #include <stdint.h> |
| 32 | #include <stdio.h> |
| 33 | #include <stdlib.h> |
| 34 | #include <string.h> |
| 35 | |
| 36 | using namespace __prof_rocm; |
| 37 | |
| 38 | /* Status-check shorthands, in the spirit of the thin HIP wrappers in |
| 39 | * InstrProfilingPlatformROCm.cpp: every HSA entry point returns |
| 40 | * hsa_status_t. hsaOkOrBreak() also accepts INFO_BREAK, which the |
| 41 | * iterate_* callbacks use to stop early and is not an error. */ |
| 42 | static inline bool hsaOk(hsa_status_t St) { return St == HSA_STATUS_SUCCESS; } |
| 43 | static inline bool hsaOkOrBreak(hsa_status_t St) { |
| 44 | return St == HSA_STATUS_SUCCESS || St == HSA_STATUS_INFO_BREAK; |
| 45 | } |
| 46 | |
| 47 | /* 0 = not attempted, 1 = ready, -1 = unavailable. Release publishes the HSA |
| 48 | * pointers to a thread that observes ready. */ |
| 49 | static int HsaRuntimeState = 0; |
| 50 | |
| 51 | typedef decltype(&hsa_iterate_agents) hsa_iterate_agents_ty; |
| 52 | typedef decltype(&hsa_agent_get_info) hsa_agent_get_info_ty; |
| 53 | typedef decltype(&hsa_executable_iterate_agent_symbols) |
| 54 | hsa_executable_iterate_agent_symbols_ty; |
| 55 | typedef decltype(&hsa_executable_symbol_get_info) |
| 56 | hsa_executable_symbol_get_info_ty; |
| 57 | typedef decltype(LoaderApi::QuerySegmentDescriptors) hsa_query_segments_ty; |
| 58 | |
| 59 | static hsa_iterate_agents_ty pHsaIterateAgents = nullptr; |
| 60 | static hsa_agent_get_info_ty pHsaAgentGetInfo = nullptr; |
| 61 | static hsa_executable_iterate_agent_symbols_ty pHsaExecIterAgentSyms = nullptr; |
| 62 | static hsa_executable_symbol_get_info_ty pHsaSymGetInfo = nullptr; |
| 63 | static hsa_query_segments_ty pQuerySegDescs = nullptr; |
| 64 | |
| 65 | static int setHsaRuntimeState(int S) { |
| 66 | __atomic_store_n(&HsaRuntimeState, S, __ATOMIC_RELEASE); |
| 67 | return S > 0 ? 0 : -1; |
| 68 | } |
| 69 | |
| 70 | /* Dlopen libhsa and call its hsa_init, then keep the loader slot this drain |
| 71 | * calls. Runs on the first drain, never from a constructor -- see the |
| 72 | * fork-safety note at end of file. */ |
| 73 | static int bindHsaRuntime(void) { |
| 74 | int State = __atomic_load_n(&HsaRuntimeState, __ATOMIC_ACQUIRE); |
| 75 | if (State) |
| 76 | return State > 0 ? 0 : -1; |
| 77 | |
| 78 | if (!__interception::DynamicLoaderAvailable()) { |
| 79 | if (isVerboseMode()) |
| 80 | PROF_NOTE("%s" , "Dynamic library loading not available - " |
| 81 | "HSA device profiling disabled\n" ); |
| 82 | return setHsaRuntimeState(-1); |
| 83 | } |
| 84 | |
| 85 | void *Hsa = __interception::OpenLibrary(SANITIZER_HSA_LIBRARY ".so" ); |
| 86 | if (!Hsa) |
| 87 | Hsa = __interception::OpenLibrary(SANITIZER_HSA_LIBRARY ".so.1" ); |
| 88 | if (!Hsa) { |
| 89 | if (isVerboseMode()) |
| 90 | PROF_NOTE("%s" , "libhsa-runtime64.so not loadable - " |
| 91 | "HSA device profiling disabled\n" ); |
| 92 | return setHsaRuntimeState(-1); |
| 93 | } |
| 94 | |
| 95 | auto *pHsaInit = |
| 96 | (decltype(&hsa_init))__interception::LookupSymbol(handle: Hsa, symbol: "hsa_init" ); |
| 97 | auto *pGetExtTable = (decltype(&hsa_system_get_major_extension_table)) |
| 98 | __interception::LookupSymbol(handle: Hsa, symbol: "hsa_system_get_major_extension_table" ); |
| 99 | pHsaIterateAgents = (hsa_iterate_agents_ty)__interception::LookupSymbol( |
| 100 | handle: Hsa, symbol: "hsa_iterate_agents" ); |
| 101 | pHsaAgentGetInfo = (hsa_agent_get_info_ty)__interception::LookupSymbol( |
| 102 | handle: Hsa, symbol: "hsa_agent_get_info" ); |
| 103 | pHsaExecIterAgentSyms = |
| 104 | (hsa_executable_iterate_agent_symbols_ty)__interception::LookupSymbol( |
| 105 | handle: Hsa, symbol: "hsa_executable_iterate_agent_symbols" ); |
| 106 | pHsaSymGetInfo = |
| 107 | (hsa_executable_symbol_get_info_ty)__interception::LookupSymbol( |
| 108 | handle: Hsa, symbol: "hsa_executable_symbol_get_info" ); |
| 109 | if (!pHsaInit || !pGetExtTable || !pHsaIterateAgents || !pHsaAgentGetInfo || |
| 110 | !pHsaExecIterAgentSyms || !pHsaSymGetInfo) { |
| 111 | PROF_WARN("%s" , |
| 112 | "required HSA symbols missing - HSA device profiling disabled\n" ); |
| 113 | return setHsaRuntimeState(-1); |
| 114 | } |
| 115 | |
| 116 | hsa_status_t St = pHsaInit(); |
| 117 | if (!hsaOkOrBreak(St)) { |
| 118 | if (isVerboseMode()) |
| 119 | PROF_NOTE("hsa_init failed (0x%x) - HSA device profiling disabled\n" , St); |
| 120 | return setHsaRuntimeState(-1); |
| 121 | } |
| 122 | |
| 123 | // Full 1.04 table. A shorter runtime may succeed and leave the tail null. |
| 124 | LoaderApi LoaderTable = {}; |
| 125 | St = pGetExtTable(HSA_EXTENSION_AMD_LOADER, 1, sizeof(LoaderTable), |
| 126 | &LoaderTable); |
| 127 | if (!hsaOk(St) || !LoaderTable.QuerySegmentDescriptors) { |
| 128 | PROF_WARN("AMD loader extension unavailable (0x%x) - " |
| 129 | "HSA device profiling disabled\n" , |
| 130 | St); |
| 131 | return setHsaRuntimeState(-1); |
| 132 | } |
| 133 | pQuerySegDescs = LoaderTable.QuerySegmentDescriptors; |
| 134 | |
| 135 | /* The device-to-host copies go through the shared HIP loader. */ |
| 136 | ensureHipLoaded(); |
| 137 | if (!hipMemcpyAvailable()) { |
| 138 | PROF_WARN("%s" , "hipMemcpy unavailable - HSA device profiling disabled\n" ); |
| 139 | return setHsaRuntimeState(-1); |
| 140 | } |
| 141 | |
| 142 | if (isVerboseMode()) |
| 143 | PROF_NOTE("%s" , "HSA + HIP runtime resolved for device profiling\n" ); |
| 144 | return setHsaRuntimeState(1); |
| 145 | } |
| 146 | |
| 147 | int __prof_rocm::hsaRuntimeAvailable(void) { return bindHsaRuntime() == 0; } |
| 148 | |
| 149 | /* The canonical device bounds-table symbol from InstrProfilingPlatformGPU.c. */ |
| 150 | static const char ProfileSectionsSymbol[] = "__llvm_profile_sections" ; |
| 151 | |
| 152 | /* Dedup of drained section-bounds tuples, shared with the host-shadow path |
| 153 | * (processDeviceOffloadPrf records here on every successful drain) so each |
| 154 | * unique counter set is drained exactly once across both paths. |
| 155 | */ |
| 156 | static ProfBoundsSet SeenBounds; |
| 157 | |
| 158 | /* Has this bounds tuple already been drained? Pure check, no state mutation. */ |
| 159 | static int profBoundsAlreadyDrained(const void *D, const void *C, |
| 160 | const void *N) { |
| 161 | return SeenBounds.contains(D, C, N); |
| 162 | } |
| 163 | |
| 164 | /* Record a drained bounds tuple. Idempotent; call only after a successful drain |
| 165 | * so a failed attempt stays retryable. */ |
| 166 | void __prof_rocm::profRecordDrainedBounds(const void *D, const void *C, |
| 167 | const void *N) { |
| 168 | SeenBounds.record(D, C, N); |
| 169 | } |
| 170 | |
| 171 | #define PROF_MAX_GPU_AGENTS 64 |
| 172 | |
| 173 | /* Buffer size for HSA agent names and symbol names we read back; both the |
| 174 | * device arch string and the __llvm_profile_sections symbol are far shorter. */ |
| 175 | #define PROF_HSA_NAME_MAX 64 |
| 176 | |
| 177 | namespace { |
| 178 | struct GpuAgent { |
| 179 | hsa_agent_t agent; |
| 180 | char arch[PROF_HSA_NAME_MAX]; |
| 181 | }; |
| 182 | |
| 183 | struct WalkState { |
| 184 | GpuAgent agents[PROF_MAX_GPU_AGENTS]; |
| 185 | int num_agents; |
| 186 | int total_found; |
| 187 | int total_drained; |
| 188 | }; |
| 189 | |
| 190 | /* Per (agent, executable) symbol-iteration state. */ |
| 191 | struct SymbolState { |
| 192 | const char *arch; |
| 193 | int found; |
| 194 | int drained; |
| 195 | }; |
| 196 | } // namespace |
| 197 | |
| 198 | /* HSA per-symbol callback: when it finds a __llvm_profile_sections variable, |
| 199 | * drain it via processDeviceOffloadPrf() unless the host-shadow path (or an |
| 200 | * earlier agent) already handled the same bounds. */ |
| 201 | static hsa_status_t onSymbol(hsa_executable_t, hsa_agent_t, |
| 202 | hsa_executable_symbol_t Sym, void *Data) { |
| 203 | SymbolState *S = (SymbolState *)Data; |
| 204 | |
| 205 | hsa_symbol_kind_t Kind; |
| 206 | if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &Kind)) || |
| 207 | Kind != HSA_SYMBOL_KIND_VARIABLE) |
| 208 | return HSA_STATUS_SUCCESS; |
| 209 | |
| 210 | uint32_t NameLen = 0; |
| 211 | if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, |
| 212 | &NameLen)) || |
| 213 | NameLen != sizeof(ProfileSectionsSymbol) - 1) |
| 214 | return HSA_STATUS_SUCCESS; |
| 215 | |
| 216 | char NameBuf[PROF_HSA_NAME_MAX]; |
| 217 | if (NameLen + 1 > sizeof(NameBuf)) |
| 218 | return HSA_STATUS_SUCCESS; |
| 219 | if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_NAME, NameBuf))) |
| 220 | return HSA_STATUS_SUCCESS; |
| 221 | NameBuf[NameLen] = '\0'; |
| 222 | |
| 223 | if (strcmp(s1: NameBuf, s2: ProfileSectionsSymbol) != 0) |
| 224 | return HSA_STATUS_SUCCESS; |
| 225 | |
| 226 | uint64_t Addr = 0; |
| 227 | if (!hsaOk(St: pHsaSymGetInfo(Sym, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, |
| 228 | &Addr)) || |
| 229 | Addr == 0) { |
| 230 | if (isVerboseMode()) |
| 231 | PROF_NOTE("%s" , "failed to read __llvm_profile_sections address\n" ); |
| 232 | return HSA_STATUS_SUCCESS; |
| 233 | } |
| 234 | |
| 235 | S->found++; |
| 236 | |
| 237 | // Read the bounds table first to dedup (and detect empty sections) before |
| 238 | // the full copy/relocate done by processDeviceOffloadPrf. |
| 239 | __llvm_profile_gpu_sections Sec; |
| 240 | if (memcpyDeviceToHost(Dst: &Sec, Src: (void *)(uintptr_t)Addr, Size: sizeof(Sec)) != 0) { |
| 241 | PROF_WARN("%s" , "failed to copy device bounds table\n" ); |
| 242 | return HSA_STATUS_SUCCESS; |
| 243 | } |
| 244 | if (profBoundsAlreadyDrained(D: Sec.DataStart, C: Sec.CountersStart, |
| 245 | N: Sec.NamesStart)) { |
| 246 | if (isVerboseMode()) |
| 247 | PROF_NOTE("%s" , "device bounds already drained, skipping\n" ); |
| 248 | return HSA_STATUS_SUCCESS; |
| 249 | } |
| 250 | |
| 251 | size_t DataBytes = (const char *)Sec.DataStop - (const char *)Sec.DataStart; |
| 252 | size_t CntsBytes = |
| 253 | (const char *)Sec.CountersStop - (const char *)Sec.CountersStart; |
| 254 | if (DataBytes == 0 || CntsBytes == 0) { |
| 255 | // Empty code object: nothing to write. Mark seen so we don't revisit it. |
| 256 | profRecordDrainedBounds(D: Sec.DataStart, C: Sec.CountersStart, N: Sec.NamesStart); |
| 257 | return HSA_STATUS_SUCCESS; |
| 258 | } |
| 259 | |
| 260 | // Name HSA-drained objects in their own ".hsaN" suffix space so they never |
| 261 | // collide with the host-shadow path's "arch"/"arch.<i>" filenames. The drain |
| 262 | // latch (HsaDrainCompleted) already prevents re-draining an object, so a |
| 263 | // plain per-drain counter is enough for uniqueness. |
| 264 | static int DrainIndex = 0; |
| 265 | char Target[96]; |
| 266 | snprintf(s: Target, maxlen: sizeof(Target), format: "%s.hsa%d" , S->arch, DrainIndex); |
| 267 | |
| 268 | // Record the bounds (and advance the index) only on a successful write so a |
| 269 | // transient error stays retryable on a later agent or collect call. |
| 270 | if (processDeviceOffloadPrf(DeviceOffloadPrf: (void *)(uintptr_t)Addr, Target, Sections: nullptr) == 0) { |
| 271 | S->drained++; |
| 272 | DrainIndex++; |
| 273 | profRecordDrainedBounds(D: Sec.DataStart, C: Sec.CountersStart, N: Sec.NamesStart); |
| 274 | } |
| 275 | |
| 276 | return HSA_STATUS_SUCCESS; |
| 277 | } |
| 278 | |
| 279 | static hsa_status_t collectAgent(hsa_agent_t Agent, void *Data) { |
| 280 | hsa_device_type_t DevType; |
| 281 | if (!hsaOk(St: pHsaAgentGetInfo(Agent, HSA_AGENT_INFO_DEVICE, &DevType)) || |
| 282 | DevType != HSA_DEVICE_TYPE_GPU) |
| 283 | return HSA_STATUS_SUCCESS; |
| 284 | |
| 285 | WalkState *W = (WalkState *)Data; |
| 286 | if (W->num_agents >= PROF_MAX_GPU_AGENTS) |
| 287 | return HSA_STATUS_SUCCESS; |
| 288 | |
| 289 | GpuAgent &GA = W->agents[W->num_agents++]; |
| 290 | GA.agent = Agent; |
| 291 | char Name[PROF_HSA_NAME_MAX]; |
| 292 | __builtin_memset(Name, 0, sizeof(Name)); |
| 293 | pHsaAgentGetInfo(Agent, HSA_AGENT_INFO_NAME, Name); |
| 294 | size_t N = strnlen(string: Name, maxlen: sizeof(GA.arch) - 1); |
| 295 | __builtin_memcpy(GA.arch, Name, N); |
| 296 | GA.arch[N] = '\0'; |
| 297 | if (!GA.arch[0]) |
| 298 | strncpy(dest: GA.arch, src: "amdgpu" , n: sizeof(GA.arch) - 1); |
| 299 | |
| 300 | if (isVerboseMode()) |
| 301 | PROF_NOTE("GPU agent %d: %s\n" , W->num_agents - 1, GA.arch); |
| 302 | return HSA_STATUS_SUCCESS; |
| 303 | } |
| 304 | |
| 305 | /* Reentrancy guard and "drained at least once" latch (both acquire/release). */ |
| 306 | static int HsaDrainInProgress = 0; |
| 307 | static int HsaDrainCompleted = 0; |
| 308 | |
| 309 | int __prof_rocm::drainDevicesViaHsa(void) { |
| 310 | if (__atomic_load_n(&HsaDrainCompleted, __ATOMIC_ACQUIRE)) |
| 311 | return 0; |
| 312 | |
| 313 | int Expected = 0; |
| 314 | if (!__atomic_compare_exchange_n(&HsaDrainInProgress, &Expected, 1, |
| 315 | /*weak=*/0, __ATOMIC_ACQ_REL, |
| 316 | __ATOMIC_ACQUIRE)) |
| 317 | return 0; |
| 318 | |
| 319 | struct InProgressGuard { |
| 320 | ~InProgressGuard() { |
| 321 | __atomic_store_n(&HsaDrainInProgress, 0, __ATOMIC_RELEASE); |
| 322 | } |
| 323 | } _Guard; |
| 324 | |
| 325 | if (bindHsaRuntime() != 0) |
| 326 | return 0; /* Runtime unavailable: stay retryable. */ |
| 327 | |
| 328 | WalkState W; |
| 329 | __builtin_memset(&W, 0, sizeof(W)); |
| 330 | hsa_status_t St = pHsaIterateAgents(collectAgent, &W); |
| 331 | if (!hsaOkOrBreak(St)) { |
| 332 | PROF_WARN("hsa_iterate_agents failed (0x%x)\n" , St); |
| 333 | return -1; |
| 334 | } |
| 335 | if (W.num_agents == 0) { |
| 336 | if (isVerboseMode()) |
| 337 | PROF_NOTE("%s" , "no GPU agents present; nothing to drain (will retry)\n" ); |
| 338 | return 0; |
| 339 | } |
| 340 | |
| 341 | /* query_segment_descriptors ships in every loader-extension version, is more |
| 342 | * permissive than iterate_executables on ROCm, and yields the loaded |
| 343 | * (agent, executable) pairs directly. */ |
| 344 | size_t NumSegs = 0; |
| 345 | St = pQuerySegDescs(nullptr, &NumSegs); |
| 346 | if (!hsaOk(St)) { |
| 347 | PROF_WARN("query_segment_descriptors(count) failed (0x%x)\n" , St); |
| 348 | return -1; |
| 349 | } |
| 350 | if (NumSegs == 0) { |
| 351 | if (isVerboseMode()) |
| 352 | PROF_NOTE("%s" , "no loaded segments; nothing to drain (will retry)\n" ); |
| 353 | return 0; |
| 354 | } |
| 355 | |
| 356 | hsa_ven_amd_loader_segment_descriptor_t *Segs = |
| 357 | (hsa_ven_amd_loader_segment_descriptor_t *)calloc(nmemb: NumSegs, size: sizeof(*Segs)); |
| 358 | if (!Segs) { |
| 359 | PROF_ERR("%s\n" , "failed to allocate segment descriptor array" ); |
| 360 | return -1; |
| 361 | } |
| 362 | UniqueFree SegsOwner(Segs); |
| 363 | |
| 364 | St = pQuerySegDescs(Segs, &NumSegs); |
| 365 | if (!hsaOk(St)) { |
| 366 | PROF_WARN("query_segment_descriptors(fetch) failed (0x%x)\n" , St); |
| 367 | return -1; |
| 368 | } |
| 369 | |
| 370 | if (isVerboseMode()) |
| 371 | PROF_NOTE("query_segment_descriptors: %zu segments\n" , NumSegs); |
| 372 | |
| 373 | // Walk each unique (agent, executable) pair once. |
| 374 | struct SeenPair { |
| 375 | uint64_t agent; |
| 376 | uint64_t exec; |
| 377 | }; |
| 378 | enum { kSeenPairsInitCap = 64 }; |
| 379 | SeenPair *Seen = nullptr; |
| 380 | int NumPairs = 0; |
| 381 | int CapPairs = 0; |
| 382 | int IterFailures = 0; |
| 383 | |
| 384 | for (size_t i = 0; i < NumSegs; ++i) { |
| 385 | if (Segs[i].executable.handle == 0 || Segs[i].agent.handle == 0) |
| 386 | continue; |
| 387 | |
| 388 | bool AlreadySeen = false; |
| 389 | for (int j = 0; j < NumPairs; ++j) |
| 390 | if (Seen[j].agent == Segs[i].agent.handle && |
| 391 | Seen[j].exec == Segs[i].executable.handle) { |
| 392 | AlreadySeen = true; |
| 393 | break; |
| 394 | } |
| 395 | if (AlreadySeen) |
| 396 | continue; |
| 397 | if (growArray(Arr: (void **)&Seen, Cap: &CapPairs, MinCount: NumPairs + 1, InitCap: kSeenPairsInitCap, |
| 398 | ElemSize: sizeof(*Seen)) == 0) { |
| 399 | Seen[NumPairs].agent = Segs[i].agent.handle; |
| 400 | Seen[NumPairs].exec = Segs[i].executable.handle; |
| 401 | NumPairs++; |
| 402 | } |
| 403 | |
| 404 | const char *Arch = nullptr; |
| 405 | for (int k = 0; k < W.num_agents; ++k) |
| 406 | if (W.agents[k].agent.handle == Segs[i].agent.handle) { |
| 407 | Arch = W.agents[k].arch; |
| 408 | break; |
| 409 | } |
| 410 | if (!Arch) |
| 411 | continue; /* not a GPU agent we collected */ |
| 412 | |
| 413 | SymbolState S; |
| 414 | __builtin_memset(&S, 0, sizeof(S)); |
| 415 | S.arch = Arch; |
| 416 | if (isVerboseMode()) |
| 417 | PROF_NOTE("walking executable 0x%llx on %s\n" , |
| 418 | (unsigned long long)Segs[i].executable.handle, Arch); |
| 419 | hsa_status_t IterSt = |
| 420 | pHsaExecIterAgentSyms(Segs[i].executable, Segs[i].agent, onSymbol, &S); |
| 421 | if (!hsaOkOrBreak(St: IterSt)) { |
| 422 | PROF_WARN("hsa_executable_iterate_agent_symbols on executable 0x%llx " |
| 423 | "failed (0x%x)\n" , |
| 424 | (unsigned long long)Segs[i].executable.handle, IterSt); |
| 425 | IterFailures++; |
| 426 | } |
| 427 | W.total_found += S.found; |
| 428 | W.total_drained += S.drained; |
| 429 | } |
| 430 | |
| 431 | if (isVerboseMode()) |
| 432 | PROF_NOTE("HSA walk complete: agents=%d pairs=%d found=%d drained=%d " |
| 433 | "iter-failures=%d\n" , |
| 434 | W.num_agents, NumPairs, W.total_found, W.total_drained, |
| 435 | IterFailures); |
| 436 | |
| 437 | free(ptr: Seen); |
| 438 | |
| 439 | /* Latch only when we actually drained data. A "found nothing new" walk is |
| 440 | * deliberately not latched: an early collect can precede any kernel launch, |
| 441 | * and latching it would suppress the real exit-time drain. No-op walks are |
| 442 | * cheap to repeat. */ |
| 443 | if (W.total_drained > 0) |
| 444 | __atomic_store_n(&HsaDrainCompleted, 1, __ATOMIC_RELEASE); |
| 445 | return (IterFailures > 0) ? -1 : 0; |
| 446 | } |
| 447 | |
| 448 | /* Fork-safety: deliberately no library constructor calling hsa_init(). */ |
| 449 | |
| 450 | #endif /* defined(__linux__) */ |
| 451 | |