1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// HSA interceptors for host-side ConcurrencySanitizer offload support.
11///
12//===----------------------------------------------------------------------===//
13
14#include <dlfcn.h>
15
16#include "csan_offload.h"
17#include "interception/interception.h"
18#include "sanitizer_common/sanitizer_atomic.h"
19#include "sanitizer_common/sanitizer_common.h"
20#include "sanitizer_common/sanitizer_flag_parser.h"
21#include "sanitizer_common/sanitizer_flags.h"
22#include "sanitizer_common/sanitizer_mutex.h"
23#include "sanitizer_common/sanitizer_offload.h"
24#include "sanitizer_common/sanitizer_platform.h"
25#include "sanitizer_common/sanitizer_symbolizer.h"
26#include "sanitizer_common/sanitizer_vector.h"
27
28#if !SANITIZER_LINUX
29#error "Offload CSan reporting is supported on Linux only"
30#endif
31
32using namespace __sanitizer;
33using namespace __csan;
34
35namespace {
36
37static StaticSpinMutex InitMutex;
38static StaticSpinMutex HsaMutex;
39static atomic_uint8_t Initialized;
40static void *HsaHandle;
41
42struct DeviceAllocation {
43 hsa_agent_t Agent;
44 void *Ptr;
45};
46
47Mutex AllocationMutex;
48InternalMmapVectorNoCtor<DeviceAllocation> Allocations;
49uptr HsaRefs;
50
51void Initialize() {
52 if (LIKELY(atomic_load(&Initialized, memory_order_acquire)))
53 return;
54 SpinMutexLock L(&InitMutex);
55 if (atomic_load(a: &Initialized, mo: memory_order_relaxed))
56 return;
57 SanitizerToolName = "ConcurrencySanitizer";
58 CacheBinaryName();
59 SetCommonFlagsDefaults();
60 {
61 CommonFlags cf;
62 cf.CopyFrom(other: *common_flags());
63 if (const char *Path = GetEnv(name: "CSAN_SYMBOLIZER_PATH"))
64 cf.external_symbolizer_path = Path;
65 cf.stack_trace_format = " #%n %f %S (%p)";
66 OverrideCommonFlags(cf);
67 }
68 FlagParser Parser;
69 RegisterCommonFlags(parser: &Parser);
70 Parser.ParseStringFromEnv(env_name: "CSAN_OPTIONS");
71 InitializeCommonFlags();
72 Offload::Get().RegisterHandler(Fn: HandleOffloadReport);
73 Atexit(function: [] { Offload::Get().UntrackImages(); });
74 AddDieCallback(callback: [] { Offload::Get().UntrackImages(); });
75 atomic_store(a: &Initialized, v: 1, mo: memory_order_release);
76 Symbolizer::LateInitialize();
77}
78
79} // namespace
80
81// DSOs with a static runtime all export these wrappers and interpose onto the
82// first, so 'RTLD_NEXT' can loop back into it. Resolve from HSA directly
83// without loading it.
84static void *HsaSymbol(const char *Name) {
85 SpinMutexLock L(&HsaMutex);
86 constexpr const char *Libs[] = {SANITIZER_HSA_LIBRARY ".so.1",
87 SANITIZER_HSA_LIBRARY ".so"};
88 for (const char *Lib : Libs)
89 if (!HsaHandle)
90 HsaHandle = dlopen(file: Lib, RTLD_LAZY | RTLD_NOLOAD);
91 return HsaHandle ? dlsym(handle: HsaHandle, name: Name) : nullptr;
92}
93
94template <typename T> static T HsaFunction(const char *Name) {
95 return reinterpret_cast<T>(HsaSymbol(Name));
96}
97
98// The shared runtime exports these to every program, act as if HSA is absent
99// when it is not loaded.
100#define CSAN_HSA_ENTER(name) \
101 Initialize(); \
102 if (UNLIKELY(!REAL(name))) { \
103 REAL(name) = HsaFunction<decltype(REAL(name))>(#name); \
104 if (UNLIKELY(!REAL(name))) { \
105 VReport(1, "%s: cannot find %s in this process\n", SanitizerToolName, \
106 #name); \
107 return HSA_STATUS_ERROR; \
108 } \
109 }
110
111#define CSAN_HSA_FORWARD(name, ...) \
112 CSAN_HSA_ENTER(name); \
113 if (UNLIKELY(!Offload::Get().Ready())) \
114 return REAL(name)(__VA_ARGS__);
115
116static bool Lookup(hsa_executable_t Executable, const char *Name,
117 hsa_agent_t Agent, u64 *Addr) {
118 auto GetSymbol = HsaFunction<decltype(&hsa_executable_get_symbol_by_name)>(
119 Name: "hsa_executable_get_symbol_by_name");
120 auto GetInfo = HsaFunction<decltype(&hsa_executable_symbol_get_info)>(
121 Name: "hsa_executable_symbol_get_info");
122 if (!GetSymbol || !GetInfo)
123 return false;
124
125 hsa_executable_symbol_t Symbol;
126 if (GetSymbol(Executable, Name, &Agent, &Symbol) != HSA_STATUS_SUCCESS)
127 return false;
128 *Addr = 0;
129 return GetInfo(Symbol, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, Addr) ==
130 HSA_STATUS_SUCCESS &&
131 *Addr;
132}
133
134static DeviceAllocation *FindAllocation(hsa_agent_t Agent) {
135 for (uptr I = 0; I < Allocations.size(); ++I)
136 if (Allocations[I].Agent.handle == Agent.handle)
137 return &Allocations[I];
138 return nullptr;
139}
140
141struct BindContext {
142 hsa_executable_t Executable;
143 bool Found;
144 bool Success;
145};
146
147// The watchpoint table is shared between all executables loaded on the device.
148static hsa_status_t BindAgent(hsa_agent_t Agent, void *Data) {
149 BindContext &Ctx = *reinterpret_cast<BindContext *>(Data);
150 auto AgentInfo =
151 HsaFunction<decltype(&hsa_agent_get_info)>(Name: "hsa_agent_get_info");
152 auto Copy = HsaFunction<decltype(&hsa_memory_copy)>(Name: "hsa_memory_copy");
153 auto Free = HsaFunction<decltype(&hsa_amd_memory_pool_free)>(
154 Name: "hsa_amd_memory_pool_free");
155 if (!AgentInfo || !Copy || !Free) {
156 Ctx.Success = false;
157 return HSA_STATUS_SUCCESS;
158 }
159
160 hsa_device_type_t Type;
161 if (AgentInfo(Agent, HSA_AGENT_INFO_DEVICE, &Type) != HSA_STATUS_SUCCESS ||
162 Type != HSA_DEVICE_TYPE_GPU)
163 return HSA_STATUS_SUCCESS;
164
165 u64 PointerAddr = 0;
166 if (!Lookup(Executable: Ctx.Executable, Name: "__csan_watchpoint_table", Agent, Addr: &PointerAddr))
167 return HSA_STATUS_SUCCESS;
168 Ctx.Found = true;
169
170 // Create an allocation for the table if one does not already exist.
171 DeviceAllocation *Allocation = FindAllocation(Agent);
172 if (!Allocation) {
173 hsa_amd_memory_pool_t Pool;
174 void *Ptr = nullptr;
175 if (!Offload::Get().GetMemoryPool(Agent, Pool: &Pool) ||
176 !Offload::Get().Allocate(Pool, Bytes: CSAN_WATCHPOINT_TABLE_BYTES, Out: &Ptr)) {
177 Ctx.Success = false;
178 return HSA_STATUS_SUCCESS;
179 }
180
181 // Fresh pages from MMap are always zero initialized, as required.
182 void *Zeros =
183 MmapOrDie(size: CSAN_WATCHPOINT_TABLE_BYTES, mem_type: "CSan watchpoint table");
184 bool Copied =
185 Copy(Ptr, Zeros, CSAN_WATCHPOINT_TABLE_BYTES) == HSA_STATUS_SUCCESS;
186 UnmapOrDie(addr: Zeros, size: CSAN_WATCHPOINT_TABLE_BYTES);
187 if (!Copied) {
188 Free(Ptr);
189 Ctx.Success = false;
190 return HSA_STATUS_SUCCESS;
191 }
192 Allocations.push_back(element: {.Agent: Agent, .Ptr: Ptr});
193 Allocation = &Allocations.back();
194 }
195
196 // Update the pointer on the device to the associated allocation.
197 if (Copy(reinterpret_cast<void *>(PointerAddr), &Allocation->Ptr,
198 sizeof(Allocation->Ptr)) != HSA_STATUS_SUCCESS)
199 Ctx.Success = false;
200 return HSA_STATUS_SUCCESS;
201}
202
203static void BindWatchpointTable(hsa_executable_t Executable) {
204 auto Iterate =
205 HsaFunction<decltype(&hsa_iterate_agents)>(Name: "hsa_iterate_agents");
206 if (!Iterate)
207 return;
208
209 // Check if we need to set up the watchpoint table.
210 Lock L(&AllocationMutex);
211 BindContext Ctx = {.Executable: Executable, .Found: false, .Success: true};
212 if (Iterate(BindAgent, &Ctx) != HSA_STATUS_SUCCESS)
213 Ctx.Success = false;
214 if (Ctx.Found && !Ctx.Success) {
215 Report(format: "ERROR: %s: cannot initialize device watchpoint table\n",
216 SanitizerToolName);
217 Die();
218 }
219}
220
221static void RetainAllocations() {
222 Lock L(&AllocationMutex);
223 ++HsaRefs;
224}
225
226static void ReleaseAllocations() {
227 Lock L(&AllocationMutex);
228 if (!HsaRefs || --HsaRefs)
229 return;
230 auto Free = HsaFunction<decltype(&hsa_amd_memory_pool_free)>(
231 Name: "hsa_amd_memory_pool_free");
232 if (Free)
233 for (uptr I = 0; I < Allocations.size(); ++I)
234 Free(Allocations[I].Ptr);
235 Allocations.clear();
236}
237
238INTERCEPTOR(hsa_status_t, hsa_init, void) {
239 CSAN_HSA_ENTER(hsa_init);
240
241 hsa_status_t Status = REAL(hsa_init)();
242 if (Status != HSA_STATUS_SUCCESS)
243 return Status;
244
245 if (!Offload::Get().Init()) {
246 Report(format: "ERROR: %s: cannot initialize HSA offload support\n",
247 SanitizerToolName);
248 Die();
249 }
250 RetainAllocations();
251 return Status;
252}
253
254INTERCEPTOR(hsa_status_t, hsa_shut_down, void) {
255 CSAN_HSA_ENTER(hsa_shut_down);
256
257 ReleaseAllocations();
258 Offload::Get().Shutdown();
259 return REAL(hsa_shut_down)();
260}
261
262INTERCEPTOR(hsa_status_t, hsa_executable_freeze, hsa_executable_t Executable,
263 const char *Options) {
264 CSAN_HSA_FORWARD(hsa_executable_freeze, Executable, Options);
265
266 hsa_status_t Status = REAL(hsa_executable_freeze)(Executable, Options);
267 if (Status == HSA_STATUS_SUCCESS) {
268 BindWatchpointTable(Executable);
269 Offload::Get().TrackExecutable(Exec: Executable);
270 }
271 return Status;
272}
273
274INTERCEPTOR(hsa_status_t, hsa_executable_destroy, hsa_executable_t Executable) {
275 CSAN_HSA_FORWARD(hsa_executable_destroy, Executable);
276
277 Offload::Get().UntrackExecutable(Exec: Executable);
278 return REAL(hsa_executable_destroy)(Executable);
279}
280
281extern "C" void __csan_offload_init() { Initialize(); }
282
283__attribute__((constructor(0))) static void CsanOffloadDynInit() {
284 __csan_offload_init();
285 WarnIfHsaNotInterposed();
286}
287