| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// HSA interceptors for host-side ConcurrencySanitizer offload support. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include <dlfcn.h> |
| 15 | |
| 16 | #include "csan_offload.h" |
| 17 | #include "interception/interception.h" |
| 18 | #include "sanitizer_common/sanitizer_atomic.h" |
| 19 | #include "sanitizer_common/sanitizer_common.h" |
| 20 | #include "sanitizer_common/sanitizer_flag_parser.h" |
| 21 | #include "sanitizer_common/sanitizer_flags.h" |
| 22 | #include "sanitizer_common/sanitizer_mutex.h" |
| 23 | #include "sanitizer_common/sanitizer_offload.h" |
| 24 | #include "sanitizer_common/sanitizer_platform.h" |
| 25 | #include "sanitizer_common/sanitizer_symbolizer.h" |
| 26 | #include "sanitizer_common/sanitizer_vector.h" |
| 27 | |
| 28 | #if !SANITIZER_LINUX |
| 29 | #error "Offload CSan reporting is supported on Linux only" |
| 30 | #endif |
| 31 | |
| 32 | using namespace __sanitizer; |
| 33 | using namespace __csan; |
| 34 | |
| 35 | namespace { |
| 36 | |
| 37 | static StaticSpinMutex InitMutex; |
| 38 | static StaticSpinMutex HsaMutex; |
| 39 | static atomic_uint8_t Initialized; |
| 40 | static void *HsaHandle; |
| 41 | |
| 42 | struct DeviceAllocation { |
| 43 | hsa_agent_t Agent; |
| 44 | void *Ptr; |
| 45 | }; |
| 46 | |
| 47 | Mutex AllocationMutex; |
| 48 | InternalMmapVectorNoCtor<DeviceAllocation> Allocations; |
| 49 | uptr HsaRefs; |
| 50 | |
| 51 | void Initialize() { |
| 52 | if (LIKELY(atomic_load(&Initialized, memory_order_acquire))) |
| 53 | return; |
| 54 | SpinMutexLock L(&InitMutex); |
| 55 | if (atomic_load(a: &Initialized, mo: memory_order_relaxed)) |
| 56 | return; |
| 57 | SanitizerToolName = "ConcurrencySanitizer" ; |
| 58 | CacheBinaryName(); |
| 59 | SetCommonFlagsDefaults(); |
| 60 | { |
| 61 | CommonFlags cf; |
| 62 | cf.CopyFrom(other: *common_flags()); |
| 63 | if (const char *Path = GetEnv(name: "CSAN_SYMBOLIZER_PATH" )) |
| 64 | cf.external_symbolizer_path = Path; |
| 65 | cf.stack_trace_format = " #%n %f %S (%p)" ; |
| 66 | OverrideCommonFlags(cf); |
| 67 | } |
| 68 | FlagParser Parser; |
| 69 | RegisterCommonFlags(parser: &Parser); |
| 70 | Parser.ParseStringFromEnv(env_name: "CSAN_OPTIONS" ); |
| 71 | InitializeCommonFlags(); |
| 72 | Offload::Get().RegisterHandler(Fn: HandleOffloadReport); |
| 73 | Atexit(function: [] { Offload::Get().UntrackImages(); }); |
| 74 | AddDieCallback(callback: [] { Offload::Get().UntrackImages(); }); |
| 75 | atomic_store(a: &Initialized, v: 1, mo: memory_order_release); |
| 76 | Symbolizer::LateInitialize(); |
| 77 | } |
| 78 | |
| 79 | } // namespace |
| 80 | |
| 81 | // DSOs with a static runtime all export these wrappers and interpose onto the |
| 82 | // first, so 'RTLD_NEXT' can loop back into it. Resolve from HSA directly |
| 83 | // without loading it. |
| 84 | static void *HsaSymbol(const char *Name) { |
| 85 | SpinMutexLock L(&HsaMutex); |
| 86 | constexpr const char *Libs[] = {SANITIZER_HSA_LIBRARY ".so.1" , |
| 87 | SANITIZER_HSA_LIBRARY ".so" }; |
| 88 | for (const char *Lib : Libs) |
| 89 | if (!HsaHandle) |
| 90 | HsaHandle = dlopen(file: Lib, RTLD_LAZY | RTLD_NOLOAD); |
| 91 | return HsaHandle ? dlsym(handle: HsaHandle, name: Name) : nullptr; |
| 92 | } |
| 93 | |
| 94 | template <typename T> static T HsaFunction(const char *Name) { |
| 95 | return reinterpret_cast<T>(HsaSymbol(Name)); |
| 96 | } |
| 97 | |
| 98 | // The shared runtime exports these to every program, act as if HSA is absent |
| 99 | // when it is not loaded. |
| 100 | #define CSAN_HSA_ENTER(name) \ |
| 101 | Initialize(); \ |
| 102 | if (UNLIKELY(!REAL(name))) { \ |
| 103 | REAL(name) = HsaFunction<decltype(REAL(name))>(#name); \ |
| 104 | if (UNLIKELY(!REAL(name))) { \ |
| 105 | VReport(1, "%s: cannot find %s in this process\n", SanitizerToolName, \ |
| 106 | #name); \ |
| 107 | return HSA_STATUS_ERROR; \ |
| 108 | } \ |
| 109 | } |
| 110 | |
| 111 | #define CSAN_HSA_FORWARD(name, ...) \ |
| 112 | CSAN_HSA_ENTER(name); \ |
| 113 | if (UNLIKELY(!Offload::Get().Ready())) \ |
| 114 | return REAL(name)(__VA_ARGS__); |
| 115 | |
| 116 | static bool Lookup(hsa_executable_t Executable, const char *Name, |
| 117 | hsa_agent_t Agent, u64 *Addr) { |
| 118 | auto GetSymbol = HsaFunction<decltype(&hsa_executable_get_symbol_by_name)>( |
| 119 | Name: "hsa_executable_get_symbol_by_name" ); |
| 120 | auto GetInfo = HsaFunction<decltype(&hsa_executable_symbol_get_info)>( |
| 121 | Name: "hsa_executable_symbol_get_info" ); |
| 122 | if (!GetSymbol || !GetInfo) |
| 123 | return false; |
| 124 | |
| 125 | hsa_executable_symbol_t Symbol; |
| 126 | if (GetSymbol(Executable, Name, &Agent, &Symbol) != HSA_STATUS_SUCCESS) |
| 127 | return false; |
| 128 | *Addr = 0; |
| 129 | return GetInfo(Symbol, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, Addr) == |
| 130 | HSA_STATUS_SUCCESS && |
| 131 | *Addr; |
| 132 | } |
| 133 | |
| 134 | static DeviceAllocation *FindAllocation(hsa_agent_t Agent) { |
| 135 | for (uptr I = 0; I < Allocations.size(); ++I) |
| 136 | if (Allocations[I].Agent.handle == Agent.handle) |
| 137 | return &Allocations[I]; |
| 138 | return nullptr; |
| 139 | } |
| 140 | |
| 141 | struct BindContext { |
| 142 | hsa_executable_t Executable; |
| 143 | bool Found; |
| 144 | bool Success; |
| 145 | }; |
| 146 | |
| 147 | // The watchpoint table is shared between all executables loaded on the device. |
| 148 | static hsa_status_t BindAgent(hsa_agent_t Agent, void *Data) { |
| 149 | BindContext &Ctx = *reinterpret_cast<BindContext *>(Data); |
| 150 | auto AgentInfo = |
| 151 | HsaFunction<decltype(&hsa_agent_get_info)>(Name: "hsa_agent_get_info" ); |
| 152 | auto Copy = HsaFunction<decltype(&hsa_memory_copy)>(Name: "hsa_memory_copy" ); |
| 153 | auto Free = HsaFunction<decltype(&hsa_amd_memory_pool_free)>( |
| 154 | Name: "hsa_amd_memory_pool_free" ); |
| 155 | if (!AgentInfo || !Copy || !Free) { |
| 156 | Ctx.Success = false; |
| 157 | return HSA_STATUS_SUCCESS; |
| 158 | } |
| 159 | |
| 160 | hsa_device_type_t Type; |
| 161 | if (AgentInfo(Agent, HSA_AGENT_INFO_DEVICE, &Type) != HSA_STATUS_SUCCESS || |
| 162 | Type != HSA_DEVICE_TYPE_GPU) |
| 163 | return HSA_STATUS_SUCCESS; |
| 164 | |
| 165 | u64 PointerAddr = 0; |
| 166 | if (!Lookup(Executable: Ctx.Executable, Name: "__csan_watchpoint_table" , Agent, Addr: &PointerAddr)) |
| 167 | return HSA_STATUS_SUCCESS; |
| 168 | Ctx.Found = true; |
| 169 | |
| 170 | // Create an allocation for the table if one does not already exist. |
| 171 | DeviceAllocation *Allocation = FindAllocation(Agent); |
| 172 | if (!Allocation) { |
| 173 | hsa_amd_memory_pool_t Pool; |
| 174 | void *Ptr = nullptr; |
| 175 | if (!Offload::Get().GetMemoryPool(Agent, Pool: &Pool) || |
| 176 | !Offload::Get().Allocate(Pool, Bytes: CSAN_WATCHPOINT_TABLE_BYTES, Out: &Ptr)) { |
| 177 | Ctx.Success = false; |
| 178 | return HSA_STATUS_SUCCESS; |
| 179 | } |
| 180 | |
| 181 | // Fresh pages from MMap are always zero initialized, as required. |
| 182 | void *Zeros = |
| 183 | MmapOrDie(size: CSAN_WATCHPOINT_TABLE_BYTES, mem_type: "CSan watchpoint table" ); |
| 184 | bool Copied = |
| 185 | Copy(Ptr, Zeros, CSAN_WATCHPOINT_TABLE_BYTES) == HSA_STATUS_SUCCESS; |
| 186 | UnmapOrDie(addr: Zeros, size: CSAN_WATCHPOINT_TABLE_BYTES); |
| 187 | if (!Copied) { |
| 188 | Free(Ptr); |
| 189 | Ctx.Success = false; |
| 190 | return HSA_STATUS_SUCCESS; |
| 191 | } |
| 192 | Allocations.push_back(element: {.Agent: Agent, .Ptr: Ptr}); |
| 193 | Allocation = &Allocations.back(); |
| 194 | } |
| 195 | |
| 196 | // Update the pointer on the device to the associated allocation. |
| 197 | if (Copy(reinterpret_cast<void *>(PointerAddr), &Allocation->Ptr, |
| 198 | sizeof(Allocation->Ptr)) != HSA_STATUS_SUCCESS) |
| 199 | Ctx.Success = false; |
| 200 | return HSA_STATUS_SUCCESS; |
| 201 | } |
| 202 | |
| 203 | static void BindWatchpointTable(hsa_executable_t Executable) { |
| 204 | auto Iterate = |
| 205 | HsaFunction<decltype(&hsa_iterate_agents)>(Name: "hsa_iterate_agents" ); |
| 206 | if (!Iterate) |
| 207 | return; |
| 208 | |
| 209 | // Check if we need to set up the watchpoint table. |
| 210 | Lock L(&AllocationMutex); |
| 211 | BindContext Ctx = {.Executable: Executable, .Found: false, .Success: true}; |
| 212 | if (Iterate(BindAgent, &Ctx) != HSA_STATUS_SUCCESS) |
| 213 | Ctx.Success = false; |
| 214 | if (Ctx.Found && !Ctx.Success) { |
| 215 | Report(format: "ERROR: %s: cannot initialize device watchpoint table\n" , |
| 216 | SanitizerToolName); |
| 217 | Die(); |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | static void RetainAllocations() { |
| 222 | Lock L(&AllocationMutex); |
| 223 | ++HsaRefs; |
| 224 | } |
| 225 | |
| 226 | static void ReleaseAllocations() { |
| 227 | Lock L(&AllocationMutex); |
| 228 | if (!HsaRefs || --HsaRefs) |
| 229 | return; |
| 230 | auto Free = HsaFunction<decltype(&hsa_amd_memory_pool_free)>( |
| 231 | Name: "hsa_amd_memory_pool_free" ); |
| 232 | if (Free) |
| 233 | for (uptr I = 0; I < Allocations.size(); ++I) |
| 234 | Free(Allocations[I].Ptr); |
| 235 | Allocations.clear(); |
| 236 | } |
| 237 | |
| 238 | INTERCEPTOR(hsa_status_t, hsa_init, void) { |
| 239 | CSAN_HSA_ENTER(hsa_init); |
| 240 | |
| 241 | hsa_status_t Status = REAL(hsa_init)(); |
| 242 | if (Status != HSA_STATUS_SUCCESS) |
| 243 | return Status; |
| 244 | |
| 245 | if (!Offload::Get().Init()) { |
| 246 | Report(format: "ERROR: %s: cannot initialize HSA offload support\n" , |
| 247 | SanitizerToolName); |
| 248 | Die(); |
| 249 | } |
| 250 | RetainAllocations(); |
| 251 | return Status; |
| 252 | } |
| 253 | |
| 254 | INTERCEPTOR(hsa_status_t, hsa_shut_down, void) { |
| 255 | CSAN_HSA_ENTER(hsa_shut_down); |
| 256 | |
| 257 | ReleaseAllocations(); |
| 258 | Offload::Get().Shutdown(); |
| 259 | return REAL(hsa_shut_down)(); |
| 260 | } |
| 261 | |
| 262 | INTERCEPTOR(hsa_status_t, hsa_executable_freeze, hsa_executable_t Executable, |
| 263 | const char *Options) { |
| 264 | CSAN_HSA_FORWARD(hsa_executable_freeze, Executable, Options); |
| 265 | |
| 266 | hsa_status_t Status = REAL(hsa_executable_freeze)(Executable, Options); |
| 267 | if (Status == HSA_STATUS_SUCCESS) { |
| 268 | BindWatchpointTable(Executable); |
| 269 | Offload::Get().TrackExecutable(Exec: Executable); |
| 270 | } |
| 271 | return Status; |
| 272 | } |
| 273 | |
| 274 | INTERCEPTOR(hsa_status_t, hsa_executable_destroy, hsa_executable_t Executable) { |
| 275 | CSAN_HSA_FORWARD(hsa_executable_destroy, Executable); |
| 276 | |
| 277 | Offload::Get().UntrackExecutable(Exec: Executable); |
| 278 | return REAL(hsa_executable_destroy)(Executable); |
| 279 | } |
| 280 | |
| 281 | extern "C" void __csan_offload_init() { Initialize(); } |
| 282 | |
| 283 | __attribute__((constructor(0))) static void CsanOffloadDynInit() { |
| 284 | __csan_offload_init(); |
| 285 | WarnIfHsaNotInterposed(); |
| 286 | } |
| 287 | |