1//===- AMDGPUArchByHIP.cpp - list AMDGPU installed ----------*- C++ -*-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a tool for detecting name of AMDGPU installed in system
10// using HIP runtime. This tool is used by AMDGPU OpenMP and HIP driver.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/Sequence.h"
16#include "llvm/Support/CommandLine.h"
17#include "llvm/Support/ConvertUTF.h"
18#include "llvm/Support/DynamicLibrary.h"
19#include "llvm/Support/Error.h"
20#include "llvm/Support/FileSystem.h"
21#include "llvm/Support/Path.h"
22#include "llvm/Support/Process.h"
23#include "llvm/Support/Program.h"
24#include "llvm/Support/VersionTuple.h"
25#include "llvm/Support/raw_ostream.h"
26#include <algorithm>
27#include <string>
28#include <vector>
29
30#ifdef _WIN32
31#include <windows.h>
32#endif
33
34using namespace llvm;
35
36// R0600 struct layout (HIP 6.x+)
37typedef struct alignas(8) {
38 char padding[1160];
39 char gcnArchName[256];
40 char padding2[56];
41} hipDeviceProp_tR0600;
42
43// R0000 struct layout (legacy)
44typedef struct alignas(8) {
45 char padding[396];
46 char gcnArchName[256];
47 char padding2[1024];
48} hipDeviceProp_tR0000;
49
50typedef enum {
51 hipSuccess = 0,
52} hipError_t;
53
54typedef hipError_t (*hipGetDeviceCount_t)(int *);
55typedef hipError_t (*hipGetDevicePropertiesR0600_t)(hipDeviceProp_tR0600 *,
56 int);
57typedef hipError_t (*hipGetDevicePropertiesR0000_t)(hipDeviceProp_tR0000 *,
58 int);
59typedef hipError_t (*hipGetDeviceProperties_t)(hipDeviceProp_tR0000 *, int);
60typedef hipError_t (*hipRuntimeGetVersion_t)(int *);
61typedef const char *(*hipGetErrorString_t)(hipError_t);
62
63extern cl::opt<bool> Verbose;
64
65cl::OptionCategory AMDGPUArchByHIPCategory("amdgpu-arch (HIP) options");
66
67enum class HipApiVersion {
68 Auto, // Automatic fallback (R0600 -> R0000 -> unversioned)
69 R0600, // Force R0600 API (HIP 6.x+)
70 R0000, // Force R0000 API (legacy HIP)
71 Unversioned // Force unversioned API (very old HIP)
72};
73
74static cl::opt<HipApiVersion> HipApi(
75 "hip-api-version", cl::desc("Select HIP API version for device properties"),
76 cl::values(clEnumValN(HipApiVersion::Auto, "auto",
77 "Auto-detect (R0600 -> R0000 -> unversioned)"),
78 clEnumValN(HipApiVersion::R0600, "r0600", "Force R0600 API"),
79 clEnumValN(HipApiVersion::R0000, "r0000", "Force R0000 API"),
80 clEnumValN(HipApiVersion::Unversioned, "unversioned",
81 "Force unversioned API")),
82 cl::init(Val: HipApiVersion::Auto), cl::cat(AMDGPUArchByHIPCategory));
83
84#ifdef _WIN32
85static std::vector<std::string> getSearchPaths() {
86 std::vector<std::string> Paths;
87
88 // Get the directory of the current executable
89 if (auto MainExe = sys::fs::getMainExecutable(nullptr, nullptr);
90 !MainExe.empty())
91 Paths.push_back(sys::path::parent_path(MainExe).str());
92
93 // Get the system directory
94 wchar_t SystemDirectory[MAX_PATH];
95 if (GetSystemDirectoryW(SystemDirectory, MAX_PATH) > 0) {
96 std::string Utf8SystemDir;
97 if (convertUTF16ToUTF8String(
98 ArrayRef<UTF16>(reinterpret_cast<const UTF16 *>(SystemDirectory),
99 wcslen(SystemDirectory)),
100 Utf8SystemDir))
101 Paths.push_back(Utf8SystemDir);
102 }
103
104 // Get the Windows directory
105 wchar_t WindowsDirectory[MAX_PATH];
106 if (GetWindowsDirectoryW(WindowsDirectory, MAX_PATH) > 0) {
107 std::string Utf8WindowsDir;
108 if (convertUTF16ToUTF8String(
109 ArrayRef<UTF16>(reinterpret_cast<const UTF16 *>(WindowsDirectory),
110 wcslen(WindowsDirectory)),
111 Utf8WindowsDir))
112 Paths.push_back(Utf8WindowsDir);
113 }
114
115 // Get the current working directory
116 SmallVector<char, 256> CWD;
117 if (sys::fs::current_path(CWD))
118 Paths.push_back(std::string(CWD.begin(), CWD.end()));
119
120 // Get the PATH environment variable
121 if (std::optional<std::string> PathEnv = sys::Process::GetEnv("PATH")) {
122 SmallVector<StringRef, 16> PathList;
123 StringRef(*PathEnv).split(PathList, sys::EnvPathSeparator);
124 for (auto &Path : PathList)
125 Paths.push_back(Path.str());
126 }
127
128 return Paths;
129}
130
131// Custom comparison function for dll name
132static bool compareVersions(StringRef A, StringRef B) {
133 auto ParseVersion = [](StringRef S) -> VersionTuple {
134 StringRef Filename = sys::path::filename(S);
135 size_t Pos = Filename.find_last_of('_');
136 if (Pos == StringRef::npos)
137 return VersionTuple();
138
139 StringRef VerStr = Filename.substr(Pos + 1);
140 size_t DotPos = VerStr.find('.');
141 if (DotPos != StringRef::npos)
142 VerStr = VerStr.substr(0, DotPos);
143
144 VersionTuple Vt;
145 (void)Vt.tryParse(VerStr);
146 return Vt;
147 };
148
149 VersionTuple VtA = ParseVersion(A);
150 VersionTuple VtB = ParseVersion(B);
151 return VtA > VtB;
152}
153#endif
154
155// On Windows, prefer amdhip64_n.dll where n is ROCm major version and greater
156// value of n takes precedence. If amdhip64_n.dll is not found, fall back to
157// amdhip64.dll. The reason is that a normal driver installation only has
158// amdhip64_n.dll but we do not know what n is since this program may be used
159// with a future version of HIP runtime.
160//
161// On Linux, always use default libamdhip64.so.
162static std::pair<std::string, bool> findNewestHIPDLL() {
163#ifdef _WIN32
164 StringRef HipDLLPrefix = "amdhip64_";
165 StringRef HipDLLSuffix = ".dll";
166
167 std::vector<std::string> SearchPaths = getSearchPaths();
168 std::vector<std::string> DLLNames;
169
170 for (const auto &Dir : SearchPaths) {
171 std::error_code EC;
172 for (sys::fs::directory_iterator DirIt(Dir, EC), DirEnd;
173 DirIt != DirEnd && !EC; DirIt.increment(EC)) {
174 StringRef Filename = sys::path::filename(DirIt->path());
175 if (Filename.starts_with(HipDLLPrefix) &&
176 Filename.ends_with(HipDLLSuffix))
177 DLLNames.push_back(sys::path::convert_to_slash(DirIt->path()));
178 }
179 }
180
181 if (DLLNames.empty())
182 return {"amdhip64.dll", true};
183
184 llvm::sort(DLLNames, compareVersions);
185 return {DLLNames[0], false};
186#else
187 // On Linux, fallback to default shared object
188 return {"libamdhip64.so", true};
189#endif
190}
191
192int printGPUsByHIP() {
193 auto [DynamicHIPPath, IsFallback] = findNewestHIPDLL();
194
195 if (Verbose) {
196 if (IsFallback)
197 outs() << "Using default HIP runtime: " << DynamicHIPPath << '\n';
198 else
199 outs() << "Found HIP runtime: " << DynamicHIPPath << '\n';
200 }
201
202 std::string ErrMsg;
203 auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
204 args: llvm::sys::DynamicLibrary::getPermanentLibrary(filename: DynamicHIPPath.c_str(),
205 errMsg: &ErrMsg));
206 if (!DynlibHandle->isValid()) {
207 if (Verbose)
208 llvm::errs() << "Failed to load " << DynamicHIPPath << ": " << ErrMsg
209 << '\n';
210 return 1;
211 }
212
213 if (Verbose)
214 outs() << "Successfully loaded HIP runtime library\n";
215
216#define DYNAMIC_INIT_HIP(SYMBOL) \
217 { \
218 void *SymbolPtr = DynlibHandle->getAddressOfSymbol(#SYMBOL); \
219 if (!SymbolPtr) { \
220 llvm::errs() << "Failed to find symbol " << #SYMBOL << '\n'; \
221 return 1; \
222 } \
223 if (Verbose) \
224 outs() << "Found symbol: " << #SYMBOL << '\n'; \
225 SYMBOL = reinterpret_cast<decltype(SYMBOL)>(SymbolPtr); \
226 }
227
228 hipGetDeviceCount_t hipGetDeviceCount;
229 hipRuntimeGetVersion_t hipRuntimeGetVersion = nullptr;
230 hipGetDevicePropertiesR0600_t hipGetDevicePropertiesR0600 = nullptr;
231 hipGetDevicePropertiesR0000_t hipGetDevicePropertiesR0000 = nullptr;
232 hipGetDeviceProperties_t hipGetDeviceProperties = nullptr;
233 hipGetErrorString_t hipGetErrorString = nullptr;
234
235 DYNAMIC_INIT_HIP(hipGetDeviceCount);
236
237#undef DYNAMIC_INIT_HIP
238
239 auto LoadSymbol = [&](const char *Name, auto &FuncPtr,
240 const char *Desc = "") {
241 void *Sym = DynlibHandle->getAddressOfSymbol(symbolName: Name);
242 if (Sym) {
243 FuncPtr = reinterpret_cast<decltype(FuncPtr)>(Sym);
244 if (Verbose)
245 outs() << "Found symbol: " << Name << (Desc[0] ? " " : "") << Desc
246 << '\n';
247 return true;
248 }
249 return false;
250 };
251
252 LoadSymbol("hipGetErrorString", hipGetErrorString);
253
254 if (LoadSymbol("hipRuntimeGetVersion", hipRuntimeGetVersion)) {
255 int RuntimeVersion = 0;
256 if (hipRuntimeGetVersion(&RuntimeVersion) == hipSuccess) {
257 int Major = RuntimeVersion / 10000000;
258 int Minor = (RuntimeVersion / 100000) % 100;
259 int Patch = RuntimeVersion % 100000;
260 if (Verbose)
261 outs() << "HIP Runtime Version: " << Major << "." << Minor << "."
262 << Patch << '\n';
263 }
264 }
265
266 LoadSymbol("hipGetDevicePropertiesR0600", hipGetDevicePropertiesR0600,
267 "(HIP 6.x+ API)");
268 LoadSymbol("hipGetDevicePropertiesR0000", hipGetDevicePropertiesR0000,
269 "(legacy API)");
270 if (!hipGetDevicePropertiesR0600 && !hipGetDevicePropertiesR0000)
271 LoadSymbol("hipGetDeviceProperties", hipGetDeviceProperties,
272 "(unversioned legacy API)");
273
274 int DeviceCount;
275 if (Verbose)
276 outs() << "Calling hipGetDeviceCount...\n";
277 hipError_t Err = hipGetDeviceCount(&DeviceCount);
278 if (Err != hipSuccess) {
279 llvm::errs() << "Failed to get device count";
280 if (hipGetErrorString) {
281 llvm::errs() << ": " << hipGetErrorString(Err);
282 }
283 llvm::errs() << " (error code: " << Err << ")\n";
284 return 1;
285 }
286
287 if (Verbose)
288 outs() << "Found " << DeviceCount << " device(s)\n";
289
290 auto TryGetProperties = [&](auto *ApiFunc, auto *DummyProp, const char *Name,
291 int DeviceId) -> std::string {
292 if (!ApiFunc)
293 return "";
294
295 if (Verbose)
296 outs() << "Using " << Name << "...\n";
297
298 using PropType = std::remove_pointer_t<decltype(DummyProp)>;
299 PropType Prop;
300 hipError_t Err = ApiFunc(&Prop, DeviceId);
301
302 if (Err == hipSuccess) {
303 if (Verbose) {
304 outs() << Name << " struct: sizeof = " << sizeof(PropType)
305 << " bytes, offsetof(gcnArchName) = "
306 << offsetof(PropType, gcnArchName) << " bytes\n";
307 }
308 return Prop.gcnArchName;
309 }
310
311 if (Verbose)
312 llvm::errs() << Name << " failed (error code: " << Err << ")\n";
313 return "";
314 };
315
316 for (auto I : llvm::seq(Size: DeviceCount)) {
317 if (Verbose)
318 outs() << "Processing device " << I << "...\n";
319
320 std::string ArchName;
321 auto TryR0600 = [&](int Dev) -> bool {
322 if (!hipGetDevicePropertiesR0600)
323 return false;
324 ArchName = TryGetProperties(hipGetDevicePropertiesR0600,
325 (hipDeviceProp_tR0600 *)nullptr,
326 "R0600 API (HIP 6.x+)", Dev);
327 return !ArchName.empty();
328 };
329 auto TryR0000 = [&](int Dev) -> bool {
330 if (!hipGetDevicePropertiesR0000)
331 return false;
332 ArchName = TryGetProperties(hipGetDevicePropertiesR0000,
333 (hipDeviceProp_tR0000 *)nullptr,
334 "R0000 API (legacy HIP)", Dev);
335 return !ArchName.empty();
336 };
337 auto TryUnversioned = [&](int Dev) -> bool {
338 if (!hipGetDeviceProperties)
339 return false;
340 ArchName = TryGetProperties(hipGetDeviceProperties,
341 (hipDeviceProp_tR0000 *)nullptr,
342 "unversioned API (very old HIP)", Dev);
343 return !ArchName.empty();
344 };
345
346 [[maybe_unused]] bool OK;
347 switch (HipApi) {
348 case HipApiVersion::Auto:
349 OK = TryR0600(I) || TryR0000(I) || TryUnversioned(I);
350 break;
351 case HipApiVersion::R0600:
352 OK = TryR0600(I);
353 break;
354 case HipApiVersion::R0000:
355 OK = TryR0000(I);
356 break;
357 case HipApiVersion::Unversioned:
358 OK = TryUnversioned(I);
359 }
360
361 if (ArchName.empty()) {
362 llvm::errs() << "Failed to get device properties for device " << I
363 << " - no APIs available or all failed\n";
364 return 1;
365 }
366
367 if (Verbose)
368 outs() << "Device " << I << " arch name: ";
369 llvm::outs() << ArchName << '\n';
370 }
371
372 return 0;
373}
374