1//===-- Main entry into the loader interface ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This utility is used to launch standard programs onto the GPU in conjunction
10// with the LLVM 'libc' project. It is designed to mimic a standard emulator
11// workflow, allowing for unit tests to be run on the GPU directly.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm-gpu-loader.h"
16
17#include "llvm/BinaryFormat/Magic.h"
18#include "llvm/Object/ELF.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/Error.h"
21#include "llvm/Support/MemoryBuffer.h"
22#include "llvm/Support/Signals.h"
23#include "llvm/Support/WithColor.h"
24#include "llvm/TargetParser/Triple.h"
25
26#include <cstdio>
27#include <cstdlib>
28#include <cstring>
29#include <string>
30
31using namespace llvm;
32
33static cl::OptionCategory LoaderCategory("loader options");
34
35static cl::opt<unsigned>
36 ThreadsX("threads-x", cl::desc("Number of threads in the 'x' dimension"),
37 cl::init(Val: 1), cl::cat(LoaderCategory));
38static cl::opt<unsigned>
39 ThreadsY("threads-y", cl::desc("Number of threads in the 'y' dimension"),
40 cl::init(Val: 1), cl::cat(LoaderCategory));
41static cl::opt<unsigned>
42 ThreadsZ("threads-z", cl::desc("Number of threads in the 'z' dimension"),
43 cl::init(Val: 1), cl::cat(LoaderCategory));
44static cl::alias threads("threads", cl::aliasopt(ThreadsX),
45 cl::desc("Alias for --threads-x"),
46 cl::cat(LoaderCategory));
47
48static cl::opt<unsigned>
49 BlocksX("blocks-x", cl::desc("Number of blocks in the 'x' dimension"),
50 cl::init(Val: 1), cl::cat(LoaderCategory));
51static cl::opt<unsigned>
52 BlocksY("blocks-y", cl::desc("Number of blocks in the 'y' dimension"),
53 cl::init(Val: 1), cl::cat(LoaderCategory));
54static cl::opt<unsigned>
55 BlocksZ("blocks-z", cl::desc("Number of blocks in the 'z' dimension"),
56 cl::init(Val: 1), cl::cat(LoaderCategory));
57static cl::alias Blocks("blocks", cl::aliasopt(BlocksX),
58 cl::desc("Alias for --blocks-x"),
59 cl::cat(LoaderCategory));
60
61static cl::list<std::string> Kernels(
62 "kernel", cl::value_desc("name"),
63 cl::desc("Launch '<name>(void)' instead of the 'main' entry point."),
64 cl::cat(LoaderCategory));
65
66static cl::opt<std::string> File(cl::Positional, cl::Required,
67 cl::desc("<gpu executable>"),
68 cl::cat(LoaderCategory));
69static cl::list<std::string> Args(cl::ConsumeAfter,
70 cl::desc("<program arguments>..."),
71 cl::cat(LoaderCategory));
72
73[[noreturn]] static void handleError(Error E) {
74 outs().flush();
75 logAllUnhandledErrors(E: std::move(E), OS&: WithColor::error(OS&: errs(), Prefix: "loader"));
76 exit(EXIT_FAILURE);
77}
78
79[[noreturn]] static void handleError(ol_result_t Err, unsigned Line) {
80 fprintf(stderr, format: "%s:%d %s\n", __FILE__, Line, Err->Details);
81 exit(EXIT_FAILURE);
82}
83
84#define OFFLOAD_ERR(X) \
85 if (ol_result_t Err = X) \
86 handleError(Err, __LINE__);
87
88static void *copyArgumentVector(int Argc, const char **Argv,
89 ol_context_handle_t Context,
90 ol_device_handle_t Device) {
91 size_t ArgSize = sizeof(char *) * (Argc + 1);
92 size_t StringLen = 0;
93 for (int i = 0; i < Argc; ++i)
94 StringLen += strlen(s: Argv[i]) + 1;
95
96 // We allocate enough space for a null terminated array and all the strings.
97 void *DevArgv;
98 OFFLOAD_ERR(olMemAllocHost(Context, Device, ArgSize + StringLen, &DevArgv));
99 if (!DevArgv)
100 handleError(
101 E: createStringError(Fmt: "Failed to allocate memory for environment."));
102
103 // Store the strings linerally in the same memory buffer.
104 void *DevString = reinterpret_cast<uint8_t *>(DevArgv) + ArgSize;
105 for (int i = 0; i < Argc; ++i) {
106 size_t size = strlen(s: Argv[i]) + 1;
107 std::memcpy(dest: DevString, src: Argv[i], n: size);
108 static_cast<void **>(DevArgv)[i] = DevString;
109 DevString = reinterpret_cast<uint8_t *>(DevString) + size;
110 }
111
112 // Ensure the vector is null terminated.
113 reinterpret_cast<void **>(DevArgv)[Argc] = nullptr;
114 return DevArgv;
115}
116
117void *copyEnvironment(const char **Envp, ol_context_handle_t Context,
118 ol_device_handle_t Device) {
119 int Envc = 0;
120 for (const char **Env = Envp; *Env != 0; ++Env)
121 ++Envc;
122
123 return copyArgumentVector(Argc: Envc, Argv: Envp, Context, Device);
124}
125
126ol_device_handle_t findDevice(MemoryBufferRef Binary) {
127 ol_device_handle_t Device = nullptr;
128 std::tuple Data = std::make_tuple(args: &Device, args: &Binary);
129 OFFLOAD_ERR(olIterateDevices(
130 [](ol_device_handle_t Device, void *UserData) {
131 auto &[Output, Binary] = *reinterpret_cast<decltype(Data) *>(UserData);
132 bool IsValid = false;
133 OFFLOAD_ERR(olIsValidBinary(Device, Binary->getBufferStart(),
134 Binary->getBufferSize(), &IsValid));
135 if (!IsValid)
136 return true;
137
138 *Output = Device;
139 return false;
140 },
141 &Data));
142 return Device;
143}
144
145ol_device_handle_t getHostDevice() {
146 ol_device_handle_t Device;
147 OFFLOAD_ERR(olIterateDevices(
148 [](ol_device_handle_t Device, void *UserData) {
149 ol_platform_handle_t Platform;
150 olGetDeviceInfo(Device, OL_DEVICE_INFO_PLATFORM, sizeof(Platform),
151 &Platform);
152 ol_platform_backend_t Backend;
153 olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND, sizeof(Backend),
154 &Backend);
155
156 auto &Output = *reinterpret_cast<decltype(Device) *>(UserData);
157 if (Backend == OL_PLATFORM_BACKEND_HOST) {
158 Output = Device;
159 return false;
160 }
161 return true;
162 },
163 &Device));
164 return Device;
165}
166
167template <typename... Args>
168void launchKernel(ol_queue_handle_t Queue, ol_device_handle_t Device,
169 ol_program_handle_t Program, const char *Name,
170 ol_kernel_launch_size_args_t LaunchArgs,
171 Args &...KernelArgs) {
172 ol_symbol_handle_t Kernel;
173 OFFLOAD_ERR(olGetSymbol(Program, Name, OL_SYMBOL_KIND_KERNEL, &Kernel));
174
175 if constexpr (sizeof...(Args) == 0) {
176 OFFLOAD_ERR(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr, 0,
177 nullptr, nullptr));
178 } else {
179 void *ArgPtrs[] = {static_cast<void *>(&KernelArgs)...};
180 size_t ArgSizes[] = {sizeof(KernelArgs)...};
181 OFFLOAD_ERR(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
182 sizeof...(Args), ArgPtrs, ArgSizes));
183 }
184}
185
186int main(int argc, const char **argv, const char **envp) {
187 sys::PrintStackTraceOnErrorSignal(Argv0: argv[0]);
188 cl::HideUnrelatedOptions(Category&: LoaderCategory);
189 cl::ParseCommandLineOptions(
190 argc, argv,
191 Overview: "A utility used to launch unit tests built for a GPU target. This is\n"
192 "intended to provide an interface similar to cross-compiling "
193 "emulators\n");
194
195 if (Error Err = loadLLVMOffload())
196 handleError(E: std::move(Err));
197
198 ErrorOr<std::unique_ptr<MemoryBuffer>> ImageOrErr =
199 MemoryBuffer::getFileOrSTDIN(Filename: File);
200 if (std::error_code EC = ImageOrErr.getError())
201 handleError(E: errorCodeToError(EC));
202 MemoryBufferRef Image = **ImageOrErr;
203
204 ol_platform_backend_t Backend = OL_PLATFORM_BACKEND_UNKNOWN;
205 ol_init_args_t InitArgs = OL_INIT_ARGS_INIT;
206
207 file_magic Magic = identify_magic(magic: Image.getBuffer());
208 if (Magic >= file_magic::elf && Magic <= file_magic::elf_core) {
209 Expected<object::ELFFile<object::ELF64LE>> ElfOrErr =
210 object::ELFFile<object::ELF64LE>::create(Object: Image.getBuffer());
211 if (!ElfOrErr)
212 handleError(E: ElfOrErr.takeError());
213
214 switch (ElfOrErr->getHeader().e_machine) {
215 case ELF::EM_AMDGPU:
216 Backend = OL_PLATFORM_BACKEND_AMDGPU;
217 break;
218 case ELF::EM_CUDA:
219 Backend = OL_PLATFORM_BACKEND_CUDA;
220 break;
221 default:
222 handleError(E: createStringError(
223 Fmt: "unhandled ELF architecture: %s",
224 Vals: ELF::convertEMachineToArchName(EMachine: ElfOrErr->getHeader().e_machine)
225 .data()));
226 }
227 }
228
229 if (Backend != OL_PLATFORM_BACKEND_UNKNOWN) {
230 InitArgs.NumPlatforms = 1;
231 InitArgs.Platforms = &Backend;
232 }
233
234 SmallVector<const char *> NewArgv = {File.c_str()};
235 llvm::transform(Range&: Args, d_first: std::back_inserter(x&: NewArgv),
236 F: [](const std::string &Arg) { return Arg.c_str(); });
237
238 OFFLOAD_ERR(olInit(&InitArgs));
239 ol_device_handle_t Device = findDevice(Binary: Image);
240 if (!Device)
241 handleError(E: createStringError(Fmt: "No compatible device was found"));
242 ol_device_handle_t Host = getHostDevice();
243 assert(Host && "Host device should always be present");
244
245 ol_context_handle_t Context;
246 OFFLOAD_ERR(olCreateContext(1, &Device, &Context));
247
248 ol_program_handle_t Program;
249 OFFLOAD_ERR(olCreateProgram(Context, Device, Image.getBufferStart(),
250 Image.getBufferSize(), &Program));
251
252 ol_queue_handle_t Queue;
253 OFFLOAD_ERR(olCreateQueue(Context, Device, &Queue));
254
255 int DevArgc = static_cast<int>(NewArgv.size());
256 void *DevArgv =
257 copyArgumentVector(Argc: NewArgv.size(), Argv: NewArgv.begin(), Context, Device);
258 void *DevEnvp = copyEnvironment(Envp: envp, Context, Device);
259
260 void *DevRet;
261 int Zero = 0;
262 OFFLOAD_ERR(
263 olMemAlloc(Context, Device, OL_ALLOC_TYPE_DEVICE, sizeof(int), &DevRet));
264 OFFLOAD_ERR(olMemcpy(Queue, DevRet, Device, &Zero, Host, sizeof(int)));
265
266 uint32_t Dims = (BlocksZ > 1) ? 3 : (BlocksY > 1) ? 2 : 1;
267 ol_kernel_launch_size_args_t StartLaunch{.Dimensions: Dims,
268 .NumGroups: {.x: BlocksX, .y: BlocksY, .z: BlocksZ},
269 .GroupSize: {.x: ThreadsX, .y: ThreadsY, .z: ThreadsZ},
270 /*SharedMemBytes=*/.DynSharedMemory: 0};
271 if (!Kernels.empty()) {
272 // Launch the user-specified kernels in order. These must take no arguments.
273 for (const std::string &Kernel : Kernels)
274 launchKernel(Queue, Device, Program, Name: Kernel.c_str(), LaunchArgs: StartLaunch);
275 } else {
276 // The '_begin' and '_end' kernels perform libc startup and teardown. Global
277 // constructors and destructors are handled automatically by the runtime.
278 ol_kernel_launch_size_args_t BeginLaunch{.Dimensions: 1, .NumGroups: {.x: 1, .y: 1, .z: 1}, .GroupSize: {.x: 1, .y: 1, .z: 1}, .DynSharedMemory: 0};
279 launchKernel(Queue, Device, Program, Name: "_begin", LaunchArgs: BeginLaunch, KernelArgs&: DevArgc,
280 KernelArgs&: DevArgv, KernelArgs&: DevEnvp);
281 OFFLOAD_ERR(olSyncQueue(Queue));
282
283 launchKernel(Queue, Device, Program, Name: "_start", LaunchArgs: StartLaunch, KernelArgs&: DevArgc,
284 KernelArgs&: DevArgv, KernelArgs&: DevEnvp, KernelArgs&: DevRet);
285
286 ol_kernel_launch_size_args_t EndLaunch{.Dimensions: 1, .NumGroups: {.x: 1, .y: 1, .z: 1}, .GroupSize: {.x: 1, .y: 1, .z: 1}, .DynSharedMemory: 0};
287 launchKernel(Queue, Device, Program, Name: "_end", LaunchArgs: EndLaunch);
288 }
289
290 int Ret;
291 OFFLOAD_ERR(olMemcpy(Queue, &Ret, Host, DevRet, Device, sizeof(int)));
292 OFFLOAD_ERR(olSyncQueue(Queue));
293
294 OFFLOAD_ERR(olMemFree(Context, DevRet));
295 OFFLOAD_ERR(olMemFree(Context, DevArgv));
296 OFFLOAD_ERR(olMemFree(Context, DevEnvp));
297 OFFLOAD_ERR(olDestroyQueue(Queue));
298 OFFLOAD_ERR(olDestroyContext(Context));
299 OFFLOAD_ERR(olDestroyProgram(Program));
300 OFFLOAD_ERR(olShutDown());
301
302 return Ret;
303}
304