1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Offloading/OffloadWrapper.h"
10#include "llvm/ADT/ArrayRef.h"
11#include "llvm/ADT/SmallVector.h"
12#include "llvm/ADT/StringRef.h"
13#include "llvm/ADT/Twine.h"
14#include "llvm/BinaryFormat/Magic.h"
15#include "llvm/Frontend/Offloading/Utility.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/LLVMContext.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Type.h"
23#include "llvm/Object/OffloadBinary.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/LineIterator.h"
27#include "llvm/Support/MemoryBufferRef.h"
28#include "llvm/TargetParser/Triple.h"
29#include "llvm/Transforms/Utils/ModuleUtils.h"
30
31#include <memory>
32#include <utility>
33
34using namespace llvm;
35using namespace llvm::object;
36using namespace llvm::offloading;
37
38namespace {
39/// Magic number that begins the section containing the CUDA fatbinary.
40constexpr unsigned CudaFatMagic = 0x466243b1;
41constexpr unsigned HIPFatMagic = 0x48495046;
42
43IntegerType *getSizeTTy(Module &M) {
44 return M.getDataLayout().getIntPtrType(C&: M.getContext());
45}
46
47/// Returns the appropriate startup section for registration functions.
48/// Mach-O uses "__TEXT,__StaticInit"; ELF/COFF use ".text.startup".
49StringRef getStartupSection(const Triple &T) {
50 return T.isOSBinFormatMachO() ? "__TEXT,__StaticInit" : ".text.startup";
51}
52
53// struct __tgt_device_image {
54// void *ImageStart;
55// void *ImageEnd;
56// __tgt_offload_entry *EntriesBegin;
57// __tgt_offload_entry *EntriesEnd;
58// };
59StructType *getDeviceImageTy(Module &M) {
60 LLVMContext &C = M.getContext();
61 StructType *ImageTy = StructType::getTypeByName(C, Name: "__tgt_device_image");
62 if (!ImageTy)
63 ImageTy =
64 StructType::create(Name: "__tgt_device_image", elt1: PointerType::getUnqual(C),
65 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C),
66 elts: PointerType::getUnqual(C));
67 return ImageTy;
68}
69
70PointerType *getDeviceImagePtrTy(Module &M) {
71 return PointerType::getUnqual(C&: M.getContext());
72}
73
74// struct __tgt_bin_desc {
75// int32_t NumDeviceImages;
76// __tgt_device_image *DeviceImages;
77// __tgt_offload_entry *HostEntriesBegin;
78// __tgt_offload_entry *HostEntriesEnd;
79// };
80StructType *getBinDescTy(Module &M) {
81 LLVMContext &C = M.getContext();
82 StructType *DescTy = StructType::getTypeByName(C, Name: "__tgt_bin_desc");
83 if (!DescTy)
84 DescTy = StructType::create(
85 Name: "__tgt_bin_desc", elt1: Type::getInt32Ty(C), elts: getDeviceImagePtrTy(M),
86 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
87 return DescTy;
88}
89
90PointerType *getBinDescPtrTy(Module &M) {
91 return PointerType::getUnqual(C&: M.getContext());
92}
93
94/// Creates binary descriptor for the given device images. Binary descriptor
95/// is an object that is passed to the offloading runtime at program startup
96/// and it describes all device images available in the executable or shared
97/// library. It is defined as follows
98///
99/// __attribute__((visibility("hidden")))
100/// extern __tgt_offload_entry *__start_llvm_offload_entries;
101/// __attribute__((visibility("hidden")))
102/// extern __tgt_offload_entry *__stop_llvm_offload_entries;
103///
104/// static const char Image0[] = { <Bufs.front() contents> };
105/// ...
106/// static const char ImageN[] = { <Bufs.back() contents> };
107///
108/// static const __tgt_device_image Images[] = {
109/// {
110/// Image0, /*ImageStart*/
111/// Image0 + sizeof(Image0), /*ImageEnd*/
112/// __start_llvm_offload_entries, /*EntriesBegin*/
113/// __stop_llvm_offload_entries /*EntriesEnd*/
114/// },
115/// ...
116/// {
117/// ImageN, /*ImageStart*/
118/// ImageN + sizeof(ImageN), /*ImageEnd*/
119/// __start_llvm_offload_entries, /*EntriesBegin*/
120/// __stop_llvm_offload_entries /*EntriesEnd*/
121/// }
122/// };
123///
124/// static const __tgt_bin_desc BinDesc = {
125/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
126/// Images, /*DeviceImages*/
127/// __start_llvm_offload_entries, /*HostEntriesBegin*/
128/// __stop_llvm_offload_entries /*HostEntriesEnd*/
129/// };
130///
131/// Global variable that represents BinDesc is returned.
132GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
133 EntryArrayTy EntryArray, StringRef Suffix,
134 bool Relocatable) {
135 LLVMContext &C = M.getContext();
136 auto [EntriesB, EntriesE] = EntryArray;
137
138 auto *Zero = ConstantInt::get(Ty: getSizeTTy(M), V: 0u);
139
140 // Create initializer for the images array.
141 SmallVector<Constant *, 4u> ImagesInits;
142 ImagesInits.reserve(N: Bufs.size());
143 for (ArrayRef<char> Buf : Bufs) {
144 // We embed the full offloading entry so the binary utilities can parse it.
145 auto *Data = ConstantDataArray::get(Context&: C, Elts: Buf);
146 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
147 GlobalVariable::InternalLinkage, Data,
148 ".omp_offloading.device_image" + Suffix);
149 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
150 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
151 : ".llvm.offloading");
152 Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
153
154 StringRef Binary(Buf.data(), Buf.size());
155
156 uint64_t BeginOffset = 0;
157 uint64_t EndOffset = Binary.size();
158
159 // Optionally use an offload binary for its offload dumping support.
160 // The device image struct contains the pointer to the beginning and end of
161 // the image stored inside of the offload binary. There should only be one
162 // of these for each buffer so we parse it out manually.
163 if (identify_magic(magic: Binary) == file_magic::offload_binary) {
164 const auto *Header =
165 reinterpret_cast<const object::OffloadBinary::Header *>(
166 Binary.bytes_begin());
167 const auto *Entry =
168 reinterpret_cast<const object::OffloadBinary::Entry *>(
169 Binary.bytes_begin() + Header->EntriesOffset);
170 BeginOffset = Entry->ImageOffset;
171 EndOffset = Entry->ImageOffset + Entry->ImageSize;
172 }
173
174 auto *Begin = ConstantInt::get(Ty: getSizeTTy(M), V: BeginOffset);
175 auto *Size = ConstantInt::get(Ty: getSizeTTy(M), V: EndOffset);
176 Constant *ZeroBegin[] = {Zero, Begin};
177 Constant *ZeroSize[] = {Zero, Size};
178
179 auto *ImageB =
180 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroBegin);
181 auto *ImageE =
182 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroSize);
183
184 ImagesInits.push_back(Elt: ConstantStruct::get(T: getDeviceImageTy(M), Vs: ImageB,
185 Vs: ImageE, Vs: EntriesB, Vs: EntriesE));
186 }
187
188 // Then create images array.
189 auto *ImagesData = ConstantArray::get(
190 T: ArrayType::get(ElementType: getDeviceImageTy(M), NumElements: ImagesInits.size()), V: ImagesInits);
191
192 auto *Images =
193 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
194 GlobalValue::InternalLinkage, ImagesData,
195 ".omp_offloading.device_images" + Suffix);
196 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
197
198 // And finally create the binary descriptor object.
199 auto *DescInit = ConstantStruct::get(
200 T: getBinDescTy(M),
201 Vs: ConstantInt::get(Ty: Type::getInt32Ty(C), V: ImagesInits.size()), Vs: Images,
202 Vs: EntriesB, Vs: EntriesE);
203
204 return new GlobalVariable(M, DescInit->getType(), /*isConstant=*/true,
205 GlobalValue::InternalLinkage, DescInit,
206 ".omp_offloading.descriptor" + Suffix);
207}
208
209Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
210 StringRef Suffix) {
211 LLVMContext &C = M.getContext();
212 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
213 auto *Func =
214 Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
215 N: ".omp_offloading.descriptor_unreg" + Suffix, M: &M);
216 Func->setSection(getStartupSection(T: M.getTargetTriple()));
217
218 // Get __tgt_unregister_lib function declaration.
219 auto *UnRegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
220 /*isVarArg*/ false);
221 FunctionCallee UnRegFuncC =
222 M.getOrInsertFunction(Name: "__tgt_unregister_lib", T: UnRegFuncTy);
223
224 // Construct function body
225 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
226 Builder.CreateCall(Callee: UnRegFuncC, Args: BinDesc);
227 Builder.CreateRetVoid();
228
229 return Func;
230}
231
232void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
233 StringRef Suffix) {
234 LLVMContext &C = M.getContext();
235 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
236 auto *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
237 N: ".omp_offloading.descriptor_reg" + Suffix, M: &M);
238 Func->setSection(getStartupSection(T: M.getTargetTriple()));
239
240 // Get __tgt_register_lib function declaration.
241 auto *RegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
242 /*isVarArg*/ false);
243 FunctionCallee RegFuncC =
244 M.getOrInsertFunction(Name: "__tgt_register_lib", T: RegFuncTy);
245
246 auto *AtExitTy = FunctionType::get(
247 Result: Type::getInt32Ty(C), Params: PointerType::getUnqual(C), /*isVarArg=*/false);
248 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
249
250 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
251
252 // Construct function body
253 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
254
255 Builder.CreateCall(Callee: RegFuncC, Args: BinDesc);
256
257 // Register the destructors with 'atexit'. This is expected by the CUDA
258 // runtime and ensures that we clean up before dynamic objects are destroyed.
259 // This needs to be done after plugin initialization to ensure that it is
260 // called before the plugin runtime is destroyed.
261 Builder.CreateCall(Callee: AtExit, Args: UnregFunc);
262 Builder.CreateRetVoid();
263
264 // Add this function to constructors.
265 appendToGlobalCtors(M, F: Func, /*Priority=*/101);
266}
267
268// struct fatbin_wrapper {
269// int32_t magic;
270// int32_t version;
271// void *image;
272// void *reserved;
273//};
274StructType *getFatbinWrapperTy(Module &M) {
275 LLVMContext &C = M.getContext();
276 StructType *FatbinTy = StructType::getTypeByName(C, Name: "fatbin_wrapper");
277 if (!FatbinTy)
278 FatbinTy = StructType::create(
279 Name: "fatbin_wrapper", elt1: Type::getInt32Ty(C), elts: Type::getInt32Ty(C),
280 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
281 return FatbinTy;
282}
283
284/// Embed the image \p Image into the module \p M so it can be found by the
285/// runtime.
286GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
287 StringRef Suffix) {
288 LLVMContext &C = M.getContext();
289 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
290 const llvm::Triple &Triple = M.getTargetTriple();
291
292 // Create the global string containing the fatbinary.
293 StringRef FatbinConstantSection =
294 IsHIP ? (Triple.isMacOSX() ? "__HIP,__hip_fatbin" : ".hip_fatbin")
295 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
296 auto *Data = ConstantDataArray::get(Context&: C, Elts: Image);
297 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
298 GlobalVariable::InternalLinkage, Data,
299 ".fatbin_image" + Suffix);
300 Fatbin->setSection(FatbinConstantSection);
301
302 // Create the fatbinary wrapper
303 StringRef FatbinWrapperSection =
304 IsHIP ? (Triple.isMacOSX() ? "__HIP,__fatbin" : ".hipFatBinSegment")
305 : (Triple.isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment");
306 Constant *FatbinWrapper[] = {
307 ConstantInt::get(Ty: Type::getInt32Ty(C), V: IsHIP ? HIPFatMagic : CudaFatMagic),
308 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1),
309 ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Fatbin, Ty: Int8PtrTy),
310 ConstantPointerNull::get(T: PointerType::getUnqual(C))};
311
312 Constant *FatbinInitializer =
313 ConstantStruct::get(T: getFatbinWrapperTy(M), V: FatbinWrapper);
314
315 auto *FatbinDesc =
316 new GlobalVariable(M, getFatbinWrapperTy(M),
317 /*isConstant*/ true, GlobalValue::InternalLinkage,
318 FatbinInitializer, ".fatbin_wrapper" + Suffix);
319 FatbinDesc->setSection(FatbinWrapperSection);
320 FatbinDesc->setAlignment(Align(8));
321 FatbinDesc->setNoSanitizeMetadata();
322
323 return FatbinDesc;
324}
325
326/// Create the register globals function. We will iterate all of the offloading
327/// entries stored at the begin / end symbols and register them according to
328/// their type. This creates the following function in IR:
329///
330/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
331/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
332///
333/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
334/// void *, void *, void *, void *, int *);
335/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
336/// int64_t, int32_t, int32_t);
337///
338/// void __cudaRegisterTest(void **fatbinHandle) {
339/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
340/// entry != &__stop_cuda_offloading_entries; ++entry) {
341/// if (entry->Kind != OFK_CUDA)
342/// continue
343///
344/// if (!entry->Size)
345/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
346/// entry->name, -1, 0, 0, 0, 0, 0);
347/// else
348/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
349/// 0, entry->size, 0, 0);
350/// }
351/// }
352Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
353 EntryArrayTy EntryArray,
354 StringRef Suffix,
355 bool EmitSurfacesAndTextures) {
356 LLVMContext &C = M.getContext();
357 auto [EntriesB, EntriesE] = EntryArray;
358
359 // Get the __cudaRegisterFunction function declaration.
360 PointerType *Int8PtrTy = PointerType::get(C, AddressSpace: 0);
361 PointerType *Int8PtrPtrTy = PointerType::get(C, AddressSpace: 0);
362 PointerType *Int32PtrTy = PointerType::get(C, AddressSpace: 0);
363 auto *RegFuncTy = FunctionType::get(
364 Result: Type::getInt32Ty(C),
365 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
366 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
367 /*isVarArg*/ false);
368 FunctionCallee RegFunc = M.getOrInsertFunction(
369 Name: IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", T: RegFuncTy);
370
371 // Get the __cudaRegisterVar function declaration.
372 auto *RegVarTy = FunctionType::get(
373 Result: Type::getVoidTy(C),
374 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
375 getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},
376 /*isVarArg*/ false);
377 FunctionCallee RegVar = M.getOrInsertFunction(
378 Name: IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", T: RegVarTy);
379
380 // Get the __cudaRegisterSurface function declaration.
381 FunctionType *RegManagedVarTy =
382 FunctionType::get(Result: Type::getVoidTy(C),
383 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
384 getSizeTTy(M), Type::getInt32Ty(C)},
385 /*isVarArg=*/false);
386 FunctionCallee RegManagedVar = M.getOrInsertFunction(
387 Name: IsHIP ? "__hipRegisterManagedVar" : "__cudaRegisterManagedVar",
388 T: RegManagedVarTy);
389
390 // Get the __cudaRegisterSurface function declaration.
391 FunctionType *RegSurfaceTy =
392 FunctionType::get(Result: Type::getVoidTy(C),
393 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
394 Type::getInt32Ty(C), Type::getInt32Ty(C)},
395 /*isVarArg=*/false);
396 FunctionCallee RegSurface = M.getOrInsertFunction(
397 Name: IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", T: RegSurfaceTy);
398
399 // Get the __cudaRegisterTexture function declaration.
400 FunctionType *RegTextureTy = FunctionType::get(
401 Result: Type::getVoidTy(C),
402 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
403 Type::getInt32Ty(C), Type::getInt32Ty(C)},
404 /*isVarArg=*/false);
405 FunctionCallee RegTexture = M.getOrInsertFunction(
406 Name: IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", T: RegTextureTy);
407
408 auto *RegGlobalsTy = FunctionType::get(Result: Type::getVoidTy(C), Params: Int8PtrPtrTy,
409 /*isVarArg*/ false);
410 auto *RegGlobalsFn =
411 Function::Create(Ty: RegGlobalsTy, Linkage: GlobalValue::InternalLinkage,
412 N: IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", M: &M);
413 RegGlobalsFn->setSection(getStartupSection(T: M.getTargetTriple()));
414
415 // Create the loop to register all the entries.
416 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: RegGlobalsFn));
417 auto *EntryBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: RegGlobalsFn);
418 auto *IfKindBB = BasicBlock::Create(Context&: C, Name: "if.kind", Parent: RegGlobalsFn);
419 auto *IfThenBB = BasicBlock::Create(Context&: C, Name: "if.then", Parent: RegGlobalsFn);
420 auto *IfElseBB = BasicBlock::Create(Context&: C, Name: "if.else", Parent: RegGlobalsFn);
421 auto *SwGlobalBB = BasicBlock::Create(Context&: C, Name: "sw.global", Parent: RegGlobalsFn);
422 auto *SwManagedBB = BasicBlock::Create(Context&: C, Name: "sw.managed", Parent: RegGlobalsFn);
423 auto *SwSurfaceBB = BasicBlock::Create(Context&: C, Name: "sw.surface", Parent: RegGlobalsFn);
424 auto *SwTextureBB = BasicBlock::Create(Context&: C, Name: "sw.texture", Parent: RegGlobalsFn);
425 auto *IfEndBB = BasicBlock::Create(Context&: C, Name: "if.end", Parent: RegGlobalsFn);
426 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: RegGlobalsFn);
427
428 auto *EntryCmp = Builder.CreateICmpNE(LHS: EntriesB, RHS: EntriesE);
429 Builder.CreateCondBr(Cond: EntryCmp, True: EntryBB, False: ExitBB);
430 Builder.SetInsertPoint(EntryBB);
431 auto *Entry = Builder.CreatePHI(Ty: PointerType::getUnqual(C), NumReservedValues: 2, Name: "entry");
432 auto *AddrPtr =
433 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
434 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
435 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4)});
436 auto *Addr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AddrPtr, Name: "addr");
437 auto *AuxAddrPtr =
438 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
439 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
440 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 8)});
441 auto *AuxAddr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AuxAddrPtr, Name: "aux_addr");
442 auto *KindPtr =
443 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
444 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
445 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 2)});
446 auto *Kind = Builder.CreateLoad(Ty: Type::getInt16Ty(C), Ptr: KindPtr, Name: "kind");
447 auto *NamePtr =
448 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
449 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
450 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5)});
451 auto *Name = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: NamePtr, Name: "name");
452 auto *SizePtr =
453 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
454 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
455 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 6)});
456 auto *Size = Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: SizePtr, Name: "size");
457 auto *FlagsPtr =
458 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
459 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
460 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3)});
461 auto *Flags = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: FlagsPtr, Name: "flags");
462 auto *DataPtr =
463 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
464 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
465 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 7)});
466 auto *Data = Builder.CreateTrunc(
467 V: Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: DataPtr, Name: "data"),
468 DestTy: Type::getInt32Ty(C));
469 auto *Type = Builder.CreateAnd(
470 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0x7), Name: "type");
471
472 // Extract the flags stored in the bit-field and convert them to C booleans.
473 auto *ExternBit = Builder.CreateAnd(
474 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
475 V: llvm::offloading::OffloadGlobalExtern));
476 auto *Extern = Builder.CreateLShr(
477 LHS: ExternBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3), Name: "extern");
478 auto *ConstantBit = Builder.CreateAnd(
479 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
480 V: llvm::offloading::OffloadGlobalConstant));
481 auto *Const = Builder.CreateLShr(
482 LHS: ConstantBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4), Name: "constant");
483 auto *NormalizedBit = Builder.CreateAnd(
484 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
485 V: llvm::offloading::OffloadGlobalNormalized));
486 auto *Normalized = Builder.CreateLShr(
487 LHS: NormalizedBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5), Name: "normalized");
488 auto *KindCond = Builder.CreateICmpEQ(
489 LHS: Kind, RHS: ConstantInt::get(Ty: Type::getInt16Ty(C),
490 V: IsHIP ? object::OffloadKind::OFK_HIP
491 : object::OffloadKind::OFK_Cuda));
492 Builder.CreateCondBr(Cond: KindCond, True: IfKindBB, False: IfEndBB);
493 Builder.SetInsertPoint(IfKindBB);
494 auto *FnCond = Builder.CreateICmpEQ(
495 LHS: Size, RHS: ConstantInt::getNullValue(Ty: Type::getInt64Ty(C)));
496 Builder.CreateCondBr(Cond: FnCond, True: IfThenBB, False: IfElseBB);
497
498 // Create kernel registration code.
499 Builder.SetInsertPoint(IfThenBB);
500 Builder.CreateCall(
501 Callee: RegFunc,
502 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
503 ConstantInt::getAllOnesValue(Ty: Type::getInt32Ty(C)),
504 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
505 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
506 ConstantPointerNull::get(T: Int32PtrTy)});
507 Builder.CreateBr(Dest: IfEndBB);
508 Builder.SetInsertPoint(IfElseBB);
509
510 auto *Switch = Builder.CreateSwitch(V: Type, Dest: IfEndBB);
511 // Create global variable registration code.
512 Builder.SetInsertPoint(SwGlobalBB);
513 Builder.CreateCall(Callee: RegVar,
514 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
515 Const, ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)});
516 Builder.CreateBr(Dest: IfEndBB);
517 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalEntry),
518 Dest: SwGlobalBB);
519
520 // Create managed variable registration code.
521 Builder.SetInsertPoint(SwManagedBB);
522 Builder.CreateCall(Callee: RegManagedVar, Args: {RegGlobalsFn->arg_begin(), AuxAddr, Addr,
523 Name, Size, Data});
524 Builder.CreateBr(Dest: IfEndBB);
525 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalManagedEntry),
526 Dest: SwManagedBB);
527 // Create surface variable registration code.
528 Builder.SetInsertPoint(SwSurfaceBB);
529 if (EmitSurfacesAndTextures)
530 Builder.CreateCall(Callee: RegSurface, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
531 Data, Extern});
532 Builder.CreateBr(Dest: IfEndBB);
533 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalSurfaceEntry),
534 Dest: SwSurfaceBB);
535
536 // Create texture variable registration code.
537 Builder.SetInsertPoint(SwTextureBB);
538 if (EmitSurfacesAndTextures)
539 Builder.CreateCall(Callee: RegTexture, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
540 Data, Normalized, Extern});
541 Builder.CreateBr(Dest: IfEndBB);
542 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalTextureEntry),
543 Dest: SwTextureBB);
544
545 Builder.SetInsertPoint(IfEndBB);
546 auto *NewEntry = Builder.CreateInBoundsGEP(
547 Ty: offloading::getEntryTy(M), Ptr: Entry, IdxList: ConstantInt::get(Ty: getSizeTTy(M), V: 1));
548 auto *Cmp = Builder.CreateICmpEQ(LHS: NewEntry, RHS: EntriesE);
549 Entry->addIncoming(V: EntriesB, BB: &RegGlobalsFn->getEntryBlock());
550 Entry->addIncoming(V: NewEntry, BB: IfEndBB);
551 Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: EntryBB);
552 Builder.SetInsertPoint(ExitBB);
553 Builder.CreateRetVoid();
554
555 return RegGlobalsFn;
556}
557
558// Create the constructor and destructor to register the fatbinary with the CUDA
559// runtime.
560void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
561 bool IsHIP, EntryArrayTy EntryArray,
562 StringRef Suffix,
563 bool EmitSurfacesAndTextures) {
564 LLVMContext &C = M.getContext();
565 auto *CtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
566 auto *CtorFunc = Function::Create(
567 Ty: CtorFuncTy, Linkage: GlobalValue::InternalLinkage,
568 N: (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, M: &M);
569 CtorFunc->setSection(getStartupSection(T: M.getTargetTriple()));
570
571 auto *DtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
572 auto *DtorFunc = Function::Create(
573 Ty: DtorFuncTy, Linkage: GlobalValue::InternalLinkage,
574 N: (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, M: &M);
575 DtorFunc->setSection(getStartupSection(T: M.getTargetTriple()));
576
577 auto *PtrTy = PointerType::getUnqual(C);
578
579 // Get the __cudaRegisterFatBinary function declaration.
580 auto *RegFatTy = FunctionType::get(Result: PtrTy, Params: PtrTy, /*isVarArg=*/false);
581 FunctionCallee RegFatbin = M.getOrInsertFunction(
582 Name: IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", T: RegFatTy);
583 // Get the __cudaRegisterFatBinaryEnd function declaration.
584 auto *RegFatEndTy =
585 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
586 FunctionCallee RegFatbinEnd =
587 M.getOrInsertFunction(Name: "__cudaRegisterFatBinaryEnd", T: RegFatEndTy);
588 // Get the __cudaUnregisterFatBinary function declaration.
589 auto *UnregFatTy =
590 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
591 FunctionCallee UnregFatbin = M.getOrInsertFunction(
592 Name: IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
593 T: UnregFatTy);
594
595 auto *AtExitTy =
596 FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false);
597 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
598
599 auto *BinaryHandleGlobal = new llvm::GlobalVariable(
600 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
601 llvm::ConstantPointerNull::get(T: PtrTy),
602 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
603
604 // Create the constructor to register this image with the runtime.
605 IRBuilder<> CtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: CtorFunc));
606 CallInst *Handle = CtorBuilder.CreateCall(
607 Callee: RegFatbin,
608 Args: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: FatbinDesc, Ty: PtrTy));
609 CtorBuilder.CreateAlignedStore(
610 Val: Handle, Ptr: BinaryHandleGlobal,
611 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
612 CtorBuilder.CreateCall(Callee: createRegisterGlobalsFunction(M, IsHIP, EntryArray,
613 Suffix,
614 EmitSurfacesAndTextures),
615 Args: Handle);
616 if (!IsHIP)
617 CtorBuilder.CreateCall(Callee: RegFatbinEnd, Args: Handle);
618 CtorBuilder.CreateCall(Callee: AtExit, Args: DtorFunc);
619 CtorBuilder.CreateRetVoid();
620
621 // Create the destructor to unregister the image with the runtime. We cannot
622 // use a standard global destructor after CUDA 9.2 so this must be called by
623 // `atexit()` instead.
624 IRBuilder<> DtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: DtorFunc));
625 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
626 Ty: PtrTy, Ptr: BinaryHandleGlobal,
627 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
628 DtorBuilder.CreateCall(Callee: UnregFatbin, Args: BinaryHandle);
629 DtorBuilder.CreateRetVoid();
630
631 // Add this function to constructors.
632 appendToGlobalCtors(M, F: CtorFunc, /*Priority=*/101);
633}
634
635/// SYCLWrapper helper class that creates all LLVM IRs wrapping given images.
636class SYCLWrapper {
637public:
638 SYCLWrapper(Module &M, const SYCLJITOptions &Options)
639 : M(M), C(M.getContext()), Options(Options) {}
640
641 /// Embeds \p Buffer (a raw OffloadBinary) as a global constant and returns
642 /// a pair of (Start, Size), where Start points to the beginning of the
643 /// embedded data and Size is its length in bytes.
644 std::pair<Constant *, Constant *> embedBinary(ArrayRef<char> Buffer) {
645 Constant *Arr = ConstantDataArray::get(Context&: C, Elts: Buffer);
646 GlobalVariable *BinaryGV = new GlobalVariable(
647 M, Arr->getType(), /*isConstant=*/true, GlobalValue::InternalLinkage,
648 Arr, ".sycl_offloading.binary");
649 BinaryGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
650 BinaryGV->setSection(".llvm.offloading");
651
652 IntegerType *Int64Ty = Type::getInt64Ty(C);
653 Constant *Zero = ConstantInt::get(Ty: Int64Ty, V: 0);
654 Constant *Size = ConstantInt::get(Ty: Int64Ty, V: Buffer.size());
655 Constant *Start = ConstantExpr::getGetElementPtr(
656 Ty: BinaryGV->getValueType(), C: BinaryGV, IdxList: ArrayRef<Constant *>{Zero, Zero});
657 return {Start, Size};
658 }
659
660 void createRegisterFatbinFunction(Constant *Start, Constant *Size) {
661 FunctionType *FuncTy =
662 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
663 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
664 N: Twine("sycl") + ".descriptor_reg", M: &M);
665 Func->setSection(getStartupSection(T: M.getTargetTriple()));
666
667 PointerType *PtrTy = PointerType::getUnqual(C);
668 IntegerType *Int64Ty = Type::getInt64Ty(C);
669 FunctionType *RegFuncTy =
670 FunctionType::get(Result: Type::getVoidTy(C), Params: {PtrTy, Int64Ty},
671 /*isVarArg=*/false);
672 FunctionCallee RegFuncC =
673 M.getOrInsertFunction(Name: "__sycl_register_lib", T: RegFuncTy);
674
675 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
676 Builder.CreateCall(Callee: RegFuncC, Args: {Start, Size});
677 Builder.CreateRetVoid();
678
679 appendToGlobalCtors(M, F: Func, /*Priority*/ 1);
680 }
681
682 void createUnregisterFunction(Constant *Start, Constant *Size) {
683 FunctionType *FuncTy =
684 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
685 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
686 N: "sycl.descriptor_unreg", M: &M);
687 Func->setSection(getStartupSection(T: M.getTargetTriple()));
688
689 PointerType *PtrTy = PointerType::getUnqual(C);
690 IntegerType *Int64Ty = Type::getInt64Ty(C);
691 FunctionType *UnRegFuncTy =
692 FunctionType::get(Result: Type::getVoidTy(C), Params: {PtrTy, Int64Ty},
693 /*isVarArg=*/false);
694 FunctionCallee UnRegFuncC =
695 M.getOrInsertFunction(Name: "__sycl_unregister_lib", T: UnRegFuncTy);
696
697 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
698 Builder.CreateCall(Callee: UnRegFuncC, Args: {Start, Size});
699 Builder.CreateRetVoid();
700
701 appendToGlobalDtors(M, F: Func, /*Priority*/ 1);
702 }
703
704private:
705 Module &M;
706 LLVMContext &C;
707 SYCLJITOptions Options;
708}; // end of SYCLWrapper
709
710} // namespace
711
712Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
713 EntryArrayTy EntryArray,
714 llvm::StringRef Suffix, bool Relocatable) {
715 GlobalVariable *Desc =
716 createBinDesc(M, Bufs: Images, EntryArray, Suffix, Relocatable);
717 if (!Desc)
718 return createStringError(EC: inconvertibleErrorCode(),
719 S: "No binary descriptors created.");
720 createRegisterFunction(M, BinDesc: Desc, Suffix);
721 return Error::success();
722}
723
724Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,
725 EntryArrayTy EntryArray,
726 llvm::StringRef Suffix,
727 bool EmitSurfacesAndTextures) {
728 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: false, Suffix);
729 if (!Desc)
730 return createStringError(EC: inconvertibleErrorCode(),
731 S: "No fatbin section created.");
732
733 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: false, EntryArray, Suffix,
734 EmitSurfacesAndTextures);
735 return Error::success();
736}
737
738Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,
739 EntryArrayTy EntryArray, llvm::StringRef Suffix,
740 bool EmitSurfacesAndTextures) {
741 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: true, Suffix);
742 if (!Desc)
743 return createStringError(EC: inconvertibleErrorCode(),
744 S: "No fatbin section created.");
745
746 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: true, EntryArray, Suffix,
747 EmitSurfacesAndTextures);
748 return Error::success();
749}
750
751Error llvm::offloading::wrapSYCLBinaries(llvm::Module &M, ArrayRef<char> Buffer,
752 SYCLJITOptions Options) {
753 SYCLWrapper W(M, Options);
754 auto [Start, Size] = W.embedBinary(Buffer);
755 W.createRegisterFatbinFunction(Start, Size);
756 W.createUnregisterFunction(Start, Size);
757 return Error::success();
758}
759