1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Offloading/OffloadWrapper.h"
10#include "llvm/ADT/ArrayRef.h"
11#include "llvm/ADT/SmallVector.h"
12#include "llvm/ADT/StringRef.h"
13#include "llvm/ADT/Twine.h"
14#include "llvm/BinaryFormat/Magic.h"
15#include "llvm/Frontend/Offloading/Utility.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/LLVMContext.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Type.h"
23#include "llvm/Object/OffloadBinary.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/TargetParser/Triple.h"
27#include "llvm/Transforms/Utils/ModuleUtils.h"
28
29#include <utility>
30
31using namespace llvm;
32using namespace llvm::object;
33using namespace llvm::offloading;
34
35namespace {
36/// Magic number that begins the section containing the CUDA fatbinary.
37constexpr unsigned CudaFatMagic = 0x466243b1;
38constexpr unsigned HIPFatMagic = 0x48495046;
39
40IntegerType *getSizeTTy(Module &M) {
41 return M.getDataLayout().getIntPtrType(C&: M.getContext());
42}
43
44/// Returns the appropriate startup section for registration functions.
45/// Mach-O uses "__TEXT,__StaticInit"; ELF/COFF use ".text.startup".
46StringRef getStartupSection(const Triple &T) {
47 return T.isOSBinFormatMachO() ? "__TEXT,__StaticInit" : ".text.startup";
48}
49
50// struct __tgt_device_image {
51// void *ImageStart;
52// void *ImageEnd;
53// __tgt_offload_entry *EntriesBegin;
54// __tgt_offload_entry *EntriesEnd;
55// };
56StructType *getDeviceImageTy(Module &M) {
57 LLVMContext &C = M.getContext();
58 StructType *ImageTy = StructType::getTypeByName(C, Name: "__tgt_device_image");
59 if (!ImageTy)
60 ImageTy =
61 StructType::create(Name: "__tgt_device_image", elt1: PointerType::getUnqual(C),
62 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C),
63 elts: PointerType::getUnqual(C));
64 return ImageTy;
65}
66
67PointerType *getDeviceImagePtrTy(Module &M) {
68 return PointerType::getUnqual(C&: M.getContext());
69}
70
71// struct __tgt_bin_desc {
72// int32_t NumDeviceImages;
73// __tgt_device_image *DeviceImages;
74// __tgt_offload_entry *HostEntriesBegin;
75// __tgt_offload_entry *HostEntriesEnd;
76// };
77StructType *getBinDescTy(Module &M) {
78 LLVMContext &C = M.getContext();
79 StructType *DescTy = StructType::getTypeByName(C, Name: "__tgt_bin_desc");
80 if (!DescTy)
81 DescTy = StructType::create(
82 Name: "__tgt_bin_desc", elt1: Type::getInt32Ty(C), elts: getDeviceImagePtrTy(M),
83 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
84 return DescTy;
85}
86
87PointerType *getBinDescPtrTy(Module &M) {
88 return PointerType::getUnqual(C&: M.getContext());
89}
90
91/// Creates binary descriptor for the given device images. Binary descriptor
92/// is an object that is passed to the offloading runtime at program startup
93/// and it describes all device images available in the executable or shared
94/// library. It is defined as follows
95///
96/// __attribute__((visibility("hidden")))
97/// extern __tgt_offload_entry *__start_llvm_offload_entries;
98/// __attribute__((visibility("hidden")))
99/// extern __tgt_offload_entry *__stop_llvm_offload_entries;
100///
101/// static const char Image0[] = { <Bufs.front() contents> };
102/// ...
103/// static const char ImageN[] = { <Bufs.back() contents> };
104///
105/// static const __tgt_device_image Images[] = {
106/// {
107/// Image0, /*ImageStart*/
108/// Image0 + sizeof(Image0), /*ImageEnd*/
109/// __start_llvm_offload_entries, /*EntriesBegin*/
110/// __stop_llvm_offload_entries /*EntriesEnd*/
111/// },
112/// ...
113/// {
114/// ImageN, /*ImageStart*/
115/// ImageN + sizeof(ImageN), /*ImageEnd*/
116/// __start_llvm_offload_entries, /*EntriesBegin*/
117/// __stop_llvm_offload_entries /*EntriesEnd*/
118/// }
119/// };
120///
121/// static const __tgt_bin_desc BinDesc = {
122/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
123/// Images, /*DeviceImages*/
124/// __start_llvm_offload_entries, /*HostEntriesBegin*/
125/// __stop_llvm_offload_entries /*HostEntriesEnd*/
126/// };
127///
128/// Global variable that represents BinDesc is returned.
129GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
130 EntryArrayTy EntryArray, StringRef Suffix,
131 bool Relocatable) {
132 LLVMContext &C = M.getContext();
133 auto [EntriesB, EntriesE] = EntryArray;
134
135 // Create initializer for the images array.
136 SmallVector<Constant *, 4u> ImagesInits;
137 ImagesInits.reserve(N: Bufs.size());
138 for (ArrayRef<char> Buf : Bufs) {
139 // We embed the full offloading entry so the binary utilities can parse it.
140 auto *Data = ConstantDataArray::get(Context&: C, Elts: Buf);
141 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
142 GlobalVariable::InternalLinkage, Data,
143 ".llvm_offloading.device_image" + Suffix);
144 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
145 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
146 : ".llvm.offloading");
147 Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
148
149 StringRef Binary(Buf.data(), Buf.size());
150
151 uint64_t BeginOffset = 0;
152 uint64_t EndOffset = Binary.size();
153
154 // Optionally use an offload binary for its offload dumping support.
155 // The device image struct contains the pointer to the beginning and end of
156 // the image stored inside of the offload binary. There should only be one
157 // of these for each buffer so we parse it out manually.
158 if (identify_magic(magic: Binary) == file_magic::offload_binary) {
159 const auto *Header =
160 reinterpret_cast<const object::OffloadBinary::Header *>(
161 Binary.bytes_begin());
162 const auto *Entry =
163 reinterpret_cast<const object::OffloadBinary::Entry *>(
164 Binary.bytes_begin() + Header->EntriesOffset);
165 BeginOffset = Entry->ImageOffset;
166 EndOffset = Entry->ImageOffset + Entry->ImageSize;
167 }
168
169 auto *Begin = ConstantInt::get(Ty: getSizeTTy(M), V: BeginOffset);
170 auto *Size = ConstantInt::get(Ty: getSizeTTy(M), V: EndOffset);
171 auto *ImageB = ConstantExpr::getPtrAdd(Ptr: Image, Offset: Begin);
172 auto *ImageE = ConstantExpr::getPtrAdd(Ptr: Image, Offset: Size);
173
174 ImagesInits.push_back(Elt: ConstantStruct::get(T: getDeviceImageTy(M), Vs: ImageB,
175 Vs: ImageE, Vs: EntriesB, Vs: EntriesE));
176 }
177
178 // Then create images array.
179 auto *ImagesData = ConstantArray::get(
180 T: ArrayType::get(ElementType: getDeviceImageTy(M), NumElements: ImagesInits.size()), V: ImagesInits);
181
182 auto *Images =
183 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
184 GlobalValue::InternalLinkage, ImagesData,
185 ".llvm_offloading.device_images" + Suffix);
186 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
187
188 // And finally create the binary descriptor object.
189 auto *DescInit = ConstantStruct::get(
190 T: getBinDescTy(M),
191 Vs: ConstantInt::get(Ty: Type::getInt32Ty(C), V: ImagesInits.size()), Vs: Images,
192 Vs: EntriesB, Vs: EntriesE);
193
194 return new GlobalVariable(M, DescInit->getType(), /*isConstant=*/true,
195 GlobalValue::InternalLinkage, DescInit,
196 ".llvm_offloading.descriptor" + Suffix);
197}
198
199Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
200 StringRef Suffix, StringRef UnregFuncName) {
201 LLVMContext &C = M.getContext();
202 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
203 auto *Func =
204 Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
205 N: ".llvm_offloading.descriptor_unreg" + Suffix, M: &M);
206 Func->setSection(getStartupSection(T: M.getTargetTriple()));
207
208 // Get the unregister function declaration.
209 auto *UnRegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
210 /*isVarArg*/ false);
211 FunctionCallee UnRegFuncC = M.getOrInsertFunction(Name: UnregFuncName, T: UnRegFuncTy);
212
213 // Construct function body
214 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
215 Builder.CreateCall(Callee: UnRegFuncC, Args: BinDesc);
216 Builder.CreateRetVoid();
217
218 return Func;
219}
220
221void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
222 StringRef Suffix, StringRef RegFuncName,
223 StringRef UnregFuncName) {
224 LLVMContext &C = M.getContext();
225 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
226 auto *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
227 N: ".llvm_offloading.descriptor_reg" + Suffix, M: &M);
228 Func->setSection(getStartupSection(T: M.getTargetTriple()));
229
230 // Get the register function declaration.
231 auto *RegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
232 /*isVarArg*/ false);
233 FunctionCallee RegFuncC = M.getOrInsertFunction(Name: RegFuncName, T: RegFuncTy);
234
235 auto *AtExitTy = FunctionType::get(
236 Result: Type::getInt32Ty(C), Params: PointerType::getUnqual(C), /*isVarArg=*/false);
237 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
238
239 Function *UnregFunc =
240 createUnregisterFunction(M, BinDesc, Suffix, UnregFuncName);
241
242 // Construct function body
243 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
244
245 Builder.CreateCall(Callee: RegFuncC, Args: BinDesc);
246
247 // Register the destructors with 'atexit'. This is expected by the CUDA
248 // runtime and ensures that we clean up before dynamic objects are destroyed.
249 // This needs to be done after plugin initialization to ensure that it is
250 // called before the plugin runtime is destroyed.
251 Builder.CreateCall(Callee: AtExit, Args: UnregFunc);
252 Builder.CreateRetVoid();
253
254 // Add this function to constructors.
255 appendToGlobalCtors(M, F: Func, /*Priority=*/101);
256}
257
258// struct fatbin_wrapper {
259// int32_t magic;
260// int32_t version;
261// void *image;
262// void *reserved;
263//};
264StructType *getFatbinWrapperTy(Module &M) {
265 LLVMContext &C = M.getContext();
266 StructType *FatbinTy = StructType::getTypeByName(C, Name: "fatbin_wrapper");
267 if (!FatbinTy)
268 FatbinTy = StructType::create(
269 Name: "fatbin_wrapper", elt1: Type::getInt32Ty(C), elts: Type::getInt32Ty(C),
270 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
271 return FatbinTy;
272}
273
274/// Embed the image \p Image into the module \p M so it can be found by the
275/// runtime.
276GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
277 StringRef Suffix) {
278 LLVMContext &C = M.getContext();
279 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
280 const llvm::Triple &Triple = M.getTargetTriple();
281
282 // Create the global string containing the fatbinary.
283 StringRef FatbinConstantSection =
284 IsHIP ? (Triple.isMacOSX() ? "__HIP,__hip_fatbin" : ".hip_fatbin")
285 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
286 auto *Data = ConstantDataArray::get(Context&: C, Elts: Image);
287 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
288 GlobalVariable::InternalLinkage, Data,
289 ".fatbin_image" + Suffix);
290 Fatbin->setSection(FatbinConstantSection);
291
292 // Create the fatbinary wrapper
293 StringRef FatbinWrapperSection =
294 IsHIP ? (Triple.isMacOSX() ? "__HIP,__fatbin" : ".hipFatBinSegment")
295 : (Triple.isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment");
296 Constant *FatbinWrapper[] = {
297 ConstantInt::get(Ty: Type::getInt32Ty(C), V: IsHIP ? HIPFatMagic : CudaFatMagic),
298 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1),
299 ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Fatbin, Ty: Int8PtrTy),
300 ConstantPointerNull::get(T: PointerType::getUnqual(C))};
301
302 Constant *FatbinInitializer =
303 ConstantStruct::get(T: getFatbinWrapperTy(M), V: FatbinWrapper);
304
305 auto *FatbinDesc =
306 new GlobalVariable(M, getFatbinWrapperTy(M),
307 /*isConstant*/ true, GlobalValue::InternalLinkage,
308 FatbinInitializer, ".fatbin_wrapper" + Suffix);
309 FatbinDesc->setSection(FatbinWrapperSection);
310 FatbinDesc->setAlignment(Align(8));
311 FatbinDesc->setNoSanitizeMetadata();
312
313 return FatbinDesc;
314}
315
316/// Create the register globals function. We will iterate all of the offloading
317/// entries stored at the begin / end symbols and register them according to
318/// their type. This creates the following function in IR:
319///
320/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
321/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
322///
323/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
324/// void *, void *, void *, void *, int *);
325/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
326/// int64_t, int32_t, int32_t);
327///
328/// void __cudaRegisterTest(void **fatbinHandle) {
329/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
330/// entry != &__stop_cuda_offloading_entries; ++entry) {
331/// if (entry->Kind != OFK_CUDA)
332/// continue
333///
334/// if (!entry->Size)
335/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
336/// entry->name, -1, 0, 0, 0, 0, 0);
337/// else
338/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
339/// 0, entry->size, 0, 0);
340/// }
341/// }
342Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
343 EntryArrayTy EntryArray,
344 StringRef Suffix,
345 bool EmitSurfacesAndTextures) {
346 LLVMContext &C = M.getContext();
347 auto [EntriesB, EntriesE] = EntryArray;
348
349 // Get the __cudaRegisterFunction function declaration.
350 PointerType *Int8PtrTy = PointerType::get(C, AddressSpace: 0);
351 PointerType *Int8PtrPtrTy = PointerType::get(C, AddressSpace: 0);
352 PointerType *Int32PtrTy = PointerType::get(C, AddressSpace: 0);
353 auto *RegFuncTy = FunctionType::get(
354 Result: Type::getInt32Ty(C),
355 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
356 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
357 /*isVarArg*/ false);
358 FunctionCallee RegFunc = M.getOrInsertFunction(
359 Name: IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", T: RegFuncTy);
360
361 // Get the __cudaRegisterVar function declaration.
362 auto *RegVarTy = FunctionType::get(
363 Result: Type::getVoidTy(C),
364 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
365 getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},
366 /*isVarArg*/ false);
367 FunctionCallee RegVar = M.getOrInsertFunction(
368 Name: IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", T: RegVarTy);
369
370 // Get the __cudaRegisterSurface function declaration.
371 FunctionType *RegManagedVarTy =
372 FunctionType::get(Result: Type::getVoidTy(C),
373 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
374 getSizeTTy(M), Type::getInt32Ty(C)},
375 /*isVarArg=*/false);
376 FunctionCallee RegManagedVar = M.getOrInsertFunction(
377 Name: IsHIP ? "__hipRegisterManagedVar" : "__cudaRegisterManagedVar",
378 T: RegManagedVarTy);
379
380 // Get the __cudaRegisterSurface function declaration.
381 FunctionType *RegSurfaceTy =
382 FunctionType::get(Result: Type::getVoidTy(C),
383 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
384 Type::getInt32Ty(C), Type::getInt32Ty(C)},
385 /*isVarArg=*/false);
386 FunctionCallee RegSurface = M.getOrInsertFunction(
387 Name: IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", T: RegSurfaceTy);
388
389 // Get the __cudaRegisterTexture function declaration.
390 FunctionType *RegTextureTy = FunctionType::get(
391 Result: Type::getVoidTy(C),
392 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
393 Type::getInt32Ty(C), Type::getInt32Ty(C)},
394 /*isVarArg=*/false);
395 FunctionCallee RegTexture = M.getOrInsertFunction(
396 Name: IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", T: RegTextureTy);
397
398 auto *RegGlobalsTy = FunctionType::get(Result: Type::getVoidTy(C), Params: Int8PtrPtrTy,
399 /*isVarArg*/ false);
400 auto *RegGlobalsFn =
401 Function::Create(Ty: RegGlobalsTy, Linkage: GlobalValue::InternalLinkage,
402 N: IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", M: &M);
403 RegGlobalsFn->setSection(getStartupSection(T: M.getTargetTriple()));
404
405 // Create the loop to register all the entries.
406 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: RegGlobalsFn));
407 Builder.disableGEPCanonicalization();
408 auto *EntryBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: RegGlobalsFn);
409 auto *IfKindBB = BasicBlock::Create(Context&: C, Name: "if.kind", Parent: RegGlobalsFn);
410 auto *IfThenBB = BasicBlock::Create(Context&: C, Name: "if.then", Parent: RegGlobalsFn);
411 auto *IfElseBB = BasicBlock::Create(Context&: C, Name: "if.else", Parent: RegGlobalsFn);
412 auto *SwGlobalBB = BasicBlock::Create(Context&: C, Name: "sw.global", Parent: RegGlobalsFn);
413 auto *SwManagedBB = BasicBlock::Create(Context&: C, Name: "sw.managed", Parent: RegGlobalsFn);
414 auto *SwSurfaceBB = BasicBlock::Create(Context&: C, Name: "sw.surface", Parent: RegGlobalsFn);
415 auto *SwTextureBB = BasicBlock::Create(Context&: C, Name: "sw.texture", Parent: RegGlobalsFn);
416 auto *IfEndBB = BasicBlock::Create(Context&: C, Name: "if.end", Parent: RegGlobalsFn);
417 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: RegGlobalsFn);
418
419 auto *EntryCmp = Builder.CreateICmpNE(LHS: EntriesB, RHS: EntriesE);
420 Builder.CreateCondBr(Cond: EntryCmp, True: EntryBB, False: ExitBB);
421 Builder.SetInsertPoint(EntryBB);
422 auto *Entry = Builder.CreatePHI(Ty: PointerType::getUnqual(C), NumReservedValues: 2, Name: "entry");
423 auto *AddrPtr =
424 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
425 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
426 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4)});
427 auto *Addr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AddrPtr, Name: "addr");
428 auto *AuxAddrPtr =
429 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
430 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
431 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 8)});
432 auto *AuxAddr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AuxAddrPtr, Name: "aux_addr");
433 auto *KindPtr =
434 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
435 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
436 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 2)});
437 auto *Kind = Builder.CreateLoad(Ty: Type::getInt16Ty(C), Ptr: KindPtr, Name: "kind");
438 auto *NamePtr =
439 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
440 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
441 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5)});
442 auto *Name = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: NamePtr, Name: "name");
443 auto *SizePtr =
444 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
445 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
446 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 6)});
447 auto *Size = Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: SizePtr, Name: "size");
448 auto *FlagsPtr =
449 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
450 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
451 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3)});
452 auto *Flags = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: FlagsPtr, Name: "flags");
453 auto *DataPtr =
454 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
455 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
456 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 7)});
457 auto *Data = Builder.CreateTrunc(
458 V: Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: DataPtr, Name: "data"),
459 DestTy: Type::getInt32Ty(C));
460 auto *Type = Builder.CreateAnd(
461 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0x7), Name: "type");
462
463 // Extract the flags stored in the bit-field and convert them to C booleans.
464 auto *ExternBit = Builder.CreateAnd(
465 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
466 V: llvm::offloading::OffloadGlobalExtern));
467 auto *Extern = Builder.CreateLShr(
468 LHS: ExternBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3), Name: "extern");
469 auto *ConstantBit = Builder.CreateAnd(
470 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
471 V: llvm::offloading::OffloadGlobalConstant));
472 auto *Const = Builder.CreateLShr(
473 LHS: ConstantBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4), Name: "constant");
474 auto *NormalizedBit = Builder.CreateAnd(
475 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
476 V: llvm::offloading::OffloadGlobalNormalized));
477 auto *Normalized = Builder.CreateLShr(
478 LHS: NormalizedBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5), Name: "normalized");
479 auto *KindCond = Builder.CreateICmpEQ(
480 LHS: Kind, RHS: ConstantInt::get(Ty: Type::getInt16Ty(C),
481 V: IsHIP ? object::OffloadKind::OFK_HIP
482 : object::OffloadKind::OFK_Cuda));
483 Builder.CreateCondBr(Cond: KindCond, True: IfKindBB, False: IfEndBB);
484 Builder.SetInsertPoint(IfKindBB);
485 auto *FnCond = Builder.CreateICmpEQ(
486 LHS: Size, RHS: ConstantInt::getNullValue(Ty: Type::getInt64Ty(C)));
487 Builder.CreateCondBr(Cond: FnCond, True: IfThenBB, False: IfElseBB);
488
489 // Create kernel registration code.
490 Builder.SetInsertPoint(IfThenBB);
491 Builder.CreateCall(
492 Callee: RegFunc,
493 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
494 ConstantInt::getAllOnesValue(Ty: Type::getInt32Ty(C)),
495 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
496 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
497 ConstantPointerNull::get(T: Int32PtrTy)});
498 Builder.CreateBr(Dest: IfEndBB);
499 Builder.SetInsertPoint(IfElseBB);
500
501 auto *Switch = Builder.CreateSwitch(V: Type, Dest: IfEndBB);
502 // Create global variable registration code.
503 Builder.SetInsertPoint(SwGlobalBB);
504 Builder.CreateCall(Callee: RegVar,
505 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
506 Const, ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)});
507 Builder.CreateBr(Dest: IfEndBB);
508 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalEntry),
509 Dest: SwGlobalBB);
510
511 // Create managed variable registration code.
512 Builder.SetInsertPoint(SwManagedBB);
513 Builder.CreateCall(Callee: RegManagedVar, Args: {RegGlobalsFn->arg_begin(), AuxAddr, Addr,
514 Name, Size, Data});
515 Builder.CreateBr(Dest: IfEndBB);
516 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalManagedEntry),
517 Dest: SwManagedBB);
518 // Create surface variable registration code.
519 Builder.SetInsertPoint(SwSurfaceBB);
520 if (EmitSurfacesAndTextures)
521 Builder.CreateCall(Callee: RegSurface, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
522 Data, Extern});
523 Builder.CreateBr(Dest: IfEndBB);
524 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalSurfaceEntry),
525 Dest: SwSurfaceBB);
526
527 // Create texture variable registration code.
528 Builder.SetInsertPoint(SwTextureBB);
529 if (EmitSurfacesAndTextures)
530 Builder.CreateCall(Callee: RegTexture, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
531 Data, Normalized, Extern});
532 Builder.CreateBr(Dest: IfEndBB);
533 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalTextureEntry),
534 Dest: SwTextureBB);
535
536 Builder.SetInsertPoint(IfEndBB);
537 auto *NewEntry = Builder.CreateInBoundsGEP(
538 Ty: offloading::getEntryTy(M), Ptr: Entry, IdxList: ConstantInt::get(Ty: getSizeTTy(M), V: 1));
539 auto *Cmp = Builder.CreateICmpEQ(LHS: NewEntry, RHS: EntriesE);
540 Entry->addIncoming(V: EntriesB, BB: &RegGlobalsFn->getEntryBlock());
541 Entry->addIncoming(V: NewEntry, BB: IfEndBB);
542 Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: EntryBB);
543 Builder.SetInsertPoint(ExitBB);
544 Builder.CreateRetVoid();
545
546 return RegGlobalsFn;
547}
548
549// Create the constructor and destructor to register the fatbinary with the CUDA
550// runtime.
551void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
552 bool IsHIP, EntryArrayTy EntryArray,
553 StringRef Suffix,
554 bool EmitSurfacesAndTextures) {
555 LLVMContext &C = M.getContext();
556 auto *CtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
557 auto *CtorFunc = Function::Create(
558 Ty: CtorFuncTy, Linkage: GlobalValue::InternalLinkage,
559 N: (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, M: &M);
560 CtorFunc->setSection(getStartupSection(T: M.getTargetTriple()));
561
562 auto *DtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
563 auto *DtorFunc = Function::Create(
564 Ty: DtorFuncTy, Linkage: GlobalValue::InternalLinkage,
565 N: (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, M: &M);
566 DtorFunc->setSection(getStartupSection(T: M.getTargetTriple()));
567
568 auto *PtrTy = PointerType::getUnqual(C);
569
570 // Get the __cudaRegisterFatBinary function declaration.
571 auto *RegFatTy = FunctionType::get(Result: PtrTy, Params: PtrTy, /*isVarArg=*/false);
572 FunctionCallee RegFatbin = M.getOrInsertFunction(
573 Name: IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", T: RegFatTy);
574 // Get the __cudaRegisterFatBinaryEnd function declaration.
575 auto *RegFatEndTy =
576 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
577 FunctionCallee RegFatbinEnd =
578 M.getOrInsertFunction(Name: "__cudaRegisterFatBinaryEnd", T: RegFatEndTy);
579 // Get the __cudaUnregisterFatBinary function declaration.
580 auto *UnregFatTy =
581 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
582 FunctionCallee UnregFatbin = M.getOrInsertFunction(
583 Name: IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
584 T: UnregFatTy);
585
586 auto *AtExitTy =
587 FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false);
588 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
589
590 auto *BinaryHandleGlobal = new llvm::GlobalVariable(
591 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
592 llvm::ConstantPointerNull::get(T: PtrTy),
593 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
594
595 // Create the constructor to register this image with the runtime.
596 IRBuilder<> CtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: CtorFunc));
597 CallInst *Handle = CtorBuilder.CreateCall(
598 Callee: RegFatbin,
599 Args: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: FatbinDesc, Ty: PtrTy));
600 CtorBuilder.CreateAlignedStore(
601 Val: Handle, Ptr: BinaryHandleGlobal,
602 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
603 CtorBuilder.CreateCall(Callee: createRegisterGlobalsFunction(M, IsHIP, EntryArray,
604 Suffix,
605 EmitSurfacesAndTextures),
606 Args: Handle);
607 if (!IsHIP)
608 CtorBuilder.CreateCall(Callee: RegFatbinEnd, Args: Handle);
609 CtorBuilder.CreateCall(Callee: AtExit, Args: DtorFunc);
610 CtorBuilder.CreateRetVoid();
611
612 // Create the destructor to unregister the image with the runtime. We cannot
613 // use a standard global destructor after CUDA 9.2 so this must be called by
614 // `atexit()` instead.
615 IRBuilder<> DtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: DtorFunc));
616 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
617 Ty: PtrTy, Ptr: BinaryHandleGlobal,
618 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
619 DtorBuilder.CreateCall(Callee: UnregFatbin, Args: BinaryHandle);
620 DtorBuilder.CreateRetVoid();
621
622 // Add this function to constructors.
623 appendToGlobalCtors(M, F: CtorFunc, /*Priority=*/101);
624}
625
626/// SYCLWrapper helper class that creates all LLVM IRs wrapping given images.
627class SYCLWrapper {
628public:
629 SYCLWrapper(Module &M, const SYCLJITOptions &Options, bool IsFinalizedImage)
630 : M(M), C(M.getContext()), Options(Options),
631 IsFinalizedImage(IsFinalizedImage) {}
632
633 /// Embeds \p Buffer (a raw OffloadBinary) as a global constant and returns
634 /// a pair of (Start, Size), where Start points to the beginning of the
635 /// embedded data and Size is its length in bytes.
636 std::pair<Constant *, Constant *> embedBinary(ArrayRef<char> Buffer) {
637 Constant *Arr = ConstantDataArray::get(Context&: C, Elts: Buffer);
638 GlobalVariable *BinaryGV = new GlobalVariable(
639 M, Arr->getType(), /*isConstant=*/true, GlobalValue::InternalLinkage,
640 Arr, ".sycl_offloading.binary");
641 BinaryGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
642 // The linker wrapper scans ".llvm.offloading" for device code to link, so
643 // an already finalized image must go elsewhere to avoid being linked again.
644 BinaryGV->setSection(IsFinalizedImage ? ".sycl_fatbin"
645 : ".llvm.offloading");
646
647 IntegerType *Int64Ty = Type::getInt64Ty(C);
648 Constant *Size = ConstantInt::get(Ty: Int64Ty, V: Buffer.size());
649 return {BinaryGV, Size};
650 }
651
652 Function *createRegisterFatbinFunction(Constant *Start, Constant *Size) {
653 FunctionType *FuncTy =
654 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
655 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
656 N: Twine("sycl") + ".descriptor_reg", M: &M);
657 Func->setSection(getStartupSection(T: M.getTargetTriple()));
658
659 PointerType *PtrTy = PointerType::getUnqual(C);
660 IntegerType *Int64Ty = Type::getInt64Ty(C);
661 FunctionType *RegFuncTy =
662 FunctionType::get(Result: Type::getVoidTy(C), Params: {PtrTy, Int64Ty},
663 /*isVarArg=*/false);
664 FunctionCallee RegFuncC =
665 M.getOrInsertFunction(Name: "__sycl_register_lib", T: RegFuncTy);
666
667 FunctionType *AtExitTy =
668 FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false);
669 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
670
671 Function *UnregFunc = createUnregisterFunction(Start, Size);
672
673 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
674 Builder.CreateCall(Callee: RegFuncC, Args: {Start, Size});
675
676 // Unregister with 'atexit'. The handler is installed after
677 // __sycl_register_lib has brought the runtime's own exit-time cleanup into
678 // the atexit chain, so it is ordered ahead of that cleanup.
679 Builder.CreateCall(Callee: AtExit, Args: UnregFunc);
680 Builder.CreateRetVoid();
681
682 return Func;
683 }
684
685private:
686 Function *createUnregisterFunction(Constant *Start, Constant *Size) {
687 FunctionType *FuncTy =
688 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
689 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
690 N: "sycl.descriptor_unreg", M: &M);
691 Func->setSection(getStartupSection(T: M.getTargetTriple()));
692
693 PointerType *PtrTy = PointerType::getUnqual(C);
694 IntegerType *Int64Ty = Type::getInt64Ty(C);
695 FunctionType *UnRegFuncTy =
696 FunctionType::get(Result: Type::getVoidTy(C), Params: {PtrTy, Int64Ty},
697 /*isVarArg=*/false);
698 FunctionCallee UnRegFuncC =
699 M.getOrInsertFunction(Name: "__sycl_unregister_lib", T: UnRegFuncTy);
700
701 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
702 Builder.CreateCall(Callee: UnRegFuncC, Args: {Start, Size});
703 Builder.CreateRetVoid();
704
705 return Func;
706 }
707
708 Module &M;
709 LLVMContext &C;
710 SYCLJITOptions Options;
711 bool IsFinalizedImage;
712}; // end of SYCLWrapper
713
714} // namespace
715
716/// Wraps the images and registers them with the runtime through the given
717/// registration entry points. Shared by the OpenMP and OpenACC wrappers, which
718/// differ only in the names of those entry points.
719static Error wrapOmpAccBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
720 offloading::EntryArrayTy EntryArray,
721 StringRef Suffix, bool Relocatable,
722 StringRef RegFuncName,
723 StringRef UnregFuncName) {
724 GlobalVariable *Desc =
725 createBinDesc(M, Bufs: Images, EntryArray, Suffix, Relocatable);
726 if (!Desc)
727 return createStringError(EC: inconvertibleErrorCode(),
728 S: "No binary descriptors created.");
729 createRegisterFunction(M, BinDesc: Desc, Suffix, RegFuncName, UnregFuncName);
730 return Error::success();
731}
732
733Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
734 EntryArrayTy EntryArray,
735 llvm::StringRef Suffix, bool Relocatable) {
736 return wrapOmpAccBinaries(M, Images, EntryArray, Suffix, Relocatable,
737 RegFuncName: "__tgt_register_lib", UnregFuncName: "__tgt_unregister_lib");
738}
739
740Error offloading::wrapOpenACCBinaries(Module &M,
741 ArrayRef<ArrayRef<char>> Images,
742 EntryArrayTy EntryArray,
743 bool Relocatable) {
744 return wrapOmpAccBinaries(M, Images, EntryArray, /*Suffix=*/"", Relocatable,
745 RegFuncName: "__tgt_acc_register_lib",
746 UnregFuncName: "__tgt_acc_unregister_lib");
747}
748
749Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,
750 EntryArrayTy EntryArray,
751 llvm::StringRef Suffix,
752 bool EmitSurfacesAndTextures) {
753 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: false, Suffix);
754 if (!Desc)
755 return createStringError(EC: inconvertibleErrorCode(),
756 S: "No fatbin section created.");
757
758 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: false, EntryArray, Suffix,
759 EmitSurfacesAndTextures);
760 return Error::success();
761}
762
763Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,
764 EntryArrayTy EntryArray, llvm::StringRef Suffix,
765 bool EmitSurfacesAndTextures) {
766 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: true, Suffix);
767 if (!Desc)
768 return createStringError(EC: inconvertibleErrorCode(),
769 S: "No fatbin section created.");
770
771 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: true, EntryArray, Suffix,
772 EmitSurfacesAndTextures);
773 return Error::success();
774}
775
776Error llvm::offloading::wrapSYCLBinaries(llvm::Module &M, ArrayRef<char> Buffer,
777 SYCLJITOptions Options,
778 bool IsFinalizedImage,
779 Function **RegistrationFunc) {
780 SYCLWrapper W(M, Options, IsFinalizedImage);
781 auto [Start, Size] = W.embedBinary(Buffer);
782 Function *RegisterFunc = W.createRegisterFatbinFunction(Start, Size);
783 if (RegistrationFunc) {
784 *RegistrationFunc = RegisterFunc;
785 return Error::success();
786 }
787
788 appendToGlobalCtors(M, F: RegisterFunc, /*Priority=*/101);
789 return Error::success();
790}
791