1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Offloading/OffloadWrapper.h"
10#include "llvm/ADT/ArrayRef.h"
11#include "llvm/ADT/SmallVector.h"
12#include "llvm/ADT/StringRef.h"
13#include "llvm/ADT/Twine.h"
14#include "llvm/BinaryFormat/Magic.h"
15#include "llvm/Frontend/Offloading/Utility.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/LLVMContext.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Type.h"
23#include "llvm/Object/OffloadBinary.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/LineIterator.h"
27#include "llvm/Support/MemoryBufferRef.h"
28#include "llvm/TargetParser/Triple.h"
29#include "llvm/Transforms/Utils/ModuleUtils.h"
30
31#include <memory>
32#include <utility>
33
34using namespace llvm;
35using namespace llvm::object;
36using namespace llvm::offloading;
37
38namespace {
39/// Magic number that begins the section containing the CUDA fatbinary.
40constexpr unsigned CudaFatMagic = 0x466243b1;
41constexpr unsigned HIPFatMagic = 0x48495046;
42
43IntegerType *getSizeTTy(Module &M) {
44 return M.getDataLayout().getIntPtrType(C&: M.getContext());
45}
46
47// struct __tgt_device_image {
48// void *ImageStart;
49// void *ImageEnd;
50// __tgt_offload_entry *EntriesBegin;
51// __tgt_offload_entry *EntriesEnd;
52// };
53StructType *getDeviceImageTy(Module &M) {
54 LLVMContext &C = M.getContext();
55 StructType *ImageTy = StructType::getTypeByName(C, Name: "__tgt_device_image");
56 if (!ImageTy)
57 ImageTy =
58 StructType::create(Name: "__tgt_device_image", elt1: PointerType::getUnqual(C),
59 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C),
60 elts: PointerType::getUnqual(C));
61 return ImageTy;
62}
63
64PointerType *getDeviceImagePtrTy(Module &M) {
65 return PointerType::getUnqual(C&: M.getContext());
66}
67
68// struct __tgt_bin_desc {
69// int32_t NumDeviceImages;
70// __tgt_device_image *DeviceImages;
71// __tgt_offload_entry *HostEntriesBegin;
72// __tgt_offload_entry *HostEntriesEnd;
73// };
74StructType *getBinDescTy(Module &M) {
75 LLVMContext &C = M.getContext();
76 StructType *DescTy = StructType::getTypeByName(C, Name: "__tgt_bin_desc");
77 if (!DescTy)
78 DescTy = StructType::create(
79 Name: "__tgt_bin_desc", elt1: Type::getInt32Ty(C), elts: getDeviceImagePtrTy(M),
80 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
81 return DescTy;
82}
83
84PointerType *getBinDescPtrTy(Module &M) {
85 return PointerType::getUnqual(C&: M.getContext());
86}
87
88/// Creates binary descriptor for the given device images. Binary descriptor
89/// is an object that is passed to the offloading runtime at program startup
90/// and it describes all device images available in the executable or shared
91/// library. It is defined as follows
92///
93/// __attribute__((visibility("hidden")))
94/// extern __tgt_offload_entry *__start_omp_offloading_entries;
95/// __attribute__((visibility("hidden")))
96/// extern __tgt_offload_entry *__stop_omp_offloading_entries;
97///
98/// static const char Image0[] = { <Bufs.front() contents> };
99/// ...
100/// static const char ImageN[] = { <Bufs.back() contents> };
101///
102/// static const __tgt_device_image Images[] = {
103/// {
104/// Image0, /*ImageStart*/
105/// Image0 + sizeof(Image0), /*ImageEnd*/
106/// __start_omp_offloading_entries, /*EntriesBegin*/
107/// __stop_omp_offloading_entries /*EntriesEnd*/
108/// },
109/// ...
110/// {
111/// ImageN, /*ImageStart*/
112/// ImageN + sizeof(ImageN), /*ImageEnd*/
113/// __start_omp_offloading_entries, /*EntriesBegin*/
114/// __stop_omp_offloading_entries /*EntriesEnd*/
115/// }
116/// };
117///
118/// static const __tgt_bin_desc BinDesc = {
119/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
120/// Images, /*DeviceImages*/
121/// __start_omp_offloading_entries, /*HostEntriesBegin*/
122/// __stop_omp_offloading_entries /*HostEntriesEnd*/
123/// };
124///
125/// Global variable that represents BinDesc is returned.
126GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
127 EntryArrayTy EntryArray, StringRef Suffix,
128 bool Relocatable) {
129 LLVMContext &C = M.getContext();
130 auto [EntriesB, EntriesE] = EntryArray;
131
132 auto *Zero = ConstantInt::get(Ty: getSizeTTy(M), V: 0u);
133 Constant *ZeroZero[] = {Zero, Zero};
134
135 // Create initializer for the images array.
136 SmallVector<Constant *, 4u> ImagesInits;
137 ImagesInits.reserve(N: Bufs.size());
138 for (ArrayRef<char> Buf : Bufs) {
139 // We embed the full offloading entry so the binary utilities can parse it.
140 auto *Data = ConstantDataArray::get(Context&: C, Elts: Buf);
141 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
142 GlobalVariable::InternalLinkage, Data,
143 ".omp_offloading.device_image" + Suffix);
144 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
145 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
146 : ".llvm.offloading");
147 Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
148
149 StringRef Binary(Buf.data(), Buf.size());
150
151 uint64_t BeginOffset = 0;
152 uint64_t EndOffset = Binary.size();
153
154 // Optionally use an offload binary for its offload dumping support.
155 // The device image struct contains the pointer to the beginning and end of
156 // the image stored inside of the offload binary. There should only be one
157 // of these for each buffer so we parse it out manually.
158 if (identify_magic(magic: Binary) == file_magic::offload_binary) {
159 const auto *Header =
160 reinterpret_cast<const object::OffloadBinary::Header *>(
161 Binary.bytes_begin());
162 const auto *Entry =
163 reinterpret_cast<const object::OffloadBinary::Entry *>(
164 Binary.bytes_begin() + Header->EntryOffset);
165 BeginOffset = Entry->ImageOffset;
166 EndOffset = Entry->ImageOffset + Entry->ImageSize;
167 }
168
169 auto *Begin = ConstantInt::get(Ty: getSizeTTy(M), V: BeginOffset);
170 auto *Size = ConstantInt::get(Ty: getSizeTTy(M), V: EndOffset);
171 Constant *ZeroBegin[] = {Zero, Begin};
172 Constant *ZeroSize[] = {Zero, Size};
173
174 auto *ImageB =
175 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroBegin);
176 auto *ImageE =
177 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroSize);
178
179 ImagesInits.push_back(Elt: ConstantStruct::get(T: getDeviceImageTy(M), Vs: ImageB,
180 Vs: ImageE, Vs: EntriesB, Vs: EntriesE));
181 }
182
183 // Then create images array.
184 auto *ImagesData = ConstantArray::get(
185 T: ArrayType::get(ElementType: getDeviceImageTy(M), NumElements: ImagesInits.size()), V: ImagesInits);
186
187 auto *Images =
188 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
189 GlobalValue::InternalLinkage, ImagesData,
190 ".omp_offloading.device_images" + Suffix);
191 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
192
193 auto *ImagesB =
194 ConstantExpr::getGetElementPtr(Ty: Images->getValueType(), C: Images, IdxList: ZeroZero);
195
196 // And finally create the binary descriptor object.
197 auto *DescInit = ConstantStruct::get(
198 T: getBinDescTy(M),
199 Vs: ConstantInt::get(Ty: Type::getInt32Ty(C), V: ImagesInits.size()), Vs: ImagesB,
200 Vs: EntriesB, Vs: EntriesE);
201
202 return new GlobalVariable(M, DescInit->getType(), /*isConstant=*/true,
203 GlobalValue::InternalLinkage, DescInit,
204 ".omp_offloading.descriptor" + Suffix);
205}
206
207Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
208 StringRef Suffix) {
209 LLVMContext &C = M.getContext();
210 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
211 auto *Func =
212 Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
213 N: ".omp_offloading.descriptor_unreg" + Suffix, M: &M);
214 Func->setSection(".text.startup");
215
216 // Get __tgt_unregister_lib function declaration.
217 auto *UnRegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
218 /*isVarArg*/ false);
219 FunctionCallee UnRegFuncC =
220 M.getOrInsertFunction(Name: "__tgt_unregister_lib", T: UnRegFuncTy);
221
222 // Construct function body
223 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
224 Builder.CreateCall(Callee: UnRegFuncC, Args: BinDesc);
225 Builder.CreateRetVoid();
226
227 return Func;
228}
229
230void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
231 StringRef Suffix) {
232 LLVMContext &C = M.getContext();
233 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
234 auto *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
235 N: ".omp_offloading.descriptor_reg" + Suffix, M: &M);
236 Func->setSection(".text.startup");
237
238 // Get __tgt_register_lib function declaration.
239 auto *RegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
240 /*isVarArg*/ false);
241 FunctionCallee RegFuncC =
242 M.getOrInsertFunction(Name: "__tgt_register_lib", T: RegFuncTy);
243
244 auto *AtExitTy = FunctionType::get(
245 Result: Type::getInt32Ty(C), Params: PointerType::getUnqual(C), /*isVarArg=*/false);
246 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
247
248 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
249
250 // Construct function body
251 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
252
253 Builder.CreateCall(Callee: RegFuncC, Args: BinDesc);
254
255 // Register the destructors with 'atexit'. This is expected by the CUDA
256 // runtime and ensures that we clean up before dynamic objects are destroyed.
257 // This needs to be done after plugin initialization to ensure that it is
258 // called before the plugin runtime is destroyed.
259 Builder.CreateCall(Callee: AtExit, Args: UnregFunc);
260 Builder.CreateRetVoid();
261
262 // Add this function to constructors.
263 appendToGlobalCtors(M, F: Func, /*Priority=*/101);
264}
265
266// struct fatbin_wrapper {
267// int32_t magic;
268// int32_t version;
269// void *image;
270// void *reserved;
271//};
272StructType *getFatbinWrapperTy(Module &M) {
273 LLVMContext &C = M.getContext();
274 StructType *FatbinTy = StructType::getTypeByName(C, Name: "fatbin_wrapper");
275 if (!FatbinTy)
276 FatbinTy = StructType::create(
277 Name: "fatbin_wrapper", elt1: Type::getInt32Ty(C), elts: Type::getInt32Ty(C),
278 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
279 return FatbinTy;
280}
281
282/// Embed the image \p Image into the module \p M so it can be found by the
283/// runtime.
284GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
285 StringRef Suffix) {
286 LLVMContext &C = M.getContext();
287 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
288 const llvm::Triple &Triple = M.getTargetTriple();
289
290 // Create the global string containing the fatbinary.
291 StringRef FatbinConstantSection =
292 IsHIP ? ".hip_fatbin"
293 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
294 auto *Data = ConstantDataArray::get(Context&: C, Elts: Image);
295 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
296 GlobalVariable::InternalLinkage, Data,
297 ".fatbin_image" + Suffix);
298 Fatbin->setSection(FatbinConstantSection);
299
300 // Create the fatbinary wrapper
301 StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment"
302 : Triple.isMacOSX() ? "__NV_CUDA,__fatbin"
303 : ".nvFatBinSegment";
304 Constant *FatbinWrapper[] = {
305 ConstantInt::get(Ty: Type::getInt32Ty(C), V: IsHIP ? HIPFatMagic : CudaFatMagic),
306 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1),
307 ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Fatbin, Ty: Int8PtrTy),
308 ConstantPointerNull::get(T: PointerType::getUnqual(C))};
309
310 Constant *FatbinInitializer =
311 ConstantStruct::get(T: getFatbinWrapperTy(M), V: FatbinWrapper);
312
313 auto *FatbinDesc =
314 new GlobalVariable(M, getFatbinWrapperTy(M),
315 /*isConstant*/ true, GlobalValue::InternalLinkage,
316 FatbinInitializer, ".fatbin_wrapper" + Suffix);
317 FatbinDesc->setSection(FatbinWrapperSection);
318 FatbinDesc->setAlignment(Align(8));
319
320 return FatbinDesc;
321}
322
323/// Create the register globals function. We will iterate all of the offloading
324/// entries stored at the begin / end symbols and register them according to
325/// their type. This creates the following function in IR:
326///
327/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
328/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
329///
330/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
331/// void *, void *, void *, void *, int *);
332/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
333/// int64_t, int32_t, int32_t);
334///
335/// void __cudaRegisterTest(void **fatbinHandle) {
336/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
337/// entry != &__stop_cuda_offloading_entries; ++entry) {
338/// if (entry->Kind != OFK_CUDA)
339/// continue
340///
341/// if (!entry->Size)
342/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
343/// entry->name, -1, 0, 0, 0, 0, 0);
344/// else
345/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
346/// 0, entry->size, 0, 0);
347/// }
348/// }
349Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
350 EntryArrayTy EntryArray,
351 StringRef Suffix,
352 bool EmitSurfacesAndTextures) {
353 LLVMContext &C = M.getContext();
354 auto [EntriesB, EntriesE] = EntryArray;
355
356 // Get the __cudaRegisterFunction function declaration.
357 PointerType *Int8PtrTy = PointerType::get(C, AddressSpace: 0);
358 PointerType *Int8PtrPtrTy = PointerType::get(C, AddressSpace: 0);
359 PointerType *Int32PtrTy = PointerType::get(C, AddressSpace: 0);
360 auto *RegFuncTy = FunctionType::get(
361 Result: Type::getInt32Ty(C),
362 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
363 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
364 /*isVarArg*/ false);
365 FunctionCallee RegFunc = M.getOrInsertFunction(
366 Name: IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", T: RegFuncTy);
367
368 // Get the __cudaRegisterVar function declaration.
369 auto *RegVarTy = FunctionType::get(
370 Result: Type::getVoidTy(C),
371 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
372 getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},
373 /*isVarArg*/ false);
374 FunctionCallee RegVar = M.getOrInsertFunction(
375 Name: IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", T: RegVarTy);
376
377 // Get the __cudaRegisterSurface function declaration.
378 FunctionType *RegManagedVarTy =
379 FunctionType::get(Result: Type::getVoidTy(C),
380 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
381 getSizeTTy(M), Type::getInt32Ty(C)},
382 /*isVarArg=*/false);
383 FunctionCallee RegManagedVar = M.getOrInsertFunction(
384 Name: IsHIP ? "__hipRegisterManagedVar" : "__cudaRegisterManagedVar",
385 T: RegManagedVarTy);
386
387 // Get the __cudaRegisterSurface function declaration.
388 FunctionType *RegSurfaceTy =
389 FunctionType::get(Result: Type::getVoidTy(C),
390 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
391 Type::getInt32Ty(C), Type::getInt32Ty(C)},
392 /*isVarArg=*/false);
393 FunctionCallee RegSurface = M.getOrInsertFunction(
394 Name: IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", T: RegSurfaceTy);
395
396 // Get the __cudaRegisterTexture function declaration.
397 FunctionType *RegTextureTy = FunctionType::get(
398 Result: Type::getVoidTy(C),
399 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
400 Type::getInt32Ty(C), Type::getInt32Ty(C)},
401 /*isVarArg=*/false);
402 FunctionCallee RegTexture = M.getOrInsertFunction(
403 Name: IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", T: RegTextureTy);
404
405 auto *RegGlobalsTy = FunctionType::get(Result: Type::getVoidTy(C), Params: Int8PtrPtrTy,
406 /*isVarArg*/ false);
407 auto *RegGlobalsFn =
408 Function::Create(Ty: RegGlobalsTy, Linkage: GlobalValue::InternalLinkage,
409 N: IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", M: &M);
410 RegGlobalsFn->setSection(".text.startup");
411
412 // Create the loop to register all the entries.
413 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: RegGlobalsFn));
414 auto *EntryBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: RegGlobalsFn);
415 auto *IfKindBB = BasicBlock::Create(Context&: C, Name: "if.kind", Parent: RegGlobalsFn);
416 auto *IfThenBB = BasicBlock::Create(Context&: C, Name: "if.then", Parent: RegGlobalsFn);
417 auto *IfElseBB = BasicBlock::Create(Context&: C, Name: "if.else", Parent: RegGlobalsFn);
418 auto *SwGlobalBB = BasicBlock::Create(Context&: C, Name: "sw.global", Parent: RegGlobalsFn);
419 auto *SwManagedBB = BasicBlock::Create(Context&: C, Name: "sw.managed", Parent: RegGlobalsFn);
420 auto *SwSurfaceBB = BasicBlock::Create(Context&: C, Name: "sw.surface", Parent: RegGlobalsFn);
421 auto *SwTextureBB = BasicBlock::Create(Context&: C, Name: "sw.texture", Parent: RegGlobalsFn);
422 auto *IfEndBB = BasicBlock::Create(Context&: C, Name: "if.end", Parent: RegGlobalsFn);
423 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: RegGlobalsFn);
424
425 auto *EntryCmp = Builder.CreateICmpNE(LHS: EntriesB, RHS: EntriesE);
426 Builder.CreateCondBr(Cond: EntryCmp, True: EntryBB, False: ExitBB);
427 Builder.SetInsertPoint(EntryBB);
428 auto *Entry = Builder.CreatePHI(Ty: PointerType::getUnqual(C), NumReservedValues: 2, Name: "entry");
429 auto *AddrPtr =
430 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
431 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
432 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4)});
433 auto *Addr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AddrPtr, Name: "addr");
434 auto *AuxAddrPtr =
435 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
436 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
437 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 8)});
438 auto *AuxAddr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AuxAddrPtr, Name: "aux_addr");
439 auto *KindPtr =
440 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
441 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
442 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 2)});
443 auto *Kind = Builder.CreateLoad(Ty: Type::getInt16Ty(C), Ptr: KindPtr, Name: "kind");
444 auto *NamePtr =
445 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
446 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
447 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5)});
448 auto *Name = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: NamePtr, Name: "name");
449 auto *SizePtr =
450 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
451 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
452 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 6)});
453 auto *Size = Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: SizePtr, Name: "size");
454 auto *FlagsPtr =
455 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
456 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
457 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3)});
458 auto *Flags = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: FlagsPtr, Name: "flags");
459 auto *DataPtr =
460 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
461 IdxList: {ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0),
462 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 7)});
463 auto *Data = Builder.CreateTrunc(
464 V: Builder.CreateLoad(Ty: Type::getInt64Ty(C), Ptr: DataPtr, Name: "data"),
465 DestTy: Type::getInt32Ty(C));
466 auto *Type = Builder.CreateAnd(
467 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0x7), Name: "type");
468
469 // Extract the flags stored in the bit-field and convert them to C booleans.
470 auto *ExternBit = Builder.CreateAnd(
471 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
472 V: llvm::offloading::OffloadGlobalExtern));
473 auto *Extern = Builder.CreateLShr(
474 LHS: ExternBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3), Name: "extern");
475 auto *ConstantBit = Builder.CreateAnd(
476 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
477 V: llvm::offloading::OffloadGlobalConstant));
478 auto *Const = Builder.CreateLShr(
479 LHS: ConstantBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4), Name: "constant");
480 auto *NormalizedBit = Builder.CreateAnd(
481 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
482 V: llvm::offloading::OffloadGlobalNormalized));
483 auto *Normalized = Builder.CreateLShr(
484 LHS: NormalizedBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5), Name: "normalized");
485 auto *KindCond = Builder.CreateICmpEQ(
486 LHS: Kind, RHS: ConstantInt::get(Ty: Type::getInt16Ty(C),
487 V: IsHIP ? object::OffloadKind::OFK_HIP
488 : object::OffloadKind::OFK_Cuda));
489 Builder.CreateCondBr(Cond: KindCond, True: IfKindBB, False: IfEndBB);
490 Builder.SetInsertPoint(IfKindBB);
491 auto *FnCond = Builder.CreateICmpEQ(
492 LHS: Size, RHS: ConstantInt::getNullValue(Ty: Type::getInt64Ty(C)));
493 Builder.CreateCondBr(Cond: FnCond, True: IfThenBB, False: IfElseBB);
494
495 // Create kernel registration code.
496 Builder.SetInsertPoint(IfThenBB);
497 Builder.CreateCall(
498 Callee: RegFunc,
499 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
500 ConstantInt::getAllOnesValue(Ty: Type::getInt32Ty(C)),
501 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
502 ConstantPointerNull::get(T: Int8PtrTy), ConstantPointerNull::get(T: Int8PtrTy),
503 ConstantPointerNull::get(T: Int32PtrTy)});
504 Builder.CreateBr(Dest: IfEndBB);
505 Builder.SetInsertPoint(IfElseBB);
506
507 auto *Switch = Builder.CreateSwitch(V: Type, Dest: IfEndBB);
508 // Create global variable registration code.
509 Builder.SetInsertPoint(SwGlobalBB);
510 Builder.CreateCall(Callee: RegVar,
511 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
512 Const, ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)});
513 Builder.CreateBr(Dest: IfEndBB);
514 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalEntry),
515 Dest: SwGlobalBB);
516
517 // Create managed variable registration code.
518 Builder.SetInsertPoint(SwManagedBB);
519 Builder.CreateCall(Callee: RegManagedVar, Args: {RegGlobalsFn->arg_begin(), AuxAddr, Addr,
520 Name, Size, Data});
521 Builder.CreateBr(Dest: IfEndBB);
522 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalManagedEntry),
523 Dest: SwManagedBB);
524 // Create surface variable registration code.
525 Builder.SetInsertPoint(SwSurfaceBB);
526 if (EmitSurfacesAndTextures)
527 Builder.CreateCall(Callee: RegSurface, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
528 Data, Extern});
529 Builder.CreateBr(Dest: IfEndBB);
530 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalSurfaceEntry),
531 Dest: SwSurfaceBB);
532
533 // Create texture variable registration code.
534 Builder.SetInsertPoint(SwTextureBB);
535 if (EmitSurfacesAndTextures)
536 Builder.CreateCall(Callee: RegTexture, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
537 Data, Normalized, Extern});
538 Builder.CreateBr(Dest: IfEndBB);
539 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalTextureEntry),
540 Dest: SwTextureBB);
541
542 Builder.SetInsertPoint(IfEndBB);
543 auto *NewEntry = Builder.CreateInBoundsGEP(
544 Ty: offloading::getEntryTy(M), Ptr: Entry, IdxList: ConstantInt::get(Ty: getSizeTTy(M), V: 1));
545 auto *Cmp = Builder.CreateICmpEQ(
546 LHS: NewEntry,
547 RHS: ConstantExpr::getInBoundsGetElementPtr(
548 Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesE,
549 IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0),
550 ConstantInt::get(Ty: getSizeTTy(M), V: 0)})));
551 Entry->addIncoming(
552 V: ConstantExpr::getInBoundsGetElementPtr(
553 Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesB,
554 IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0),
555 ConstantInt::get(Ty: getSizeTTy(M), V: 0)})),
556 BB: &RegGlobalsFn->getEntryBlock());
557 Entry->addIncoming(V: NewEntry, BB: IfEndBB);
558 Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: EntryBB);
559 Builder.SetInsertPoint(ExitBB);
560 Builder.CreateRetVoid();
561
562 return RegGlobalsFn;
563}
564
565// Create the constructor and destructor to register the fatbinary with the CUDA
566// runtime.
567void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
568 bool IsHIP, EntryArrayTy EntryArray,
569 StringRef Suffix,
570 bool EmitSurfacesAndTextures) {
571 LLVMContext &C = M.getContext();
572 auto *CtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
573 auto *CtorFunc = Function::Create(
574 Ty: CtorFuncTy, Linkage: GlobalValue::InternalLinkage,
575 N: (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, M: &M);
576 CtorFunc->setSection(".text.startup");
577
578 auto *DtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
579 auto *DtorFunc = Function::Create(
580 Ty: DtorFuncTy, Linkage: GlobalValue::InternalLinkage,
581 N: (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, M: &M);
582 DtorFunc->setSection(".text.startup");
583
584 auto *PtrTy = PointerType::getUnqual(C);
585
586 // Get the __cudaRegisterFatBinary function declaration.
587 auto *RegFatTy = FunctionType::get(Result: PtrTy, Params: PtrTy, /*isVarArg=*/false);
588 FunctionCallee RegFatbin = M.getOrInsertFunction(
589 Name: IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", T: RegFatTy);
590 // Get the __cudaRegisterFatBinaryEnd function declaration.
591 auto *RegFatEndTy =
592 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
593 FunctionCallee RegFatbinEnd =
594 M.getOrInsertFunction(Name: "__cudaRegisterFatBinaryEnd", T: RegFatEndTy);
595 // Get the __cudaUnregisterFatBinary function declaration.
596 auto *UnregFatTy =
597 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
598 FunctionCallee UnregFatbin = M.getOrInsertFunction(
599 Name: IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
600 T: UnregFatTy);
601
602 auto *AtExitTy =
603 FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false);
604 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
605
606 auto *BinaryHandleGlobal = new llvm::GlobalVariable(
607 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
608 llvm::ConstantPointerNull::get(T: PtrTy),
609 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
610
611 // Create the constructor to register this image with the runtime.
612 IRBuilder<> CtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: CtorFunc));
613 CallInst *Handle = CtorBuilder.CreateCall(
614 Callee: RegFatbin,
615 Args: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: FatbinDesc, Ty: PtrTy));
616 CtorBuilder.CreateAlignedStore(
617 Val: Handle, Ptr: BinaryHandleGlobal,
618 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
619 CtorBuilder.CreateCall(Callee: createRegisterGlobalsFunction(M, IsHIP, EntryArray,
620 Suffix,
621 EmitSurfacesAndTextures),
622 Args: Handle);
623 if (!IsHIP)
624 CtorBuilder.CreateCall(Callee: RegFatbinEnd, Args: Handle);
625 CtorBuilder.CreateCall(Callee: AtExit, Args: DtorFunc);
626 CtorBuilder.CreateRetVoid();
627
628 // Create the destructor to unregister the image with the runtime. We cannot
629 // use a standard global destructor after CUDA 9.2 so this must be called by
630 // `atexit()` instead.
631 IRBuilder<> DtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: DtorFunc));
632 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
633 Ty: PtrTy, Ptr: BinaryHandleGlobal,
634 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
635 DtorBuilder.CreateCall(Callee: UnregFatbin, Args: BinaryHandle);
636 DtorBuilder.CreateRetVoid();
637
638 // Add this function to constructors.
639 appendToGlobalCtors(M, F: CtorFunc, /*Priority=*/101);
640}
641
642/// SYCLWrapper helper class that creates all LLVM IRs wrapping given images.
643class SYCLWrapper {
644public:
645 SYCLWrapper(Module &M, const SYCLJITOptions &Options)
646 : M(M), C(M.getContext()), Options(Options) {
647 EntryTy = offloading::getEntryTy(M);
648 SyclDeviceImageTy = getSyclDeviceImageTy();
649 SyclBinDescTy = getSyclBinDescTy();
650 }
651
652 /// Creates binary descriptor for the given device images. Binary descriptor
653 /// is an object that is passed to the offloading runtime at program startup
654 /// and it describes all device images available in the executable or shared
655 /// library. It is defined as follows:
656 ///
657 /// \code
658 /// __attribute__((visibility("hidden")))
659 /// __tgt_offload_entry *__sycl_offload_entries_arr0[];
660 /// ...
661 /// __attribute__((visibility("hidden")))
662 /// __tgt_offload_entry *__sycl_offload_entries_arrN[];
663 ///
664 /// __attribute__((visibility("hidden")))
665 /// extern const char *CompileOptions = "...";
666 /// ...
667 /// __attribute__((visibility("hidden")))
668 /// extern const char *LinkOptions = "...";
669 /// ...
670 ///
671 /// static const char Image0[] = { ... };
672 /// ...
673 /// static const char ImageN[] = { ... };
674 ///
675 /// static const __sycl.tgt_device_image Images[] = {
676 /// {
677 /// Version, // Version
678 /// OffloadKind, // OffloadKind
679 /// Format, // Format of the image.
680 // TripleString, // Arch
681 /// CompileOptions, // CompileOptions
682 /// LinkOptions, // LinkOptions
683 /// Image0, // ImageStart
684 /// Image0 + IMAGE0_SIZE, // ImageEnd
685 /// __sycl_offload_entries_arr0, // EntriesBegin
686 /// __sycl_offload_entries_arr0 + ENTRIES0_SIZE, // EntriesEnd
687 /// NULL, // PropertiesBegin
688 /// NULL, // PropertiesEnd
689 /// },
690 /// ...
691 /// };
692 ///
693 /// static const __sycl.tgt_bin_desc FatbinDesc = {
694 /// Version, //Version
695 /// sizeof(Images) / sizeof(Images[0]), //NumDeviceImages
696 /// Images, //DeviceImages
697 /// NULL, //HostEntriesBegin
698 /// NULL //HostEntriesEnd
699 /// };
700 /// \endcode
701 ///
702 /// \returns Global variable that represents FatbinDesc.
703 GlobalVariable *createFatbinDesc(ArrayRef<OffloadFile> OffloadFiles) {
704 StringRef OffloadKindTag = ".sycl_offloading.";
705 SmallVector<Constant *> WrappedImages;
706 WrappedImages.reserve(N: OffloadFiles.size());
707 for (size_t I = 0, E = OffloadFiles.size(); I != E; ++I)
708 WrappedImages.push_back(
709 Elt: wrapImage(OB: *OffloadFiles[I].getBinary(), ImageID: Twine(I), OffloadKindTag));
710
711 return combineWrappedImages(WrappedImages, OffloadKindTag);
712 }
713
714 void createRegisterFatbinFunction(GlobalVariable *FatbinDesc) {
715 FunctionType *FuncTy =
716 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
717 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
718 N: Twine("sycl") + ".descriptor_reg", M: &M);
719 Func->setSection(".text.startup");
720
721 // Get RegFuncName function declaration.
722 FunctionType *RegFuncTy =
723 FunctionType::get(Result: Type::getVoidTy(C), Params: PointerType::getUnqual(C),
724 /*isVarArg=*/false);
725 FunctionCallee RegFuncC =
726 M.getOrInsertFunction(Name: "__sycl_register_lib", T: RegFuncTy);
727
728 // Construct function body.
729 IRBuilder Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
730 Builder.CreateCall(Callee: RegFuncC, Args: FatbinDesc);
731 Builder.CreateRetVoid();
732
733 // Add this function to constructors.
734 appendToGlobalCtors(M, F: Func, /*Priority*/ 1);
735 }
736
737 void createUnregisterFunction(GlobalVariable *FatbinDesc) {
738 FunctionType *FuncTy =
739 FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
740 Function *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
741 N: "sycl.descriptor_unreg", M: &M);
742 Func->setSection(".text.startup");
743
744 // Get UnregFuncName function declaration.
745 FunctionType *UnRegFuncTy =
746 FunctionType::get(Result: Type::getVoidTy(C), Params: PointerType::getUnqual(C),
747 /*isVarArg=*/false);
748 FunctionCallee UnRegFuncC =
749 M.getOrInsertFunction(Name: "__sycl_unregister_lib", T: UnRegFuncTy);
750
751 // Construct function body
752 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
753 Builder.CreateCall(Callee: UnRegFuncC, Args: FatbinDesc);
754 Builder.CreateRetVoid();
755
756 // Add this function to global destructors.
757 appendToGlobalDtors(M, F: Func, /*Priority*/ 1);
758 }
759
760private:
761 IntegerType *getSizeTTy() {
762 switch (M.getDataLayout().getPointerSize()) {
763 case 4:
764 return Type::getInt32Ty(C);
765 case 8:
766 return Type::getInt64Ty(C);
767 }
768 llvm_unreachable("unsupported pointer type size");
769 }
770
771 SmallVector<Constant *, 2> getSizetConstPair(size_t First, size_t Second) {
772 IntegerType *SizeTTy = getSizeTTy();
773 return SmallVector<Constant *, 2>{ConstantInt::get(Ty: SizeTTy, V: First),
774 ConstantInt::get(Ty: SizeTTy, V: Second)};
775 }
776
777 /// Note: Properties aren't supported and the support is going
778 /// to be added later.
779 /// Creates a structure corresponding to:
780 /// SYCL specific image descriptor type.
781 /// \code
782 /// struct __sycl.tgt_device_image {
783 /// // Version of this structure - for backward compatibility;
784 /// // all modifications which change order/type/offsets of existing fields
785 /// // should increment the version.
786 /// uint16_t Version;
787 /// // The kind of offload model the image employs.
788 /// uint8_t OffloadKind;
789 /// // Format of the image data - SPIRV, LLVMIR bitcode, etc.
790 /// uint8_t Format;
791 /// // Null-terminated string representation of the device's target
792 /// // architecture.
793 /// const char *Arch;
794 /// // A null-terminated string; target- and compiler-specific options
795 /// // which are passed to the device compiler at runtime.
796 /// const char *CompileOptions;
797 /// // A null-terminated string; target- and compiler-specific options
798 /// // which are passed to the device linker at runtime.
799 /// const char *LinkOptions;
800 /// // Pointer to the device binary image start.
801 /// void *ImageStart;
802 /// // Pointer to the device binary image end.
803 /// void *ImageEnd;
804 /// // The entry table.
805 /// __tgt_offload_entry *EntriesBegin;
806 /// __tgt_offload_entry *EntriesEnd;
807 /// const char *PropertiesBegin;
808 /// const char *PropertiesEnd;
809 /// };
810 /// \endcode
811 StructType *getSyclDeviceImageTy() {
812 return StructType::create(
813 Elements: {
814 Type::getInt16Ty(C), // Version
815 Type::getInt8Ty(C), // OffloadKind
816 Type::getInt8Ty(C), // Format
817 PointerType::getUnqual(C), // Arch
818 PointerType::getUnqual(C), // CompileOptions
819 PointerType::getUnqual(C), // LinkOptions
820 PointerType::getUnqual(C), // ImageStart
821 PointerType::getUnqual(C), // ImageEnd
822 PointerType::getUnqual(C), // EntriesBegin
823 PointerType::getUnqual(C), // EntriesEnd
824 PointerType::getUnqual(C), // PropertiesBegin
825 PointerType::getUnqual(C) // PropertiesEnd
826 },
827 Name: "__sycl.tgt_device_image");
828 }
829
830 /// Creates a structure for SYCL specific binary descriptor type. Corresponds
831 /// to:
832 ///
833 /// \code
834 /// struct __sycl.tgt_bin_desc {
835 /// // version of this structure - for backward compatibility;
836 /// // all modifications which change order/type/offsets of existing fields
837 /// // should increment the version.
838 /// uint16_t Version;
839 /// uint16_t NumDeviceImages;
840 /// __sycl.tgt_device_image *DeviceImages;
841 /// // the offload entry table
842 /// __tgt_offload_entry *HostEntriesBegin;
843 /// __tgt_offload_entry *HostEntriesEnd;
844 /// };
845 /// \endcode
846 StructType *getSyclBinDescTy() {
847 return StructType::create(
848 Elements: {Type::getInt16Ty(C), Type::getInt16Ty(C), PointerType::getUnqual(C),
849 PointerType::getUnqual(C), PointerType::getUnqual(C)},
850 Name: "__sycl.tgt_bin_desc");
851 }
852
853 /// Adds a global readonly variable that is initialized by given
854 /// \p Initializer to the module.
855 GlobalVariable *addGlobalArrayVariable(const Twine &Name,
856 ArrayRef<char> Initializer,
857 const Twine &Section = "") {
858 Constant *Arr = ConstantDataArray::get(Context&: M.getContext(), Elts: Initializer);
859 GlobalVariable *Var =
860 new GlobalVariable(M, Arr->getType(), /*isConstant*/ true,
861 GlobalVariable::InternalLinkage, Arr, Name);
862 Var->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
863
864 SmallVector<char, 32> NameBuf;
865 StringRef SectionName = Section.toStringRef(Out&: NameBuf);
866 if (!SectionName.empty())
867 Var->setSection(SectionName);
868 return Var;
869 }
870
871 /// Adds given \p Buf as a global variable into the module.
872 /// \returns Pair of pointers that point at the beginning and the end of the
873 /// variable.
874 std::pair<Constant *, Constant *>
875 addArrayToModule(ArrayRef<char> Buf, const Twine &Name,
876 const Twine &Section = "") {
877 GlobalVariable *Var = addGlobalArrayVariable(Name, Initializer: Buf, Section);
878 Constant *ImageB = ConstantExpr::getGetElementPtr(Ty: Var->getValueType(), C: Var,
879 IdxList: getSizetConstPair(First: 0, Second: 0));
880 Constant *ImageE = ConstantExpr::getGetElementPtr(
881 Ty: Var->getValueType(), C: Var, IdxList: getSizetConstPair(First: 0, Second: Buf.size()));
882 return std::make_pair(x&: ImageB, y&: ImageE);
883 }
884
885 /// Adds given \p Data as constant byte array in the module.
886 /// \returns Constant pointer to the added data. The pointer type does not
887 /// carry size information.
888 Constant *addRawDataToModule(ArrayRef<char> Data, const Twine &Name) {
889 GlobalVariable *Var = addGlobalArrayVariable(Name, Initializer: Data);
890 Constant *DataPtr = ConstantExpr::getGetElementPtr(Ty: Var->getValueType(), C: Var,
891 IdxList: getSizetConstPair(First: 0, Second: 0));
892 return DataPtr;
893 }
894
895 /// Creates a global variable of const char* type and creates an
896 /// initializer that initializes it with \p Str.
897 ///
898 /// \returns Link-time constant pointer (constant expr) to that
899 /// variable.
900 Constant *addStringToModule(StringRef Str, const Twine &Name) {
901 Constant *Arr = ConstantDataArray::getString(Context&: C, Initializer: Str);
902 GlobalVariable *Var =
903 new GlobalVariable(M, Arr->getType(), /*isConstant*/ true,
904 GlobalVariable::InternalLinkage, Arr, Name);
905 Var->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
906 ConstantInt *Zero = ConstantInt::get(Ty: getSizeTTy(), V: 0);
907 Constant *ZeroZero[] = {Zero, Zero};
908 return ConstantExpr::getGetElementPtr(Ty: Var->getValueType(), C: Var, IdxList: ZeroZero);
909 }
910
911 /// Each image contains its own set of symbols, which may contain different
912 /// symbols than other images. This function constructs an array of
913 /// symbol entries for a particular image.
914 ///
915 /// \returns Pointers to the beginning and end of the array.
916 std::pair<Constant *, Constant *>
917 initOffloadEntriesPerImage(StringRef Entries, const Twine &OffloadKindTag) {
918 SmallVector<Constant *> EntriesInits;
919 std::unique_ptr<MemoryBuffer> MB = MemoryBuffer::getMemBuffer(
920 InputData: Entries, /*BufferName*/ "", /*RequiresNullTerminator*/ false);
921 for (line_iterator LI(*MB); !LI.is_at_eof(); ++LI) {
922 GlobalVariable *GV =
923 emitOffloadingEntry(M, /*Kind*/ OffloadKind::OFK_SYCL,
924 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C)),
925 /*Name*/ *LI, /*Size*/ 0,
926 /*Flags*/ 0, /*Data*/ 0);
927 EntriesInits.push_back(Elt: GV->getInitializer());
928 }
929
930 Constant *Arr = ConstantArray::get(
931 T: ArrayType::get(ElementType: EntryTy, NumElements: EntriesInits.size()), V: EntriesInits);
932 GlobalVariable *EntriesGV = new GlobalVariable(
933 M, Arr->getType(), /*isConstant*/ true, GlobalVariable::InternalLinkage,
934 Arr, OffloadKindTag + "entries_arr");
935
936 Constant *EntriesB = ConstantExpr::getGetElementPtr(
937 Ty: EntriesGV->getValueType(), C: EntriesGV, IdxList: getSizetConstPair(First: 0, Second: 0));
938 Constant *EntriesE = ConstantExpr::getGetElementPtr(
939 Ty: EntriesGV->getValueType(), C: EntriesGV,
940 IdxList: getSizetConstPair(First: 0, Second: EntriesInits.size()));
941 return std::make_pair(x&: EntriesB, y&: EntriesE);
942 }
943
944 Constant *wrapImage(const OffloadBinary &OB, const Twine &ImageID,
945 StringRef OffloadKindTag) {
946 // Note: Intel DPC++ compiler had 2 versions of this structure
947 // and clang++ has a third different structure. To avoid ABI incompatibility
948 // between generated device images the Version here starts from 3.
949 constexpr uint16_t DeviceImageStructVersion = 3;
950 Constant *Version =
951 ConstantInt::get(Ty: Type::getInt16Ty(C), V: DeviceImageStructVersion);
952 Constant *OffloadKindConstant = ConstantInt::get(
953 Ty: Type::getInt8Ty(C), V: static_cast<uint8_t>(OB.getOffloadKind()));
954 Constant *ImageKindConstant = ConstantInt::get(
955 Ty: Type::getInt8Ty(C), V: static_cast<uint8_t>(OB.getImageKind()));
956 StringRef Triple = OB.getString(Key: "triple");
957 Constant *TripleConstant =
958 addStringToModule(Str: Triple, Name: Twine(OffloadKindTag) + "target." + ImageID);
959 Constant *CompileOptions =
960 addStringToModule(Str: Options.CompileOptions,
961 Name: Twine(OffloadKindTag) + "opts.compile." + ImageID);
962 Constant *LinkOptions = addStringToModule(
963 Str: Options.LinkOptions, Name: Twine(OffloadKindTag) + "opts.link." + ImageID);
964
965 // Note: NULL for now.
966 std::pair<Constant *, Constant *> PropertiesConstants = {
967 Constant::getNullValue(Ty: PointerType::getUnqual(C)),
968 Constant::getNullValue(Ty: PointerType::getUnqual(C))};
969
970 StringRef RawImage = OB.getImage();
971 std::pair<Constant *, Constant *> Binary = addArrayToModule(
972 Buf: ArrayRef<char>(RawImage.begin(), RawImage.end()),
973 Name: Twine(OffloadKindTag) + ImageID + ".data", Section: ".llvm.offloading");
974
975 // For SYCL images offload entries are defined here per image.
976 std::pair<Constant *, Constant *> ImageEntriesPtrs =
977 initOffloadEntriesPerImage(Entries: OB.getString(Key: "symbols"), OffloadKindTag);
978
979 // .first and .second arguments below correspond to start and end pointers
980 // respectively.
981 Constant *WrappedBinary = ConstantStruct::get(
982 T: SyclDeviceImageTy, Vs: Version, Vs: OffloadKindConstant, Vs: ImageKindConstant,
983 Vs: TripleConstant, Vs: CompileOptions, Vs: LinkOptions, Vs: Binary.first,
984 Vs: Binary.second, Vs: ImageEntriesPtrs.first, Vs: ImageEntriesPtrs.second,
985 Vs: PropertiesConstants.first, Vs: PropertiesConstants.second);
986
987 return WrappedBinary;
988 }
989
990 GlobalVariable *combineWrappedImages(ArrayRef<Constant *> WrappedImages,
991 StringRef OffloadKindTag) {
992 Constant *ImagesData = ConstantArray::get(
993 T: ArrayType::get(ElementType: SyclDeviceImageTy, NumElements: WrappedImages.size()), V: WrappedImages);
994 GlobalVariable *ImagesGV =
995 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
996 GlobalValue::InternalLinkage, ImagesData,
997 Twine(OffloadKindTag) + "device_images");
998 ImagesGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
999
1000 ConstantInt *Zero = ConstantInt::get(Ty: getSizeTTy(), V: 0);
1001 Constant *ZeroZero[] = {Zero, Zero};
1002 Constant *ImagesB = ConstantExpr::getGetElementPtr(Ty: ImagesGV->getValueType(),
1003 C: ImagesGV, IdxList: ZeroZero);
1004
1005 Constant *EntriesB = Constant::getNullValue(Ty: PointerType::getUnqual(C));
1006 Constant *EntriesE = Constant::getNullValue(Ty: PointerType::getUnqual(C));
1007 static constexpr uint16_t BinDescStructVersion = 1;
1008 Constant *DescInit = ConstantStruct::get(
1009 T: SyclBinDescTy,
1010 Vs: ConstantInt::get(Ty: Type::getInt16Ty(C), V: BinDescStructVersion),
1011 Vs: ConstantInt::get(Ty: Type::getInt16Ty(C), V: WrappedImages.size()), Vs: ImagesB,
1012 Vs: EntriesB, Vs: EntriesE);
1013
1014 return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,
1015 GlobalValue::InternalLinkage, DescInit,
1016 Twine(OffloadKindTag) + "descriptor");
1017 }
1018
1019 Module &M;
1020 LLVMContext &C;
1021 SYCLJITOptions Options;
1022
1023 StructType *EntryTy = nullptr;
1024 StructType *SyclDeviceImageTy = nullptr;
1025 StructType *SyclBinDescTy = nullptr;
1026}; // end of SYCLWrapper
1027
1028} // namespace
1029
1030Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
1031 EntryArrayTy EntryArray,
1032 llvm::StringRef Suffix, bool Relocatable) {
1033 GlobalVariable *Desc =
1034 createBinDesc(M, Bufs: Images, EntryArray, Suffix, Relocatable);
1035 if (!Desc)
1036 return createStringError(EC: inconvertibleErrorCode(),
1037 S: "No binary descriptors created.");
1038 createRegisterFunction(M, BinDesc: Desc, Suffix);
1039 return Error::success();
1040}
1041
1042Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,
1043 EntryArrayTy EntryArray,
1044 llvm::StringRef Suffix,
1045 bool EmitSurfacesAndTextures) {
1046 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: false, Suffix);
1047 if (!Desc)
1048 return createStringError(EC: inconvertibleErrorCode(),
1049 S: "No fatbin section created.");
1050
1051 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: false, EntryArray, Suffix,
1052 EmitSurfacesAndTextures);
1053 return Error::success();
1054}
1055
1056Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,
1057 EntryArrayTy EntryArray, llvm::StringRef Suffix,
1058 bool EmitSurfacesAndTextures) {
1059 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: true, Suffix);
1060 if (!Desc)
1061 return createStringError(EC: inconvertibleErrorCode(),
1062 S: "No fatbin section created.");
1063
1064 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: true, EntryArray, Suffix,
1065 EmitSurfacesAndTextures);
1066 return Error::success();
1067}
1068
1069Error llvm::offloading::wrapSYCLBinaries(llvm::Module &M, ArrayRef<char> Buffer,
1070 SYCLJITOptions Options) {
1071 SYCLWrapper W(M, Options);
1072 MemoryBufferRef MBR(StringRef(Buffer.begin(), Buffer.size()),
1073 /*Identifier*/ "");
1074 SmallVector<OffloadFile> OffloadFiles;
1075 if (Error E = extractOffloadBinaries(Buffer: MBR, Binaries&: OffloadFiles))
1076 return E;
1077
1078 GlobalVariable *Desc = W.createFatbinDesc(OffloadFiles);
1079 if (!Desc)
1080 return createStringError(EC: inconvertibleErrorCode(),
1081 S: "No binary descriptors created.");
1082
1083 W.createRegisterFatbinFunction(FatbinDesc: Desc);
1084 W.createUnregisterFunction(FatbinDesc: Desc);
1085 return Error::success();
1086}
1087