1//===----- CGCUDANV.cpp - Interface to NVIDIA CUDA Runtime ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for CUDA code generation targeting the NVIDIA CUDA
10// runtime library.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "clang/AST/CharUnits.h"
19#include "clang/AST/Decl.h"
20#include "clang/Basic/Cuda.h"
21#include "clang/CodeGen/CodeGenABITypes.h"
22#include "clang/CodeGen/ConstantInitBuilder.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Frontend/Offloading/Utility.h"
25#include "llvm/IR/BasicBlock.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/ReplaceConstant.h"
30#include "llvm/ProfileData/InstrProf.h"
31#include "llvm/Support/Format.h"
32#include "llvm/Support/MD5.h"
33#include "llvm/Support/VirtualFileSystem.h"
34
35using namespace clang;
36using namespace CodeGen;
37
38namespace {
39constexpr unsigned CudaFatMagic = 0x466243b1;
40constexpr unsigned HIPFatMagic = 0x48495046; // "HIPF"
41
42class CGNVCUDARuntime : public CGCUDARuntime {
43
44 /// The prefix used for function calls and section names (CUDA, HIP, LLVM)
45 StringRef Prefix;
46
47private:
48 llvm::IntegerType *IntTy, *SizeTy;
49 llvm::Type *VoidTy;
50 llvm::PointerType *PtrTy;
51
52 /// Convenience reference to LLVM Context
53 llvm::LLVMContext &Context;
54 /// Convenience reference to the current module
55 llvm::Module &TheModule;
56 /// Keeps track of kernel launch stubs and handles emitted in this module
57 struct KernelInfo {
58 llvm::Function *Kernel; // stub function to help launch kernel
59 const Decl *D;
60 };
61 llvm::SmallVector<KernelInfo, 16> EmittedKernels;
62 // Map a kernel mangled name to a symbol for identifying kernel in host code
63 // For CUDA, the symbol for identifying the kernel is the same as the device
64 // stub function. For HIP, they are different.
65 llvm::DenseMap<StringRef, llvm::GlobalValue *> KernelHandles;
66 // Map a kernel handle to the kernel stub.
67 llvm::DenseMap<llvm::GlobalValue *, llvm::Function *> KernelStubs;
68 struct VarInfo {
69 llvm::GlobalVariable *Var;
70 const VarDecl *D;
71 DeviceVarFlags Flags;
72 };
73 llvm::SmallVector<VarInfo, 16> DeviceVars;
74 /// Keeps track of variable containing handle of GPU binary. Populated by
75 /// ModuleCtorFunction() and used to create corresponding cleanup calls in
76 /// ModuleDtorFunction()
77 llvm::GlobalVariable *GpuBinaryHandle = nullptr;
78 /// Host-side shadow for the per-TU __llvm_profile_sections_<CUID> global,
79 /// emitted only for HIP host compiles when PGO is on. Registered via
80 /// __hipRegisterVar (non-RDC) or an offloading entry (RDC) so the runtime
81 /// can locate the device-side table by name.
82 llvm::GlobalVariable *OffloadProfShadow = nullptr;
83 /// Windows needs a registered handle for the device raw-version symbol.
84 llvm::GlobalVariable *OffloadProfVersionShadow = nullptr;
85 struct OffloadProfSectionShadowInfo {
86 llvm::GlobalVariable *Shadow;
87 std::string DeviceName;
88 };
89 llvm::SmallVector<OffloadProfSectionShadowInfo, 16> OffloadProfSectionShadows;
90 /// Whether we generate relocatable device code.
91 bool RelocatableDeviceCode;
92 /// Mangle context for device.
93 std::unique_ptr<MangleContext> DeviceMC;
94
95 llvm::FunctionCallee getSetupArgumentFn() const;
96 llvm::FunctionCallee getLaunchFn() const;
97
98 llvm::FunctionType *getRegisterGlobalsFnTy() const;
99 llvm::FunctionType *getCallbackFnTy() const;
100 llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
101 std::string addPrefixToName(StringRef FuncName) const;
102 std::string addUnderscoredPrefixToName(StringRef FuncName) const;
103
104 /// Creates a function to register all kernel stubs generated in this module.
105 llvm::Function *makeRegisterGlobalsFn();
106
107 /// Helper function that generates a constant string and returns a pointer to
108 /// the start of the string. The result of this function can be used anywhere
109 /// where the C code specifies const char*.
110 llvm::Constant *makeConstantString(const std::string &Str,
111 const std::string &Name = "") {
112 return CGM.GetAddrOfConstantCString(Str, GlobalName: Name).getPointer();
113 }
114
115 /// Helper function which generates an initialized constant array from Str,
116 /// and optionally sets section name and alignment. AddNull specifies whether
117 /// the array should nave NUL termination.
118 llvm::Constant *makeConstantArray(StringRef Str,
119 StringRef Name = "",
120 StringRef SectionName = "",
121 unsigned Alignment = 0,
122 bool AddNull = false) {
123 llvm::Constant *Value =
124 llvm::ConstantDataArray::getString(Context, Initializer: Str, AddNull);
125 auto *GV = new llvm::GlobalVariable(
126 TheModule, Value->getType(), /*isConstant=*/true,
127 llvm::GlobalValue::PrivateLinkage, Value, Name);
128 if (!SectionName.empty()) {
129 GV->setSection(SectionName);
130 // Mark the address as used which make sure that this section isn't
131 // merged and we will really have it in the object file.
132 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None);
133 }
134 if (Alignment)
135 GV->setAlignment(llvm::Align(Alignment));
136 return GV;
137 }
138
139 /// Helper function that generates an empty dummy function returning void.
140 llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) {
141 assert(FnTy->getReturnType()->isVoidTy() &&
142 "Can only generate dummy functions returning void!");
143 llvm::Function *DummyFunc = llvm::Function::Create(
144 Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage, N: "dummy", M: &TheModule);
145
146 llvm::BasicBlock *DummyBlock =
147 llvm::BasicBlock::Create(Context, Name: "", Parent: DummyFunc);
148 CGBuilderTy FuncBuilder(CGM, Context);
149 FuncBuilder.SetInsertPoint(DummyBlock);
150 FuncBuilder.CreateRetVoid();
151
152 return DummyFunc;
153 }
154
155 Address prepareKernelArgs(CodeGenFunction &CGF, FunctionArgList &Args);
156 Address prepareKernelArgsLLVMOffload(CodeGenFunction &CGF,
157 FunctionArgList &Args);
158 void emitDeviceStubBodyLegacy(CodeGenFunction &CGF, FunctionArgList &Args);
159 void emitDeviceStubBodyNew(CodeGenFunction &CGF, FunctionArgList &Args);
160 std::string getDeviceSideName(const NamedDecl *ND) override;
161
162 void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var,
163 bool Extern, bool Constant) {
164 DeviceVars.push_back(Elt: {.Var: &Var,
165 .D: VD,
166 .Flags: {DeviceVarFlags::Variable, Extern, Constant,
167 VD->hasAttr<HIPManagedAttr>(),
168 /*Normalized*/ false, 0}});
169 }
170 void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var,
171 bool Extern, int Type) {
172 DeviceVars.push_back(Elt: {.Var: &Var,
173 .D: VD,
174 .Flags: {DeviceVarFlags::Surface, Extern, /*Constant*/ false,
175 /*Managed*/ false,
176 /*Normalized*/ false, Type}});
177 }
178 void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var,
179 bool Extern, int Type, bool Normalized) {
180 DeviceVars.push_back(Elt: {.Var: &Var,
181 .D: VD,
182 .Flags: {DeviceVarFlags::Texture, Extern, /*Constant*/ false,
183 /*Managed*/ false, Normalized, Type}});
184 }
185
186 /// Creates module constructor function
187 llvm::Function *makeModuleCtorFunction();
188 /// Creates module destructor function
189 llvm::Function *makeModuleDtorFunction();
190 /// Transform managed variables for device compilation.
191 void transformManagedVars();
192 /// Create offloading entries to register globals in RDC mode.
193 void createOffloadingEntries();
194 /// For HIP+PGO, emit the per-TU __llvm_profile_sections_<CUID> global.
195 /// On the device side, InstrProfiling emits the populated section-bounds
196 /// table only when the TU has real profile data. On the host side it is a
197 /// placeholder void* shadow stored in
198 /// OffloadProfShadow, registered later by makeRegisterGlobalsFn (non-RDC)
199 /// or createOffloadingEntries (RDC) so the runtime can locate the
200 /// device-side table by name.
201 void emitOffloadProfilingSections();
202
203public:
204 CGNVCUDARuntime(CodeGenModule &CGM);
205
206 llvm::GlobalValue *getKernelHandle(llvm::Function *F, GlobalDecl GD) override;
207 llvm::Function *getKernelStub(llvm::GlobalValue *Handle) override {
208 auto Loc = KernelStubs.find(Val: Handle);
209 assert(Loc != KernelStubs.end());
210 return Loc->second;
211 }
212 void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
213 void handleVarRegistration(const VarDecl *VD,
214 llvm::GlobalVariable &Var) override;
215 void
216 internalizeDeviceSideVar(const VarDecl *D,
217 llvm::GlobalValue::LinkageTypes &Linkage) override;
218
219 llvm::Function *finalizeModule() override;
220};
221
222} // end anonymous namespace
223
224std::string CGNVCUDARuntime::addPrefixToName(StringRef FuncName) const {
225 return (Prefix + FuncName).str();
226}
227std::string
228CGNVCUDARuntime::addUnderscoredPrefixToName(StringRef FuncName) const {
229 return ("__" + Prefix + FuncName).str();
230}
231
232static std::unique_ptr<MangleContext> InitDeviceMC(CodeGenModule &CGM) {
233 // If the host and device have different C++ ABIs, mark it as the device
234 // mangle context so that the mangling needs to retrieve the additional
235 // device lambda mangling number instead of the regular host one.
236 if (CGM.getContext().getAuxTargetInfo() &&
237 CGM.getContext().getTargetInfo().getCXXABI().isMicrosoft() &&
238 CGM.getContext().getAuxTargetInfo()->getCXXABI().isItaniumFamily()) {
239 return std::unique_ptr<MangleContext>(
240 CGM.getContext().createDeviceMangleContext(
241 T: *CGM.getContext().getAuxTargetInfo()));
242 }
243
244 return std::unique_ptr<MangleContext>(CGM.getContext().createMangleContext(
245 T: CGM.getContext().getAuxTargetInfo()));
246}
247
248CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
249 : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
250 TheModule(CGM.getModule()),
251 RelocatableDeviceCode(CGM.getLangOpts().GPURelocatableDeviceCode),
252 DeviceMC(InitDeviceMC(CGM)) {
253 IntTy = CGM.IntTy;
254 SizeTy = CGM.SizeTy;
255 VoidTy = CGM.VoidTy;
256 PtrTy = CGM.DefaultPtrTy;
257
258 if (CGM.getLangOpts().OffloadViaLLVM)
259 Prefix = "llvm";
260 else if (CGM.getLangOpts().HIP)
261 Prefix = "hip";
262 else
263 Prefix = "cuda";
264}
265
266llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const {
267 // cudaError_t cudaSetupArgument(void *, size_t, size_t)
268 llvm::Type *Params[] = {PtrTy, SizeTy, SizeTy};
269 return CGM.CreateRuntimeFunction(
270 Ty: llvm::FunctionType::get(Result: IntTy, Params, isVarArg: false),
271 Name: addPrefixToName(FuncName: "SetupArgument"));
272}
273
274llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const {
275 if (CGM.getLangOpts().HIP) {
276 // hipError_t hipLaunchByPtr(char *);
277 return CGM.CreateRuntimeFunction(
278 Ty: llvm::FunctionType::get(Result: IntTy, Params: PtrTy, isVarArg: false), Name: "hipLaunchByPtr");
279 }
280 // cudaError_t cudaLaunch(char *);
281 return CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: IntTy, Params: PtrTy, isVarArg: false),
282 Name: "cudaLaunch");
283}
284
285llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const {
286 return llvm::FunctionType::get(Result: VoidTy, Params: PtrTy, isVarArg: false);
287}
288
289llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const {
290 return llvm::FunctionType::get(Result: VoidTy, Params: PtrTy, isVarArg: false);
291}
292
293llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const {
294 llvm::Type *Params[] = {llvm::PointerType::getUnqual(C&: Context), PtrTy, PtrTy,
295 llvm::PointerType::getUnqual(C&: Context)};
296 return llvm::FunctionType::get(Result: VoidTy, Params, isVarArg: false);
297}
298
299std::string CGNVCUDARuntime::getDeviceSideName(const NamedDecl *ND) {
300 GlobalDecl GD;
301 // D could be either a kernel or a variable.
302 if (auto *FD = dyn_cast<FunctionDecl>(Val: ND))
303 GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
304 else
305 GD = GlobalDecl(ND);
306 std::string DeviceSideName;
307 MangleContext *MC;
308 if (CGM.getLangOpts().CUDAIsDevice)
309 MC = &CGM.getCXXABI().getMangleContext();
310 else
311 MC = DeviceMC.get();
312 if (MC->shouldMangleDeclName(D: ND)) {
313 SmallString<256> Buffer;
314 llvm::raw_svector_ostream Out(Buffer);
315 MC->mangleName(GD, Out);
316 DeviceSideName = std::string(Out.str());
317 } else
318 DeviceSideName = std::string(ND->getIdentifier()->getName());
319
320 // Make unique name for device side static file-scope variable for HIP.
321 if (CGM.getContext().shouldExternalize(D: ND) &&
322 CGM.getLangOpts().GPURelocatableDeviceCode) {
323 SmallString<256> Buffer;
324 llvm::raw_svector_ostream Out(Buffer);
325 Out << DeviceSideName;
326 CGM.printPostfixForExternalizedDecl(OS&: Out, D: ND);
327 DeviceSideName = std::string(Out.str());
328 }
329 return DeviceSideName;
330}
331
332void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
333 FunctionArgList &Args) {
334 EmittedKernels.push_back(Elt: {.Kernel: CGF.CurFn, .D: CGF.CurFuncDecl});
335 if (auto *GV =
336 dyn_cast<llvm::GlobalVariable>(Val: KernelHandles[CGF.CurFn->getName()])) {
337 GV->setLinkage(CGF.CurFn->getLinkage());
338 GV->setInitializer(CGF.CurFn);
339 CGM.setDSOLocal(GV);
340 }
341 if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
342 CudaFeature::CUDA_USES_NEW_LAUNCH) ||
343 (CGF.getLangOpts().HIP && CGF.getLangOpts().HIPUseNewLaunchAPI) ||
344 (CGF.getLangOpts().OffloadViaLLVM))
345 emitDeviceStubBodyNew(CGF, Args);
346 else
347 emitDeviceStubBodyLegacy(CGF, Args);
348}
349
350/// CUDA passes the arguments with a level of indirection. For example, a
351/// (void*, short, void*) is passed as {void **, short *, void **} to the launch
352/// function. For the LLVM/Offload launch we include the number of arguments and
353/// their size. Thus, we pass {{void **, short*, void **}, 3, {sizeof(void*),
354/// sizeof(short), sizeof(void*)}}.
355Address CGNVCUDARuntime::prepareKernelArgsLLVMOffload(CodeGenFunction &CGF,
356 FunctionArgList &Args) {
357 SmallVector<llvm::Type *> KernelLaunchParamsTypes;
358
359 auto *Int64Ty = CGF.Builder.getInt64Ty();
360 KernelLaunchParamsTypes.push_back(Elt: PtrTy);
361 KernelLaunchParamsTypes.push_back(Elt: Int64Ty);
362 KernelLaunchParamsTypes.push_back(Elt: PtrTy);
363
364 llvm::StructType *KernelLaunchParamsTy =
365 llvm::StructType::create(Elements: KernelLaunchParamsTypes);
366 Address KernelLaunchParams = CGF.CreateTempAllocaWithoutCast(
367 Ty: KernelLaunchParamsTy, align: CharUnits::fromQuantity(Quantity: 16),
368 Name: "kernel_launch_params");
369 Address KernelArgs = CGF.CreateTempAlloca(
370 Ty: PtrTy, UseAddrSpace: LangAS::Default, align: CharUnits::fromQuantity(Quantity: 16), Name: "kernel_args",
371 ArraySize: llvm::ConstantInt::get(Ty: SizeTy, V: std::max<size_t>(a: 1, b: Args.size())));
372 Address KernelArgSizes = CGF.CreateTempAlloca(
373 Ty: SizeTy, UseAddrSpace: LangAS::Default, align: CharUnits::fromQuantity(Quantity: 16), Name: "kernel_arg_sizes",
374 ArraySize: llvm::ConstantInt::get(Ty: SizeTy, V: std::max<size_t>(a: 1, b: Args.size())));
375
376 CGF.Builder.CreateStore(Val: KernelArgs.emitRawPointer(CGF),
377 Addr: CGF.Builder.CreateStructGEP(Addr: KernelLaunchParams, Index: 0));
378 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: Int64Ty, V: Args.size()),
379 Addr: CGF.Builder.CreateStructGEP(Addr: KernelLaunchParams, Index: 1));
380 CGF.Builder.CreateStore(Val: KernelArgSizes.emitRawPointer(CGF),
381 Addr: CGF.Builder.CreateStructGEP(Addr: KernelLaunchParams, Index: 2));
382
383 for (unsigned i = 0; i < Args.size(); ++i) {
384 llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(VD: Args[i]).emitRawPointer(CGF);
385 llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(V: VarPtr, DestTy: PtrTy);
386 CGF.Builder.CreateDefaultAlignedStore(
387 Val: VoidVarPtr, Addr: CGF.Builder.CreateConstGEP1_32(
388 Ty: PtrTy, Ptr: KernelArgs.emitRawPointer(CGF), Idx0: i));
389
390 auto ArgSize = CGM.getDataLayout().getTypeAllocSize(
391 Ty: CGM.getTypes().ConvertType(T: Args[i]->getType()));
392 CGF.Builder.CreateDefaultAlignedStore(
393 Val: llvm::ConstantInt::get(Ty: SizeTy, V: ArgSize),
394 Addr: CGF.Builder.CreateConstGEP1_32(Ty: SizeTy,
395 Ptr: KernelArgSizes.emitRawPointer(CGF), Idx0: i));
396 }
397
398 return KernelLaunchParams;
399}
400
401Address CGNVCUDARuntime::prepareKernelArgs(CodeGenFunction &CGF,
402 FunctionArgList &Args) {
403 // Calculate amount of space we will need for all arguments. If we have no
404 // args, allocate a single pointer so we still have a valid pointer to the
405 // argument array that we can pass to runtime, even if it will be unused.
406 Address KernelArgs = CGF.CreateTempAlloca(
407 Ty: PtrTy, UseAddrSpace: LangAS::Default, align: CharUnits::fromQuantity(Quantity: 16), Name: "kernel_args",
408 ArraySize: llvm::ConstantInt::get(Ty: SizeTy, V: std::max<size_t>(a: 1, b: Args.size())));
409 // Store pointers to the arguments in a locally allocated launch_args.
410 for (unsigned i = 0; i < Args.size(); ++i) {
411 llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(VD: Args[i]).emitRawPointer(CGF);
412 llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(V: VarPtr, DestTy: PtrTy);
413 CGF.Builder.CreateDefaultAlignedStore(
414 Val: VoidVarPtr, Addr: CGF.Builder.CreateConstGEP1_32(
415 Ty: PtrTy, Ptr: KernelArgs.emitRawPointer(CGF), Idx0: i));
416 }
417 return KernelArgs;
418}
419
420// CUDA 9.0+ uses new way to launch kernels. Parameters are packed in a local
421// array and kernels are launched using cudaLaunchKernel().
422void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
423 FunctionArgList &Args) {
424 bool UsesLLVMOffloading = CGF.getLangOpts().OffloadViaLLVM;
425 // Build the shadow stack entry at the very start of the function.
426 Address KernelArgs = UsesLLVMOffloading
427 ? prepareKernelArgsLLVMOffload(CGF, Args)
428 : prepareKernelArgs(CGF, Args);
429
430 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "setup.end");
431
432 // Lookup cudaLaunchKernel/hipLaunchKernel function.
433 // HIP kernel launching API name depends on -fgpu-default-stream option. For
434 // the default value 'legacy', it is hipLaunchKernel. For 'per-thread',
435 // it is hipLaunchKernel_spt.
436 // cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
437 // void **args, size_t sharedMem,
438 // cudaStream_t stream);
439 // hipError_t hipLaunchKernel[_spt](const void *func, dim3 gridDim,
440 // dim3 blockDim, void **args,
441 // size_t sharedMem, hipStream_t stream);
442 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
443 DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
444 std::string KernelLaunchAPI = "LaunchKernel";
445 if (CGF.getLangOpts().GPUDefaultStream ==
446 LangOptions::GPUDefaultStreamKind::PerThread) {
447 if (CGF.getLangOpts().HIP)
448 KernelLaunchAPI = KernelLaunchAPI + "_spt";
449 else if (CGF.getLangOpts().CUDA)
450 KernelLaunchAPI = KernelLaunchAPI + "_ptsz";
451 }
452 auto LaunchKernelName = addPrefixToName(FuncName: KernelLaunchAPI);
453 const IdentifierInfo &cudaLaunchKernelII =
454 CGM.getContext().Idents.get(Name: LaunchKernelName);
455 FunctionDecl *cudaLaunchKernelFD = nullptr;
456 for (auto *Result : DC->lookup(Name: &cudaLaunchKernelII)) {
457 if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Result))
458 cudaLaunchKernelFD = FD;
459 }
460
461 if (cudaLaunchKernelFD == nullptr) {
462 CGM.Error(loc: CGF.CurFuncDecl->getLocation(),
463 error: "Can't find declaration for " + LaunchKernelName);
464 return;
465 }
466 // Create temporary dim3 grid_dim, block_dim.
467 ParmVarDecl *GridDimParam = cudaLaunchKernelFD->getParamDecl(i: 1);
468 QualType Dim3Ty = GridDimParam->getType();
469 Address GridDim = CGF.CreateMemTempWithoutCast(
470 T: Dim3Ty, Align: CharUnits::fromQuantity(Quantity: 8), Name: "grid_dim");
471 Address BlockDim = CGF.CreateMemTempWithoutCast(
472 T: Dim3Ty, Align: CharUnits::fromQuantity(Quantity: 8), Name: "block_dim");
473 Address ShmemSize = CGF.CreateTempAlloca(Ty: SizeTy, UseAddrSpace: LangAS::Default,
474 align: CGM.getSizeAlign(), Name: "shmem_size");
475 Address Stream = CGF.CreateTempAlloca(Ty: PtrTy, UseAddrSpace: LangAS::Default,
476 align: CGM.getPointerAlign(), Name: "stream");
477 llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction(
478 Ty: llvm::FunctionType::get(Result: IntTy,
479 Params: {/*gridDim=*/GridDim.getType(),
480 /*blockDim=*/BlockDim.getType(),
481 /*ShmemSize=*/ShmemSize.getType(),
482 /*Stream=*/Stream.getType()},
483 /*isVarArg=*/false),
484 Name: addUnderscoredPrefixToName(FuncName: "PopCallConfiguration"));
485
486 CGF.EmitRuntimeCallOrInvoke(callee: cudaPopConfigFn, args: {GridDim.emitRawPointer(CGF),
487 BlockDim.emitRawPointer(CGF),
488 ShmemSize.emitRawPointer(CGF),
489 Stream.emitRawPointer(CGF)});
490
491 // Emit the call to cudaLaunch
492 llvm::Value *Kernel =
493 CGF.Builder.CreatePointerCast(V: KernelHandles[CGF.CurFn->getName()], DestTy: PtrTy);
494 CallArgList LaunchKernelArgs;
495 LaunchKernelArgs.add(rvalue: RValue::get(V: Kernel),
496 type: cudaLaunchKernelFD->getParamDecl(i: 0)->getType());
497 LaunchKernelArgs.add(rvalue: RValue::getAggregate(addr: GridDim), type: Dim3Ty);
498 LaunchKernelArgs.add(rvalue: RValue::getAggregate(addr: BlockDim), type: Dim3Ty);
499 LaunchKernelArgs.add(rvalue: RValue::get(Addr: KernelArgs, CGF),
500 type: cudaLaunchKernelFD->getParamDecl(i: 3)->getType());
501 LaunchKernelArgs.add(rvalue: RValue::get(V: CGF.Builder.CreateLoad(Addr: ShmemSize)),
502 type: cudaLaunchKernelFD->getParamDecl(i: 4)->getType());
503 LaunchKernelArgs.add(rvalue: RValue::get(V: CGF.Builder.CreateLoad(Addr: Stream)),
504 type: cudaLaunchKernelFD->getParamDecl(i: 5)->getType());
505
506 QualType QT = cudaLaunchKernelFD->getType();
507 QualType CQT = QT.getCanonicalType();
508 llvm::Type *Ty = CGM.getTypes().ConvertType(T: CQT);
509 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Val: Ty);
510
511 const CGFunctionInfo &FI =
512 CGM.getTypes().arrangeFunctionDeclaration(GD: cudaLaunchKernelFD);
513 llvm::FunctionCallee cudaLaunchKernelFn =
514 CGM.CreateRuntimeFunction(Ty: FTy, Name: LaunchKernelName);
515 CGF.EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: cudaLaunchKernelFn), ReturnValue: ReturnValueSlot(),
516 Args: LaunchKernelArgs);
517
518 // To prevent CUDA device stub functions from being merged by ICF in MSVC
519 // environment, create an unique global variable for each kernel and write to
520 // the variable in the device stub.
521 if (CGM.getContext().getTargetInfo().getCXXABI().isMicrosoft() &&
522 !CGF.getLangOpts().HIP) {
523 llvm::Function *KernelFunction = llvm::cast<llvm::Function>(Val: Kernel);
524 std::string GlobalVarName = (KernelFunction->getName() + ".id").str();
525
526 llvm::GlobalVariable *HandleVar =
527 CGM.getModule().getNamedGlobal(Name: GlobalVarName);
528 if (!HandleVar) {
529 HandleVar = new llvm::GlobalVariable(
530 CGM.getModule(), CGM.Int8Ty,
531 /*Constant=*/false, KernelFunction->getLinkage(),
532 llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 0), GlobalVarName);
533 HandleVar->setDSOLocal(KernelFunction->isDSOLocal());
534 HandleVar->setVisibility(KernelFunction->getVisibility());
535 if (KernelFunction->hasComdat())
536 HandleVar->setComdat(CGM.getModule().getOrInsertComdat(Name: GlobalVarName));
537 }
538
539 CGF.Builder.CreateAlignedStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
540 Addr: HandleVar, Align: CharUnits::One(),
541 /*IsVolatile=*/true);
542 }
543
544 CGF.EmitBranch(Block: EndBlock);
545
546 CGF.EmitBlock(BB: EndBlock);
547}
548
549void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF,
550 FunctionArgList &Args) {
551 // Emit a call to cudaSetupArgument for each arg in Args.
552 llvm::FunctionCallee cudaSetupArgFn = getSetupArgumentFn();
553 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "setup.end");
554 CharUnits Offset = CharUnits::Zero();
555 for (const VarDecl *A : Args) {
556 auto TInfo = CGM.getContext().getTypeInfoInChars(T: A->getType());
557 Offset = Offset.alignTo(Align: TInfo.Align);
558 llvm::Value *Args[] = {
559 CGF.Builder.CreatePointerCast(
560 V: CGF.GetAddrOfLocalVar(VD: A).emitRawPointer(CGF), DestTy: PtrTy),
561 llvm::ConstantInt::get(Ty: SizeTy, V: TInfo.Width.getQuantity()),
562 llvm::ConstantInt::get(Ty: SizeTy, V: Offset.getQuantity()),
563 };
564 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: cudaSetupArgFn, args: Args);
565 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
566 llvm::Value *CBZero = CGF.Builder.CreateICmpEQ(LHS: CB, RHS: Zero);
567 llvm::BasicBlock *NextBlock = CGF.createBasicBlock(name: "setup.next");
568 CGF.Builder.CreateCondBr(Cond: CBZero, True: NextBlock, False: EndBlock);
569 CGF.EmitBlock(BB: NextBlock);
570 Offset += TInfo.Width;
571 }
572
573 // Emit the call to cudaLaunch
574 llvm::FunctionCallee cudaLaunchFn = getLaunchFn();
575 llvm::Value *Arg =
576 CGF.Builder.CreatePointerCast(V: KernelHandles[CGF.CurFn->getName()], DestTy: PtrTy);
577 CGF.EmitRuntimeCallOrInvoke(callee: cudaLaunchFn, args: Arg);
578 CGF.EmitBranch(Block: EndBlock);
579
580 CGF.EmitBlock(BB: EndBlock);
581}
582
583// Replace the original variable Var with the address loaded from variable
584// ManagedVar populated by HIP runtime.
585static void replaceManagedVar(llvm::GlobalVariable *Var,
586 llvm::GlobalVariable *ManagedVar) {
587 SmallVector<SmallVector<llvm::User *, 8>, 8> WorkList;
588 for (auto &&VarUse : Var->uses()) {
589 WorkList.push_back(Elt: {VarUse.getUser()});
590 }
591 while (!WorkList.empty()) {
592 auto &&WorkItem = WorkList.pop_back_val();
593 auto *U = WorkItem.back();
594 if (isa<llvm::ConstantExpr>(Val: U)) {
595 for (auto &&UU : U->uses()) {
596 WorkItem.push_back(Elt: UU.getUser());
597 WorkList.push_back(Elt: WorkItem);
598 WorkItem.pop_back();
599 }
600 continue;
601 }
602 if (auto *I = dyn_cast<llvm::Instruction>(Val: U)) {
603 llvm::Value *OldV = Var;
604 llvm::Instruction *NewV =
605 new llvm::LoadInst(Var->getType(), ManagedVar, "ld.managed", false,
606 Var->getAlign().valueOrOne(), I->getIterator());
607 WorkItem.pop_back();
608 // Replace constant expressions directly or indirectly using the managed
609 // variable with instructions.
610 for (auto &&Op : WorkItem) {
611 auto *CE = cast<llvm::ConstantExpr>(Val: Op);
612 auto *NewInst = CE->getAsInstruction();
613 NewInst->insertBefore(BB&: *I->getParent(), InsertPos: I->getIterator());
614 NewInst->replaceUsesOfWith(From: OldV, To: NewV);
615 OldV = CE;
616 NewV = NewInst;
617 }
618 I->replaceUsesOfWith(From: OldV, To: NewV);
619 } else {
620 llvm_unreachable("Invalid use of managed variable");
621 }
622 }
623}
624
625/// Creates a function that sets up state on the host side for CUDA objects that
626/// have a presence on both the host and device sides. Specifically, registers
627/// the host side of kernel functions and device global variables with the CUDA
628/// runtime.
629/// \code
630/// void __cuda_register_globals(void** GpuBinaryHandle) {
631/// __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...);
632/// ...
633/// __cudaRegisterFunction(GpuBinaryHandle,KernelM,...);
634/// __cudaRegisterVar(GpuBinaryHandle, GlobalVar0, ...);
635/// ...
636/// __cudaRegisterVar(GpuBinaryHandle, GlobalVarN, ...);
637/// }
638/// \endcode
639llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
640 // No need to register anything
641 if (EmittedKernels.empty() && DeviceVars.empty())
642 return nullptr;
643
644 llvm::Function *RegisterKernelsFunc = llvm::Function::Create(
645 Ty: getRegisterGlobalsFnTy(), Linkage: llvm::GlobalValue::InternalLinkage,
646 N: addUnderscoredPrefixToName(FuncName: "_register_globals"), M: &TheModule);
647 llvm::BasicBlock *EntryBB =
648 llvm::BasicBlock::Create(Context, Name: "entry", Parent: RegisterKernelsFunc);
649 CGBuilderTy Builder(CGM, Context);
650 Builder.SetInsertPoint(EntryBB);
651
652 // void __cudaRegisterFunction(void **, const char *, char *, const char *,
653 // int, uint3*, uint3*, dim3*, dim3*, int*)
654 llvm::Type *RegisterFuncParams[] = {
655 PtrTy, PtrTy, PtrTy, PtrTy, IntTy,
656 PtrTy, PtrTy, PtrTy, PtrTy, llvm::PointerType::getUnqual(C&: Context)};
657 llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction(
658 Ty: llvm::FunctionType::get(Result: IntTy, Params: RegisterFuncParams, isVarArg: false),
659 Name: addUnderscoredPrefixToName(FuncName: "RegisterFunction"));
660
661 // Extract GpuBinaryHandle passed as the first argument passed to
662 // __cuda_register_globals() and generate __cudaRegisterFunction() call for
663 // each emitted kernel.
664 llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin();
665 for (auto &&I : EmittedKernels) {
666 llvm::Constant *KernelName =
667 makeConstantString(Str: getDeviceSideName(ND: cast<NamedDecl>(Val: I.D)));
668 llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(T: PtrTy);
669 llvm::Value *Args[] = {
670 &GpuBinaryHandlePtr,
671 KernelHandles[I.Kernel->getName()],
672 KernelName,
673 KernelName,
674 llvm::ConstantInt::getAllOnesValue(Ty: IntTy),
675 NullPtr,
676 NullPtr,
677 NullPtr,
678 NullPtr,
679 llvm::ConstantPointerNull::get(T: llvm::PointerType::getUnqual(C&: Context))};
680 Builder.CreateCall(Callee: RegisterFunc, Args);
681 }
682
683 llvm::Type *VarSizeTy = IntTy;
684 // For HIP or CUDA 9.0+, device variable size is type of `size_t`.
685 if (CGM.getLangOpts().HIP ||
686 ToCudaVersion(CGM.getTarget().getSDKVersion()) >= CudaVersion::CUDA_90)
687 VarSizeTy = SizeTy;
688
689 // void __cudaRegisterVar(void **, char *, char *, const char *,
690 // int, int, int, int)
691 llvm::Type *RegisterVarParams[] = {PtrTy, PtrTy, PtrTy, PtrTy,
692 IntTy, VarSizeTy, IntTy, IntTy};
693 llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction(
694 Ty: llvm::FunctionType::get(Result: VoidTy, Params: RegisterVarParams, isVarArg: false),
695 Name: addUnderscoredPrefixToName(FuncName: "RegisterVar"));
696 // void __hipRegisterManagedVar(void **, char *, char *, const char *,
697 // size_t, unsigned)
698 llvm::Type *RegisterManagedVarParams[] = {PtrTy, PtrTy, PtrTy,
699 PtrTy, VarSizeTy, IntTy};
700 llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction(
701 Ty: llvm::FunctionType::get(Result: VoidTy, Params: RegisterManagedVarParams, isVarArg: false),
702 Name: addUnderscoredPrefixToName(FuncName: "RegisterManagedVar"));
703 // void __cudaRegisterSurface(void **, const struct surfaceReference *,
704 // const void **, const char *, int, int);
705 llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction(
706 Ty: llvm::FunctionType::get(
707 Result: VoidTy, Params: {PtrTy, PtrTy, PtrTy, PtrTy, IntTy, IntTy}, isVarArg: false),
708 Name: addUnderscoredPrefixToName(FuncName: "RegisterSurface"));
709 // void __cudaRegisterTexture(void **, const struct textureReference *,
710 // const void **, const char *, int, int, int)
711 llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction(
712 Ty: llvm::FunctionType::get(
713 Result: VoidTy, Params: {PtrTy, PtrTy, PtrTy, PtrTy, IntTy, IntTy, IntTy}, isVarArg: false),
714 Name: addUnderscoredPrefixToName(FuncName: "RegisterTexture"));
715 for (auto &&Info : DeviceVars) {
716 llvm::GlobalVariable *Var = Info.Var;
717 assert((!Var->isDeclaration() || Info.Flags.isManaged()) &&
718 "External variables should not show up here, except HIP managed "
719 "variables");
720 llvm::Constant *VarName = makeConstantString(Str: getDeviceSideName(ND: Info.D));
721 switch (Info.Flags.getKind()) {
722 case DeviceVarFlags::Variable: {
723 uint64_t VarSize =
724 CGM.getDataLayout().getTypeAllocSize(Ty: Var->getValueType());
725 if (Info.Flags.isManaged()) {
726 assert(Var->getName().ends_with(".managed") &&
727 "HIP managed variables not transformed");
728 auto *ManagedVar = CGM.getModule().getNamedGlobal(
729 Name: Var->getName().drop_back(N: StringRef(".managed").size()));
730 llvm::Value *Args[] = {
731 &GpuBinaryHandlePtr,
732 ManagedVar,
733 Var,
734 VarName,
735 llvm::ConstantInt::get(Ty: VarSizeTy, V: VarSize),
736 llvm::ConstantInt::get(Ty: IntTy,
737 V: Var->getAlign().valueOrOne().value())};
738 if (!Var->isDeclaration())
739 Builder.CreateCall(Callee: RegisterManagedVar, Args);
740 } else {
741 llvm::Value *Args[] = {
742 &GpuBinaryHandlePtr,
743 Var,
744 VarName,
745 VarName,
746 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.isExtern()),
747 llvm::ConstantInt::get(Ty: VarSizeTy, V: VarSize),
748 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.isConstant()),
749 llvm::ConstantInt::get(Ty: IntTy, V: 0)};
750 Builder.CreateCall(Callee: RegisterVar, Args);
751 }
752 break;
753 }
754 case DeviceVarFlags::Surface:
755 Builder.CreateCall(
756 Callee: RegisterSurf,
757 Args: {&GpuBinaryHandlePtr, Var, VarName, VarName,
758 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.getSurfTexType()),
759 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.isExtern())});
760 break;
761 case DeviceVarFlags::Texture:
762 Builder.CreateCall(
763 Callee: RegisterTex,
764 Args: {&GpuBinaryHandlePtr, Var, VarName, VarName,
765 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.getSurfTexType()),
766 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.isNormalized()),
767 llvm::ConstantInt::get(Ty: IntTy, V: Info.Flags.isExtern())});
768 break;
769 }
770 }
771
772 auto RegisterProfileShadow = [&](llvm::GlobalVariable *Shadow,
773 StringRef DeviceName) {
774 llvm::Constant *Name = makeConstantString(Str: DeviceName.str());
775 llvm::Constant *IntZero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
776 uint64_t Size =
777 CGM.getDataLayout().getTypeAllocSize(Ty: Shadow->getValueType());
778 llvm::Value *Args[] = {&GpuBinaryHandlePtr,
779 Shadow,
780 Name,
781 Name,
782 IntZero,
783 llvm::ConstantInt::get(Ty: VarSizeTy, V: Size),
784 IntZero,
785 IntZero};
786 Builder.CreateCall(Callee: RegisterVar, Args);
787 };
788
789 // Register the per-TU offload-profiling shadow so the host runtime can
790 // locate the matching device-side __llvm_profile_sections_<CUID>. We
791 // emit both __hipRegisterVar (so the HIP runtime can map the host
792 // shadow to the device symbol) and
793 // __llvm_profile_offload_register_shadow_variable (so the profile
794 // runtime adds the shadow to its drain list).
795 if (OffloadProfShadow) {
796 RegisterProfileShadow(OffloadProfShadow, OffloadProfShadow->getName());
797
798 llvm::FunctionCallee RegisterShadow = CGM.CreateRuntimeFunction(
799 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
800 Name: "__llvm_profile_offload_register_shadow_variable");
801 Builder.CreateCall(Callee: RegisterShadow, Args: {OffloadProfShadow});
802 }
803
804 if (OffloadProfVersionShadow) {
805 RegisterProfileShadow(OffloadProfVersionShadow,
806 "__llvm_profile_raw_version");
807 llvm::FunctionCallee RegisterVersionShadow = CGM.CreateRuntimeFunction(
808 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
809 Name: "__llvm_profile_offload_register_version_shadow_variable");
810 Builder.CreateCall(Callee: RegisterVersionShadow, Args: {OffloadProfVersionShadow});
811 }
812
813 if (!OffloadProfSectionShadows.empty()) {
814 llvm::FunctionCallee RegisterSectionShadow = CGM.CreateRuntimeFunction(
815 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
816 Name: "__llvm_profile_offload_register_section_shadow_variable");
817 for (const auto &Info : OffloadProfSectionShadows) {
818 RegisterProfileShadow(Info.Shadow, Info.DeviceName);
819 Builder.CreateCall(Callee: RegisterSectionShadow, Args: {Info.Shadow});
820 }
821 }
822
823 Builder.CreateRetVoid();
824 return RegisterKernelsFunc;
825}
826
827/// Creates a global constructor function for the module:
828///
829/// For CUDA:
830/// \code
831/// void __cuda_module_ctor() {
832/// Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
833/// __cuda_register_globals(Handle);
834/// }
835/// \endcode
836///
837/// For HIP:
838/// \code
839/// void __hip_module_ctor() {
840/// if (__hip_gpubin_handle == 0) {
841/// __hip_gpubin_handle = __hipRegisterFatBinary(GpuBinaryBlob);
842/// __hip_register_globals(__hip_gpubin_handle);
843/// }
844/// }
845/// \endcode
846llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
847 bool IsHIP = CGM.getLangOpts().HIP;
848 bool IsCUDA = CGM.getLangOpts().CUDA;
849 // No need to generate ctors/dtors if there is no GPU binary.
850 StringRef CudaGpuBinaryFileName =
851 CGM.getCodeGenOpts().OffloadBinaryToEmbedFile;
852 if (CudaGpuBinaryFileName.empty() && !IsHIP)
853 return nullptr;
854 if ((IsHIP || (IsCUDA && !RelocatableDeviceCode)) && EmittedKernels.empty() &&
855 DeviceVars.empty())
856 return nullptr;
857
858 // void __{cuda|hip}_register_globals(void* handle);
859 llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn();
860 // We always need a function to pass in as callback. Create a dummy
861 // implementation if we don't need to register anything.
862 if (RelocatableDeviceCode && !RegisterGlobalsFunc)
863 RegisterGlobalsFunc = makeDummyFunction(FnTy: getRegisterGlobalsFnTy());
864
865 // void ** __{cuda|hip}RegisterFatBinary(void *);
866 llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction(
867 Ty: llvm::FunctionType::get(Result: PtrTy, Params: PtrTy, isVarArg: false),
868 Name: addUnderscoredPrefixToName(FuncName: "RegisterFatBinary"));
869 // struct { int magic, int version, void * gpu_binary, void * dont_care };
870 llvm::StructType *FatbinWrapperTy =
871 llvm::StructType::get(elt1: IntTy, elts: IntTy, elts: PtrTy, elts: PtrTy);
872
873 // Register GPU binary with the CUDA runtime, store returned handle in a
874 // global variable and save a reference in GpuBinaryHandle to be cleaned up
875 // in destructor on exit. Then associate all known kernels with the GPU binary
876 // handle so CUDA runtime can figure out what to call on the GPU side.
877 std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary = nullptr;
878 if (!CudaGpuBinaryFileName.empty()) {
879 auto VFS = CGM.getFileSystem();
880 auto CudaGpuBinaryOrErr =
881 VFS->getBufferForFile(Name: CudaGpuBinaryFileName, FileSize: -1, RequiresNullTerminator: false);
882 if (std::error_code EC = CudaGpuBinaryOrErr.getError()) {
883 CGM.getDiags().Report(DiagID: diag::err_cannot_open_file)
884 << CudaGpuBinaryFileName << EC.message();
885 return nullptr;
886 }
887 CudaGpuBinary = std::move(CudaGpuBinaryOrErr.get());
888 }
889
890 llvm::Function *ModuleCtorFunc = llvm::Function::Create(
891 Ty: llvm::FunctionType::get(Result: VoidTy, isVarArg: false),
892 Linkage: llvm::GlobalValue::InternalLinkage,
893 N: addUnderscoredPrefixToName(FuncName: "_module_ctor"), M: &TheModule);
894 llvm::BasicBlock *CtorEntryBB =
895 llvm::BasicBlock::Create(Context, Name: "entry", Parent: ModuleCtorFunc);
896 CGBuilderTy CtorBuilder(CGM, Context);
897
898 CtorBuilder.SetInsertPoint(CtorEntryBB);
899
900 const char *FatbinConstantName;
901 const char *FatbinSectionName;
902 const char *ModuleIDSectionName;
903 StringRef ModuleIDPrefix;
904 llvm::Constant *FatBinStr;
905 unsigned FatMagic;
906 if (IsHIP) {
907 // On macOS (Mach-O), section names must be in "segment,section" format.
908 FatbinConstantName =
909 CGM.getTriple().isMacOSX() ? "__HIP,__hip_fatbin" : ".hip_fatbin";
910 FatbinSectionName =
911 CGM.getTriple().isMacOSX() ? "__HIP,__fatbin" : ".hipFatBinSegment";
912
913 ModuleIDSectionName =
914 CGM.getTriple().isMacOSX() ? "__HIP,__module_id" : "__hip_module_id";
915 ModuleIDPrefix = "__hip_";
916
917 if (CudaGpuBinary) {
918 // If fatbin is available from early finalization, create a string
919 // literal containing the fat binary loaded from the given file.
920 const unsigned HIPCodeObjectAlign = 4096;
921 FatBinStr = makeConstantArray(Str: std::string(CudaGpuBinary->getBuffer()), Name: "",
922 SectionName: FatbinConstantName, Alignment: HIPCodeObjectAlign);
923 } else {
924 // If fatbin is not available, create an external symbol
925 // __hip_fatbin in section .hip_fatbin. The external symbol is supposed
926 // to contain the fat binary but will be populated somewhere else,
927 // e.g. by lld through link script.
928 FatBinStr = new llvm::GlobalVariable(
929 CGM.getModule(), CGM.Int8Ty,
930 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr,
931 "__hip_fatbin" + (CGM.getLangOpts().CUID.empty()
932 ? ""
933 : "_" + CGM.getContext().getCUIDHash()),
934 nullptr, llvm::GlobalVariable::NotThreadLocal);
935 cast<llvm::GlobalVariable>(Val: FatBinStr)->setSection(FatbinConstantName);
936 }
937
938 FatMagic = HIPFatMagic;
939 } else {
940 if (RelocatableDeviceCode)
941 FatbinConstantName = CGM.getTriple().isMacOSX()
942 ? "__NV_CUDA,__nv_relfatbin"
943 : "__nv_relfatbin";
944 else
945 FatbinConstantName =
946 CGM.getTriple().isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin";
947 // NVIDIA's cuobjdump looks for fatbins in this section.
948 FatbinSectionName =
949 CGM.getTriple().isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment";
950
951 ModuleIDSectionName = CGM.getTriple().isMacOSX()
952 ? "__NV_CUDA,__nv_module_id"
953 : "__nv_module_id";
954 ModuleIDPrefix = "__nv_";
955
956 // For CUDA, create a string literal containing the fat binary loaded from
957 // the given file.
958 FatBinStr = makeConstantArray(Str: std::string(CudaGpuBinary->getBuffer()), Name: "",
959 SectionName: FatbinConstantName, Alignment: 8);
960 FatMagic = CudaFatMagic;
961 }
962
963 // Create initialized wrapper structure that points to the loaded GPU binary
964 ConstantInitBuilder Builder(CGM);
965 auto Values = Builder.beginStruct(structTy: FatbinWrapperTy);
966 // Fatbin wrapper magic.
967 Values.addInt(intTy: IntTy, value: FatMagic);
968 // Fatbin version.
969 Values.addInt(intTy: IntTy, value: 1);
970 // Data.
971 Values.add(value: FatBinStr);
972 // Unused in fatbin v1.
973 Values.add(value: llvm::ConstantPointerNull::get(T: PtrTy));
974 llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal(
975 args: addUnderscoredPrefixToName(FuncName: "_fatbin_wrapper"), args: CGM.getPointerAlign(),
976 /*constant*/ args: true);
977 FatbinWrapper->setSection(FatbinSectionName);
978 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: FatbinWrapper);
979
980 // There is only one HIP fat binary per linked module, however there are
981 // multiple constructor functions. Make sure the fat binary is registered
982 // only once. The constructor functions are executed by the dynamic loader
983 // before the program gains control. The dynamic loader cannot execute the
984 // constructor functions concurrently since doing that would not guarantee
985 // thread safety of the loaded program. Therefore we can assume sequential
986 // execution of constructor functions here.
987 if (IsHIP) {
988 auto Linkage = RelocatableDeviceCode ? llvm::GlobalValue::ExternalLinkage
989 : llvm::GlobalValue::InternalLinkage;
990 llvm::BasicBlock *IfBlock =
991 llvm::BasicBlock::Create(Context, Name: "if", Parent: ModuleCtorFunc);
992 llvm::BasicBlock *ExitBlock =
993 llvm::BasicBlock::Create(Context, Name: "exit", Parent: ModuleCtorFunc);
994 // The name, size, and initialization pattern of this variable is part
995 // of HIP ABI.
996 GpuBinaryHandle = new llvm::GlobalVariable(
997 TheModule, PtrTy, /*isConstant=*/false, Linkage,
998 /*Initializer=*/
999 !RelocatableDeviceCode ? llvm::ConstantPointerNull::get(T: PtrTy)
1000 : nullptr,
1001 "__hip_gpubin_handle" + (CGM.getLangOpts().CUID.empty()
1002 ? ""
1003 : "_" + CGM.getContext().getCUIDHash()));
1004 GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign());
1005 // Prevent the weak symbol in different shared libraries being merged.
1006 if (Linkage != llvm::GlobalValue::InternalLinkage)
1007 GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility);
1008 Address GpuBinaryAddr(
1009 GpuBinaryHandle, PtrTy,
1010 CharUnits::fromQuantity(Quantity: GpuBinaryHandle->getAlign().valueOrOne()));
1011 {
1012 auto *HandleValue = CtorBuilder.CreateLoad(Addr: GpuBinaryAddr);
1013 llvm::Constant *Zero =
1014 llvm::Constant::getNullValue(Ty: HandleValue->getType());
1015 llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(LHS: HandleValue, RHS: Zero);
1016 CtorBuilder.CreateCondBr(Cond: EQZero, True: IfBlock, False: ExitBlock);
1017 }
1018 {
1019 CtorBuilder.SetInsertPoint(IfBlock);
1020 // GpuBinaryHandle = __hipRegisterFatBinary(&FatbinWrapper);
1021 llvm::CallInst *RegisterFatbinCall =
1022 CtorBuilder.CreateCall(Callee: RegisterFatbinFunc, Args: FatbinWrapper);
1023 CtorBuilder.CreateStore(Val: RegisterFatbinCall, Addr: GpuBinaryAddr);
1024 CtorBuilder.CreateBr(Dest: ExitBlock);
1025 }
1026 {
1027 CtorBuilder.SetInsertPoint(ExitBlock);
1028 // Call __hip_register_globals(GpuBinaryHandle);
1029 if (RegisterGlobalsFunc) {
1030 auto *HandleValue = CtorBuilder.CreateLoad(Addr: GpuBinaryAddr);
1031 CtorBuilder.CreateCall(Callee: RegisterGlobalsFunc, Args: HandleValue);
1032 }
1033 }
1034 } else if (!RelocatableDeviceCode) {
1035 // Register binary with CUDA runtime. This is substantially different in
1036 // default mode vs. separate compilation!
1037 // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper);
1038 llvm::CallInst *RegisterFatbinCall =
1039 CtorBuilder.CreateCall(Callee: RegisterFatbinFunc, Args: FatbinWrapper);
1040 GpuBinaryHandle = new llvm::GlobalVariable(
1041 TheModule, PtrTy, false, llvm::GlobalValue::InternalLinkage,
1042 llvm::ConstantPointerNull::get(T: PtrTy), "__cuda_gpubin_handle");
1043 GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign());
1044 CtorBuilder.CreateAlignedStore(Val: RegisterFatbinCall, Addr: GpuBinaryHandle,
1045 Align: CGM.getPointerAlign());
1046
1047 // Call __cuda_register_globals(GpuBinaryHandle);
1048 if (RegisterGlobalsFunc)
1049 CtorBuilder.CreateCall(Callee: RegisterGlobalsFunc, Args: RegisterFatbinCall);
1050
1051 // Call __cudaRegisterFatBinaryEnd(Handle) if this CUDA version needs it.
1052 if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
1053 CudaFeature::CUDA_USES_FATBIN_REGISTER_END)) {
1054 // void __cudaRegisterFatBinaryEnd(void **);
1055 llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction(
1056 Ty: llvm::FunctionType::get(Result: VoidTy, Params: PtrTy, isVarArg: false),
1057 Name: "__cudaRegisterFatBinaryEnd");
1058 CtorBuilder.CreateCall(Callee: RegisterFatbinEndFunc, Args: RegisterFatbinCall);
1059 }
1060 } else {
1061 // Generate a unique module ID.
1062 // Note that this is unique in a build (with some collision probability
1063 // inherent to MD5 hashing) as long as each compilation sees modules with
1064 // different `SourceFileName`s. Builds using absolute paths or paths
1065 // relative to the same base path should be OK. This is similar to the
1066 // guarantees for ThinLTO and GlobalValue's GUID.
1067 // If desired, a stronger uniqueness guarantee could be computed (with a
1068 // small refactoring) with `llvm::getUniqueModuleId`, which hashes the
1069 // module content (and, therefore, a compile-time tradeoff).
1070 SmallString<64> ModuleID;
1071 llvm::raw_svector_ostream OS(ModuleID);
1072 OS << ModuleIDPrefix
1073 << llvm::format(Fmt: "%" PRIx64,
1074 Vals: llvm::MD5Hash(Str: TheModule.getSourceFileName()));
1075 llvm::Constant *ModuleIDConstant = makeConstantArray(
1076 Str: std::string(ModuleID), Name: "", SectionName: ModuleIDSectionName, Alignment: 32, /*AddNull=*/true);
1077
1078 // Create an alias for the FatbinWrapper that nvcc will look for.
1079 llvm::GlobalAlias::create(Linkage: llvm::GlobalValue::ExternalLinkage,
1080 Name: Twine("__fatbinwrap") + ModuleID, Aliasee: FatbinWrapper);
1081
1082 // void __cudaRegisterLinkedBinary%ModuleID%(void (*)(void *), void *,
1083 // void *, void (*)(void **))
1084 SmallString<128> RegisterLinkedBinaryName("__cudaRegisterLinkedBinary");
1085 RegisterLinkedBinaryName += ModuleID;
1086 llvm::FunctionCallee RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction(
1087 Ty: getRegisterLinkedBinaryFnTy(), Name: RegisterLinkedBinaryName);
1088
1089 assert(RegisterGlobalsFunc && "Expecting at least dummy function!");
1090 llvm::Value *Args[] = {RegisterGlobalsFunc, FatbinWrapper, ModuleIDConstant,
1091 makeDummyFunction(FnTy: getCallbackFnTy())};
1092 CtorBuilder.CreateCall(Callee: RegisterLinkedBinaryFunc, Args);
1093 }
1094
1095 // Create destructor and register it with atexit() the way NVCC does it. Doing
1096 // it during regular destructor phase worked in CUDA before 9.2 but results in
1097 // double-free in 9.2.
1098 if (llvm::Function *CleanupFn = makeModuleDtorFunction()) {
1099 // extern "C" int atexit(void (*f)(void));
1100 llvm::FunctionType *AtExitTy =
1101 llvm::FunctionType::get(Result: IntTy, Params: CleanupFn->getType(), isVarArg: false);
1102 llvm::FunctionCallee AtExitFunc =
1103 CGM.CreateRuntimeFunction(Ty: AtExitTy, Name: "atexit", ExtraAttrs: llvm::AttributeList(),
1104 /*Local=*/true);
1105 CtorBuilder.CreateCall(Callee: AtExitFunc, Args: CleanupFn);
1106 }
1107
1108 CtorBuilder.CreateRetVoid();
1109 return ModuleCtorFunc;
1110}
1111
1112/// Creates a global destructor function that unregisters the GPU code blob
1113/// registered by constructor.
1114///
1115/// For CUDA:
1116/// \code
1117/// void __cuda_module_dtor() {
1118/// __cudaUnregisterFatBinary(Handle);
1119/// }
1120/// \endcode
1121///
1122/// For HIP:
1123/// \code
1124/// void __hip_module_dtor() {
1125/// if (__hip_gpubin_handle) {
1126/// __hipUnregisterFatBinary(__hip_gpubin_handle);
1127/// __hip_gpubin_handle = 0;
1128/// }
1129/// }
1130/// \endcode
1131llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
1132 // No need for destructor if we don't have a handle to unregister.
1133 if (!GpuBinaryHandle)
1134 return nullptr;
1135
1136 // void __cudaUnregisterFatBinary(void ** handle);
1137 llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction(
1138 Ty: llvm::FunctionType::get(Result: VoidTy, Params: PtrTy, isVarArg: false),
1139 Name: addUnderscoredPrefixToName(FuncName: "UnregisterFatBinary"));
1140
1141 llvm::Function *ModuleDtorFunc = llvm::Function::Create(
1142 Ty: llvm::FunctionType::get(Result: VoidTy, isVarArg: false),
1143 Linkage: llvm::GlobalValue::InternalLinkage,
1144 N: addUnderscoredPrefixToName(FuncName: "_module_dtor"), M: &TheModule);
1145
1146 llvm::BasicBlock *DtorEntryBB =
1147 llvm::BasicBlock::Create(Context, Name: "entry", Parent: ModuleDtorFunc);
1148 CGBuilderTy DtorBuilder(CGM, Context);
1149 DtorBuilder.SetInsertPoint(DtorEntryBB);
1150
1151 Address GpuBinaryAddr(
1152 GpuBinaryHandle, GpuBinaryHandle->getValueType(),
1153 CharUnits::fromQuantity(Quantity: GpuBinaryHandle->getAlign().valueOrOne()));
1154 auto *HandleValue = DtorBuilder.CreateLoad(Addr: GpuBinaryAddr);
1155 // There is only one HIP fat binary per linked module, however there are
1156 // multiple destructor functions. Make sure the fat binary is unregistered
1157 // only once.
1158 if (CGM.getLangOpts().HIP) {
1159 llvm::BasicBlock *IfBlock =
1160 llvm::BasicBlock::Create(Context, Name: "if", Parent: ModuleDtorFunc);
1161 llvm::BasicBlock *ExitBlock =
1162 llvm::BasicBlock::Create(Context, Name: "exit", Parent: ModuleDtorFunc);
1163 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: HandleValue->getType());
1164 llvm::Value *NEZero = DtorBuilder.CreateICmpNE(LHS: HandleValue, RHS: Zero);
1165 DtorBuilder.CreateCondBr(Cond: NEZero, True: IfBlock, False: ExitBlock);
1166
1167 DtorBuilder.SetInsertPoint(IfBlock);
1168 DtorBuilder.CreateCall(Callee: UnregisterFatbinFunc, Args: HandleValue);
1169 DtorBuilder.CreateStore(Val: Zero, Addr: GpuBinaryAddr);
1170 DtorBuilder.CreateBr(Dest: ExitBlock);
1171
1172 DtorBuilder.SetInsertPoint(ExitBlock);
1173 } else {
1174 DtorBuilder.CreateCall(Callee: UnregisterFatbinFunc, Args: HandleValue);
1175 }
1176 DtorBuilder.CreateRetVoid();
1177 return ModuleDtorFunc;
1178}
1179
1180CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
1181 return new CGNVCUDARuntime(CGM);
1182}
1183
1184void CGNVCUDARuntime::internalizeDeviceSideVar(
1185 const VarDecl *D, llvm::GlobalValue::LinkageTypes &Linkage) {
1186 // For -fno-gpu-rdc, host-side shadows of external declarations of device-side
1187 // global variables become internal definitions. These have to be internal in
1188 // order to prevent name conflicts with global host variables with the same
1189 // name in a different TUs.
1190 //
1191 // For -fgpu-rdc, the shadow variables should not be internalized because
1192 // they may be accessed by different TU.
1193 if (CGM.getLangOpts().GPURelocatableDeviceCode)
1194 return;
1195
1196 // __shared__ variables are odd. Shadows do get created, but
1197 // they are not registered with the CUDA runtime, so they
1198 // can't really be used to access their device-side
1199 // counterparts. It's not clear yet whether it's nvcc's bug or
1200 // a feature, but we've got to do the same for compatibility.
1201 if (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
1202 D->hasAttr<CUDASharedAttr>() ||
1203 D->getType()->isCUDADeviceBuiltinSurfaceType() ||
1204 D->getType()->isCUDADeviceBuiltinTextureType()) {
1205 Linkage = llvm::GlobalValue::InternalLinkage;
1206 }
1207}
1208
1209void CGNVCUDARuntime::handleVarRegistration(const VarDecl *D,
1210 llvm::GlobalVariable &GV) {
1211 if (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>()) {
1212 // Shadow variables and their properties must be registered with CUDA
1213 // runtime. Skip Extern global variables, which will be registered in
1214 // the TU where they are defined.
1215 //
1216 // Don't register a C++17 inline variable. The local symbol can be
1217 // discarded and referencing a discarded local symbol from outside the
1218 // comdat (__cuda_register_globals) is disallowed by the ELF spec.
1219 //
1220 // HIP managed variables need to be always recorded in device and host
1221 // compilations for transformation.
1222 //
1223 // HIP managed variables and variables in CUDADeviceVarODRUsedByHost are
1224 // added to llvm.compiler-used, therefore they are safe to be registered.
1225 if ((!D->hasExternalStorage() && !D->isInline()) ||
1226 CGM.getContext().CUDADeviceVarODRUsedByHost.contains(key: D) ||
1227 D->hasAttr<HIPManagedAttr>()) {
1228 registerDeviceVar(VD: D, Var&: GV, Extern: !D->hasDefinition(),
1229 Constant: D->hasAttr<CUDAConstantAttr>());
1230 }
1231 } else if (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
1232 D->getType()->isCUDADeviceBuiltinTextureType()) {
1233 // Builtin surfaces and textures and their template arguments are
1234 // also registered with CUDA runtime.
1235 const auto *TD = cast<ClassTemplateSpecializationDecl>(
1236 Val: D->getType()->castAsCXXRecordDecl());
1237 const TemplateArgumentList &Args = TD->getTemplateArgs();
1238 if (TD->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>()) {
1239 assert(Args.size() == 2 &&
1240 "Unexpected number of template arguments of CUDA device "
1241 "builtin surface type.");
1242 auto SurfType = Args[1].getAsIntegral();
1243 if (!D->hasExternalStorage())
1244 registerDeviceSurf(VD: D, Var&: GV, Extern: !D->hasDefinition(), Type: SurfType.getSExtValue());
1245 } else {
1246 assert(Args.size() == 3 &&
1247 "Unexpected number of template arguments of CUDA device "
1248 "builtin texture type.");
1249 auto TexType = Args[1].getAsIntegral();
1250 auto Normalized = Args[2].getAsIntegral();
1251 if (!D->hasExternalStorage())
1252 registerDeviceTex(VD: D, Var&: GV, Extern: !D->hasDefinition(), Type: TexType.getSExtValue(),
1253 Normalized: Normalized.getZExtValue());
1254 }
1255 }
1256}
1257
1258// Transform managed variables to pointers to managed variables in device code.
1259// Each use of the original managed variable is replaced by a load from the
1260// transformed managed variable. The transformed managed variable contains
1261// the address of managed memory which will be allocated by the runtime.
1262void CGNVCUDARuntime::transformManagedVars() {
1263 for (auto &&Info : DeviceVars) {
1264 llvm::GlobalVariable *Var = Info.Var;
1265 if (Info.Flags.getKind() == DeviceVarFlags::Variable &&
1266 Info.Flags.isManaged()) {
1267 auto *ManagedVar = new llvm::GlobalVariable(
1268 CGM.getModule(), Var->getType(),
1269 /*isConstant=*/false, Var->getLinkage(),
1270 /*Init=*/Var->isDeclaration()
1271 ? nullptr
1272 : llvm::ConstantPointerNull::get(T: Var->getType()),
1273 /*Name=*/"", /*InsertBefore=*/nullptr,
1274 llvm::GlobalVariable::NotThreadLocal,
1275 CGM.getContext().getTargetAddressSpace(AS: CGM.getLangOpts().CUDAIsDevice
1276 ? LangAS::cuda_device
1277 : LangAS::Default));
1278 ManagedVar->setDSOLocal(Var->isDSOLocal());
1279 ManagedVar->setVisibility(Var->getVisibility());
1280 ManagedVar->setExternallyInitialized(true);
1281 replaceManagedVar(Var, ManagedVar);
1282 ManagedVar->takeName(V: Var);
1283 Var->setName(Twine(ManagedVar->getName()) + ".managed");
1284 // Keep managed variables even if they are not used in device code since
1285 // they need to be allocated by the runtime.
1286 if (CGM.getLangOpts().CUDAIsDevice && !Var->isDeclaration()) {
1287 assert(!ManagedVar->isDeclaration());
1288 CGM.addCompilerUsedGlobal(GV: Var);
1289 CGM.addCompilerUsedGlobal(GV: ManagedVar);
1290 }
1291 }
1292 }
1293}
1294
1295// Creates offloading entries for all the kernels and globals that must be
1296// registered. The linker will provide a pointer to this section so we can
1297// register the symbols with the linked device image.
1298void CGNVCUDARuntime::createOffloadingEntries() {
1299 llvm::object::OffloadKind Kind = CGM.getLangOpts().HIP
1300 ? llvm::object::OffloadKind::OFK_HIP
1301 : llvm::object::OffloadKind::OFK_Cuda;
1302
1303 llvm::Module &M = CGM.getModule();
1304 for (KernelInfo &I : EmittedKernels)
1305 llvm::offloading::emitOffloadingEntry(
1306 M, Kind, Addr: KernelHandles[I.Kernel->getName()],
1307 Name: getDeviceSideName(ND: cast<NamedDecl>(Val: I.D)), /*Flags=*/Size: 0, /*Data=*/Flags: 0,
1308 Data: llvm::offloading::OffloadGlobalEntry);
1309
1310 for (VarInfo &I : DeviceVars) {
1311 uint64_t VarSize =
1312 CGM.getDataLayout().getTypeAllocSize(Ty: I.Var->getValueType());
1313 int32_t Flags =
1314 (I.Flags.isExtern()
1315 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalExtern)
1316 : 0) |
1317 (I.Flags.isConstant()
1318 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalConstant)
1319 : 0) |
1320 (I.Flags.isNormalized()
1321 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalNormalized)
1322 : 0);
1323 if (I.Flags.getKind() == DeviceVarFlags::Variable) {
1324 if (I.Flags.isManaged()) {
1325 assert(I.Var->getName().ends_with(".managed") &&
1326 "HIP managed variables not transformed");
1327
1328 auto *ManagedVar = M.getNamedGlobal(
1329 Name: I.Var->getName().drop_back(N: StringRef(".managed").size()));
1330 llvm::offloading::emitOffloadingEntry(
1331 M, Kind, Addr: I.Var, Name: getDeviceSideName(ND: I.D), Size: VarSize,
1332 Flags: llvm::offloading::OffloadGlobalManagedEntry | Flags,
1333 /*Data=*/I.Var->getAlign().valueOrOne().value(), AuxAddr: ManagedVar);
1334 } else {
1335 llvm::offloading::emitOffloadingEntry(
1336 M, Kind, Addr: I.Var, Name: getDeviceSideName(ND: I.D), Size: VarSize,
1337 Flags: llvm::offloading::OffloadGlobalEntry | Flags,
1338 /*Data=*/0);
1339 }
1340 } else if (I.Flags.getKind() == DeviceVarFlags::Surface) {
1341 llvm::offloading::emitOffloadingEntry(
1342 M, Kind, Addr: I.Var, Name: getDeviceSideName(ND: I.D), Size: VarSize,
1343 Flags: llvm::offloading::OffloadGlobalSurfaceEntry | Flags,
1344 Data: I.Flags.getSurfTexType());
1345 } else if (I.Flags.getKind() == DeviceVarFlags::Texture) {
1346 llvm::offloading::emitOffloadingEntry(
1347 M, Kind, Addr: I.Var, Name: getDeviceSideName(ND: I.D), Size: VarSize,
1348 Flags: llvm::offloading::OffloadGlobalTextureEntry | Flags,
1349 Data: I.Flags.getSurfTexType());
1350 }
1351 }
1352
1353 // Register the per-TU offload-profiling shadow. The offloading entry
1354 // makes the linker-wrapper emit the host __hipRegisterVar call in the
1355 // combined ctor. Separately emit a per-TU ctor that registers the
1356 // shadow with the profile runtime's drain list.
1357 if (OffloadProfShadow) {
1358 llvm::offloading::emitOffloadingEntry(
1359 M, Kind, Addr: OffloadProfShadow, Name: OffloadProfShadow->getName(),
1360 Size: CGM.getDataLayout().getPointerSize(/*AS=*/0),
1361 Flags: llvm::offloading::OffloadGlobalEntry, /*Data=*/0);
1362
1363 llvm::LLVMContext &Ctx = M.getContext();
1364 auto *PtrTy = llvm::PointerType::getUnqual(C&: Ctx);
1365 llvm::FunctionCallee RegisterShadow = CGM.CreateRuntimeFunction(
1366 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
1367 Name: "__llvm_profile_offload_register_shadow_variable");
1368 llvm::FunctionCallee RegisterSectionShadow = CGM.CreateRuntimeFunction(
1369 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
1370 Name: "__llvm_profile_offload_register_section_shadow_variable");
1371 auto *CtorFn = llvm::Function::Create(
1372 Ty: llvm::FunctionType::get(Result: VoidTy, isVarArg: false),
1373 Linkage: llvm::GlobalValue::InternalLinkage,
1374 N: "__llvm_profile_register_shadow." + CGM.getContext().getCUIDHash(), M: &M);
1375 auto *Entry = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: CtorFn);
1376 llvm::IRBuilder<> B(Entry);
1377 B.CreateCall(Callee: RegisterShadow, Args: {OffloadProfShadow});
1378 if (OffloadProfVersionShadow) {
1379 uint64_t VersionSize = CGM.getDataLayout().getTypeAllocSize(
1380 Ty: OffloadProfVersionShadow->getValueType());
1381 auto *VersionEntry = llvm::offloading::emitOffloadingEntry(
1382 M, Kind, Addr: OffloadProfVersionShadow, Name: "__llvm_profile_raw_version",
1383 Size: VersionSize, Flags: llvm::offloading::OffloadGlobalEntry, /*Data=*/0);
1384 // The device symbol is shared, but each TU needs its own host handle.
1385 // Keep the weak offloading entries from coalescing at the host link.
1386 VersionEntry->setName(VersionEntry->getName() + "." +
1387 CGM.getContext().getCUIDHash());
1388 llvm::FunctionCallee RegisterVersionShadow = CGM.CreateRuntimeFunction(
1389 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false),
1390 Name: "__llvm_profile_offload_register_version_shadow_variable");
1391 B.CreateCall(Callee: RegisterVersionShadow, Args: {OffloadProfVersionShadow});
1392 }
1393 for (const auto &Info : OffloadProfSectionShadows) {
1394 llvm::offloading::emitOffloadingEntry(
1395 M, Kind, Addr: Info.Shadow, Name: Info.DeviceName,
1396 Size: CGM.getDataLayout().getPointerSize(/*AS=*/0),
1397 Flags: llvm::offloading::OffloadGlobalEntry, /*Data=*/0);
1398 B.CreateCall(Callee: RegisterSectionShadow, Args: {Info.Shadow});
1399 }
1400 B.CreateRetVoid();
1401 CGM.AddGlobalCtor(Ctor: CtorFn);
1402 }
1403}
1404
1405// For HIP host+device compiles with PGO enabled, emit the host-side shadow for
1406// the per-TU __llvm_profile_sections_<CUID> global. Device-side section table
1407// emission is owned by InstrProfiling so it can be gated on real profile data.
1408void CGNVCUDARuntime::emitOffloadProfilingSections() {
1409 if (!CGM.getLangOpts().HIP)
1410 return;
1411 if (!CGM.getCodeGenOpts().hasProfileInstr())
1412 return;
1413
1414 StringRef CUIDHash = CGM.getContext().getCUIDHash();
1415 if (CUIDHash.empty())
1416 return;
1417
1418 llvm::Module &M = CGM.getModule();
1419 llvm::LLVMContext &Ctx = M.getContext();
1420 std::string Name = ("__llvm_profile_sections_" + CUIDHash).str();
1421
1422 // If the global already exists (e.g. another TU was merged in), don't
1423 // duplicate it.
1424 if (M.getNamedValue(Name))
1425 return;
1426
1427 if (CGM.getLangOpts().CUDAIsDevice) {
1428 // Device side: emit only the per-TU names postfix marker. The sections
1429 // struct is emitted later by the InstrProfiling pass, which emits it only
1430 // when the TU has profile data, avoiding dangling section references.
1431 unsigned GlobalAS = M.getDataLayout().getDefaultGlobalsAddressSpace();
1432 std::string NamesVarPostfixVarName =
1433 std::string(llvm::getInstrProfNamesVarPostfixVarName());
1434 if (!M.getNamedValue(Name: NamesVarPostfixVarName)) {
1435 auto *NamesVarPostfix = llvm::ConstantDataArray::getString(
1436 Context&: Ctx, Initializer: (llvm::Twine("_") + CUIDHash).str(), AddNull: true);
1437 auto *NamesGV = new llvm::GlobalVariable(
1438 M, NamesVarPostfix->getType(), /*isConstant=*/true,
1439 llvm::GlobalValue::PrivateLinkage, NamesVarPostfix,
1440 NamesVarPostfixVarName,
1441 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1442 GlobalAS);
1443 CGM.addCompilerUsedGlobal(GV: NamesGV);
1444 }
1445 return;
1446 }
1447
1448 // Host side: emit an opaque void* shadow. Layout doesn't matter — the
1449 // runtime locates it by name via hipGetSymbolAddress and treats it as
1450 // the address of the device-side struct. Registration with the HIP
1451 // runtime is added by makeRegisterGlobalsFn (non-RDC) or
1452 // createOffloadingEntries (RDC).
1453 auto *PtrTy = llvm::PointerType::getUnqual(C&: Ctx);
1454 OffloadProfShadow = new llvm::GlobalVariable(
1455 M, PtrTy, /*isConstant=*/false, llvm::GlobalValue::ExternalLinkage,
1456 llvm::ConstantPointerNull::get(T: PtrTy), Name);
1457 CGM.addCompilerUsedGlobal(GV: OffloadProfShadow);
1458
1459 // Linux collects the raw version through HSA. Windows resolves it through
1460 // HIP's registered-symbol API, using a distinct handle for each TU.
1461 if (CGM.getTriple().isOSWindows()) {
1462 auto *VersionTy = llvm::Type::getInt64Ty(C&: Ctx);
1463 OffloadProfVersionShadow = new llvm::GlobalVariable(
1464 M, VersionTy, /*isConstant=*/false, llvm::GlobalValue::ExternalLinkage,
1465 llvm::ConstantInt::get(Ty: VersionTy, V: 0),
1466 ("__llvm_profile_shadow_version_" + CUIDHash).str());
1467 CGM.addCompilerUsedGlobal(GV: OffloadProfVersionShadow);
1468 }
1469
1470 auto AddSectionShadow = [&](StringRef Kind, const Twine &DeviceName) {
1471 std::string ShadowName =
1472 (Twine("__llvm_profile_shadow_") + Kind + "_" + CUIDHash + "_" +
1473 Twine(OffloadProfSectionShadows.size()))
1474 .str();
1475 auto *Shadow = new llvm::GlobalVariable(
1476 M, PtrTy, /*isConstant=*/false, llvm::GlobalValue::ExternalLinkage,
1477 llvm::ConstantPointerNull::get(T: PtrTy), ShadowName);
1478 CGM.addCompilerUsedGlobal(GV: Shadow);
1479 OffloadProfSectionShadows.push_back(Elt: {.Shadow: Shadow, .DeviceName: DeviceName.str()});
1480 };
1481
1482 // Keep this order in sync with the runtime: data, counters, uniform counters,
1483 // then names.
1484 for (auto &&I : EmittedKernels) {
1485 std::string KernelName = getDeviceSideName(ND: cast<NamedDecl>(Val: I.D));
1486 AddSectionShadow("data", Twine("__profd_") + KernelName);
1487 AddSectionShadow("cnts", Twine("__profc_") + KernelName);
1488 AddSectionShadow("ucnts", Twine("__llvm_prf_unifcnt_") + KernelName);
1489 AddSectionShadow("names",
1490 Twine(llvm::getInstrProfNamesVarName()) + "_" + CUIDHash);
1491 }
1492}
1493
1494// Returns module constructor to be added.
1495llvm::Function *CGNVCUDARuntime::finalizeModule() {
1496 transformManagedVars();
1497 emitOffloadProfilingSections();
1498 if (CGM.getLangOpts().CUDAIsDevice) {
1499 // Mark ODR-used device variables as compiler used to prevent it from being
1500 // eliminated by optimization. This is necessary for device variables
1501 // ODR-used by host functions. Sema correctly marks them as ODR-used no
1502 // matter whether they are ODR-used by device or host functions.
1503 //
1504 // We do not need to do this if the variable has used attribute since it
1505 // has already been added.
1506 //
1507 // Static device variables have been externalized at this point, therefore
1508 // variables with LLVM private or internal linkage need not be added.
1509 for (auto &&Info : DeviceVars) {
1510 auto Kind = Info.Flags.getKind();
1511 if (!Info.Var->isDeclaration() &&
1512 !llvm::GlobalValue::isLocalLinkage(Linkage: Info.Var->getLinkage()) &&
1513 (Kind == DeviceVarFlags::Variable ||
1514 Kind == DeviceVarFlags::Surface ||
1515 Kind == DeviceVarFlags::Texture) &&
1516 Info.D->isUsed() && !Info.D->hasAttr<UsedAttr>()) {
1517 CGM.addCompilerUsedGlobal(GV: Info.Var);
1518 }
1519 }
1520 return nullptr;
1521 }
1522 if (!CGM.getLangOpts().CUDANVCCABI &&
1523 (CGM.getLangOpts().OffloadViaLLVM || RelocatableDeviceCode))
1524 createOffloadingEntries();
1525 else
1526 return makeModuleCtorFunction();
1527
1528 return nullptr;
1529}
1530
1531llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F,
1532 GlobalDecl GD) {
1533 auto Loc = KernelHandles.find(Val: F->getName());
1534 if (Loc != KernelHandles.end()) {
1535 auto OldHandle = Loc->second;
1536 if (KernelStubs[OldHandle] == F)
1537 return OldHandle;
1538
1539 // We've found the function name, but F itself has changed, so we need to
1540 // update the references.
1541 if (CGM.getLangOpts().HIP) {
1542 // For HIP compilation the handle itself does not change, so we only need
1543 // to update the Stub value.
1544 KernelStubs[OldHandle] = F;
1545 return OldHandle;
1546 }
1547 // For non-HIP compilation, erase the old Stub and fall-through to creating
1548 // new entries.
1549 KernelStubs.erase(Val: OldHandle);
1550 }
1551
1552 if (!CGM.getLangOpts().HIP) {
1553 KernelHandles[F->getName()] = F;
1554 KernelStubs[F] = F;
1555 return F;
1556 }
1557
1558 auto *Var = new llvm::GlobalVariable(
1559 TheModule, F->getType(), /*isConstant=*/true, F->getLinkage(),
1560 /*Initializer=*/nullptr,
1561 CGM.getMangledName(
1562 GD: GD.getWithKernelReferenceKind(Kind: KernelReferenceKind::Kernel)));
1563 Var->setAlignment(CGM.getPointerAlign().getAsAlign());
1564 Var->setVisibility(F->getVisibility());
1565 CGM.setDSOLocal(Var);
1566 auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
1567 auto *FT = FD->getPrimaryTemplate();
1568 if (!FT || FT->isThisDeclarationADefinition())
1569 CGM.maybeSetTrivialComdat(D: *FD, GO&: *Var);
1570 KernelHandles[F->getName()] = Var;
1571 KernelStubs[Var] = F;
1572 return Var;
1573}
1574