1//========- CIRGenCUDANV.cpp - Interface to NVIDIA CUDA Runtime -----=========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for CUDA code generation targeting the NVIDIA CUDA
10// runtime library.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CIRGenCUDARuntime.h"
15#include "CIRGenCXXABI.h"
16#include "CIRGenFunction.h"
17#include "CIRGenModule.h"
18#include "mlir/IR/Operation.h"
19#include "clang/AST/ASTContext.h"
20#include "clang/AST/Attrs.inc"
21#include "clang/AST/Decl.h"
22#include "clang/AST/GlobalDecl.h"
23#include "clang/Basic/AddressSpaces.h"
24#include "clang/Basic/Cuda.h"
25#include "clang/Basic/DiagnosticFrontend.h"
26#include "clang/Basic/FileManager.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/CIR/Dialect/IR/CIRDialect.h"
29#include "clang/CIR/Dialect/IR/CIRTypes.h"
30#include "llvm/Support/Casting.h"
31#include "llvm/Support/MemoryBuffer.h"
32#include "llvm/Support/VirtualFileSystem.h"
33
34using namespace clang;
35using namespace clang::CIRGen;
36
37namespace {
38
39class CIRGenNVCUDARuntime : public CIRGenCUDARuntime {
40protected:
41 StringRef prefix;
42
43 // Map a device stub function to a symbol for identifying kernel in host
44 // code. For CUDA, the symbol for identifying the kernel is the same as the
45 // device stub function. For HIP, they are different.
46 llvm::StringMap<mlir::Operation *> kernelHandles;
47
48 // Map a kernel handle to the kernel stub.
49 llvm::DenseMap<mlir::Operation *, mlir::Operation *> kernelStubs;
50
51 struct VarInfo {
52 cir::GlobalOp var;
53 const VarDecl *d;
54 cir::CUDADeviceVarKind flags;
55 };
56 llvm::SmallVector<VarInfo, 16> deviceVars;
57
58 // Mangle context for device.
59 std::unique_ptr<MangleContext> deviceMC;
60
61private:
62 void emitDeviceStubBodyNew(CIRGenFunction &cgf, cir::FuncOp fn,
63 FunctionArgList &args);
64 void recordDeviceBinary();
65 void transformManagedVars();
66 mlir::Value prepareKernelArgs(CIRGenFunction &cgf, mlir::Location loc,
67 FunctionArgList &args);
68 mlir::Operation *getKernelHandle(cir::FuncOp fn, GlobalDecl gd) override;
69
70 mlir::Operation *getKernelStub(mlir::Operation *handle) override {
71 auto it = kernelStubs.find(Val: handle);
72 assert(it != kernelStubs.end());
73 return it->second;
74 }
75 std::string addPrefixToName(StringRef funcName) const;
76 std::string addUnderscoredPrefixToName(StringRef funcName) const;
77
78public:
79 CIRGenNVCUDARuntime(CIRGenModule &cgm);
80 ~CIRGenNVCUDARuntime();
81
82 void emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
83 FunctionArgList &args) override;
84
85 void handleVarRegistration(const VarDecl *vd, cir::GlobalOp var) override;
86 void finalizeModule() override;
87 void handleGlobalReplace(cir::GlobalOp oldGV, cir::GlobalOp newGV) override;
88
89 void internalizeDeviceSideVar(const VarDecl *d,
90 cir::GlobalLinkageKind &linkage) override;
91
92 std::string getDeviceSideName(const NamedDecl *nd) override;
93
94 void registerDeviceVar(const VarDecl *vd, cir::GlobalOp &var, bool isExtern,
95 bool isConstant) {
96 // Attach the device var attribute to the GlobalOp
97 auto &builder = cgm.getBuilder();
98 var->setAttr(name: cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
99 value: cir::CUDAVarRegistrationInfoAttr::get(
100 context: builder.getContext(),
101 device_side_name: getDeviceSideName(nd: cast<NamedDecl>(Val: vd)),
102 kind: cir::CUDADeviceVarKind::Variable, isExtern, isConstant,
103 isManaged: vd->hasAttr<HIPManagedAttr>(),
104 /*surfTexType=*/0,
105 /*normalized=*/false));
106 deviceVars.push_back(Elt: {
107 .var: var,
108 .d: vd,
109 .flags: cir::CUDADeviceVarKind::Variable,
110 });
111 }
112
113 void registerDeviceSurf(const VarDecl *vd, cir::GlobalOp &var, bool isExtern,
114 int32_t surfaceType) {
115 auto &builder = cgm.getBuilder();
116
117 var->setAttr(name: cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
118 value: cir::CUDAVarRegistrationInfoAttr::get(
119 context: builder.getContext(),
120 device_side_name: getDeviceSideName(nd: cast<NamedDecl>(Val: vd)),
121 kind: cir::CUDADeviceVarKind::Surface, isExtern,
122 /*isConstant=*/false,
123 /*isManaged=*/false, surfTexType: surfaceType,
124 /*normalized=*/false));
125
126 deviceVars.push_back(Elt: {
127 .var: var,
128 .d: vd,
129 .flags: cir::CUDADeviceVarKind::Surface,
130 });
131 }
132
133 void registerDeviceTex(const VarDecl *vd, cir::GlobalOp &var, bool isExtern,
134 int32_t textureType, bool normalized) {
135 auto &builder = cgm.getBuilder();
136
137 var->setAttr(name: cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
138 value: cir::CUDAVarRegistrationInfoAttr::get(
139 context: builder.getContext(),
140 device_side_name: getDeviceSideName(nd: cast<NamedDecl>(Val: vd)),
141 kind: cir::CUDADeviceVarKind::Texture, isExtern,
142 /*isConstant=*/false,
143 /*isManaged=*/false, surfTexType: textureType, normalized));
144
145 deviceVars.push_back(Elt: {
146 .var: var,
147 .d: vd,
148 .flags: cir::CUDADeviceVarKind::Texture,
149 });
150 }
151};
152
153} // namespace
154
155std::string CIRGenNVCUDARuntime::addPrefixToName(StringRef funcName) const {
156 return (prefix + funcName).str();
157}
158
159std::string
160CIRGenNVCUDARuntime::addUnderscoredPrefixToName(StringRef funcName) const {
161 return ("__" + prefix + funcName).str();
162}
163
164CIRGenNVCUDARuntime::CIRGenNVCUDARuntime(CIRGenModule &cgm)
165 : CIRGenCUDARuntime(cgm),
166 deviceMC(cgm.getASTContext().cudaNVInitDeviceMC()) {
167 if (cgm.getLangOpts().OffloadViaLLVM)
168 cgm.errorNYI(feature: "CIRGenNVCUDARuntime: Offload via LLVM");
169 else if (cgm.getLangOpts().HIP)
170 prefix = "hip";
171 else
172 prefix = "cuda";
173}
174
175mlir::Value CIRGenNVCUDARuntime::prepareKernelArgs(CIRGenFunction &cgf,
176 mlir::Location loc,
177 FunctionArgList &args) {
178 CIRGenBuilderTy &builder = cgm.getBuilder();
179
180 // Build void *args[] and populate with the addresses of kernel arguments.
181 auto voidPtrArrayTy = cir::ArrayType::get(elementType: cgm.voidPtrTy, size: args.size());
182 mlir::Value kernelArgs =
183 builder.createAlloca(loc, addrType: cir::PointerType::get(pointee: voidPtrArrayTy),
184 name: "kernel_args", alignment: CharUnits::fromQuantity(Quantity: 16));
185
186 mlir::Value kernelArgsDecayed =
187 builder.createCast(kind: cir::CastKind::array_to_ptrdecay, src: kernelArgs,
188 newTy: cir::PointerType::get(pointee: cgm.voidPtrTy));
189
190 for (const auto &[i, arg] : llvm::enumerate(First&: args)) {
191 mlir::Value index =
192 builder.getConstInt(loc, intVal: llvm::APInt(/*numBits=*/32, i));
193 mlir::Value storePos =
194 builder.createPtrStride(loc, base: kernelArgsDecayed, stride: index);
195 mlir::Value argAddr = cgf.getAddrOfLocalVar(vd: arg).getPointer();
196 mlir::Value argAsVoid = builder.createBitcast(src: argAddr, newTy: cgm.voidPtrTy);
197
198 builder.CIRBaseBuilderTy::createStore(loc, val: argAsVoid, dst: storePos);
199 }
200
201 return kernelArgsDecayed;
202}
203
204// CUDA 9.0+ uses new way to launch kernels. Parameters are packed in a local
205// array and kernels are launched using cudaLaunchKernel().
206void CIRGenNVCUDARuntime::emitDeviceStubBodyNew(CIRGenFunction &cgf,
207 cir::FuncOp fn,
208 FunctionArgList &args) {
209
210 // This requires arguments to be sent to kernels in a different way.
211 if (cgm.getLangOpts().OffloadViaLLVM)
212 cgm.errorNYI(feature: "CIRGenNVCUDARuntime: Offload via LLVM");
213
214 CIRGenBuilderTy &builder = cgm.getBuilder();
215 mlir::Location loc = fn.getLoc();
216
217 // For [cuda|hip]LaunchKernel, we must add another layer of indirection
218 // to arguments. For example, for function `add(int a, float b)`,
219 // we need to pass it as `void *args[2] = { &a, &b }`.
220 mlir::Value kernelArgs = prepareKernelArgs(cgf, loc, args);
221
222 // Lookup cudaLaunchKernel/hipLaunchKernel function.
223 // HIP kernel launching API name depends on -fgpu-default-stream option. For
224 // the default value 'legacy', it is hipLaunchKernel. For 'per-thread',
225 // it is hipLaunchKernel_spt.
226 // cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
227 // void **args, size_t sharedMem,
228 // cudaStream_t stream);
229 // hipError_t hipLaunchKernel[_spt](const void *func, dim3 gridDim,
230 // dim3 blockDim, void **args,
231 // size_t sharedMem, hipStream_t stream);
232 TranslationUnitDecl *tuDecl = cgm.getASTContext().getTranslationUnitDecl();
233 DeclContext *dc = TranslationUnitDecl::castToDeclContext(D: tuDecl);
234
235 // The default stream is usually stream 0 (the legacy default stream).
236 // For per-thread default stream, we need a different LaunchKernel function.
237 std::string kernelLaunchAPI = "LaunchKernel";
238 if (cgm.getLangOpts().GPUDefaultStream ==
239 LangOptions::GPUDefaultStreamKind::PerThread) {
240 if (cgm.getLangOpts().HIP)
241 kernelLaunchAPI += "_spt";
242 else if (cgm.getLangOpts().CUDA)
243 kernelLaunchAPI += "_ptsz";
244 }
245
246 std::string launchKernelName = addPrefixToName(funcName: kernelLaunchAPI);
247 const IdentifierInfo &launchII =
248 cgm.getASTContext().Idents.get(Name: launchKernelName);
249 FunctionDecl *cudaLaunchKernelFD = nullptr;
250 for (NamedDecl *result : dc->lookup(Name: &launchII)) {
251 if (FunctionDecl *fd = dyn_cast<FunctionDecl>(Val: result))
252 cudaLaunchKernelFD = fd;
253 }
254
255 if (cudaLaunchKernelFD == nullptr) {
256 cgm.error(loc: cgf.curFuncDecl->getLocation(),
257 error: "Can't find declaration for " + launchKernelName);
258 return;
259 }
260
261 // Use this function to retrieve arguments for cudaLaunchKernel:
262 // int __[cuda|hip]PopCallConfiguration(dim3 *gridDim, dim3 *blockDim, size_t
263 // *sharedMem, cudaStream_t *stream)
264 //
265 // Here [cuda|hip]Stream_t, while also being the 6th argument of
266 // [cuda|hip]LaunchKernel, is a pointer to some opaque struct.
267
268 mlir::Type dim3Ty = cgf.getTypes().convertType(
269 type: cudaLaunchKernelFD->getParamDecl(i: 1)->getType());
270 mlir::Type streamTy = cgf.getTypes().convertType(
271 type: cudaLaunchKernelFD->getParamDecl(i: 5)->getType());
272
273 mlir::Value gridDim =
274 builder.createAlloca(loc, addrType: cir::PointerType::get(pointee: dim3Ty), name: "grid_dim",
275 alignment: CharUnits::fromQuantity(Quantity: 8));
276 mlir::Value blockDim =
277 builder.createAlloca(loc, addrType: cir::PointerType::get(pointee: dim3Ty), name: "block_dim",
278 alignment: CharUnits::fromQuantity(Quantity: 8));
279 mlir::Value sharedMem = builder.createAlloca(
280 loc, addrType: cir::PointerType::get(pointee: cgm.sizeTy), name: "shared_mem", alignment: cgm.getSizeAlign());
281 mlir::Value stream = builder.createAlloca(
282 loc, addrType: cir::PointerType::get(pointee: streamTy), name: "stream", alignment: cgm.getPointerAlign());
283
284 cir::FuncOp popConfig = cgm.createRuntimeFunction(
285 ty: cir::FuncType::get(inputs: {gridDim.getType(), blockDim.getType(),
286 sharedMem.getType(), stream.getType()},
287 returnType: cgm.sInt32Ty),
288 name: addUnderscoredPrefixToName(funcName: "PopCallConfiguration"));
289 cgf.emitRuntimeCall(loc, callee: popConfig, args: {gridDim, blockDim, sharedMem, stream});
290
291 // Now emit the call to cudaLaunchKernel
292 // [cuda|hip]Error_t [cuda|hip]LaunchKernel(const void *func, dim3 gridDim,
293 // dim3 blockDim,
294 // void **args, size_t sharedMem,
295 // [cuda|hip]Stream_t stream);
296
297 // We now either pick the function or the stub global for cuda, hip
298 // respectively.
299 mlir::Value kernel = [&]() -> mlir::Value {
300 if (cir::GlobalOp globalOp = llvm::dyn_cast_or_null<cir::GlobalOp>(
301 Val: kernelHandles[fn.getSymName()])) {
302 cir::PointerType kernelTy = cir::PointerType::get(pointee: globalOp.getSymType());
303 mlir::Value kernelVal = cir::GetGlobalOp::create(builder, location: loc, addr: kernelTy,
304 name: globalOp.getSymName());
305 mlir::Value func = builder.createBitcast(src: kernelVal, newTy: cgm.voidPtrTy);
306 return func;
307 }
308 if (cir::FuncOp funcOp = llvm::dyn_cast_or_null<cir::FuncOp>(
309 Val: kernelHandles[fn.getSymName()])) {
310 cir::PointerType kernelTy =
311 cir::PointerType::get(pointee: funcOp.getFunctionType());
312 mlir::Value kernelVal =
313 cir::GetGlobalOp::create(builder, location: loc, addr: kernelTy, name: funcOp.getSymName());
314 mlir::Value func = builder.createBitcast(src: kernelVal, newTy: cgm.voidPtrTy);
315 return func;
316 }
317 llvm_unreachable("Expected stub handle to be cir::GlobalOp or FuncOp");
318 }();
319
320 CallArgList launchArgs;
321 launchArgs.add(rvalue: RValue::get(v: kernel),
322 type: cudaLaunchKernelFD->getParamDecl(i: 0)->getType());
323 launchArgs.add(
324 rvalue: RValue::getAggregate(addr: Address(gridDim, CharUnits::fromQuantity(Quantity: 8))),
325 type: cudaLaunchKernelFD->getParamDecl(i: 1)->getType());
326 launchArgs.add(
327 rvalue: RValue::getAggregate(addr: Address(blockDim, CharUnits::fromQuantity(Quantity: 8))),
328 type: cudaLaunchKernelFD->getParamDecl(i: 2)->getType());
329 launchArgs.add(rvalue: RValue::get(v: kernelArgs),
330 type: cudaLaunchKernelFD->getParamDecl(i: 3)->getType());
331 launchArgs.add(
332 rvalue: RValue::get(v: builder.CIRBaseBuilderTy::createLoad(loc, ptr: sharedMem)),
333 type: cudaLaunchKernelFD->getParamDecl(i: 4)->getType());
334 launchArgs.add(rvalue: RValue::get(v: builder.CIRBaseBuilderTy::createLoad(loc, ptr: stream)),
335 type: cudaLaunchKernelFD->getParamDecl(i: 5)->getType());
336
337 mlir::Type launchTy =
338 cgm.getTypes().convertType(type: cudaLaunchKernelFD->getType());
339 mlir::Operation *cudaKernelLauncherFn = cgm.createRuntimeFunction(
340 ty: cast<cir::FuncType>(Val&: launchTy), name: launchKernelName);
341 const CIRGenFunctionInfo &callInfo =
342 cgm.getTypes().arrangeFunctionDeclaration(fd: cudaLaunchKernelFD);
343 cgf.emitCall(funcInfo: callInfo, callee: CIRGenCallee::forDirect(funcPtr: cudaKernelLauncherFn),
344 returnValue: ReturnValueSlot(), args: launchArgs, /*isMustTail=*/false);
345
346 if (cgm.getASTContext().getTargetInfo().getCXXABI().isMicrosoft() &&
347 !cgf.getLangOpts().HIP)
348 cgm.errorNYI(feature: "MSVC CUDA stub handling");
349}
350
351void CIRGenNVCUDARuntime::emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
352 FunctionArgList &args) {
353 CIRGenBuilderTy &builder = cgm.getBuilder();
354 StringRef kernelName = cgm.getMangledName(
355 gd: cgf.curGD.getWithKernelReferenceKind(Kind: KernelReferenceKind::Kernel));
356 auto kernelNameAttr = cir::CUDAKernelNameAttr::get(
357 context: builder.getContext(), kernel_name: builder.getStringAttr(bytes: kernelName));
358 fn->setDiscardableAttr(name: cir::CUDAKernelNameAttr::getMnemonic(),
359 value: kernelNameAttr);
360
361 if (auto globalOp =
362 llvm::dyn_cast<cir::GlobalOp>(Val: kernelHandles[fn.getSymName()])) {
363 mlir::Type fnPtrTy = globalOp.getSymType();
364 auto sym = mlir::FlatSymbolRefAttr::get(value: fn.getSymNameAttr());
365 auto gv = cir::GlobalViewAttr::get(type: fnPtrTy, symbol: sym);
366
367 globalOp->setAttr(name: "initial_value", value: gv);
368 globalOp->setAttr(name: "alignment", value: builder.getI64IntegerAttr(
369 value: cgm.getPointerAlign().getQuantity()));
370
371 // The stub's linkage is only final once it is defined, so the handle
372 // takes it over here.
373 globalOp.setLinkage(fn.getLinkage());
374 mlir::SymbolTable::setSymbolVisibility(
375 symbol: globalOp, vis: cgm.getMLIRVisibilityFromCIRLinkage(GLK: fn.getLinkage()));
376 cgm.setDSOLocal(static_cast<mlir::Operation *>(globalOp));
377 }
378
379 // CUDA 9.0 changed the way to launch kernels.
380 if (CudaFeatureEnabled(cgm.getTarget().getSDKVersion(),
381 CudaFeature::CUDA_USES_NEW_LAUNCH) ||
382 (cgm.getLangOpts().HIP && cgm.getLangOpts().HIPUseNewLaunchAPI) ||
383 cgm.getLangOpts().OffloadViaLLVM)
384 emitDeviceStubBodyNew(cgf, fn, args);
385 else
386 cgm.errorNYI(feature: "Emit Stub Body Legacy");
387}
388
389CIRGenCUDARuntime *clang::CIRGen::createNVCUDARuntime(CIRGenModule &cgm) {
390 return new CIRGenNVCUDARuntime(cgm);
391}
392
393CIRGenNVCUDARuntime::~CIRGenNVCUDARuntime() {}
394
395mlir::Operation *CIRGenNVCUDARuntime::getKernelHandle(cir::FuncOp fn,
396 GlobalDecl gd) {
397
398 // Check if we already have a kernel handle for this function
399 auto it = kernelHandles.find(Key: fn.getSymName());
400 if (it != kernelHandles.end()) {
401 mlir::Operation *oldHandle = it->second;
402 // Here we know that the fn did not change. Return it
403 if (kernelStubs[oldHandle] == fn)
404 return oldHandle;
405
406 // We've found the function name, but F itself has changed, so we need to
407 // update the references.
408 if (cgm.getLangOpts().HIP) {
409 // For HIP compilation the handle itself does not change, so we only need
410 // to update the Stub value.
411 kernelStubs[oldHandle] = fn;
412 return oldHandle;
413 }
414 // For non-HIP compilation, erase the old Stub and fall-through to creating
415 // new entries.
416 kernelStubs.erase(Val: oldHandle);
417 }
418
419 // If not targeting HIP, store the function itself
420 if (!cgm.getLangOpts().HIP) {
421 kernelHandles[fn.getSymName()] = fn;
422 kernelStubs[fn] = fn;
423 return fn;
424 }
425
426 // Create a new CIR global variable to represent the kernel handle
427 CIRGenBuilderTy &builder = cgm.getBuilder();
428 StringRef globalName = cgm.getMangledName(
429 gd: gd.getWithKernelReferenceKind(Kind: KernelReferenceKind::Kernel));
430 cir::PointerType fnPtrTy = builder.getPointerTo(ty: fn.getFunctionType());
431 cir::GlobalOp globalOp =
432 cgm.createGlobalOp(loc: fn.getLoc(), name: globalName, t: fnPtrTy, /*isConstant=*/true);
433
434 globalOp->setAttr(name: "alignment", value: builder.getI64IntegerAttr(
435 value: cgm.getPointerAlign().getQuantity()));
436
437 // Inherit visibility and comdat from the kernel's declaration.
438 auto *fd = cast<FunctionDecl>(Val: gd.getDecl());
439 cgm.setGVProperties(op: globalOp, d: fd);
440 FunctionTemplateDecl *ft = fd->getPrimaryTemplate();
441 if (!ft || ft->isThisDeclarationADefinition())
442 cgm.maybeSetTrivialComdat(d: *fd, op: globalOp);
443
444 // Store references
445 kernelHandles[fn.getSymName()] = globalOp;
446 kernelStubs[globalOp] = fn;
447
448 return globalOp;
449}
450
451void CIRGenNVCUDARuntime::internalizeDeviceSideVar(
452 const VarDecl *d, cir::GlobalLinkageKind &linkage) {
453 if (cgm.getLangOpts().GPURelocatableDeviceCode)
454 cgm.errorNYI(d->getSourceRange(),
455 "internalizeDeviceSideVar: GPU Relocatable Device Code (RDC)");
456
457 // __shared__ variables are odd. Shadows do get created, but
458 // they are not registered with the CUDA runtime, so they
459 // can't really be used to access their device-side
460 // counterparts. It's not clear yet whether it's nvcc's bug or
461 // a feature, but we've got to do the same for compatibility.
462 if (d->hasAttr<CUDADeviceAttr>() || d->hasAttr<CUDAConstantAttr>() ||
463 d->hasAttr<CUDASharedAttr>() ||
464 d->getType()->isCUDADeviceBuiltinSurfaceType() ||
465 d->getType()->isCUDADeviceBuiltinTextureType()) {
466 linkage = cir::GlobalLinkageKind::InternalLinkage;
467 }
468}
469
470std::string CIRGenNVCUDARuntime::getDeviceSideName(const NamedDecl *nd) {
471 GlobalDecl gd;
472 // nd could be either a kernel or a variable.
473 if (auto *fd = dyn_cast<FunctionDecl>(Val: nd))
474 gd = GlobalDecl(fd, KernelReferenceKind::Kernel);
475 else
476 gd = GlobalDecl(nd);
477 std::string deviceSideName;
478 MangleContext *mc;
479 if (cgm.getLangOpts().CUDAIsDevice)
480 mc = &cgm.getCXXABI().getMangleContext();
481 else
482 mc = deviceMC.get();
483 if (mc->shouldMangleDeclName(D: nd)) {
484 SmallString<256> buffer;
485 llvm::raw_svector_ostream out(buffer);
486 mc->mangleName(GD: gd, out);
487 deviceSideName = std::string(out.str());
488 } else
489 deviceSideName = std::string(nd->getIdentifier()->getName());
490
491 // Make unique name for device side static file-scope variable for HIP.
492 if (cgm.getASTContext().shouldExternalize(D: nd) &&
493 cgm.getLangOpts().GPURelocatableDeviceCode) {
494 SmallString<256> buffer;
495 llvm::raw_svector_ostream out(buffer);
496 out << deviceSideName;
497 cgm.printPostfixForExternalizedDecl(os&: out, d: nd);
498 deviceSideName = std::string(out.str());
499 }
500 return deviceSideName;
501}
502
503void CIRGenNVCUDARuntime::handleVarRegistration(const VarDecl *vd,
504 cir::GlobalOp var) {
505 if (vd->hasAttr<CUDADeviceAttr>() || vd->hasAttr<CUDAConstantAttr>()) {
506 // Shadow variables and their properties must be registered with CUDA
507 // runtime. Skip Extern global variables, which will be registered in
508 // the TU where they are defined.
509 //
510 // Don't register a C++17 inline variable. The local symbol can be
511 // discarded and referencing a discarded local symbol from outside the
512 // comdat (__cuda_register_globals) is disallowed by the ELF spec.
513 //
514 // HIP managed variables need to be always recorded in device and host
515 // compilations for transformation.
516 //
517 // HIP managed variables and variables in CUDADeviceVarODRUsedByHost are
518 // added to llvm.compiler-used, therefore they are safe to be registered.
519 if ((!vd->hasExternalStorage() && !vd->isInline()) ||
520 cgm.getASTContext().CUDADeviceVarODRUsedByHost.contains(key: vd) ||
521 vd->hasAttr<HIPManagedAttr>()) {
522 registerDeviceVar(vd, var, isExtern: !vd->hasDefinition(),
523 isConstant: vd->hasAttr<CUDAConstantAttr>());
524 }
525 } else if (vd->getType()->isCUDADeviceBuiltinSurfaceType()) {
526 // Builtin surfaces and their template arguments are also registered
527 // with CUDA runtime.
528 const auto *td = cast<ClassTemplateSpecializationDecl>(
529 Val: vd->getType()->castAsCXXRecordDecl());
530 const TemplateArgumentList &args = td->getTemplateArgs();
531
532 assert(args.size() == 2 &&
533 "Unexpected number of template arguments of CUDA device "
534 "builtin surface type.");
535
536 llvm::APSInt surfaceType = args[1].getAsIntegral();
537
538 if (!vd->hasExternalStorage())
539 registerDeviceSurf(vd, var, isExtern: !vd->hasDefinition(),
540 surfaceType: surfaceType.getSExtValue());
541
542 } else if (vd->getType()->isCUDADeviceBuiltinTextureType()) {
543 // Builtin textures and their template arguments are also registered
544 // with CUDA runtime.
545 const auto *td = cast<ClassTemplateSpecializationDecl>(
546 Val: vd->getType()->castAsCXXRecordDecl());
547 const TemplateArgumentList &args = td->getTemplateArgs();
548
549 assert(args.size() == 3 &&
550 "Unexpected number of template arguments of CUDA device "
551 "builtin texture type.");
552
553 llvm::APSInt textureType = args[1].getAsIntegral();
554 llvm::APSInt normalized = args[2].getAsIntegral();
555
556 if (!vd->hasExternalStorage())
557 registerDeviceTex(vd, var, isExtern: !vd->hasDefinition(),
558 textureType: textureType.getSExtValue(), normalized: normalized.getZExtValue());
559 }
560}
561
562void CIRGenNVCUDARuntime::handleGlobalReplace(cir::GlobalOp oldGV,
563 cir::GlobalOp newGV) {
564 for (auto &info : deviceVars) {
565 if (info.var == oldGV)
566 info.var = newGV;
567 }
568}
569
570/// Whether this translation unit has anything for the CUDA runtime to register.
571/// Like LoweringPrepare, count only kernel stub definitions and device
572/// variables carrying registration metadata, not kernel declarations or
573/// launch-site calls.
574static bool hasEntitiesToRegister(mlir::ModuleOp module) {
575 // A walk, not a scan of the module body, so this stays in agreement with the
576 // recursive walk LoweringPrepare collects them with.
577 return module
578 ->walk(callback: [](mlir::Operation *op) {
579 if (auto fn = mlir::dyn_cast<cir::FuncOp>(Val: op);
580 fn && !fn.isDeclaration() &&
581 fn->hasAttr(name: cir::CUDAKernelNameAttr::getMnemonic()))
582 return mlir::WalkResult::interrupt();
583 if (op->hasAttr(name: cir::CUDAVarRegistrationInfoAttr::getMnemonic()))
584 return mlir::WalkResult::interrupt();
585 return mlir::WalkResult::advance();
586 })
587 .wasInterrupted();
588}
589
590/// Read the device-side fat binary and record its contents on the module as
591/// `cir.cu.device_binary`, for LoweringPrepare to build the fatbin global from.
592///
593/// This mirrors the read in CGNVCUDARuntime::makeModuleCtorFunction, guards
594/// included: nothing is read in a compilation that would build no module
595/// constructor.
596void CIRGenNVCUDARuntime::recordDeviceBinary() {
597 StringRef binaryName = cgm.getCodeGenOpts().OffloadBinaryToEmbedFile;
598 if (binaryName.empty())
599 return;
600
601 const LangOptions &langOpts = cgm.getLangOpts();
602 if ((langOpts.HIP || !langOpts.GPURelocatableDeviceCode) &&
603 !hasEntitiesToRegister(module: cgm.getModule()))
604 return;
605
606 llvm::vfs::FileSystem &fs = cgm.getASTContext()
607 .getSourceManager()
608 .getFileManager()
609 .getVirtualFileSystem();
610 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> binaryOrErr =
611 fs.getBufferForFile(Name: binaryName, /*FileSize=*/-1,
612 /*RequiresNullTerminator=*/false);
613 if (std::error_code ec = binaryOrErr.getError()) {
614 cgm.getDiags().Report(DiagID: diag::err_cannot_open_file)
615 << binaryName << ec.message();
616 return;
617 }
618
619 // Typed as the fatbin global's array type so LoweringPrepare can use this
620 // attribute as the initializer as-is: attributes are uniqued on {value, type}
621 // and never freed, so building a second, typed copy there would keep the fat
622 // binary in memory twice.
623 StringRef bytes = binaryOrErr.get()->getBuffer();
624 mlir::MLIRContext &ctx = cgm.getMLIRContext();
625 auto charTy = cir::IntType::get(context: &ctx, width: cgm.getTarget().getCharWidth(),
626 /*isSigned=*/is_signed: false);
627 auto fatbinTy = cir::ArrayType::get(elementType: charTy, size: bytes.size());
628 cgm.getModule()->setAttr(name: cir::CIRDialect::getCUDADeviceBinaryAttrName(),
629 value: mlir::StringAttr::get(bytes, type: fatbinTy));
630}
631
632// Uses of a managed variable load its address from a pointer the runtime fills.
633void CIRGenNVCUDARuntime::transformManagedVars() {
634 CIRGenBuilderTy &builder = cgm.getBuilder();
635 mlir::ModuleOp module = cgm.getModule();
636 for (auto &&info : deviceVars) {
637 cir::GlobalOp var = info.var;
638 if (info.flags != cir::CUDADeviceVarKind::Variable ||
639 !info.d->hasAttr<HIPManagedAttr>())
640 continue;
641
642 cir::PointerType varPtrTy =
643 builder.getPointerTo(ty: var.getSymType(), as: var.getAddrSpaceAttr());
644 mlir::ptr::MemorySpaceAttrInterface managedAS = cir::toCIRAddressSpaceAttr(
645 ctx&: cgm.getMLIRContext(),
646 langAS: cgm.getLangOpts().CUDAIsDevice ? LangAS::cuda_device : LangAS::Default);
647 std::optional<mlir::SymbolTable::UseRange> uses = var.getSymbolUses(from: module);
648
649 std::string name = var.getSymName().str();
650 // An empty comdat would follow the rename.
651 std::optional<llvm::StringRef> comdat = var.getComdat();
652 if (comdat && comdat->empty())
653 var.setComdat(llvm::StringRef(name));
654 cgm.eraseGlobalSymbol(op: var);
655 var.setSymName(name + ".managed");
656 cgm.insertGlobalSymbol(op: var);
657
658 cir::GlobalOp managedVar =
659 cgm.createGlobalOp(loc: var.getLoc(), name, t: varPtrTy,
660 /*isConstant=*/false, addrSpace: managedAS);
661 managedVar.setLinkage(var.getLinkage());
662 if (!var.isDeclaration())
663 managedVar.setInitialValueAttr(builder.getConstNullPtrAttr(t: varPtrTy));
664 managedVar.setDsoLocal(var.getDsoLocal());
665 managedVar.setGlobalVisibility(var.getGlobalVisibility());
666 mlir::SymbolTable::setSymbolVisibility(
667 symbol: managedVar, vis: mlir::SymbolTable::getSymbolVisibility(symbol: var));
668 managedVar->setAttr(
669 name: cir::CUDAExternallyInitializedAttr::getMnemonic(),
670 value: cir::CUDAExternallyInitializedAttr::get(ctx: &cgm.getMLIRContext()));
671
672 // The uses now name the pointer global.
673 cir::PointerType managedPtrTy =
674 builder.getPointerTo(ty: varPtrTy, as: managedVar.getAddrSpaceAttr());
675 for (mlir::SymbolTable::SymbolUse use : *uses) {
676 auto getGlobalOp = dyn_cast<cir::GetGlobalOp>(Val: use.getUser());
677 if (!getGlobalOp) {
678 cgm.errorNYI(info.d->getSourceRange(),
679 "transformManagedVars: non-instruction use");
680 continue;
681 }
682 mlir::Value addr = getGlobalOp.getAddr();
683 addr.setType(managedPtrTy);
684 mlir::OpBuilder::InsertionGuard guard(builder);
685 builder.setInsertionPointAfter(getGlobalOp);
686 mlir::Value ld = builder.createAlignedLoad(
687 loc: getGlobalOp.getLoc(), ty: varPtrTy, ptr: addr,
688 align: CharUnits::fromQuantity(Quantity: var.getAlignment().value_or(u: 1)));
689 addr.replaceAllUsesExcept(newValue: ld, exceptedUser: ld.getDefiningOp());
690 }
691
692 // The runtime allocates them even if device code never uses them.
693 if (cgm.getLangOpts().CUDAIsDevice && !var.isDeclaration()) {
694 cgm.addCompilerUsedGlobal(gv: var);
695 cgm.addCompilerUsedGlobal(gv: managedVar);
696 }
697 }
698}
699
700void CIRGenNVCUDARuntime::finalizeModule() {
701 transformManagedVars();
702 if (!cgm.getLangOpts().CUDAIsDevice) {
703 recordDeviceBinary();
704 return;
705 }
706
707 // Mark ODR-used device variables as compiler used to prevent them from being
708 // eliminated by optimization. This is necessary for device variables
709 // ODR-used by host functions. Sema correctly marks them as ODR-used no
710 // matter whether they are ODR-used by device or host functions.
711 //
712 // We do not need to do this if the variable has used attribute since it
713 // has already been added.
714 //
715 // Static device variables have been externalized at this point, therefore
716 // variables with private or internal linkage need not be added.
717 for (auto &&info : deviceVars) {
718 auto kind = info.flags;
719 bool isDecl = info.var.isDeclaration();
720 bool isLocalLinkage = cir::isLocalLinkage(linkage: info.var.getLinkage());
721 bool isVarOrSurfaceOrTexture = (kind == cir::CUDADeviceVarKind::Variable ||
722 kind == cir::CUDADeviceVarKind::Surface ||
723 kind == cir::CUDADeviceVarKind::Texture);
724 bool isUsed = info.d->isUsed();
725 bool hasUsedAttr = info.d->hasAttr<UsedAttr>();
726 if (!isDecl && !isLocalLinkage && isVarOrSurfaceOrTexture && isUsed &&
727 !hasUsedAttr) {
728 if (auto globalValue = mlir::dyn_cast<cir::CIRGlobalValueInterface>(
729 Val: info.var.getOperation())) {
730 cgm.addCompilerUsedGlobal(gv: globalValue);
731 }
732 }
733 }
734}
735