1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "TargetInfo.h"
20#include "clang/AST/ASTContext.h"
21#include "clang/AST/Decl.h"
22#include "clang/AST/RecursiveASTVisitor.h"
23#include "clang/AST/Type.h"
24#include "clang/Basic/TargetOptions.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/IR/Value.h"
35#include "llvm/Support/Alignment.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/FormatVariadic.h"
38
39using namespace clang;
40using namespace CodeGen;
41using namespace clang::hlsl;
42using namespace llvm;
43
44using llvm::hlsl::CBufferRowSizeInBytes;
45
46namespace {
47
48void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
49 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
50 // Assume ValVersionStr is legal here.
51 VersionTuple Version;
52 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
53 Version.getSubminor() || !Version.getMinor()) {
54 return;
55 }
56
57 uint64_t Major = Version.getMajor();
58 uint64_t Minor = *Version.getMinor();
59
60 auto &Ctx = M.getContext();
61 IRBuilder<> B(M.getContext());
62 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
63 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
64 StringRef DXILValKey = "dx.valver";
65 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
66 DXILValMD->addOperand(M: Val);
67}
68
69void addRootSignature(llvm::dxbc::RootSignatureVersion RootSigVer,
70 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
71 llvm::Function *Fn, llvm::Module &M) {
72 auto &Ctx = M.getContext();
73
74 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
75 MDNode *RootSignature = RSBuilder.BuildRootSignature();
76
77 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
78 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
79 MDNode *MDVals =
80 MDNode::get(Context&: Ctx, MDs: {ValueAsMetadata::get(V: Fn), RootSignature, Version});
81
82 StringRef RootSignatureValKey = "dx.rootsignatures";
83 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
84 RootSignatureValMD->addOperand(M: MDVals);
85}
86
87} // namespace
88
89llvm::Type *
90CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
91 SmallVector<int32_t> *Packoffsets) {
92 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
93
94 // Check if the target has a specific translation for this type first.
95 if (llvm::Type *TargetTy =
96 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, Packoffsets))
97 return TargetTy;
98
99 llvm_unreachable("Generic handling of HLSL types is not supported.");
100}
101
102llvm::Triple::ArchType CGHLSLRuntime::getArch() {
103 return CGM.getTarget().getTriple().getArch();
104}
105
106// Returns true if the type is an HLSL resource class or an array of them
107static bool isResourceRecordTypeOrArrayOf(const clang::Type *Ty) {
108 while (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Val: Ty))
109 Ty = CAT->getArrayElementTypeNoTypeQual();
110 return Ty->isHLSLResourceRecord();
111}
112
113// Emits constant global variables for buffer constants declarations
114// and creates metadata linking the constant globals with the buffer global.
115void CGHLSLRuntime::emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
116 llvm::GlobalVariable *BufGV) {
117 LLVMContext &Ctx = CGM.getLLVMContext();
118
119 // get the layout struct from constant buffer target type
120 llvm::Type *BufType = BufGV->getValueType();
121 llvm::Type *BufLayoutType =
122 cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0);
123 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
124 Val: cast<llvm::TargetExtType>(Val: BufLayoutType)->getTypeParameter(i: 0));
125
126 // Start metadata list associating the buffer global variable with its
127 // constatns
128 SmallVector<llvm::Metadata *> BufGlobals;
129 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
130
131 const auto *ElemIt = LayoutStruct->element_begin();
132 for (Decl *D : BufDecl->buffer_decls()) {
133 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
134 // Nothing to do for this declaration.
135 continue;
136 if (isa<FunctionDecl>(Val: D)) {
137 // A function within an cbuffer is effectively a top-level function.
138 CGM.EmitTopLevelDecl(D);
139 continue;
140 }
141 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
142 if (!VD)
143 continue;
144
145 QualType VDTy = VD->getType();
146 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
147 if (VD->getStorageClass() == SC_Static ||
148 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
149 isResourceRecordTypeOrArrayOf(Ty: VDTy.getTypePtr())) {
150 // Emit static and groupshared variables and resource classes inside
151 // cbuffer as regular globals
152 CGM.EmitGlobal(D: VD);
153 } else {
154 // Anything else that is not in the hlsl_constant address space must be
155 // an empty struct or a zero-sized array and can be ignored
156 assert(BufDecl->getASTContext().getTypeSize(VDTy) == 0 &&
157 "constant buffer decl with non-zero sized type outside of "
158 "hlsl_constant address space");
159 }
160 continue;
161 }
162
163 assert(ElemIt != LayoutStruct->element_end() &&
164 "number of elements in layout struct does not match");
165 llvm::Type *LayoutType = *ElemIt++;
166
167 // FIXME: handle resources inside user defined structs
168 // (llvm/wg-hlsl#175)
169
170 // create global variable for the constant and to metadata list
171 GlobalVariable *ElemGV =
172 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
173 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
174 }
175 assert(ElemIt == LayoutStruct->element_end() &&
176 "number of elements in layout struct does not match");
177
178 // add buffer metadata to the module
179 CGM.getModule()
180 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
181 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
182}
183
184// Creates resource handle type for the HLSL buffer declaration
185static const clang::HLSLAttributedResourceType *
186createBufferHandleType(const HLSLBufferDecl *BufDecl) {
187 ASTContext &AST = BufDecl->getASTContext();
188 QualType QT = AST.getHLSLAttributedResourceType(
189 Wrapped: AST.HLSLResourceTy,
190 Contained: QualType(BufDecl->getLayoutStruct()->getTypeForDecl(), 0),
191 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
192 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
193}
194
195// Iterates over all declarations in the HLSL buffer and based on the
196// packoffset or register(c#) annotations it fills outs the Layout
197// vector with the user-specified layout offsets.
198// The buffer offsets can be specified 2 ways:
199// 1. declarations in cbuffer {} block can have a packoffset annotation
200// (translates to HLSLPackOffsetAttr)
201// 2. default constant buffer declarations at global scope can have
202// register(c#) annotations (translates to HLSLResourceBindingAttr with
203// RegisterType::C)
204// It is not guaranteed that all declarations in a buffer have an annotation.
205// For those where it is not specified a -1 value is added to the Layout
206// vector. In the final layout these declarations will be placed at the end
207// of the HLSL buffer after all of the elements with specified offset.
208static void fillPackoffsetLayout(const HLSLBufferDecl *BufDecl,
209 SmallVector<int32_t> &Layout) {
210 assert(Layout.empty() && "expected empty vector for layout");
211 assert(BufDecl->hasValidPackoffset());
212
213 for (Decl *D : BufDecl->buffer_decls()) {
214 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
215 continue;
216 }
217 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
218 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
219 continue;
220
221 if (!VD->hasAttrs()) {
222 Layout.push_back(Elt: -1);
223 continue;
224 }
225
226 int32_t Offset = -1;
227 for (auto *Attr : VD->getAttrs()) {
228 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
229 Offset = POA->getOffsetInBytes();
230 break;
231 }
232 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
233 if (RBA &&
234 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
235 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
236 break;
237 }
238 }
239 Layout.push_back(Elt: Offset);
240 }
241}
242
243// Codegen for HLSLBufferDecl
244void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
245
246 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
247
248 // create resource handle type for the buffer
249 const clang::HLSLAttributedResourceType *ResHandleTy =
250 createBufferHandleType(BufDecl);
251
252 // empty constant buffer is ignored
253 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
254 return;
255
256 // create global variable for the constant buffer
257 SmallVector<int32_t> Layout;
258 if (BufDecl->hasValidPackoffset())
259 fillPackoffsetLayout(BufDecl, Layout);
260
261 llvm::TargetExtType *TargetTy =
262 cast<llvm::TargetExtType>(Val: convertHLSLSpecificType(
263 T: ResHandleTy, Packoffsets: BufDecl->hasValidPackoffset() ? &Layout : nullptr));
264 llvm::GlobalVariable *BufGV = new GlobalVariable(
265 TargetTy, /*isConstant*/ false,
266 GlobalValue::LinkageTypes::ExternalLinkage, PoisonValue::get(T: TargetTy),
267 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
268 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
269 GlobalValue::NotThreadLocal);
270 CGM.getModule().insertGlobalVariable(GV: BufGV);
271
272 // Add globals for constant buffer elements and create metadata nodes
273 emitBufferGlobalsAndMetadata(BufDecl, BufGV);
274
275 // Initialize cbuffer from binding (implicit or explicit)
276 HLSLResourceBindingAttr *RBA = BufDecl->getAttr<HLSLResourceBindingAttr>();
277 assert(RBA &&
278 "cbuffer/tbuffer should always have resource binding attribute");
279 initializeBufferFromBinding(BufDecl, GV: BufGV, RBA);
280}
281
282llvm::TargetExtType *
283CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
284 const auto Entry = LayoutTypes.find(Val: StructType);
285 if (Entry != LayoutTypes.end())
286 return Entry->getSecond();
287 return nullptr;
288}
289
290void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
291 llvm::TargetExtType *LayoutTy) {
292 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
293 "layout type for this struct already exist");
294 LayoutTypes[StructType] = LayoutTy;
295}
296
297void CGHLSLRuntime::finishCodeGen() {
298 auto &TargetOpts = CGM.getTarget().getTargetOpts();
299 auto &CodeGenOpts = CGM.getCodeGenOpts();
300 auto &LangOpts = CGM.getLangOpts();
301 llvm::Module &M = CGM.getModule();
302 Triple T(M.getTargetTriple());
303 if (T.getArch() == Triple::ArchType::dxil)
304 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
305 if (CodeGenOpts.ResMayAlias)
306 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
307
308 // NativeHalfType corresponds to the -fnative-half-type clang option which is
309 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
310 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
311 if (LangOpts.NativeHalfType)
312 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
313 Val: 1);
314
315 generateGlobalCtorDtorCalls();
316}
317
318void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
319 const FunctionDecl *FD, llvm::Function *Fn) {
320 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
321 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
322 const StringRef ShaderAttrKindStr = "hlsl.shader";
323 Fn->addFnAttr(Kind: ShaderAttrKindStr,
324 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
325 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
326 const StringRef NumThreadsKindStr = "hlsl.numthreads";
327 std::string NumThreadsStr =
328 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
329 Vals: NumThreadsAttr->getZ());
330 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
331 }
332 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
333 const StringRef WaveSizeKindStr = "hlsl.wavesize";
334 std::string WaveSizeStr =
335 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
336 Vals: WaveSizeAttr->getPreferred());
337 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
338 }
339 // HLSL entry functions are materialized for module functions with
340 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
341 // later in the compiler-flow for such module functions is not aware of and
342 // hence not able to set attributes of the newly materialized entry functions.
343 // So, set attributes of entry function here, as appropriate.
344 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
345 Fn->addFnAttr(Kind: llvm::Attribute::OptimizeNone);
346 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
347}
348
349static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
350 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
351 Value *Result = PoisonValue::get(T: Ty);
352 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
353 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
354 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
355 }
356 return Result;
357 }
358 return B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)});
359}
360
361static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
362 unsigned BuiltIn) {
363 LLVMContext &Ctx = GV->getContext();
364 IRBuilder<> B(GV->getContext());
365 MDNode *Operands = MDNode::get(
366 Context&: Ctx,
367 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
368 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
369 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
370 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
371}
372
373static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
374 llvm::Type *Ty, const Twine &Name,
375 unsigned BuiltInID) {
376 auto *GV = new llvm::GlobalVariable(
377 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
378 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
379 llvm::GlobalVariable::GeneralDynamicTLSModel,
380 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
381 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
382 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
383 return B.CreateLoad(Ty, Ptr: GV);
384}
385
386llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B,
387 const ParmVarDecl &D,
388 llvm::Type *Ty) {
389 assert(D.hasAttrs() && "Entry parameter missing annotation attribute!");
390 if (D.hasAttr<HLSLSV_GroupIndexAttr>()) {
391 llvm::Function *GroupIndex =
392 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
393 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
394 }
395 if (D.hasAttr<HLSLSV_DispatchThreadIDAttr>()) {
396 llvm::Function *ThreadIDIntrinsic =
397 CGM.getIntrinsic(IID: getThreadIdIntrinsic());
398 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty);
399 }
400 if (D.hasAttr<HLSLSV_GroupThreadIDAttr>()) {
401 llvm::Function *GroupThreadIDIntrinsic =
402 CGM.getIntrinsic(IID: getGroupThreadIdIntrinsic());
403 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty);
404 }
405 if (D.hasAttr<HLSLSV_GroupIDAttr>()) {
406 llvm::Function *GroupIDIntrinsic = CGM.getIntrinsic(IID: getGroupIdIntrinsic());
407 return buildVectorInput(B, F: GroupIDIntrinsic, Ty);
408 }
409 if (D.hasAttr<HLSLSV_PositionAttr>()) {
410 if (getArch() == llvm::Triple::spirv)
411 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty, Name: "sv_position",
412 /* BuiltIn::Position */ BuiltInID: 0);
413 llvm_unreachable("SV_Position semantic not implemented for this target.");
414 }
415 assert(false && "Unhandled parameter attribute");
416 return nullptr;
417}
418
419void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
420 llvm::Function *Fn) {
421 llvm::Module &M = CGM.getModule();
422 llvm::LLVMContext &Ctx = M.getContext();
423 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
424 Function *EntryFn =
425 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
426
427 // Copy function attributes over, we have no argument or return attributes
428 // that can be valid on the real entry.
429 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
430 Attrs: Fn->getAttributes().getFnAttrs());
431 EntryFn->setAttributes(NewAttrs);
432 setHLSLEntryAttributes(FD, Fn: EntryFn);
433
434 // Set the called function as internal linkage.
435 Fn->setLinkage(GlobalValue::InternalLinkage);
436
437 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
438 IRBuilder<> B(BB);
439 llvm::SmallVector<Value *> Args;
440
441 SmallVector<OperandBundleDef, 1> OB;
442 if (CGM.shouldEmitConvergenceTokens()) {
443 assert(EntryFn->isConvergent());
444 llvm::Value *I =
445 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
446 llvm::Value *bundleArgs[] = {I};
447 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
448 }
449
450 // FIXME: support struct parameters where semantics are on members.
451 // See: https://github.com/llvm/llvm-project/issues/57874
452 unsigned SRetOffset = 0;
453 for (const auto &Param : Fn->args()) {
454 if (Param.hasStructRetAttr()) {
455 // FIXME: support output.
456 // See: https://github.com/llvm/llvm-project/issues/57874
457 SRetOffset = 1;
458 Args.emplace_back(Args: PoisonValue::get(T: Param.getType()));
459 continue;
460 }
461 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
462 Args.push_back(Elt: emitInputSemantic(B, D: *PD, Ty: Param.getType()));
463 }
464
465 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
466 CI->setCallingConv(Fn->getCallingConv());
467 // FIXME: Handle codegen for return type semantics.
468 // See: https://github.com/llvm/llvm-project/issues/57875
469 B.CreateRetVoid();
470
471 // Add and identify root signature to function, if applicable
472 for (const Attr *Attr : FD->getAttrs()) {
473 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
474 auto *RSDecl = RSAttr->getSignatureDecl();
475 addRootSignature(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(), Fn: EntryFn,
476 M);
477 }
478 }
479}
480
481static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
482 bool CtorOrDtor) {
483 const auto *GV =
484 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
485 if (!GV)
486 return;
487 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
488 if (!CA)
489 return;
490 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
491 // HLSL neither supports priorities or COMDat values, so we will check those
492 // in an assert but not handle them.
493
494 for (const auto &Ctor : CA->operands()) {
495 if (isa<ConstantAggregateZero>(Val: Ctor))
496 continue;
497 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
498
499 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
500 "HLSL doesn't support setting priority for global ctors.");
501 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
502 "HLSL doesn't support COMDat for global ctors.");
503 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
504 }
505}
506
507void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
508 llvm::Module &M = CGM.getModule();
509 SmallVector<Function *> CtorFns;
510 SmallVector<Function *> DtorFns;
511 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
512 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
513
514 // Insert a call to the global constructor at the beginning of the entry block
515 // to externally exported functions. This is a bit of a hack, but HLSL allows
516 // global constructors, but doesn't support driver initialization of globals.
517 for (auto &F : M.functions()) {
518 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
519 continue;
520 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
521 Instruction *IP = &*F.getEntryBlock().begin();
522 SmallVector<OperandBundleDef, 1> OB;
523 if (Token) {
524 llvm::Value *bundleArgs[] = {Token};
525 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
526 IP = Token->getNextNode();
527 }
528 IRBuilder<> B(IP);
529 for (auto *Fn : CtorFns) {
530 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
531 CI->setCallingConv(Fn->getCallingConv());
532 }
533
534 // Insert global dtors before the terminator of the last instruction
535 B.SetInsertPoint(F.back().getTerminator());
536 for (auto *Fn : DtorFns) {
537 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
538 CI->setCallingConv(Fn->getCallingConv());
539 }
540 }
541
542 // No need to keep global ctors/dtors for non-lib profile after call to
543 // ctors/dtors added for entry.
544 Triple T(M.getTargetTriple());
545 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
546 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
547 GV->eraseFromParent();
548 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
549 GV->eraseFromParent();
550 }
551}
552
553static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
554 Intrinsic::ID IntrID,
555 ArrayRef<llvm::Value *> Args) {
556
557 LLVMContext &Ctx = CGM.getLLVMContext();
558 llvm::Function *InitResFunc = llvm::Function::Create(
559 Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
560 Linkage: llvm::GlobalValue::InternalLinkage,
561 N: ("_init_buffer_" + GV->getName()).str(), M&: CGM.getModule());
562 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
563
564 llvm::BasicBlock *EntryBB =
565 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
566 CGBuilderTy Builder(CGM, Ctx);
567 const DataLayout &DL = CGM.getModule().getDataLayout();
568 Builder.SetInsertPoint(EntryBB);
569
570 // Make sure the global variable is buffer resource handle
571 llvm::Type *HandleTy = GV->getValueType();
572 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
573
574 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
575 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
576 Name: Twine(GV->getName()).concat(Suffix: "_h"));
577
578 llvm::Value *HandleRef = Builder.CreateStructGEP(Ty: GV->getValueType(), Ptr: GV, Idx: 0);
579 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: HandleRef,
580 Align: HandleRef->getPointerAlignment(DL));
581 Builder.CreateRetVoid();
582
583 CGM.AddCXXGlobalInit(F: InitResFunc);
584}
585
586void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
587 llvm::GlobalVariable *GV,
588 HLSLResourceBindingAttr *RBA) {
589 assert(RBA && "expect a nonnull binding attribute");
590 llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGM.getLLVMContext());
591 auto *NonUniform = llvm::ConstantInt::get(Ty: Int1Ty, V: false);
592 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
593 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
594 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: RBA->getSpaceNumber());
595 Value *Name = nullptr;
596
597 llvm::Intrinsic::ID IntrinsicID =
598 RBA->hasRegisterSlot()
599 ? CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic()
600 : CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
601
602 std::string Str(BufDecl->getName());
603 std::string GlobalName(Str + ".str");
604 Name = CGM.GetAddrOfConstantCString(Str, GlobalName: GlobalName.c_str()).getPointer();
605
606 // buffer with explicit binding
607 if (RBA->hasRegisterSlot()) {
608 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: RBA->getSlotNumber());
609 SmallVector<Value *> Args{Space, RegSlot, RangeSize,
610 Index, NonUniform, Name};
611 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
612 } else {
613 // buffer with implicit binding
614 auto *OrderID =
615 llvm::ConstantInt::get(Ty: CGM.IntTy, V: RBA->getImplicitBindingOrderID());
616 SmallVector<Value *> Args{OrderID, Space, RangeSize,
617 Index, NonUniform, Name};
618 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
619 }
620}
621
622void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
623 llvm::GlobalVariable *GV) {
624 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
625 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
626}
627
628llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
629 if (!CGM.shouldEmitConvergenceTokens())
630 return nullptr;
631
632 auto E = BB.end();
633 for (auto I = BB.begin(); I != E; ++I) {
634 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
635 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
636 return II;
637 }
638 }
639 llvm_unreachable("Convergence token should have been emitted.");
640 return nullptr;
641}
642
643class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
644public:
645 llvm::SmallPtrSet<OpaqueValueExpr *, 8> OVEs;
646 OpaqueValueVisitor() {}
647
648 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
649 OVEs.insert(Ptr: E);
650 return true;
651 }
652};
653
654void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
655 InitListExpr *E) {
656
657 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
658 OpaqueValueVisitor Visitor;
659 Visitor.TraverseStmt(S: E);
660 for (auto *OVE : Visitor.OVEs) {
661 if (CGF.isOpaqueValueEmitted(E: OVE))
662 continue;
663 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
664 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
665 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
666 } else {
667 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
668 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
669 }
670 }
671}
672