1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/RecursiveASTVisitor.h"
28#include "clang/AST/Type.h"
29#include "clang/Basic/DiagnosticDriver.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetOptions.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/STLExtras.h"
35#include "llvm/ADT/ScopeExit.h"
36#include "llvm/ADT/SmallString.h"
37#include "llvm/ADT/SmallVector.h"
38#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/Type.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/Alignment.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Support/FormatVariadic.h"
51#include "llvm/Support/Path.h"
52#include "llvm/Transforms/Utils/ModuleUtils.h"
53#include <cstdint>
54#include <optional>
55
56using namespace clang;
57using namespace CodeGen;
58using namespace clang::hlsl;
59using namespace llvm;
60
61using llvm::hlsl::CBufferRowSizeInBytes;
62
63namespace {
64
65void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
66 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
67 // Assume ValVersionStr is legal here.
68 VersionTuple Version;
69 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
70 Version.getSubminor() || !Version.getMinor()) {
71 return;
72 }
73
74 uint64_t Major = Version.getMajor();
75 uint64_t Minor = *Version.getMinor();
76
77 auto &Ctx = M.getContext();
78 IRBuilder<> B(M.getContext());
79 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
80 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
81 StringRef DXILValKey = "dx.valver";
82 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
83 DXILValMD->addOperand(M: Val);
84}
85
86void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
87 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
88 llvm::Function *Fn, llvm::Module &M) {
89 auto &Ctx = M.getContext();
90
91 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
92 MDNode *RootSignature = RSBuilder.BuildRootSignature();
93
94 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
95 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
96 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
97 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
98
99 StringRef RootSignatureValKey = "dx.rootsignatures";
100 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
101 RootSignatureValMD->addOperand(M: MDVals);
102}
103
104static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
105 AggValueSlot &DestSlot) {
106 GlobalVariable *ResGV =
107 cast<GlobalVariable>(Val: CGF.CGM.GetAddrOfGlobalVar(D: ResourceVD));
108 assert(ResGV && "expected valid global variable");
109 CGF.Builder.CreateStore(Val: ResGV, Addr: DestSlot.getAddress());
110}
111
112// Given a MemberExpr of a resource or resource array type, find the parent
113// VarDecl of the struct or class instance that contains this resource and
114// build the full resource name based on the member access path.
115//
116// For example, for a member access like "myStructArray[0].memberA",
117// this function will find the VarDecl of "myStructArray" and use the
118// EmbeddedResourceNameBuilder to build the resource name
119// "myStructArray.0.memberA".
120//
121// This also works for a record type expression that has some embedded
122// resources. It finds the parent VarDecl of that record and builds a partial
123// name which is the prefix of the resource globals associated with the
124// declaration.
125static const VarDecl *findStructResourceParentDeclAndBuildName(
126 const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
127
128 SmallVector<const Expr *> WorkList;
129 const VarDecl *VD = nullptr;
130
131 for (;;) {
132 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
133 assert(isa<VarDecl>(DRE->getDecl()) &&
134 "member expr base is not a var decl");
135 VD = cast<VarDecl>(Val: DRE->getDecl());
136 NameBuilder.pushName(N: VD->getName());
137 break;
138 }
139
140 WorkList.push_back(Elt: E);
141 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
142 E = MExp->getBase();
143 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
144 E = ICE->getSubExpr();
145 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
146 E = ASE->getBase();
147 else if (isa<CXXThisExpr>(Val: E))
148 // Resource member access on "this" pointer not yet implemented
149 // (llvm/llvm-project#190299)
150 return nullptr;
151 else
152 llvm_unreachable("unexpected expr type in resource member access");
153
154 assert(E && "expected valid expression");
155 }
156
157 while (!WorkList.empty()) {
158 E = WorkList.pop_back_val();
159 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
160 NameBuilder.pushName(
161 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
162 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
163 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
164 CXXRecordDecl *DerivedRD =
165 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
166 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
167 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
168 }
169 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
170 const Expr *IdxExpr = ASE->getIdx();
171 std::optional<llvm::APSInt> Value =
172 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
173 assert(Value &&
174 "expected constant index in struct with resource array access");
175 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
176 } else {
177 llvm_unreachable("unexpected expr type in resource member access");
178 }
179 }
180 return VD;
181}
182
183// Given a MemberExpr of a resource or resource array type, find the
184// corresponding global resource declaration associated with the owning struct
185// or class instance via HLSLAssociatedResourceDeclAttr.
186static const VarDecl *
187findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
188
189 EmbeddedResourceNameBuilder NameBuilder;
190 const VarDecl *ParentVD =
191 findStructResourceParentDeclAndBuildName(E: ME, NameBuilder);
192 if (!ParentVD)
193 return nullptr;
194
195 if (!ParentVD->hasGlobalStorage())
196 return nullptr;
197
198 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
199 for (const Attr *A : ParentVD->getAttrs()) {
200 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
201 VarDecl *AssocResVD = ADA->getResDecl();
202 if (AssocResVD->getIdentifier() == II)
203 return AssocResVD;
204 }
205 }
206 return nullptr;
207}
208
209void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
210 auto &SM = CGM.getContext().getSourceManager();
211 auto &Macros = CGM.getPreprocessorOpts().Macros;
212 auto &CodeGenOpts = CGM.getCodeGenOpts();
213 auto &Ctx = M.getContext();
214
215 // Names and content of shader source code files.
216 llvm::NamedMDNode *DXContents =
217 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
218 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
219 llvm::MDTuple *FileInfo =
220 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
221 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
222 DXContents->addOperand(M: FileInfo);
223 };
224
225 bool Invalid = false;
226 const SrcMgr::SLocEntry *MainLocEntry =
227 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
228 assert(!Invalid && "Main file SLocEntry must not be invalid!");
229 const SrcMgr::ContentCache &MainCCEntry =
230 MainLocEntry->getFile().getContentCache();
231
232 SmallVector<std::pair<std::string, StringRef>> Files;
233 std::optional<SmallString<256>> MainFileName;
234 Files.reserve(N: SM.local_sloc_entry_size());
235 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
236 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
237 if (!LocEntry.isFile())
238 continue;
239
240 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
241 if (isSystem(CK: FInfo.getFileCharacteristic()))
242 continue;
243
244 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
245 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
246 if (!FEntry)
247 continue;
248
249 llvm::SmallString<256> Path = FEntry->getName();
250 llvm::sys::path::native(path&: Path);
251 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
252 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
253 if (!Buffer) {
254 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
255 << Path;
256 continue;
257 }
258
259 if (&MainCCEntry != &CCEntry) {
260 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
261 } else {
262 // Main file should be at first position.
263 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
264 MainFileName.emplace(args&: Path);
265 }
266 }
267 assert(MainFileName && "Main file not found.");
268
269 // Files other that main one should be sorted by name.
270 llvm::sort(C&: Files);
271#ifndef NDEBUG
272 for (unsigned I = 1; I < Files.size(); ++I)
273 assert((Files[I - 1].first != Files[I].first) &&
274 "duplicate files in dx.source.contents");
275#endif
276 llvm::for_each(Range&: Files, F: addFile);
277
278 SmallVector<llvm::Metadata *> Defines;
279 Defines.reserve(N: Macros.size());
280 for (const auto &Macro : Macros) {
281 // Ignore undefs.
282 if (!Macro.second)
283 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
284 }
285 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
286 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
287
288 if (!CodeGenOpts.MainFileName.empty())
289 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
290 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
291 ->addOperand(
292 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
293
294 SmallVector<llvm::Metadata *> Args;
295 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
296 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
297 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
298 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
299 M.getOrInsertNamedMetadata(Name: "dx.source.args")
300 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
301}
302
303// Find array variable declaration from DeclRef expression
304static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
305 E = E->IgnoreImpCasts();
306 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
307 return DRE->getDecl();
308 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
309 E = OVE->getSourceExpr()->IgnoreImpCasts();
310 if (isa<MemberExpr>(Val: E))
311 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
312 return nullptr;
313}
314
315// Find array variable declaration from nested array subscript AST nodes
316static const ValueDecl *getArrayDecl(ASTContext &AST,
317 const ArraySubscriptExpr *ASE) {
318 const Expr *E = nullptr;
319 while (ASE != nullptr) {
320 E = ASE->getBase()->IgnoreImpCasts();
321 if (!E)
322 return nullptr;
323 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
324 }
325 return getArrayDecl(AST, E);
326}
327
328// Get the total size of the array, or 0 if the array is unbounded.
329static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
330 Ty = Ty->getUnqualifiedDesugaredType();
331 assert(Ty->isArrayType() && "expected array type");
332 if (Ty->isIncompleteArrayType())
333 return 0;
334 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
335}
336
337static Value *buildNameForResource(llvm::StringRef BaseName,
338 CodeGenModule &CGM) {
339 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
340 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
341 .getPointer();
342}
343
344static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
345 StorageClass SC = SC_None) {
346 for (auto *Method : Record->methods()) {
347 if (Method->getStorageClass() == SC && Method->getName() == Name)
348 return Method;
349 }
350 return nullptr;
351}
352
353static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
354 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
355 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
356 CallArgList &Args) {
357 assert(Binding.hasBinding() && "at least one binding attribute expected");
358
359 ASTContext &AST = CGM.getContext();
360 CXXMethodDecl *CreateMethod = nullptr;
361 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
362 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
363
364 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
365 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
366 "resources with counter handle must have a binding with counter "
367 "implicit order ID");
368 if (Binding.isExplicit()) {
369 // explicit binding
370 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
371 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
372 const char *Name = Binding.hasCounterImplicitOrderID()
373 ? "__createFromBindingWithImplicitCounter"
374 : "__createFromBinding";
375 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
376 } else {
377 // implicit binding
378 auto *OrderID =
379 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
380 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
381 const char *Name = Binding.hasCounterImplicitOrderID()
382 ? "__createFromImplicitBindingWithImplicitCounter"
383 : "__createFromImplicitBinding";
384 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
385 }
386 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
387 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
388 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
389 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
390 if (HasCounter) {
391 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
392 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
393 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
394 }
395
396 return CreateMethod;
397}
398
399static void callResourceInitMethod(CodeGenFunction &CGF,
400 CXXMethodDecl *CreateMethod,
401 CallArgList &Args, Address ReturnAddress) {
402 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
403 const FunctionProtoType *Proto =
404 CreateMethod->getType()->getAs<FunctionProtoType>();
405 // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
406 // caller declaration is needed for x86 SysV ABI selection.
407 const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
408 Args, Ty: Proto, ChainCall: false, /*ABIInfoFD=*/nullptr);
409 ReturnValueSlot ReturnValue(ReturnAddress, false);
410 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
411 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
412}
413
414// Initializes local resource array variable with global resource array
415// elements. For multi-dimensional arrays it calls itself recursively to
416// initialize its sub-arrays. The Index used in the resource constructor calls
417// will begin at StartIndex and will be incremented for each array element. The
418// last used resource Index is returned to the caller. If the function returns
419// std::nullopt, it indicates an error.
420static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
421 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
422 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
423 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
424 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
425
426 ASTContext &AST = CGF.getContext();
427 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
428 llvm::Value *Index = StartIndex;
429 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
430 const uint64_t ArraySize = ArrayTy->getSExtSize();
431 QualType ElemType = ArrayTy->getElementType();
432 Address TmpArrayAddr = ValueSlot.getAddress();
433
434 // Add additional index to the getelementptr call indices.
435 // This index will be updated for each array element in the loops below.
436 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
437 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
438
439 // For array of arrays, recursively initialize the sub-arrays.
440 if (ElemType->isArrayType()) {
441 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
442 for (uint64_t I = 0; I < ArraySize; I++) {
443 if (I > 0) {
444 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
445 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
446 }
447 std::optional<llvm::Value *> MaybeIndex =
448 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
449 ValueSlot, Range, StartIndex: Index,
450 ResourceName, Binding, PrevGEPIndices: GEPIndices);
451 if (!MaybeIndex)
452 return std::nullopt;
453 Index = *MaybeIndex;
454 }
455 return Index;
456 }
457
458 // For array of resources, initialize each resource in the array.
459 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
460 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
461 CharUnits Align =
462 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
463
464 for (uint64_t I = 0; I < ArraySize; I++) {
465 if (I > 0) {
466 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
467 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
468 }
469 Address ReturnAddress =
470 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
471
472 CallArgList Args;
473 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
474 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
475
476 if (!CreateMethod)
477 // This can happen if someone creates an array of structs that looks like
478 // an HLSL resource record array but it does not have the required static
479 // create method. No binding will be generated for it.
480 return std::nullopt;
481
482 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
483 }
484 return Index;
485}
486
487/// Utility for emitting copies following the HLSL buffer layout rules (ie,
488/// copying out of a cbuffer).
489class HLSLBufferCopyEmitter {
490 CodeGenFunction &CGF;
491 Address DstPtr;
492 Address SrcPtr;
493 llvm::Type *LayoutTy = nullptr;
494
495 SmallVector<llvm::Value *> CurStoreIndices;
496 SmallVector<llvm::Value *> CurLoadIndices;
497
498 using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
499
500 // Creates & returns either a structured.gep or a ptradd/gep depending on
501 // langopts.
502 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
503 ArrayRef<llvm::Value *> Indices) {
504 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
505 if (EmitLogical)
506 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
507
508 llvm::SmallVector<llvm::Value *> GEPIndices;
509 GEPIndices.reserve(N: Indices.size() + 1);
510 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
511 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
512 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
513 }
514
515 bool isBufferLayoutArray(llvm::StructType *ST) {
516 // A buffer layout array is a struct with two elements: the padded array,
517 // and the last element. That is, is should look something like this:
518 //
519 // { [%n x { %type, %padding }], %type }
520 //
521 if (!ST || ST->getNumElements() != 2)
522 return false;
523
524 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
525 if (!PaddedEltsTy)
526 return false;
527
528 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
529 if (!PaddedTy || PaddedTy->getNumElements() != 2)
530 return false;
531
532 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
533 Ty: PaddedTy->getElementType(N: 1)))
534 return false;
535
536 llvm::Type *ElementTy = ST->getElementType(N: 1);
537 if (PaddedTy->getElementType(N: 0) != ElementTy)
538 return false;
539 return true;
540 }
541
542 // Returns true if the type is either a struct representing a resource record,
543 // or an array of structs that are resource records. This assumes a struct is
544 // a resource record if the first element is a target type (resource handle).
545 // This is the case for all target types used by HLSL except the padding type
546 // ("{dx|spirv.Padding"), but padding will never be the first element of a
547 // struct.
548 bool isResourceOrResourceArray(llvm::Type *Ty) {
549 while (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
550 Ty = AT->getElementType();
551
552 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
553 if (!ST || ST->getNumElements() < 1)
554 return false;
555
556 auto *TargetTy = dyn_cast<llvm::TargetExtType>(Val: ST->getElementType(N: 0));
557 return TargetTy != nullptr;
558 }
559
560 void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
561 EmitResourceFnTy EmitResFn) {
562 CharUnits DstAlign =
563 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
564 Address DstAddr(Dst, DstTy, DstAlign);
565 AggValueSlot Slot = AggValueSlot::forAddr(
566 addr: DstAddr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
567 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
568 mayOverlap: AggValueSlot::DoesNotOverlap);
569
570 EmitResFn(Slot);
571 }
572
573 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
574 llvm::ArrayType *DstTy,
575 EmitResourceFnTy EmitResFn) {
576 // Those assumptions are checked by isBufferLayoutArray.
577 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
578 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
579 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
580 ->getElementType(0) == SrcTy->getElementType(1));
581
582 auto *SrcDataTy = SrcTy->getElementType(N: 1);
583 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
584
585 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
586 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
587 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
588 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
589 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
590 EmitResFn);
591 }
592
593 auto *SrcElt =
594 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
595 auto *DstElt = emitAccessChain(
596 BaseTy: DstTy, Base: Dst,
597 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
598 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
599 EmitResFn);
600 }
601
602 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
603 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
604 assert(!isResourceOrResourceArray(DstTy) &&
605 "direct access to resources or resource arrays should be handled "
606 "separately");
607
608 if (isBufferLayoutArray(ST: SrcTy))
609 return emitBufferLayoutCopy(Src, SrcTy, Dst, DstTy: cast<llvm::ArrayType>(Val: DstTy),
610 EmitResFn);
611
612 unsigned SrcIndex = 0;
613 unsigned DstIndex = 0;
614
615 // DstTy layout is in default address space and can include resource types.
616 // SrcTy is in cbuffer layout where resources are filtered out, so the
617 // number of elements in SrcTy can be less than the number of elements in
618 // DstTy.
619 auto *DstST = cast<llvm::StructType>(Val: DstTy);
620 while (DstIndex < DstST->getNumElements()) {
621 llvm::Type *DstEltTy = DstST->getElementType(N: DstIndex);
622 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: DstEltTy)) {
623 DstIndex += 1;
624 continue;
625 }
626 if (isResourceOrResourceArray(Ty: DstEltTy)) {
627 auto *DstElt = emitAccessChain(
628 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
629 emitResourceOrResourceArray(Dst: DstElt, DstTy: DstEltTy, EmitResFn);
630 DstIndex += 1;
631 continue;
632 }
633
634 assert(SrcIndex < SrcTy->getNumElements());
635 llvm::Type *SrcEltTy = SrcTy->getElementType(N: SrcIndex);
636 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: SrcEltTy)) {
637 SrcIndex += 1;
638 continue;
639 }
640
641 auto *SrcElt = emitAccessChain(
642 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
643 auto *DstElt = emitAccessChain(
644 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
645 emitElementCopy(Src: SrcElt, SrcTy: SrcEltTy, Dst: DstElt, DstTy: DstEltTy, EmitResFn);
646 DstIndex += 1;
647 SrcIndex += 1;
648 }
649 }
650
651 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
652 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
653 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
654 auto *SrcElt =
655 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
656 auto *DstElt =
657 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
658 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
659 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType(),
660 EmitResFn);
661 }
662 }
663
664 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
665 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
666 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
667 return emitCopy(Src, SrcTy: AT, Dst, DstTy, EmitResFn);
668 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
669 return emitCopy(Src, SrcTy: ST, Dst, DstTy, EmitResFn);
670
671 // When we have a scalar or vector element we can emit the copy.
672 CharUnits SrcAlign =
673 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
674 CharUnits DstAlign =
675 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
676 Address SrcAddr(Src, SrcTy, SrcAlign);
677 Address DstAddr(Dst, DstTy, DstAlign);
678 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
679 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
680 }
681
682public:
683 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
684 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
685
686 bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
687 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
688
689 // TODO: We should be able to fall back to a regular memcpy if the layout
690 // type doesn't have any padding, but that runs into issues in the backend
691 // currently.
692 //
693 // See https://github.com/llvm/wg-hlsl/issues/351
694 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
695 DstTy: DstPtr.getElementType(), EmitResFn);
696 return true;
697 }
698};
699
700// Represents a list resources associated with a global struct whose name
701// starts with the specified prefix.
702// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
703// order of the depth-first traversal of the corresponding fields in the struct.
704// The resources are always returned in that order, which is the same order
705// we need when a struct is copied element-by-element.
706class AssociatedResourcesList {
707 // Iterator pointers for the associated resource attributes that match the
708 // prefix. Begin = begin of the range of attributes that match the prefix End
709 // = end of the range of attributes that match the prefix Next = the current
710 // attribute in the iteration to be returned by getNextResource
711 specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
712
713public:
714 AssociatedResourcesList(const VarDecl *StructVD,
715 StringRef ResourceNamePrefix) {
716 auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
717 auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
718
719 // Skip over associated resources that don't match the prefix.
720 while (I != E &&
721 !I->getResDecl()->getName().starts_with(Prefix: ResourceNamePrefix))
722 ++I;
723 assert(I != E && "expected associated resource not found");
724 Begin = End = I;
725
726 // Scan over associated resources that do match the prefix to find the end
727 // of the range.
728 while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
729 ->getResDecl()
730 ->getName()
731 .starts_with(Prefix: ResourceNamePrefix))
732 End = ++I;
733
734 Next = Begin;
735 }
736
737 const VarDecl *getNextResource() {
738 if (Next == End)
739 return nullptr;
740
741 const VarDecl *Res = Next->getResDecl();
742 ++Next;
743 return Res;
744 }
745};
746
747} // namespace
748
749llvm::Type *
750CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
751 const CGHLSLOffsetInfo &OffsetInfo) {
752 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
753
754 // Check if the target has a specific translation for this type first.
755 if (llvm::Type *TargetTy =
756 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
757 return TargetTy;
758
759 llvm_unreachable("Generic handling of HLSL types is not supported.");
760}
761
762llvm::Triple::ArchType CGHLSLRuntime::getArch() {
763 return CGM.getTarget().getTriple().getArch();
764}
765
766// Emits constant global variables for buffer constants declarations
767// and creates metadata linking the constant globals with the buffer global.
768void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
769 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
770 const CGHLSLOffsetInfo &OffsetInfo) {
771 LLVMContext &Ctx = CGM.getLLVMContext();
772
773 // get the layout struct from constant buffer target type
774 llvm::Type *BufType = BufGV->getValueType();
775 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
776 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
777
778 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
779 size_t OffsetIdx = 0;
780 for (Decl *D : BufDecl->buffer_decls()) {
781 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
782 // Nothing to do for this declaration.
783 continue;
784 if (isa<FunctionDecl>(Val: D)) {
785 // A function within an cbuffer is effectively a top-level function.
786 CGM.EmitTopLevelDecl(D);
787 continue;
788 }
789 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
790 if (!VD)
791 continue;
792
793 QualType VDTy = VD->getType();
794 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
795 if (VD->getStorageClass() == SC_Static ||
796 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
797 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
798 // Emit static and groupshared variables and resource classes inside
799 // cbuffer as regular globals
800 CGM.EmitGlobal(D: VD);
801 }
802 continue;
803 }
804
805 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
806 }
807
808 if (!OffsetInfo.empty())
809 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
810 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
811 });
812
813 // Associate the buffer global variable with its constants
814 SmallVector<llvm::Metadata *> BufGlobals;
815 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
816 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
817
818 auto ElemIt = LayoutStruct->element_begin();
819 for (auto &[VD, _] : DeclsWithOffset) {
820 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
821 ++ElemIt;
822
823 assert(ElemIt != LayoutStruct->element_end() &&
824 "number of elements in layout struct does not match");
825 llvm::Type *LayoutType = *ElemIt++;
826
827 GlobalVariable *ElemGV =
828 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
829 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
830 }
831 assert(ElemIt == LayoutStruct->element_end() &&
832 "number of elements in layout struct does not match");
833
834 // add buffer metadata to the module
835 CGM.getModule()
836 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
837 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
838}
839
840// Creates resource handle type for the HLSL buffer declaration
841static const clang::HLSLAttributedResourceType *
842createBufferHandleType(const HLSLBufferDecl *BufDecl) {
843 ASTContext &AST = BufDecl->getASTContext();
844 QualType QT = AST.getHLSLAttributedResourceType(
845 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
846 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
847 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
848}
849
850CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
851 CGHLSLOffsetInfo Result;
852
853 // If we don't have packoffset info, just return an empty result.
854 if (!BufDecl.hasValidPackoffset())
855 return Result;
856
857 for (Decl *D : BufDecl.buffer_decls()) {
858 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
859 continue;
860 }
861 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
862 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
863 continue;
864
865 if (!VD->hasAttrs()) {
866 Result.Offsets.push_back(Elt: Unspecified);
867 continue;
868 }
869
870 uint32_t Offset = Unspecified;
871 for (auto *Attr : VD->getAttrs()) {
872 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
873 Offset = POA->getOffsetInBytes();
874 break;
875 }
876 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
877 if (RBA &&
878 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
879 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
880 break;
881 }
882 }
883 Result.Offsets.push_back(Elt: Offset);
884 }
885 return Result;
886}
887
888// Codegen for HLSLBufferDecl
889void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
890
891 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
892
893 // create resource handle type for the buffer
894 const clang::HLSLAttributedResourceType *ResHandleTy =
895 createBufferHandleType(BufDecl);
896
897 // empty constant buffer is ignored
898 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
899 return;
900
901 // create global variable for the constant buffer
902 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
903 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
904 llvm::GlobalVariable *BufGV = new GlobalVariable(
905 LayoutTy, /*isConstant*/ false,
906 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
907 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
908 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
909 GlobalValue::NotThreadLocal);
910
911 llvm::Module &M = CGM.getModule();
912 M.insertGlobalVariable(GV: BufGV);
913
914 // Add the global variable to the compiler used list so it does not
915 // get optimized away by GlobalOptPass before it reaches
916 // {DXIL|SPIRV}CBufferAccess pass.
917 llvm::appendToCompilerUsed(M, Values: {BufGV});
918
919 // Add globals for constant buffer elements and create metadata nodes
920 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
921
922 // Initialize cbuffer from binding (implicit or explicit)
923 initializeBufferFromBinding(BufDecl, GV: BufGV);
924}
925
926void CGHLSLRuntime::addRootSignature(
927 const HLSLRootSignatureDecl *SignatureDecl) {
928 llvm::Module &M = CGM.getModule();
929 Triple T(M.getTargetTriple());
930
931 // Generated later with the function decl if not targeting root signature
932 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
933 return;
934
935 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
936 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
937}
938
939llvm::StructType *
940CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
941 const auto Entry = LayoutTypes.find(Val: StructType);
942 if (Entry != LayoutTypes.end())
943 return Entry->getSecond();
944 return nullptr;
945}
946
947void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
948 llvm::StructType *LayoutTy) {
949 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
950 "layout type for this struct already exist");
951 LayoutTypes[StructType] = LayoutTy;
952}
953
954void CGHLSLRuntime::finishCodeGen() {
955 auto &TargetOpts = CGM.getTarget().getTargetOpts();
956 auto &CodeGenOpts = CGM.getCodeGenOpts();
957 auto &LangOpts = CGM.getLangOpts();
958 llvm::Module &M = CGM.getModule();
959 Triple T(M.getTargetTriple());
960 if (T.getArch() == Triple::ArchType::dxil)
961 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
962 if (!CodeGenOpts.DisableDXSourceMetadata &&
963 CodeGenOpts.getDebugInfo() >=
964 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
965 addSourceInfo(CGM, M);
966 if (CodeGenOpts.ResMayAlias)
967 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
968 if (CodeGenOpts.AllResourcesBound)
969 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
970 Key: "dx.allresourcesbound", Val: 1);
971 if (CodeGenOpts.OptimizationLevel == 0)
972 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
973 Key: "dx.disable_optimizations", Val: 1);
974
975 // NativeHalfType corresponds to the -fnative-half-type clang option which is
976 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
977 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
978 if (LangOpts.NativeHalfType)
979 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
980 Val: 1);
981
982 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
983 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
984 const ASTContext &Ctx = CGM.getContext();
985 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
986 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
987 SmallVector<GlobalValue *, 8> InterfaceVars;
988 for (GlobalVariable &GV : M.globals()) {
989 unsigned AS = GV.getAddressSpace();
990 if (AS == InputAS || AS == OutputAS)
991 InterfaceVars.push_back(Elt: &GV);
992 }
993 if (!InterfaceVars.empty())
994 appendToCompilerUsed(M, Values: InterfaceVars);
995 }
996
997 generateGlobalCtorDtorCalls();
998}
999
1000void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
1001 const FunctionDecl *FD, llvm::Function *Fn) {
1002 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1003 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
1004 const StringRef ShaderAttrKindStr = "hlsl.shader";
1005 Fn->addFnAttr(Kind: ShaderAttrKindStr,
1006 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
1007 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
1008 const StringRef NumThreadsKindStr = "hlsl.numthreads";
1009 std::string NumThreadsStr =
1010 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
1011 Vals: NumThreadsAttr->getZ());
1012 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
1013 }
1014 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
1015 const StringRef WaveSizeKindStr = "hlsl.wavesize";
1016 std::string WaveSizeStr =
1017 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
1018 Vals: WaveSizeAttr->getPreferred());
1019 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
1020 }
1021 // HLSL entry functions are materialized for module functions with
1022 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
1023 // later in the compiler-flow for such module functions is not aware of and
1024 // hence not able to set attributes of the newly materialized entry functions.
1025 // So, set attributes of entry function here, as appropriate.
1026 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
1027
1028 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
1029 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
1030 }
1031}
1032
1033static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
1034 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1035 Value *Result = PoisonValue::get(T: Ty);
1036 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
1037 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
1038 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
1039 }
1040 return Result;
1041 }
1042 return B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)});
1043}
1044
1045static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
1046 unsigned BuiltIn) {
1047 LLVMContext &Ctx = GV->getContext();
1048 IRBuilder<> B(GV->getContext());
1049 MDNode *Operands = MDNode::get(
1050 Context&: Ctx,
1051 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
1052 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
1053 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1054 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1055}
1056
1057static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
1058 LLVMContext &Ctx = GV->getContext();
1059 IRBuilder<> B(GV->getContext());
1060 MDNode *Operands =
1061 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Location */ C: 30)),
1062 ConstantAsMetadata::get(C: B.getInt32(C: Location))});
1063 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1064 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1065}
1066
1067static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
1068 llvm::Type *Ty, const Twine &Name,
1069 unsigned BuiltInID) {
1070 auto *GV = new llvm::GlobalVariable(
1071 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1072 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1073 llvm::GlobalVariable::GeneralDynamicTLSModel,
1074 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1075 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1076 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1077 return B.CreateLoad(Ty, Ptr: GV);
1078}
1079
1080static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
1081 llvm::Type *Ty, unsigned Location,
1082 StringRef Name) {
1083 auto *GV = new llvm::GlobalVariable(
1084 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1085 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1086 llvm::GlobalVariable::GeneralDynamicTLSModel,
1087 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1088 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1089 addLocationDecoration(GV, Location);
1090 return B.CreateLoad(Ty, Ptr: GV);
1091}
1092
1093llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
1094 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1095 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {
1096 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1097 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1098
1099 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
1100 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1101 Location = L->getLocation();
1102
1103 // DXC completely ignores the semantic/index pair. Location are assigned from
1104 // the first semantic to the last.
1105 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
1106 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1107 SPIRVLastAssignedInputSemanticLocation += ElementCount;
1108
1109 return createSPIRVLocationLoad(B, M&: CGM.getModule(), Ty: Type, Location,
1110 Name: VariableName.str());
1111}
1112
1113static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
1114 llvm::Value *Source, unsigned Location,
1115 StringRef Name) {
1116 auto *GV = new llvm::GlobalVariable(
1117 M, Source->getType(), /* isConstant= */ false,
1118 llvm::GlobalValue::ExternalLinkage,
1119 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1120 llvm::GlobalVariable::GeneralDynamicTLSModel,
1121 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1122 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1123 addLocationDecoration(GV, Location);
1124 B.CreateStore(Val: Source, Ptr: GV);
1125}
1126
1127void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1128 llvm::IRBuilder<> &B, llvm::Value *Source,
1129 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1130 std::optional<unsigned> Index) {
1131 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1132 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1133
1134 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1135 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1136 Location = L->getLocation();
1137
1138 // DXC completely ignores the semantic/index pair. Location are assigned from
1139 // the first semantic to the last.
1140 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1141 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1142 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1143 createSPIRVLocationStore(B, M&: CGM.getModule(), Source, Location,
1144 Name: VariableName.str());
1145}
1146
1147llvm::Value *
1148CGHLSLRuntime::emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
1149 HLSLAppliedSemanticAttr *Semantic,
1150 std::optional<unsigned> Index) {
1151 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1152 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1153
1154 // DXIL packing rules etc shall be handled here.
1155 // FIXME: generate proper sigpoint, index, col, row values.
1156 // FIXME: also DXIL loads vectors element by element.
1157 SmallVector<Value *> Args{B.getInt32(C: 4), B.getInt32(C: 0), B.getInt32(C: 0),
1158 B.getInt8(C: 0),
1159 llvm::PoisonValue::get(T: B.getInt32Ty())};
1160
1161 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_load_input;
1162
1163 SmallVector<OperandBundleDef, 1> OB;
1164 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1165 llvm::Value *bundleArgs[] = {Token};
1166 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1167 }
1168
1169 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1170 M: B.GetInsertBlock()->getModule(), id: IntrinsicID, OverloadTys: {Type});
1171 llvm::Value *Value = B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: VariableName);
1172 return Value;
1173}
1174
1175void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1176 llvm::Value *Source,
1177 HLSLAppliedSemanticAttr *Semantic,
1178 std::optional<unsigned> Index) {
1179 // DXIL packing rules etc shall be handled here.
1180 // FIXME: generate proper sigpoint, index, col, row values.
1181 SmallVector<Value *> Args{B.getInt32(C: 4),
1182 B.getInt32(C: 0),
1183 B.getInt32(C: 0),
1184 B.getInt8(C: 0),
1185 llvm::PoisonValue::get(T: B.getInt32Ty()),
1186 Source};
1187
1188 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_store_output;
1189
1190 SmallVector<OperandBundleDef, 1> OB;
1191 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1192 llvm::Value *bundleArgs[] = {Token};
1193 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1194 }
1195
1196 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1197 M: B.GetInsertBlock()->getModule(), id: IntrinsicID, OverloadTys: {Source->getType()});
1198 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1199}
1200
1201llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1202 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1203 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {
1204 if (CGM.getTarget().getTriple().isSPIRV())
1205 return emitSPIRVUserSemanticLoad(B, Type, Decl, Semantic, Index);
1206
1207 if (CGM.getTarget().getTriple().isDXIL())
1208 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1209
1210 llvm_unreachable("Unsupported target for user-semantic load.");
1211}
1212
1213void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1214 const clang::DeclaratorDecl *Decl,
1215 HLSLAppliedSemanticAttr *Semantic,
1216 std::optional<unsigned> Index) {
1217 if (CGM.getTarget().getTriple().isSPIRV())
1218 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
1219
1220 if (CGM.getTarget().getTriple().isDXIL())
1221 return emitDXILUserSemanticStore(B, Source, Semantic, Index);
1222
1223 llvm_unreachable("Unsupported target for user-semantic load.");
1224}
1225
1226llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1227 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1228 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1229 std::optional<unsigned> Index) {
1230
1231 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1232 if (SemanticName == "SV_GROUPINDEX") {
1233 llvm::Function *GroupIndex =
1234 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1235 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1236 }
1237
1238 if (SemanticName == "SV_DISPATCHTHREADID") {
1239 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1240 llvm::Function *ThreadIDIntrinsic =
1241 llvm::Intrinsic::isOverloaded(id: IntrinID)
1242 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1243 : CGM.getIntrinsic(IID: IntrinID);
1244 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1245 }
1246
1247 if (SemanticName == "SV_GROUPTHREADID") {
1248 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1249 llvm::Function *GroupThreadIDIntrinsic =
1250 llvm::Intrinsic::isOverloaded(id: IntrinID)
1251 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1252 : CGM.getIntrinsic(IID: IntrinID);
1253 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1254 }
1255
1256 if (SemanticName == "SV_GROUPID") {
1257 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1258 llvm::Function *GroupIDIntrinsic =
1259 llvm::Intrinsic::isOverloaded(id: IntrinID)
1260 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1261 : CGM.getIntrinsic(IID: IntrinID);
1262 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1263 }
1264
1265 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1266 assert(ShaderAttr && "Entry point has no shader attribute");
1267 llvm::Triple::EnvironmentType ST = ShaderAttr->getType();
1268
1269 if (SemanticName == "SV_POSITION") {
1270 if (ST == Triple::EnvironmentType::Pixel) {
1271 if (CGM.getTarget().getTriple().isSPIRV())
1272 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1273 Name: Semantic->getAttrName()->getName(),
1274 /* BuiltIn::FragCoord */ BuiltInID: 15);
1275 if (CGM.getTarget().getTriple().isDXIL())
1276 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1277 }
1278
1279 if (ST == Triple::EnvironmentType::Vertex) {
1280 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);
1281 }
1282 }
1283
1284 if (SemanticName == "SV_VERTEXID") {
1285 if (ST == Triple::EnvironmentType::Vertex) {
1286 if (CGM.getTarget().getTriple().isSPIRV())
1287 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1288 Name: Semantic->getAttrName()->getName(),
1289 /* BuiltIn::VertexIndex */ BuiltInID: 42);
1290 else
1291 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1292 }
1293 }
1294
1295 llvm_unreachable(
1296 "Load hasn't been implemented yet for this system semantic. FIXME");
1297}
1298
1299static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
1300 llvm::Value *Source, const Twine &Name,
1301 unsigned BuiltInID) {
1302 auto *GV = new llvm::GlobalVariable(
1303 M, Source->getType(), /* isConstant= */ false,
1304 llvm::GlobalValue::ExternalLinkage,
1305 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1306 llvm::GlobalVariable::GeneralDynamicTLSModel,
1307 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1308 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1309 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1310 B.CreateStore(Val: Source, Ptr: GV);
1311}
1312
1313void CGHLSLRuntime::emitSystemSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1314 const clang::DeclaratorDecl *Decl,
1315 HLSLAppliedSemanticAttr *Semantic,
1316 std::optional<unsigned> Index) {
1317
1318 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1319 if (SemanticName == "SV_POSITION") {
1320 if (CGM.getTarget().getTriple().isDXIL()) {
1321 emitDXILUserSemanticStore(B, Source, Semantic, Index);
1322 return;
1323 }
1324
1325 if (CGM.getTarget().getTriple().isSPIRV()) {
1326 createSPIRVBuiltinStore(B, M&: CGM.getModule(), Source,
1327 Name: Semantic->getAttrName()->getName(),
1328 /* BuiltIn::Position */ BuiltInID: 0);
1329 return;
1330 }
1331 }
1332
1333 if (SemanticName == "SV_TARGET") {
1334 emitUserSemanticStore(B, Source, Decl, Semantic, Index);
1335 return;
1336 }
1337
1338 llvm_unreachable(
1339 "Store hasn't been implemented yet for this system semantic. FIXME");
1340}
1341
1342llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1343 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1344 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {
1345
1346 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1347 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1348 return emitSystemSemanticLoad(B, FD, Type, Decl, Semantic, Index);
1349 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);
1350}
1351
1352void CGHLSLRuntime::handleScalarSemanticStore(
1353 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1354 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {
1355 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1356 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1357 emitSystemSemanticStore(B, Source, Decl, Semantic, Index);
1358 else
1359 emitUserSemanticStore(B, Source, Decl, Semantic, Index);
1360}
1361
1362std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1363CGHLSLRuntime::handleStructSemanticLoad(
1364 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1365 const clang::DeclaratorDecl *Decl,
1366 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1367 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1368 const llvm::StructType *ST = cast<StructType>(Val: Type);
1369 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
1370
1371 assert(RD->getNumFields() == ST->getNumElements());
1372
1373 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
1374 auto FieldDecl = RD->field_begin();
1375 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
1376 auto [ChildValue, NextAttr] = handleSemanticLoad(
1377 B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin, end: AttrEnd);
1378 AttrBegin = NextAttr;
1379 assert(ChildValue);
1380 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
1381 ++FieldDecl;
1382 }
1383
1384 return std::make_pair(x&: Aggregate, y&: AttrBegin);
1385}
1386
1387specific_attr_iterator<HLSLAppliedSemanticAttr>
1388CGHLSLRuntime::handleStructSemanticStore(
1389 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1390 const clang::DeclaratorDecl *Decl,
1391 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1392 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1393
1394 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
1395
1396 const clang::RecordDecl *RD = nullptr;
1397 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
1398 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
1399 else
1400 RD = Decl->getType()->getAsRecordDecl();
1401 assert(RD);
1402
1403 assert(RD->getNumFields() == ST->getNumElements());
1404
1405 auto FieldDecl = RD->field_begin();
1406 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
1407 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
1408 AttrBegin =
1409 handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin, AttrEnd);
1410 }
1411
1412 return AttrBegin;
1413}
1414
1415std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1416CGHLSLRuntime::handleSemanticLoad(
1417 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1418 const clang::DeclaratorDecl *Decl,
1419 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1420 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1421 assert(AttrBegin != AttrEnd);
1422 if (Type->isStructTy())
1423 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd);
1424
1425 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1426 ++AttrBegin;
1427 return std::make_pair(x: handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr),
1428 y&: AttrBegin);
1429}
1430
1431specific_attr_iterator<HLSLAppliedSemanticAttr>
1432CGHLSLRuntime::handleSemanticStore(
1433 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1434 const clang::DeclaratorDecl *Decl,
1435 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1436 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1437 assert(AttrBegin != AttrEnd);
1438 if (Source->getType()->isStructTy())
1439 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd);
1440
1441 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1442 ++AttrBegin;
1443 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr);
1444 return AttrBegin;
1445}
1446
1447void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
1448 llvm::Function *Fn) {
1449 llvm::Module &M = CGM.getModule();
1450 llvm::LLVMContext &Ctx = M.getContext();
1451 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
1452 Function *EntryFn =
1453 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
1454
1455 // Copy function attributes over, we have no argument or return attributes
1456 // that can be valid on the real entry.
1457 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
1458 Attrs: Fn->getAttributes().getFnAttrs());
1459 EntryFn->setAttributes(NewAttrs);
1460 setHLSLEntryAttributes(FD, Fn: EntryFn);
1461
1462 // Set the called function as internal linkage.
1463 Fn->setLinkage(GlobalValue::InternalLinkage);
1464
1465 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
1466 IRBuilder<> B(BB);
1467 llvm::SmallVector<Value *> Args;
1468
1469 SmallVector<OperandBundleDef, 1> OB;
1470 if (CGM.shouldEmitConvergenceTokens()) {
1471 assert(EntryFn->isConvergent());
1472 llvm::Value *I =
1473 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
1474 llvm::Value *bundleArgs[] = {I};
1475 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1476 }
1477
1478 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
1479
1480 unsigned SRetOffset = 0;
1481 for (const auto &Param : Fn->args()) {
1482 if (Param.hasStructRetAttr()) {
1483 SRetOffset = 1;
1484 llvm::Type *VarType = Param.getParamStructRetType();
1485 llvm::Value *Var =
1486 CGM.getLangOpts().EmitLogicalPointer
1487 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
1488 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
1489 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
1490 Args.push_back(Elt: Var);
1491 continue;
1492 }
1493
1494 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
1495 llvm::Value *SemanticValue = nullptr;
1496 // FIXME: support inout/out parameters for semantics.
1497 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
1498 PD->getAttr<HLSLParamModifierAttr>()) {
1499 llvm_unreachable("Not handled yet");
1500 } else {
1501 llvm::Type *ParamType = nullptr;
1502 if (Param.hasByValAttr())
1503 ParamType = Param.getParamByValType();
1504 else if (PD->getType()->isRecordType())
1505 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
1506 else
1507 ParamType = Param.getType();
1508
1509 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1510 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
1511 auto Result =
1512 handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd);
1513 SemanticValue = Result.first;
1514 if (!SemanticValue)
1515 return;
1516 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
1517 llvm::Value *Var =
1518 CGM.getLangOpts().EmitLogicalPointer
1519 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
1520 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
1521 B.CreateStore(Val: SemanticValue, Ptr: Var);
1522 SemanticValue = Var;
1523 }
1524 }
1525
1526 assert(SemanticValue);
1527 Args.push_back(Elt: SemanticValue);
1528 }
1529
1530 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
1531 CI->setCallingConv(Fn->getCallingConv());
1532
1533 if (Fn->getReturnType() != CGM.VoidTy)
1534 // Element type is unused, so set to dummy value (NULL).
1535 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
1536
1537 for (auto &SourcePair : OutputSemantic) {
1538 llvm::Value *Source = SourcePair.first;
1539 llvm::Type *ElementType = SourcePair.second;
1540 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
1541 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
1542
1543 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1544 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
1545 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd);
1546 }
1547
1548 B.CreateRetVoid();
1549
1550 // Add and identify root signature to function, if applicable
1551 for (const Attr *Attr : FD->getAttrs()) {
1552 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
1553 auto *RSDecl = RSAttr->getSignatureDecl();
1554 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
1555 Fn: EntryFn, M);
1556 }
1557 }
1558}
1559
1560static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
1561 bool CtorOrDtor) {
1562 const auto *GV =
1563 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
1564 if (!GV)
1565 return;
1566 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1567 if (!CA)
1568 return;
1569 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
1570 // HLSL neither supports priorities or COMDat values, so we will check those
1571 // in an assert but not handle them.
1572
1573 for (const auto &Ctor : CA->operands()) {
1574 if (isa<ConstantAggregateZero>(Val: Ctor))
1575 continue;
1576 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
1577
1578 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
1579 "HLSL doesn't support setting priority for global ctors.");
1580 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
1581 "HLSL doesn't support COMDat for global ctors.");
1582 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
1583 }
1584}
1585
1586void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
1587 llvm::Module &M = CGM.getModule();
1588 SmallVector<Function *> CtorFns;
1589 SmallVector<Function *> DtorFns;
1590 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
1591 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
1592
1593 // Insert a call to the global constructor at the beginning of the entry block
1594 // to externally exported functions. This is a bit of a hack, but HLSL allows
1595 // global constructors, but doesn't support driver initialization of globals.
1596 for (auto &F : M.functions()) {
1597 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
1598 continue;
1599 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
1600 Instruction *IP = &*F.getEntryBlock().begin();
1601 SmallVector<OperandBundleDef, 1> OB;
1602 if (Token) {
1603 llvm::Value *bundleArgs[] = {Token};
1604 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1605 IP = Token->getNextNode();
1606 }
1607 IRBuilder<> B(IP);
1608 for (auto *Fn : CtorFns) {
1609 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1610 CI->setCallingConv(Fn->getCallingConv());
1611 }
1612
1613 // Insert global dtors before the terminator of the last instruction
1614 B.SetInsertPoint(F.back().getTerminator());
1615 for (auto *Fn : DtorFns) {
1616 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1617 CI->setCallingConv(Fn->getCallingConv());
1618 }
1619 }
1620
1621 // No need to keep global ctors/dtors for non-lib profile after call to
1622 // ctors/dtors added for entry.
1623 Triple T(M.getTargetTriple());
1624 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
1625 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
1626 GV->eraseFromParent();
1627 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
1628 GV->eraseFromParent();
1629 }
1630}
1631
1632static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
1633 Intrinsic::ID IntrID,
1634 ArrayRef<llvm::Value *> Args) {
1635
1636 LLVMContext &Ctx = CGM.getLLVMContext();
1637 llvm::Function *InitResFunc =
1638 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
1639 Linkage: llvm::GlobalValue::InternalLinkage,
1640 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
1641 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1642
1643 llvm::BasicBlock *EntryBB =
1644 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
1645 CGBuilderTy Builder(CGM, Ctx);
1646 const DataLayout &DL = CGM.getModule().getDataLayout();
1647 Builder.SetInsertPoint(EntryBB);
1648
1649 // Make sure the global variable is buffer resource handle
1650 llvm::Type *HandleTy = GV->getValueType();
1651 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
1652
1653 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
1654 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
1655 Name: Twine(GV->getName()).concat(Suffix: "_h"));
1656
1657 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
1658 Builder.CreateRetVoid();
1659
1660 CGM.AddCXXGlobalInit(F: InitResFunc);
1661}
1662
1663void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
1664 llvm::GlobalVariable *GV) {
1665 ResourceBindingAttrs Binding(BufDecl);
1666 assert(Binding.hasBinding() &&
1667 "cbuffer/tbuffer should always have resource binding attribute");
1668
1669 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
1670 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
1671 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
1672 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
1673
1674 // buffer with explicit binding
1675 if (Binding.isExplicit()) {
1676 llvm::Intrinsic::ID IntrinsicID =
1677 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1678 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
1679 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
1680 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1681 } else {
1682 // buffer with implicit binding
1683 llvm::Intrinsic::ID IntrinsicID =
1684 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1685 auto *OrderID =
1686 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
1687 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
1688 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1689 }
1690}
1691
1692void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
1693 llvm::GlobalVariable *GV) {
1694 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
1695 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1696 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
1697 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1698}
1699
1700llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
1701 if (!CGM.shouldEmitConvergenceTokens())
1702 return nullptr;
1703
1704 auto E = BB.end();
1705 for (auto I = BB.begin(); I != E; ++I) {
1706 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
1707 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
1708 return II;
1709 }
1710 }
1711 llvm_unreachable("Convergence token should have been emitted.");
1712 return nullptr;
1713}
1714
1715class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
1716public:
1717 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
1718 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
1719 OpaqueValueVisitor() {}
1720
1721 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
1722 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
1723 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
1724 // traversal, the temporary containing the copy out will not have
1725 // been created yet.
1726 return false;
1727 }
1728
1729 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
1730 // Traverse the source expression first.
1731 if (E->getSourceExpr())
1732 TraverseStmt(S: E->getSourceExpr());
1733
1734 // Then add this OVE if we haven't seen it before.
1735 if (Visited.insert(Ptr: E).second)
1736 OVEs.push_back(Elt: E);
1737
1738 return true;
1739 }
1740};
1741
1742void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
1743 InitListExpr *E) {
1744
1745 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
1746 OpaqueValueVisitor Visitor;
1747 Visitor.TraverseStmt(S: E);
1748 for (auto *OVE : Visitor.OVEs) {
1749 if (CGF.isOpaqueValueEmitted(E: OVE))
1750 continue;
1751 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
1752 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
1753 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
1754 } else {
1755 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
1756 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
1757 }
1758 }
1759}
1760
1761std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
1762 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
1763 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
1764 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
1765 "expected resource array subscript expression");
1766
1767 // Let clang codegen handle local and static resource array subscripts,
1768 // or when the subscript references on opaque expression (as part of
1769 // ArrayInitLoopExpr AST node).
1770 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
1771 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
1772 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
1773 ArrayDecl->getStorageClass() == SC_Static)
1774 return std::nullopt;
1775
1776 // get the resource array type
1777 ASTContext &AST = ArrayDecl->getASTContext();
1778 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
1779 assert(ResArrayTy->isHLSLResourceRecordArray() &&
1780 "expected array of resource classes");
1781
1782 // Iterate through all nested array subscript expressions to calculate
1783 // the index in the flattened resource array (if this is a multi-
1784 // dimensional array). The index is calculated as a sum of all indices
1785 // multiplied by the total size of the array at that level.
1786 Value *Index = nullptr;
1787 const ArraySubscriptExpr *ASE = ArraySubsExpr;
1788 while (ASE != nullptr) {
1789 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
1790 if (const auto *ArrayTy =
1791 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
1792 Value *Multiplier = llvm::ConstantInt::get(
1793 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
1794 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
1795 }
1796 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
1797 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
1798 }
1799
1800 // Find binding info for the resource array. For implicit binding
1801 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
1802 ResourceBindingAttrs Binding(ArrayDecl);
1803 assert(Binding.hasBinding() &&
1804 "resource array must have a binding attribute");
1805
1806 // Find the individual resource type.
1807 QualType ResultTy = ArraySubsExpr->getType();
1808 QualType ResourceTy =
1809 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
1810
1811 // Create a temporary variable for the result, which is either going
1812 // to be a single resource instance or a local array of resources (we need to
1813 // return an LValue).
1814 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
1815 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
1816 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
1817 kind: NormalEHLifetimeMarker, A: TmpVar);
1818
1819 AggValueSlot ValueSlot = AggValueSlot::forAddr(
1820 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
1821 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
1822 mayOverlap: AggValueSlot::DoesNotOverlap);
1823
1824 // Calculate total array size (= range size).
1825 llvm::Value *Range = llvm::ConstantInt::getSigned(
1826 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
1827
1828 // If the result of the subscript operation is a single resource, call the
1829 // constructor.
1830 if (ResultTy == ResourceTy) {
1831 CallArgList Args;
1832 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
1833 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
1834 Name: ArrayDecl->getName(), Binding, Args);
1835
1836 if (!CreateMethod) {
1837 // This can happen if someone creates an array of structs that looks like
1838 // an HLSL resource record array but it does not have the required static
1839 // create method. No binding will be generated for it.
1840 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
1841 "create method lookup should always succeed for built-in resource "
1842 "records");
1843 return std::nullopt;
1844 }
1845
1846 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
1847
1848 } else {
1849 // The result of the subscript operation is a local resource array which
1850 // needs to be initialized.
1851 const ConstantArrayType *ArrayTy =
1852 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
1853 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
1854 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
1855 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
1856 if (!EndIndex)
1857 return std::nullopt;
1858 }
1859 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
1860}
1861
1862// Initialize all resources of a global resource array into provided slot.
1863bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
1864 const VarDecl *ArrayDecl,
1865 AggValueSlot &DestSlot) {
1866 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
1867 ArrayDecl->hasGlobalStorage() &&
1868 ArrayDecl->getStorageClass() != SC_Static &&
1869 "expected global non-static resource array");
1870
1871 // Find binding info for the resource array. For implicit binding
1872 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
1873 ResourceBindingAttrs Binding(ArrayDecl);
1874 assert(Binding.hasBinding() &&
1875 "resource array must have a binding attribute");
1876
1877 // Find the individual resource type.
1878 ASTContext &AST = ArrayDecl->getASTContext();
1879 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
1880 const auto *ResArrayTy =
1881 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
1882
1883 // Create Value for index and total array size (= range size).
1884 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
1885 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
1886 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
1887
1888 // Initialize individual resources in the array into DestSlot.
1889 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
1890 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
1891 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
1892 return EndIndex.has_value();
1893}
1894
1895// If the expression is a global resource array, initialize all of its resources
1896// into Dest. Returns false if no initialization has been performed and the
1897// array copy should be handled by the default codegen.
1898bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
1899 AggValueSlot &DestSlot) {
1900 assert(E->getType()->isHLSLResourceRecordArray() &&
1901 "expected resource array");
1902
1903 // Find the array declaration for the expression. Fallback to the default
1904 // handling if it's not a global resource array.
1905 const VarDecl *ArrayDecl =
1906 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
1907 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
1908 ArrayDecl->getStorageClass() == SC_Static)
1909 return false;
1910
1911 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
1912}
1913
1914// If the expression is a global resource array, create a temporary and
1915// initialize all of its resources, and return it as an LValue. Returns nullopt
1916// if no initialization has been performed and the handling should follow the
1917// default path.
1918std::optional<LValue>
1919CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
1920 const VarDecl *ArrayDecl) {
1921 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
1922 "expected resource array declaration");
1923
1924 if (!ArrayDecl->hasGlobalStorage() ||
1925 ArrayDecl->getStorageClass() == SC_Static)
1926 return std::nullopt;
1927
1928 AggValueSlot TmpArraySlot =
1929 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
1930 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
1931 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
1932 Source: AlignmentSource::Decl);
1933 return std::nullopt;
1934}
1935
1936RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
1937 CodeGenFunction &CGF) {
1938
1939 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
1940 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
1941 "expected cbuffer matrix");
1942
1943 QualType MatQualTy = LV.getType();
1944 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
1945 Address SrcAddr = LV.getAddress();
1946
1947 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
1948 return SrcAddr;
1949
1950 RawAddress DestAlloca =
1951 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
1952 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
1953 return DestAlloca;
1954}
1955
1956std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
1957 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
1958 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
1959 // Find the element type to index by first padding the element type per HLSL
1960 // buffer rules, and then padding out to a 16-byte register boundary if
1961 // necessary.
1962 llvm::Type *LayoutTy =
1963 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
1964 uint64_t LayoutSizeInBits =
1965 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
1966 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
1967 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
1968 if (RowAlignedSize > ElementSize) {
1969 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
1970 CGM, NumBytes: RowAlignedSize - ElementSize);
1971 assert(Padding && "No padding type for target?");
1972 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
1973 /*isPacked=*/true);
1974 }
1975
1976 // If the layout type doesn't introduce any padding, we don't need to do
1977 // anything special.
1978 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
1979 if (LayoutTy == OrigTy)
1980 return std::nullopt;
1981
1982 LValueBaseInfo EltBaseInfo;
1983 TBAAAccessInfo EltTBAAInfo;
1984
1985 // Index into the object as-if we have an array of the padded element type,
1986 // and then dereference the element itself to avoid reading padding that may
1987 // be past the end of the in-memory object.
1988 SmallVector<llvm::Value *, 2> Indices;
1989 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
1990 Indices.push_back(Elt: Idx);
1991 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1992
1993 if (CGF.getLangOpts().EmitLogicalPointer) {
1994 // The fact that we emit an array-to-pointer decay might be an oversight,
1995 // but for now, we simply ignore it (see #179951).
1996 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
1997 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
1998
1999 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
2000 Address Addr = LV.getAddress();
2001 LayoutTy = llvm::ArrayType::get(
2002 ElementType: LayoutTy,
2003 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
2004 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
2005 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
2006 Addr =
2007 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
2008 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2009 }
2010
2011 Address Addr =
2012 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
2013 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
2014 IdxList: Indices, Name: "cbufferidx");
2015 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
2016 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2017}
2018
2019std::optional<LValue>
2020CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
2021 const MemberExpr *ME) {
2022 assert((ME->getType()->isHLSLResourceRecord() ||
2023 ME->getType()->isHLSLResourceRecordArray()) &&
2024 "expected resource member expression");
2025
2026 const VarDecl *ResourceVD =
2027 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
2028 if (!ResourceVD)
2029 return std::nullopt;
2030
2031 // Handle member of resource array type.
2032 if (ResourceVD->getType()->isHLSLResourceRecordArray())
2033 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
2034
2035 GlobalVariable *ResGV =
2036 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
2037 const DataLayout &DL = CGM.getDataLayout();
2038 llvm::Type *Ty = ResGV->getValueType();
2039 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
2040 Address Addr = Address(ResGV, Ty, Align);
2041 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
2042 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2043 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
2044 return LV;
2045}
2046
2047bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
2048 const LValue &SrcLV,
2049 AggValueSlot &DestSlot) {
2050 assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
2051 "expected expression in HLSL constant address space");
2052 assert(!E->getType()->isHLSLResourceRecord() &&
2053 !E->getType()->isHLSLResourceRecordArray() &&
2054 "direct accesses to resource types should be handled separately");
2055
2056 if (DestSlot.isIgnored())
2057 return false;
2058
2059 QualType Ty = E->getType();
2060 Address DstPtr = DestSlot.getAddress();
2061 Address SrcPtr = SrcLV.getAddress();
2062
2063 // If there are no intangible types, we don't need to lookup associated
2064 // resources.
2065 if (!Ty->isHLSLIntangibleType())
2066 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty);
2067
2068 // Handle structs with intangible types by setting the resource fields
2069 // of the destination struct with the resources associated with the global
2070 // struct.
2071 EmbeddedResourceNameBuilder NameBuilder;
2072 const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
2073 AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
2074
2075 // Callback to fill in the associated resource.
2076 auto EmitResFn = [&](AggValueSlot &ResSlot) {
2077 const VarDecl *ResDecl = AssociatedResources.getNextResource();
2078 assert(ResDecl && "associated resource declaration not found");
2079
2080 // Check that the resource type of dest and src matches.
2081 [[maybe_unused]] llvm::Type *DestType =
2082 ResSlot.getAddress().getElementType();
2083 [[maybe_unused]] llvm::Type *SrcConvertedType =
2084 CGM.getTypes().ConvertTypeForMem(T: ResDecl->getType());
2085 assert(DestType == SrcConvertedType && "resource slot type mismatch");
2086
2087 if (ResDecl->getType()->isHLSLResourceRecord())
2088 copyGlobalResource(CGF, ResourceVD: ResDecl, DestSlot&: ResSlot);
2089 else
2090 initializeGlobalResourceArray(CGF, ArrayDecl: ResDecl, DestSlot&: ResSlot);
2091 };
2092
2093 auto Result =
2094 HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty, EmitResFn);
2095 assert(AssociatedResources.getNextResource() == nullptr &&
2096 "expected all associated resources to be processed");
2097 return Result;
2098}
2099
2100LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
2101 const MemberExpr *E) {
2102 LValue Base =
2103 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
2104 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
2105 assert(Field && "Unexpected access into HLSL buffer");
2106
2107 const RecordDecl *Rec = Field->getParent();
2108
2109 // Work out the buffer layout type to index into.
2110 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
2111 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
2112 // Since this is a member of an object in the buffer and not the buffer's
2113 // struct/class itself, we shouldn't have any offsets on the members we need
2114 // to contend with.
2115 CGHLSLOffsetInfo EmptyOffsets;
2116 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
2117 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
2118
2119 // Get the field index for the layout struct, accounting for padding.
2120 unsigned FieldIdx =
2121 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
2122 assert(FieldIdx < LayoutTy->getNumElements() &&
2123 "Layout struct is smaller than member struct");
2124 unsigned Skipped = 0;
2125 for (unsigned I = 0; I <= FieldIdx;) {
2126 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
2127 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
2128 ++Skipped;
2129 else
2130 ++I;
2131 }
2132 FieldIdx += Skipped;
2133 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
2134
2135 // Now index into the struct, making sure that the type we return is the
2136 // buffer layout type rather than the original type in the AST.
2137 QualType FieldType = Field->getType();
2138 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
2139 CharUnits Align = CharUnits::fromQuantity(
2140 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
2141
2142 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
2143 ? CGF.Builder.CreateStructuredGEP(
2144 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
2145 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
2146 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
2147 Idx: FieldIdx, Name: Field->getName());
2148 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
2149
2150 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
2151 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2152 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
2153 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
2154
2155 return LV;
2156}
2157