1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/RecursiveASTVisitor.h"
28#include "clang/AST/Type.h"
29#include "clang/Basic/DiagnosticDriver.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetOptions.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/Enum.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/ScopeExit.h"
37#include "llvm/ADT/SmallString.h"
38#include "llvm/ADT/SmallVector.h"
39#include "llvm/Frontend/HLSL/HLSLResource.h"
40#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
41#include "llvm/Frontend/HLSL/SemanticSignaturePacking.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/GlobalVariable.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/LLVMContext.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
51#include "llvm/Support/Alignment.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/FormatVariadic.h"
54#include "llvm/Support/Path.h"
55#include "llvm/Transforms/Utils/ModuleUtils.h"
56#include <array>
57#include <cstdint>
58#include <optional>
59
60using namespace clang;
61using namespace CodeGen;
62using namespace clang::hlsl;
63using namespace llvm;
64
65using llvm::hlsl::CBufferRowSizeInBytes;
66
67namespace {
68
69void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
70 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
71 // Assume ValVersionStr is legal here.
72 VersionTuple Version;
73 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
74 Version.getSubminor() || !Version.getMinor()) {
75 return;
76 }
77
78 uint64_t Major = Version.getMajor();
79 uint64_t Minor = *Version.getMinor();
80
81 auto &Ctx = M.getContext();
82 IRBuilder<> B(M);
83 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
84 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
85 StringRef DXILValKey = "dx.valver";
86 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
87 DXILValMD->addOperand(M: Val);
88}
89
90void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
91 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
92 llvm::Function *Fn, llvm::Module &M) {
93 auto &Ctx = M.getContext();
94
95 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
96 MDNode *RootSignature = RSBuilder.BuildRootSignature();
97
98 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
99 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
100 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
101 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
102
103 StringRef RootSignatureValKey = "dx.rootsignatures";
104 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
105 RootSignatureValMD->addOperand(M: MDVals);
106}
107
108Expected<unsigned> packSemanticSignature(
109 MutableArrayRef<llvm::hlsl::SemanticSignatureElement> Elements,
110 llvm::Triple::EnvironmentType Stage, llvm::hlsl::IOType IOTy,
111 CodeGenOptions::HLSLSemanticSignaturePackingMode Mode,
112 bool UseNative16BitTypes) {
113 using namespace llvm::hlsl;
114 // Vertex inputs and pixel outputs have fixed packing rules, independent of
115 // the packing mode used between programmable stages.
116 if (Stage == llvm::Triple::Vertex && IOTy == IOType::In)
117 return packSignatureStacked(Elements, ShaderStage: Stage, IOTy);
118 if (Stage == llvm::Triple::Pixel && IOTy == IOType::Out)
119 return packSignatureIndexed(Elements, ShaderStage: Stage, IOTy);
120 if (Mode == CodeGenOptions::HLSLSemanticSignaturePackingMode::Optimized)
121 return packSignatureOptimized(Elements, ShaderStage: Stage, IOTy, UseNative16BitTypes);
122 return packSignaturePrefixStable(Elements, ShaderStage: Stage, IOTy, UseNative16BitTypes);
123}
124
125MDNode *buildSemanticSignatureMD(
126 ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
127 if (Elements.empty())
128 return nullptr;
129
130 SmallVector<Metadata *> ElementMD;
131 for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
132 ElementMD.push_back(Elt: Element.toMetadata(Ctx));
133 return MDNode::get(Context&: Ctx, MDs: ElementMD);
134}
135
136void addSemanticSignatureMD(
137 ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
138 ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
139 llvm::Function *Fn, llvm::Module &M) {
140 if (InputElements.empty() && OutputElements.empty())
141 return;
142
143 LLVMContext &Ctx = M.getContext();
144 MDNode *InputSignature = buildSemanticSignatureMD(Elements: InputElements, Ctx);
145 MDNode *OutputSignature = buildSemanticSignatureMD(Elements: OutputElements, Ctx);
146 MDNode *MDVals = MDNode::get(
147 Context&: Ctx, MDs: {ValueAsMetadata::get(V: Fn), InputSignature, OutputSignature});
148
149 M.getOrInsertNamedMetadata(Name: "dx.semantic.signatures")->addOperand(M: MDVals);
150}
151
152static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
153 AggValueSlot &DestSlot) {
154 GlobalVariable *ResGV =
155 cast<GlobalVariable>(Val: CGF.CGM.GetAddrOfGlobalVar(D: ResourceVD));
156 assert(ResGV && "expected valid global variable");
157 CGF.Builder.CreateStore(Val: ResGV, Addr: DestSlot.getAddress());
158}
159
160// Given a MemberExpr of a resource or resource array type, find the parent
161// VarDecl of the struct or class instance that contains this resource and
162// build the full resource name based on the member access path.
163//
164// For example, for a member access like "myStructArray[0].memberA",
165// this function will find the VarDecl of "myStructArray" and use the
166// EmbeddedResourceNameBuilder to build the resource name
167// "myStructArray.0.memberA".
168//
169// This also works for a record type expression that has some embedded
170// resources. It finds the parent VarDecl of that record and builds a partial
171// name which is the prefix of the resource globals associated with the
172// declaration.
173static const VarDecl *findStructResourceParentDeclAndBuildName(
174 const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
175
176 SmallVector<const Expr *> WorkList;
177 const VarDecl *VD = nullptr;
178
179 for (;;) {
180 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
181 assert(isa<VarDecl>(DRE->getDecl()) &&
182 "member expr base is not a var decl");
183 VD = cast<VarDecl>(Val: DRE->getDecl());
184 NameBuilder.pushName(N: VD->getName());
185 break;
186 }
187
188 WorkList.push_back(Elt: E);
189 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
190 E = MExp->getBase();
191 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
192 E = ICE->getSubExpr();
193 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
194 E = ASE->getBase();
195 else if (isa<CXXThisExpr>(Val: E))
196 // Resource member access on "this" pointer not yet implemented
197 // (llvm/llvm-project#190299)
198 return nullptr;
199 else
200 llvm_unreachable("unexpected expr type in resource member access");
201
202 assert(E && "expected valid expression");
203 }
204
205 while (!WorkList.empty()) {
206 E = WorkList.pop_back_val();
207 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
208 NameBuilder.pushName(
209 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
210 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
211 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
212 CXXRecordDecl *DerivedRD =
213 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
214 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
215 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
216 }
217 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
218 const Expr *IdxExpr = ASE->getIdx();
219 std::optional<llvm::APSInt> Value =
220 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
221 assert(Value &&
222 "expected constant index in struct with resource array access");
223 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
224 } else {
225 llvm_unreachable("unexpected expr type in resource member access");
226 }
227 }
228 return VD;
229}
230
231// Given a MemberExpr of a resource or resource array type, find the
232// corresponding global resource declaration associated with the owning struct
233// or class instance via HLSLAssociatedResourceDeclAttr.
234static const VarDecl *
235findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
236
237 EmbeddedResourceNameBuilder NameBuilder;
238 const VarDecl *ParentVD =
239 findStructResourceParentDeclAndBuildName(E: ME, NameBuilder);
240 if (!ParentVD)
241 return nullptr;
242
243 if (!ParentVD->hasGlobalStorage())
244 return nullptr;
245
246 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
247 for (const Attr *A : ParentVD->getAttrs()) {
248 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
249 VarDecl *AssocResVD = ADA->getResDecl();
250 if (AssocResVD->getIdentifier() == II)
251 return AssocResVD;
252 }
253 }
254 return nullptr;
255}
256
257void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
258 auto &SM = CGM.getContext().getSourceManager();
259 auto &Macros = CGM.getPreprocessorOpts().Macros;
260 auto &CodeGenOpts = CGM.getCodeGenOpts();
261 auto &Ctx = M.getContext();
262
263 // Names and content of shader source code files.
264 llvm::NamedMDNode *DXContents =
265 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
266 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
267 llvm::MDTuple *FileInfo =
268 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
269 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
270 DXContents->addOperand(M: FileInfo);
271 };
272
273 bool Invalid = false;
274 const SrcMgr::SLocEntry *MainLocEntry =
275 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
276 assert(!Invalid && "Main file SLocEntry must not be invalid!");
277 const SrcMgr::ContentCache &MainCCEntry =
278 MainLocEntry->getFile().getContentCache();
279
280 SmallVector<std::pair<std::string, StringRef>> Files;
281 std::optional<SmallString<256>> MainFileName;
282 Files.reserve(N: SM.local_sloc_entry_size());
283 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
284 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
285 if (!LocEntry.isFile())
286 continue;
287
288 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
289 if (isSystem(CK: FInfo.getFileCharacteristic()))
290 continue;
291
292 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
293 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
294 if (!FEntry)
295 continue;
296
297 llvm::SmallString<256> Path = FEntry->getName();
298 llvm::sys::path::native(path&: Path);
299 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
300 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
301 if (!Buffer) {
302 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
303 << Path;
304 continue;
305 }
306
307 if (&MainCCEntry != &CCEntry) {
308 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
309 } else {
310 // Main file should be at first position.
311 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
312 MainFileName.emplace(args&: Path);
313 }
314 }
315 assert(MainFileName && "Main file not found.");
316
317 // Files other that main one should be sorted by name.
318 llvm::sort(C&: Files);
319#ifndef NDEBUG
320 for (unsigned I = 1; I < Files.size(); ++I)
321 assert((Files[I - 1].first != Files[I].first) &&
322 "duplicate files in dx.source.contents");
323#endif
324 llvm::for_each(Range&: Files, F: addFile);
325
326 SmallVector<llvm::Metadata *> Defines;
327 Defines.reserve(N: Macros.size());
328 for (const auto &Macro : Macros) {
329 // Ignore undefs.
330 if (!Macro.second)
331 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
332 }
333 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
334 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
335
336 if (!CodeGenOpts.MainFileName.empty())
337 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
338 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
339 ->addOperand(
340 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
341
342 SmallVector<llvm::Metadata *> Args;
343 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
344 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
345 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
346 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
347 M.getOrInsertNamedMetadata(Name: "dx.source.args")
348 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
349}
350
351// Find array variable declaration from DeclRef expression
352static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
353 E = E->IgnoreImpCasts();
354 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
355 return DRE->getDecl();
356 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
357 E = OVE->getSourceExpr()->IgnoreImpCasts();
358 if (isa<MemberExpr>(Val: E))
359 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
360 return nullptr;
361}
362
363// Find array variable declaration from nested array subscript AST nodes
364static const ValueDecl *getArrayDecl(ASTContext &AST,
365 const ArraySubscriptExpr *ASE) {
366 const Expr *E = nullptr;
367 while (ASE != nullptr) {
368 E = ASE->getBase()->IgnoreImpCasts();
369 if (!E)
370 return nullptr;
371 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
372 }
373 return getArrayDecl(AST, E);
374}
375
376// Get the total size of the array, or 0 if the array is unbounded.
377static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
378 Ty = Ty->getUnqualifiedDesugaredType();
379 assert(Ty->isArrayType() && "expected array type");
380 if (Ty->isIncompleteArrayType())
381 return 0;
382 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
383}
384
385static Value *buildNameForResource(llvm::StringRef BaseName,
386 CodeGenModule &CGM) {
387 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
388 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
389 .getPointer();
390}
391
392static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
393 StorageClass SC = SC_None) {
394 for (auto *Method : Record->methods()) {
395 if (Method->getStorageClass() == SC && Method->getName() == Name)
396 return Method;
397 }
398 return nullptr;
399}
400
401static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
402 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
403 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
404 CallArgList &Args) {
405 assert(Binding.hasBinding() && "at least one binding attribute expected");
406
407 ASTContext &AST = CGM.getContext();
408 CXXMethodDecl *CreateMethod = nullptr;
409 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
410 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
411
412 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
413 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
414 "resources with counter handle must have a binding with counter "
415 "implicit order ID");
416 if (Binding.isExplicit()) {
417 // explicit binding
418 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
419 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
420 const char *Name = Binding.hasCounterImplicitOrderID()
421 ? "__createFromBindingWithImplicitCounter"
422 : "__createFromBinding";
423 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
424 } else {
425 // implicit binding
426 auto *OrderID =
427 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
428 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
429 const char *Name = Binding.hasCounterImplicitOrderID()
430 ? "__createFromImplicitBindingWithImplicitCounter"
431 : "__createFromImplicitBinding";
432 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
433 }
434 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
435 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
436 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
437 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
438 if (HasCounter) {
439 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
440 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
441 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
442 }
443
444 return CreateMethod;
445}
446
447static void callResourceInitMethod(CodeGenFunction &CGF,
448 CXXMethodDecl *CreateMethod,
449 CallArgList &Args, Address ReturnAddress) {
450 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
451 const FunctionProtoType *Proto =
452 CreateMethod->getType()->getAs<FunctionProtoType>();
453 // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
454 // caller declaration is needed for x86 SysV ABI selection.
455 const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
456 Args, Ty: Proto, ChainCall: false, /*ABIInfoFD=*/nullptr);
457 ReturnValueSlot ReturnValue(ReturnAddress, false);
458 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
459 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
460}
461
462// Initializes local resource array variable with global resource array
463// elements. For multi-dimensional arrays it calls itself recursively to
464// initialize its sub-arrays. The Index used in the resource constructor calls
465// will begin at StartIndex and will be incremented for each array element. The
466// last used resource Index is returned to the caller. If the function returns
467// std::nullopt, it indicates an error.
468static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
469 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
470 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
471 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
472 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
473
474 ASTContext &AST = CGF.getContext();
475 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
476 llvm::Value *Index = StartIndex;
477 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
478 const uint64_t ArraySize = ArrayTy->getSExtSize();
479 QualType ElemType = ArrayTy->getElementType();
480 Address TmpArrayAddr = ValueSlot.getAddress();
481
482 // Add additional index to the getelementptr call indices.
483 // This index will be updated for each array element in the loops below.
484 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
485 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
486
487 // For array of arrays, recursively initialize the sub-arrays.
488 if (ElemType->isArrayType()) {
489 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
490 for (uint64_t I = 0; I < ArraySize; I++) {
491 if (I > 0) {
492 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
493 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
494 }
495 std::optional<llvm::Value *> MaybeIndex =
496 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
497 ValueSlot, Range, StartIndex: Index,
498 ResourceName, Binding, PrevGEPIndices: GEPIndices);
499 if (!MaybeIndex)
500 return std::nullopt;
501 Index = *MaybeIndex;
502 }
503 return Index;
504 }
505
506 // For array of resources, initialize each resource in the array.
507 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
508 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
509 CharUnits Align =
510 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
511
512 for (uint64_t I = 0; I < ArraySize; I++) {
513 if (I > 0) {
514 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
515 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
516 }
517 Address ReturnAddress =
518 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
519
520 CallArgList Args;
521 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
522 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
523
524 if (!CreateMethod)
525 // This can happen if someone creates an array of structs that looks like
526 // an HLSL resource record array but it does not have the required static
527 // create method. No binding will be generated for it.
528 return std::nullopt;
529
530 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
531 }
532 return Index;
533}
534
535/// Utility for emitting copies following the HLSL buffer layout rules (ie,
536/// copying out of a cbuffer).
537class HLSLBufferCopyEmitter {
538 CodeGenFunction &CGF;
539 Address DstPtr;
540 Address SrcPtr;
541 llvm::Type *LayoutTy = nullptr;
542
543 SmallVector<llvm::Value *> CurStoreIndices;
544 SmallVector<llvm::Value *> CurLoadIndices;
545
546 using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
547
548 // Creates & returns either a structured.gep or a ptradd/gep depending on
549 // langopts.
550 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
551 ArrayRef<llvm::Value *> Indices) {
552 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
553 if (EmitLogical)
554 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
555
556 llvm::SmallVector<llvm::Value *> GEPIndices;
557 GEPIndices.reserve(N: Indices.size() + 1);
558 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
559 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
560 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
561 }
562
563 bool isBufferLayoutArray(llvm::StructType *ST) {
564 // A buffer layout array is a struct with two elements: the padded array,
565 // and the last element. That is, is should look something like this:
566 //
567 // { [%n x { %type, %padding }], %type }
568 //
569 if (!ST || ST->getNumElements() != 2)
570 return false;
571
572 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
573 if (!PaddedEltsTy)
574 return false;
575
576 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
577 if (!PaddedTy || PaddedTy->getNumElements() != 2)
578 return false;
579
580 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
581 Ty: PaddedTy->getElementType(N: 1)))
582 return false;
583
584 llvm::Type *ElementTy = ST->getElementType(N: 1);
585 if (PaddedTy->getElementType(N: 0) != ElementTy)
586 return false;
587 return true;
588 }
589
590 // Returns true if the type is either a struct representing a resource record,
591 // or an array of structs that are resource records. This assumes a struct is
592 // a resource record if the first element is a target type (resource handle).
593 // This is the case for all target types used by HLSL except the padding type
594 // ("{dx|spirv.Padding"), but padding will never be the first element of a
595 // struct.
596 bool isResourceOrResourceArray(llvm::Type *Ty) {
597 while (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
598 Ty = AT->getElementType();
599
600 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
601 if (!ST || ST->getNumElements() < 1)
602 return false;
603
604 auto *TargetTy = dyn_cast<llvm::TargetExtType>(Val: ST->getElementType(N: 0));
605 return TargetTy != nullptr;
606 }
607
608 void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
609 EmitResourceFnTy EmitResFn) {
610 CharUnits DstAlign =
611 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
612 Address DstAddr(Dst, DstTy, DstAlign);
613 AggValueSlot Slot = AggValueSlot::forAddr(
614 addr: DstAddr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
615 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
616 mayOverlap: AggValueSlot::DoesNotOverlap);
617
618 EmitResFn(Slot);
619 }
620
621 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
622 llvm::ArrayType *DstTy,
623 EmitResourceFnTy EmitResFn) {
624 // Those assumptions are checked by isBufferLayoutArray.
625 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
626 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
627 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
628 ->getElementType(0) == SrcTy->getElementType(1));
629
630 auto *SrcDataTy = SrcTy->getElementType(N: 1);
631 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
632
633 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
634 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
635 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
636 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
637 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
638 EmitResFn);
639 }
640
641 auto *SrcElt =
642 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
643 auto *DstElt = emitAccessChain(
644 BaseTy: DstTy, Base: Dst,
645 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
646 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
647 EmitResFn);
648 }
649
650 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
651 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
652 assert(!isResourceOrResourceArray(DstTy) &&
653 "direct access to resources or resource arrays should be handled "
654 "separately");
655
656 if (isBufferLayoutArray(ST: SrcTy))
657 return emitBufferLayoutCopy(Src, SrcTy, Dst, DstTy: cast<llvm::ArrayType>(Val: DstTy),
658 EmitResFn);
659
660 unsigned SrcIndex = 0;
661 unsigned DstIndex = 0;
662
663 // DstTy layout is in default address space and can include resource types.
664 // SrcTy is in cbuffer layout where resources are filtered out, so the
665 // number of elements in SrcTy can be less than the number of elements in
666 // DstTy.
667 auto *DstST = cast<llvm::StructType>(Val: DstTy);
668 while (DstIndex < DstST->getNumElements()) {
669 llvm::Type *DstEltTy = DstST->getElementType(N: DstIndex);
670 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: DstEltTy)) {
671 DstIndex += 1;
672 continue;
673 }
674 if (isResourceOrResourceArray(Ty: DstEltTy)) {
675 auto *DstElt = emitAccessChain(
676 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
677 emitResourceOrResourceArray(Dst: DstElt, DstTy: DstEltTy, EmitResFn);
678 DstIndex += 1;
679 continue;
680 }
681
682 assert(SrcIndex < SrcTy->getNumElements());
683 llvm::Type *SrcEltTy = SrcTy->getElementType(N: SrcIndex);
684 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: SrcEltTy)) {
685 SrcIndex += 1;
686 continue;
687 }
688
689 auto *SrcElt = emitAccessChain(
690 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
691 auto *DstElt = emitAccessChain(
692 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
693 emitElementCopy(Src: SrcElt, SrcTy: SrcEltTy, Dst: DstElt, DstTy: DstEltTy, EmitResFn);
694 DstIndex += 1;
695 SrcIndex += 1;
696 }
697 }
698
699 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
700 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
701 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
702 auto *SrcElt =
703 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
704 auto *DstElt =
705 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
706 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
707 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType(),
708 EmitResFn);
709 }
710 }
711
712 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
713 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
714 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
715 return emitCopy(Src, SrcTy: AT, Dst, DstTy, EmitResFn);
716 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
717 return emitCopy(Src, SrcTy: ST, Dst, DstTy, EmitResFn);
718
719 // When we have a scalar or vector element we can emit the copy.
720 CharUnits SrcAlign =
721 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
722 CharUnits DstAlign =
723 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
724 Address SrcAddr(Src, SrcTy, SrcAlign);
725 Address DstAddr(Dst, DstTy, DstAlign);
726 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
727 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
728 }
729
730public:
731 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
732 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
733
734 bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
735 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
736
737 // TODO: We should be able to fall back to a regular memcpy if the layout
738 // type doesn't have any padding, but that runs into issues in the backend
739 // currently.
740 //
741 // See https://github.com/llvm/wg-hlsl/issues/351
742 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
743 DstTy: DstPtr.getElementType(), EmitResFn);
744 return true;
745 }
746};
747
748// Represents a list resources associated with a global struct whose name
749// starts with the specified prefix.
750// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
751// order of the depth-first traversal of the corresponding fields in the struct.
752// The resources are always returned in that order, which is the same order
753// we need when a struct is copied element-by-element.
754class AssociatedResourcesList {
755 // Iterator pointers for the associated resource attributes that match the
756 // prefix. Begin = begin of the range of attributes that match the prefix End
757 // = end of the range of attributes that match the prefix Next = the current
758 // attribute in the iteration to be returned by getNextResource
759 specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
760
761public:
762 AssociatedResourcesList(const VarDecl *StructVD,
763 StringRef ResourceNamePrefix) {
764 auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
765 auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
766
767 // Skip over associated resources that don't match the prefix.
768 while (I != E &&
769 !I->getResDecl()->getName().starts_with(Prefix: ResourceNamePrefix))
770 ++I;
771 assert(I != E && "expected associated resource not found");
772 Begin = End = I;
773
774 // Scan over associated resources that do match the prefix to find the end
775 // of the range.
776 while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
777 ->getResDecl()
778 ->getName()
779 .starts_with(Prefix: ResourceNamePrefix))
780 End = ++I;
781
782 Next = Begin;
783 }
784
785 const VarDecl *getNextResource() {
786 if (Next == End)
787 return nullptr;
788
789 const VarDecl *Res = Next->getResDecl();
790 ++Next;
791 return Res;
792 }
793};
794
795} // namespace
796
797llvm::Type *
798CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
799 const CGHLSLOffsetInfo &OffsetInfo) {
800 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
801
802 // Check if the target has a specific translation for this type first.
803 if (llvm::Type *TargetTy =
804 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
805 return TargetTy;
806
807 llvm_unreachable("Generic handling of HLSL types is not supported.");
808}
809
810llvm::Triple::ArchType CGHLSLRuntime::getArch() {
811 return CGM.getTarget().getTriple().getArch();
812}
813
814// Emits constant global variables for buffer constants declarations
815// and creates metadata linking the constant globals with the buffer global.
816void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
817 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
818 const CGHLSLOffsetInfo &OffsetInfo) {
819 LLVMContext &Ctx = CGM.getLLVMContext();
820
821 // get the layout struct from constant buffer target type
822 llvm::Type *BufType = BufGV->getValueType();
823 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
824 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
825
826 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
827 size_t OffsetIdx = 0;
828 for (Decl *D : BufDecl->buffer_decls()) {
829 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
830 // Nothing to do for this declaration.
831 continue;
832 if (isa<FunctionDecl>(Val: D)) {
833 // A function within an cbuffer is effectively a top-level function.
834 CGM.EmitTopLevelDecl(D);
835 continue;
836 }
837 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
838 if (!VD)
839 continue;
840
841 QualType VDTy = VD->getType();
842 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
843 if (VD->getStorageClass() == SC_Static ||
844 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
845 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
846 // Emit static and groupshared variables and resource classes inside
847 // cbuffer as regular globals
848 CGM.EmitGlobal(D: VD);
849 }
850 continue;
851 }
852
853 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
854 }
855
856 if (!OffsetInfo.empty())
857 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
858 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
859 });
860
861 // Associate the buffer global variable with its constants
862 SmallVector<llvm::Metadata *> BufGlobals;
863 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
864 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
865
866 auto ElemIt = LayoutStruct->element_begin();
867 for (auto &[VD, _] : DeclsWithOffset) {
868 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
869 ++ElemIt;
870
871 assert(ElemIt != LayoutStruct->element_end() &&
872 "number of elements in layout struct does not match");
873 llvm::Type *LayoutType = *ElemIt++;
874
875 GlobalVariable *ElemGV =
876 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
877 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
878 }
879 assert(ElemIt == LayoutStruct->element_end() &&
880 "number of elements in layout struct does not match");
881
882 // add buffer metadata to the module
883 CGM.getModule()
884 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
885 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
886}
887
888// Creates resource handle type for the HLSL buffer declaration
889static const clang::HLSLAttributedResourceType *
890createBufferHandleType(const HLSLBufferDecl *BufDecl) {
891 ASTContext &AST = BufDecl->getASTContext();
892 QualType QT = AST.getHLSLAttributedResourceType(
893 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
894 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
895 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
896}
897
898CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
899 CGHLSLOffsetInfo Result;
900
901 // If we don't have packoffset info, just return an empty result.
902 if (!BufDecl.hasValidPackoffset())
903 return Result;
904
905 for (Decl *D : BufDecl.buffer_decls()) {
906 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
907 continue;
908 }
909 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
910 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
911 continue;
912
913 if (!VD->hasAttrs()) {
914 Result.Offsets.push_back(Elt: Unspecified);
915 continue;
916 }
917
918 uint32_t Offset = Unspecified;
919 for (auto *Attr : VD->getAttrs()) {
920 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
921 Offset = POA->getOffsetInBytes();
922 break;
923 }
924 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
925 if (RBA &&
926 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
927 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
928 break;
929 }
930 }
931 Result.Offsets.push_back(Elt: Offset);
932 }
933 return Result;
934}
935
936// Codegen for HLSLBufferDecl
937void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
938
939 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
940
941 // create resource handle type for the buffer
942 const clang::HLSLAttributedResourceType *ResHandleTy =
943 createBufferHandleType(BufDecl);
944
945 // empty constant buffer is ignored
946 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
947 return;
948
949 // create global variable for the constant buffer
950 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
951 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
952 llvm::GlobalVariable *BufGV = new GlobalVariable(
953 LayoutTy, /*isConstant*/ false,
954 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
955 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
956 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
957 GlobalValue::NotThreadLocal);
958
959 llvm::Module &M = CGM.getModule();
960 M.insertGlobalVariable(GV: BufGV);
961
962 // Add the global variable to the compiler used list so it does not
963 // get optimized away by GlobalOptPass before it reaches
964 // {DXIL|SPIRV}CBufferAccess pass.
965 llvm::appendToCompilerUsed(M, Values: {BufGV});
966
967 // Add globals for constant buffer elements and create metadata nodes
968 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
969
970 // Initialize cbuffer from binding (implicit or explicit)
971 initializeBufferFromBinding(BufDecl, GV: BufGV);
972}
973
974void CGHLSLRuntime::addRootSignature(
975 const HLSLRootSignatureDecl *SignatureDecl) {
976 llvm::Module &M = CGM.getModule();
977 Triple T(M.getTargetTriple());
978
979 // Generated later with the function decl if not targeting root signature
980 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
981 return;
982
983 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
984 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
985}
986
987llvm::StructType *
988CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
989 const auto Entry = LayoutTypes.find(Val: StructType);
990 if (Entry != LayoutTypes.end())
991 return Entry->getSecond();
992 return nullptr;
993}
994
995void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
996 llvm::StructType *LayoutTy) {
997 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
998 "layout type for this struct already exist");
999 LayoutTypes[StructType] = LayoutTy;
1000}
1001
1002void CGHLSLRuntime::finishCodeGen() {
1003 auto &TargetOpts = CGM.getTarget().getTargetOpts();
1004 auto &CodeGenOpts = CGM.getCodeGenOpts();
1005 auto &LangOpts = CGM.getLangOpts();
1006 llvm::Module &M = CGM.getModule();
1007 Triple T(M.getTargetTriple());
1008 if (T.getArch() == Triple::ArchType::dxil)
1009 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
1010 if (!CodeGenOpts.DisableDXSourceMetadata &&
1011 CodeGenOpts.getDebugInfo() >=
1012 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
1013 addSourceInfo(CGM, M);
1014 if (CodeGenOpts.ResMayAlias)
1015 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
1016 if (CodeGenOpts.AllResourcesBound)
1017 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
1018 Key: "dx.allresourcesbound", Val: 1);
1019 if (CodeGenOpts.OptimizationLevel == 0)
1020 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
1021 Key: "dx.disable_optimizations", Val: 1);
1022
1023 // NativeHalfType corresponds to the -fnative-half-type clang option which is
1024 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
1025 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
1026 if (LangOpts.NativeHalfType)
1027 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
1028 Val: 1);
1029
1030 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
1031 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
1032 const ASTContext &Ctx = CGM.getContext();
1033 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
1034 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
1035 SmallVector<GlobalValue *, 8> InterfaceVars;
1036 for (GlobalVariable &GV : M.globals()) {
1037 unsigned AS = GV.getAddressSpace();
1038 if (AS == InputAS || AS == OutputAS)
1039 InterfaceVars.push_back(Elt: &GV);
1040 }
1041 if (!InterfaceVars.empty())
1042 appendToCompilerUsed(M, Values: InterfaceVars);
1043 }
1044
1045 generateGlobalCtorDtorCalls();
1046}
1047
1048void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
1049 const FunctionDecl *FD, llvm::Function *Fn) {
1050 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1051 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
1052 const StringRef ShaderAttrKindStr = "hlsl.shader";
1053 Fn->addFnAttr(Kind: ShaderAttrKindStr,
1054 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
1055 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
1056 const StringRef NumThreadsKindStr = "hlsl.numthreads";
1057 std::string NumThreadsStr =
1058 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
1059 Vals: NumThreadsAttr->getZ());
1060 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
1061 }
1062 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
1063 const StringRef WaveSizeKindStr = "hlsl.wavesize";
1064 std::string WaveSizeStr =
1065 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
1066 Vals: WaveSizeAttr->getPreferred());
1067 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
1068 }
1069 // HLSL entry functions are materialized for module functions with
1070 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
1071 // later in the compiler-flow for such module functions is not aware of and
1072 // hence not able to set attributes of the newly materialized entry functions.
1073 // So, set attributes of entry function here, as appropriate.
1074 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
1075
1076 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
1077 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
1078 }
1079}
1080
1081static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
1082 // Compute ID intrinsics return i32 components, but the semantic may use
1083 // 16-bit integers. Narrow each component before assembling the input.
1084 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1085 Value *Result = PoisonValue::get(T: Ty);
1086 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
1087 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
1088 Elt = B.CreateTrunc(V: Elt, DestTy: VT->getElementType());
1089 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
1090 }
1091 return Result;
1092 }
1093 return B.CreateTrunc(V: B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)}), DestTy: Ty);
1094}
1095
1096static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
1097 unsigned BuiltIn) {
1098 LLVMContext &Ctx = GV->getContext();
1099 IRBuilder<> B(*GV->getParent());
1100 MDNode *Operands = MDNode::get(
1101 Context&: Ctx,
1102 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
1103 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
1104 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1105 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1106}
1107
1108static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
1109 LLVMContext &Ctx = GV->getContext();
1110 IRBuilder<> B(*GV->getParent());
1111 MDNode *Operands =
1112 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Location */ C: 30)),
1113 ConstantAsMetadata::get(C: B.getInt32(C: Location))});
1114 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1115 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1116}
1117
1118// A fragment shader input interface variable whose base type is an integer or
1119// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
1120// specification requires these variables to be decorated with Flat (see
1121// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
1122// their base scalar type.
1123static bool inputRequiresFlatDecoration(llvm::Type *Ty) {
1124 while (true) {
1125 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
1126 Ty = AT->getElementType();
1127 continue;
1128 }
1129 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: Ty)) {
1130 Ty = VT->getElementType();
1131 continue;
1132 }
1133 break;
1134 }
1135 return Ty->isIntegerTy() || Ty->isDoubleTy();
1136}
1137
1138static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
1139 llvm::Type *Ty, const Twine &Name,
1140 unsigned BuiltInID) {
1141 auto *GV = new llvm::GlobalVariable(
1142 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1143 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1144 llvm::GlobalVariable::GeneralDynamicTLSModel,
1145 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1146 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1147 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1148 return B.CreateLoad(Ty, Ptr: GV);
1149}
1150
1151static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
1152 llvm::Type *Ty, unsigned Location,
1153 StringRef Name, bool NeedsFlat) {
1154 auto *GV = new llvm::GlobalVariable(
1155 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1156 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1157 llvm::GlobalVariable::GeneralDynamicTLSModel,
1158 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1159 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1160
1161 // Emit all decorations as a single `spirv.Decorations` node. Attaching
1162 // multiple `spirv.Decorations` metadata nodes to the same global is not
1163 // supported by the SPIR-V backend and results in all but one being dropped.
1164 LLVMContext &Ctx = GV->getContext();
1165 SmallVector<Metadata *, 2> Decorations;
1166 Decorations.push_back(
1167 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1168 C: B.getInt32(/* SPIRV::Decoration::Location */ C: 30)),
1169 ConstantAsMetadata::get(C: B.getInt32(C: Location))}));
1170 if (NeedsFlat)
1171 Decorations.push_back(
1172 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1173 C: B.getInt32(/* SPIRV::Decoration::Flat */ C: 14))}));
1174 GV->addMetadata(Kind: "spirv.Decorations", MD&: *MDNode::get(Context&: Ctx, MDs: Decorations));
1175
1176 return B.CreateLoad(Ty, Ptr: GV);
1177}
1178
1179llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
1180 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1181 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1182 std::optional<unsigned> Index) {
1183 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1184 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1185
1186 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
1187 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1188 Location = L->getLocation();
1189
1190 // DXC completely ignores the semantic/index pair. Location are assigned from
1191 // the first semantic to the last.
1192 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
1193 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1194 SPIRVLastAssignedInputSemanticLocation += ElementCount;
1195
1196 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1197 bool NeedsFlat =
1198 ShaderAttr &&
1199 ShaderAttr->getType() == llvm::Triple::EnvironmentType::Pixel &&
1200 inputRequiresFlatDecoration(Ty: Type);
1201
1202 return createSPIRVLocationLoad(B, M&: CGM.getModule(), Ty: Type, Location,
1203 Name: VariableName.str(), NeedsFlat);
1204}
1205
1206static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
1207 llvm::Value *Source, unsigned Location,
1208 StringRef Name) {
1209 auto *GV = new llvm::GlobalVariable(
1210 M, Source->getType(), /* isConstant= */ false,
1211 llvm::GlobalValue::ExternalLinkage,
1212 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1213 llvm::GlobalVariable::GeneralDynamicTLSModel,
1214 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1215 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1216 addLocationDecoration(GV, Location);
1217 B.CreateStore(Val: Source, Ptr: GV);
1218}
1219
1220void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1221 llvm::IRBuilder<> &B, llvm::Value *Source,
1222 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1223 std::optional<unsigned> Index) {
1224 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1225 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1226
1227 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1228 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1229 Location = L->getLocation();
1230
1231 // DXC completely ignores the semantic/index pair. Location are assigned from
1232 // the first semantic to the last.
1233 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1234 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1235 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1236 createSPIRVLocationStore(B, M&: CGM.getModule(), Source, Location,
1237 Name: VariableName.str());
1238}
1239
1240namespace {
1241// Describes how a semantic leaf lowers to signature rows
1242struct SemanticShape {
1243 SmallVector<unsigned> Dimensions; // Empty dims denotes a scalar
1244 unsigned Cols;
1245 QualType RowType;
1246
1247 unsigned getNumRows() const {
1248 unsigned Rows = 1;
1249 for (unsigned Dimension : Dimensions)
1250 Rows *= Dimension;
1251 return Rows;
1252 }
1253
1254 SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
1255 assert(Row < getNumRows() && "row exceeds semantic shape");
1256
1257 SmallVector<unsigned> Indices(Dimensions.size());
1258 for (auto [Index, Dimension] :
1259 llvm::zip_equal(t: llvm::reverse(C&: Indices), u: llvm::reverse(C: Dimensions))) {
1260 Index = Row % Dimension;
1261 Row /= Dimension;
1262 }
1263 return Indices;
1264 }
1265};
1266} // namespace
1267
1268// Returns the QualType of a semantic leaf declarator. For a function the
1269// declared return type is used, otherwise the declared type.
1270static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
1271 if (const auto *FD = dyn_cast<clang::FunctionDecl>(Val: Decl))
1272 return FD->getDeclaredReturnType();
1273 return Decl->getType();
1274}
1275
1276// Walks through the surrounding constant array types of \p Ty, collecting their
1277// dimensions until reaching a scalar, vector, or matrix leaf.
1278static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
1279 SemanticShape Shape{.Dimensions: {}, .Cols: 1, .RowType: Ty};
1280 while (const ConstantArrayType *CAT =
1281 Ctx.getAsConstantArrayType(T: Shape.RowType)) {
1282 Shape.Dimensions.push_back(Elt: CAT->getSize().getZExtValue());
1283 Shape.RowType = CAT->getElementType();
1284 }
1285
1286 if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
1287 Shape.Cols = VT->getNumElements();
1288 } else if (const auto *MT =
1289 Shape.RowType->getAs<clang::ConstantMatrixType>()) {
1290 // FIXME: a matrix leaf lowers to one row per matrix row but if column_major
1291 // is specified we transpose the num rows and num cols, this depends on
1292 // #211977 to resolve
1293 Shape.Cols = MT->getNumColumns();
1294 }
1295
1296 return Shape;
1297}
1298
1299static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
1300 QualType Ty) {
1301 if (const auto *VT = Ty->getAs<clang::VectorType>())
1302 Ty = VT->getElementType();
1303 else if (const auto *MT = Ty->getAs<clang::ConstantMatrixType>())
1304 Ty = MT->getElementType();
1305
1306 llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
1307 bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
1308 return llvm::hlsl::getDXILElementType(Ty: IRTy, IsSigned);
1309}
1310
1311static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
1312 CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
1313 std::optional<unsigned> Index, const SemanticShape &Shape) {
1314 StringRef Name = Semantic->getAttrName()->getName();
1315
1316 // One semantic index per row, starting from the declared index.
1317 SmallVector<uint32_t> SemanticIndices;
1318 uint32_t FirstSemanticIndex = Index.value_or(u: 0);
1319 for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
1320 SemanticIndices.push_back(Elt: FirstSemanticIndex + I);
1321
1322 // The remaining members keep their default value and will be filled at a
1323 // later stage, either during packing or analysis of usage. Interpolation
1324 // is set by the load/store traversal after visiting each leaf.
1325 return llvm::hlsl::SemanticSignatureElement(
1326 SigId, Name, getSignatureComponentType(CGM, Ty: Shape.RowType),
1327 llvm::hlsl::getSemanticKind(SemanticName: Name), SemanticIndices,
1328 static_cast<uint8_t>(Shape.Cols));
1329}
1330
1331llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
1332 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1333 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1334 SemanticSignatures &Signature) {
1335 StringRef Name = Semantic->getAttrName()->getName();
1336 SemanticShape Shape =
1337 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1338
1339 uint32_t SigId = Signature.size();
1340 Signature.push_back(
1341 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1342
1343 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1344
1345 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1346 M: B.getModule(), id: llvm::Intrinsic::dx_load_input, OverloadTys: {RowTy});
1347
1348 SmallVector<OperandBundleDef, 1> OB;
1349 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1350 llvm::Value *bundleArgs[] = {Token};
1351 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1352 }
1353
1354 llvm::Type *LeafTy = CGM.getTypes().ConvertType(T: Shape.RowType);
1355 llvm::Value *Result = llvm::PoisonValue::get(T: Type);
1356
1357 const unsigned NumRows = Shape.getNumRows();
1358
1359 for (unsigned Row = 0; Row < NumRows; ++Row) {
1360 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1361 std::array<Value *, 4> Args{
1362 /*SigElementId=*/B.getInt32(C: SigId),
1363 /*RowIndex=*/B.getInt32(C: Row),
1364 /*ColIndex=*/B.getInt8(C: 0),
1365 /*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(T: B.getInt32Ty())};
1366 llvm::Value *Value =
1367 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: Twine(Name).concat(Suffix: Twine(Row)));
1368 // Booleans use their memory representation in DXIL signatures, but
1369 // function parameters use their value representation.
1370 if (Value->getType() != LeafTy) {
1371 assert(Shape.RowType->hasBooleanRepresentation() &&
1372 "unexpected semantic load type mismatch");
1373 Value = B.CreateICmpNE(
1374 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
1375 }
1376
1377 Result =
1378 Indices.empty() ? Value : B.CreateInsertValue(Agg: Result, Val: Value, Idxs: Indices);
1379 }
1380 return Result;
1381}
1382
1383void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1384 llvm::Value *Source,
1385 const clang::DeclaratorDecl *Decl,
1386 HLSLAppliedSemanticAttr *Semantic,
1387 std::optional<unsigned> Index,
1388 SemanticSignatures &Signature) {
1389 SemanticShape Shape =
1390 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1391
1392 uint32_t SigId = Signature.size();
1393 Signature.push_back(
1394 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1395
1396 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1397
1398 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1399 M: B.getModule(), id: llvm::Intrinsic::dx_store_output, OverloadTys: {RowTy});
1400
1401 SmallVector<OperandBundleDef, 1> OB;
1402 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1403 llvm::Value *bundleArgs[] = {Token};
1404 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1405 }
1406
1407 const unsigned NumRows = Shape.getNumRows();
1408 for (unsigned Row = 0; Row < NumRows; ++Row) {
1409 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1410 llvm::Value *Val =
1411 Indices.empty() ? Source : B.CreateExtractValue(Agg: Source, Idxs: Indices);
1412
1413 // Booleans use their memory representation in DXIL signatures, but direct
1414 // function results use their value representation.
1415 if (Val->getType() != RowTy) {
1416 assert(Shape.RowType->hasBooleanRepresentation() &&
1417 "unexpected semantic store type mismatch");
1418 Val = B.CreateZExt(V: Val, DestTy: RowTy, Name: "storedv");
1419 }
1420
1421 std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(C: SigId),
1422 /*RowIndex=*/B.getInt32(C: Row),
1423 /*ColIndex=*/B.getInt8(C: 0), /*Value=*/Val};
1424 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1425 }
1426}
1427
1428llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1429 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1430 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1431 std::optional<unsigned> Index, SemanticSignatures &Signature) {
1432 if (CGM.getTarget().getTriple().isSPIRV())
1433 return emitSPIRVUserSemanticLoad(B, FD, Type, Decl, Semantic, Index);
1434
1435 if (CGM.getTarget().getTriple().isDXIL())
1436 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
1437
1438 llvm_unreachable("Unsupported target for user-semantic load.");
1439}
1440
1441void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1442 const clang::DeclaratorDecl *Decl,
1443 HLSLAppliedSemanticAttr *Semantic,
1444 std::optional<unsigned> Index,
1445 SemanticSignatures &Signature) {
1446 if (CGM.getTarget().getTriple().isSPIRV())
1447 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
1448
1449 if (CGM.getTarget().getTriple().isDXIL())
1450 return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
1451 Signature);
1452
1453 llvm_unreachable("Unsupported target for user-semantic load.");
1454}
1455
1456llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1457 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1458 HLSLAppliedSemanticAttr *Semantic,
1459 llvm::dxbc::PSV::SemanticKind SemanticKind,
1460 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1461 SemanticSignatures &Signature) {
1462 switch (SemanticKind) {
1463 case llvm::dxbc::PSV::SemanticKind::GroupIndex: {
1464 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1465 llvm::Triple::Amplification},
1466 Stage) &&
1467 "SV_GroupIndex is in an unavailable stage and should have been "
1468 "diagnosed by Sema");
1469 assert(Stage != llvm::Triple::Mesh &&
1470 Stage != llvm::Triple::Amplification &&
1471 "FIXME: SV_GroupIndex is not yet implemented for this shader "
1472 "stage");
1473 llvm::Function *GroupIndex =
1474 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1475 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1476 }
1477 case llvm::dxbc::PSV::SemanticKind::DispatchThreadID: {
1478 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1479 llvm::Triple::Amplification},
1480 Stage) &&
1481 "SV_DispatchThreadID is in an unavailable stage and should have "
1482 "been diagnosed by Sema");
1483 assert(Stage != llvm::Triple::Mesh &&
1484 Stage != llvm::Triple::Amplification &&
1485 "FIXME: SV_DispatchThreadID is not yet implemented for this "
1486 "shader stage");
1487 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1488 llvm::Function *ThreadIDIntrinsic =
1489 llvm::Intrinsic::isOverloaded(id: IntrinID)
1490 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1491 : CGM.getIntrinsic(IID: IntrinID);
1492 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1493 }
1494 case llvm::dxbc::PSV::SemanticKind::GroupThreadID: {
1495 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1496 llvm::Triple::Amplification},
1497 Stage) &&
1498 "SV_GroupThreadID is in an unavailable stage and should have been "
1499 "diagnosed by Sema");
1500 assert(Stage != llvm::Triple::Mesh &&
1501 Stage != llvm::Triple::Amplification &&
1502 "FIXME: SV_GroupThreadID is not yet implemented for this shader "
1503 "stage");
1504 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1505 llvm::Function *GroupThreadIDIntrinsic =
1506 llvm::Intrinsic::isOverloaded(id: IntrinID)
1507 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1508 : CGM.getIntrinsic(IID: IntrinID);
1509 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1510 }
1511 case llvm::dxbc::PSV::SemanticKind::GroupID: {
1512 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1513 llvm::Triple::Amplification},
1514 Stage) &&
1515 "SV_GroupID is in an unavailable stage and should have been "
1516 "diagnosed by Sema");
1517 assert(Stage != llvm::Triple::Mesh &&
1518 Stage != llvm::Triple::Amplification &&
1519 "FIXME: SV_GroupID is not yet implemented for this shader stage");
1520 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1521 llvm::Function *GroupIDIntrinsic =
1522 llvm::Intrinsic::isOverloaded(id: IntrinID)
1523 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1524 : CGM.getIntrinsic(IID: IntrinID);
1525 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1526 }
1527 case llvm::dxbc::PSV::SemanticKind::Position:
1528 assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
1529 llvm::Triple::Geometry, llvm::Triple::Pixel},
1530 Stage) &&
1531 "SV_Position is in an unavailable stage and should have been "
1532 "diagnosed by Sema");
1533 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1534 Stage != llvm::Triple::Geometry &&
1535 "FIXME: loading SV_Position is not yet implemented for this "
1536 "shader stage");
1537 if (CGM.getTarget().getTriple().isSPIRV())
1538 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1539 Name: Semantic->getAttrName()->getName(),
1540 /* BuiltIn::FragCoord */ BuiltInID: 15);
1541 if (CGM.getTarget().getTriple().isDXIL())
1542 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1543 Signature);
1544 break;
1545 case llvm::dxbc::PSV::SemanticKind::VertexID:
1546 assert(Stage == llvm::Triple::Vertex &&
1547 "SV_VertexID is in an unavailable stage and should have been "
1548 "diagnosed by Sema");
1549 if (CGM.getTarget().getTriple().isSPIRV())
1550 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1551 Name: Semantic->getAttrName()->getName(),
1552 /* BuiltIn::VertexIndex */ BuiltInID: 42);
1553 if (CGM.getTarget().getTriple().isDXIL())
1554 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1555 Signature);
1556 break;
1557 case llvm::dxbc::PSV::SemanticKind::InstanceID:
1558 assert(Stage == llvm::Triple::Vertex &&
1559 "SV_InstanceID is in an unavailable stage and should have been "
1560 "diagnosed by Sema");
1561 if (CGM.getTarget().getTriple().isSPIRV())
1562 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1563 Name: Semantic->getAttrName()->getName(),
1564 /* BuiltIn::InstanceIndex */ BuiltInID: 43);
1565 if (CGM.getTarget().getTriple().isDXIL())
1566 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1567 Signature);
1568 break;
1569 default:
1570 break;
1571 }
1572
1573 llvm_unreachable(
1574 "Load hasn't been implemented yet for this system semantic. FIXME");
1575}
1576
1577static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
1578 llvm::Value *Source, const Twine &Name,
1579 unsigned BuiltInID) {
1580 auto *GV = new llvm::GlobalVariable(
1581 M, Source->getType(), /* isConstant= */ false,
1582 llvm::GlobalValue::ExternalLinkage,
1583 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1584 llvm::GlobalVariable::GeneralDynamicTLSModel,
1585 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1586 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1587 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1588 B.CreateStore(Val: Source, Ptr: GV);
1589}
1590
1591void CGHLSLRuntime::emitSystemSemanticStore(
1592 IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
1593 HLSLAppliedSemanticAttr *Semantic,
1594 llvm::dxbc::PSV::SemanticKind SemanticKind,
1595 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1596 SemanticSignatures &Signature) {
1597 switch (SemanticKind) {
1598 case llvm::dxbc::PSV::SemanticKind::Position:
1599 assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
1600 llvm::Triple::Domain, llvm::Triple::Geometry,
1601 llvm::Triple::Mesh},
1602 Stage) &&
1603 "SV_Position is in an unavailable stage and should have been "
1604 "diagnosed by Sema");
1605 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1606 Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
1607 "FIXME: storing SV_Position is not yet implemented for this "
1608 "shader stage");
1609 if (CGM.getTarget().getTriple().isDXIL()) {
1610 emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1611 return;
1612 }
1613 if (CGM.getTarget().getTriple().isSPIRV()) {
1614 createSPIRVBuiltinStore(B, M&: CGM.getModule(), Source,
1615 Name: Semantic->getAttrName()->getName(),
1616 /* BuiltIn::Position */ BuiltInID: 0);
1617 return;
1618 }
1619 break;
1620 case llvm::dxbc::PSV::SemanticKind::Target:
1621 assert(Stage == llvm::Triple::Pixel &&
1622 "SV_Target is in an unavailable stage and should have been "
1623 "diagnosed by Sema");
1624 emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1625 return;
1626 default:
1627 break;
1628 }
1629
1630 llvm_unreachable(
1631 "Store hasn't been implemented yet for this system semantic. FIXME");
1632}
1633
1634llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1635 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1636 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1637 SemanticSignatures &Signature) {
1638
1639 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1640 llvm::dxbc::PSV::SemanticKind SemanticKind =
1641 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
1642 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1643 assert(ShaderAttr && "Entry point has no shader attribute");
1644 llvm::hlsl::SemanticInterpretation Interpretation =
1645 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
1646 IOTy: llvm::hlsl::IOType::In);
1647 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
1648 "invalid semantic should have been diagnosed by Sema");
1649 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
1650 return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index, Signature);
1651 return emitSystemSemanticLoad(B, Type, Decl, Semantic, SemanticKind,
1652 Stage: ShaderAttr->getType(), Index, Signature);
1653}
1654
1655void CGHLSLRuntime::handleScalarSemanticStore(IRBuilder<> &B,
1656 const FunctionDecl *FD,
1657 llvm::Value *Source,
1658 const clang::DeclaratorDecl *Decl,
1659 HLSLAppliedSemanticAttr *Semantic,
1660 SemanticSignatures &Signature) {
1661 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1662 llvm::dxbc::PSV::SemanticKind SemanticKind =
1663 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
1664 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1665 assert(ShaderAttr && "Entry point has no shader attribute");
1666
1667 llvm::hlsl::SemanticInterpretation Interpretation =
1668 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
1669 IOTy: llvm::hlsl::IOType::Out);
1670 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
1671 "invalid semantic should have been diagnosed by Sema");
1672
1673 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
1674 return emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1675 emitSystemSemanticStore(B, Source, Decl, Semantic, SemanticKind,
1676 Stage: ShaderAttr->getType(), Index, Signature);
1677}
1678
1679std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1680CGHLSLRuntime::handleStructSemanticLoad(
1681 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1682 const clang::DeclaratorDecl *Decl,
1683 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1684 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1685 SemanticSignatures &Signature,
1686 llvm::hlsl::InterpolationModifier Modifiers) {
1687 const llvm::StructType *ST = cast<StructType>(Val: Type);
1688 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
1689
1690 assert(RD->getNumFields() == ST->getNumElements());
1691
1692 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
1693 auto FieldDecl = RD->field_begin();
1694 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
1695 auto [ChildValue, NextAttr] =
1696 handleSemanticLoad(B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin,
1697 end: AttrEnd, Signature, Modifiers);
1698 AttrBegin = NextAttr;
1699 assert(ChildValue);
1700 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
1701 ++FieldDecl;
1702 }
1703
1704 return std::make_pair(x&: Aggregate, y&: AttrBegin);
1705}
1706
1707specific_attr_iterator<HLSLAppliedSemanticAttr>
1708CGHLSLRuntime::handleStructSemanticStore(
1709 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1710 const clang::DeclaratorDecl *Decl,
1711 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1712 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1713 SemanticSignatures &Signature,
1714 llvm::hlsl::InterpolationModifier Modifiers) {
1715
1716 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
1717
1718 const clang::RecordDecl *RD = nullptr;
1719 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
1720 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
1721 else
1722 RD = Decl->getType()->getAsRecordDecl();
1723 assert(RD);
1724
1725 assert(RD->getNumFields() == ST->getNumElements());
1726
1727 auto FieldDecl = RD->field_begin();
1728 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
1729 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
1730 AttrBegin = handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin,
1731 AttrEnd, Signature, Modifiers);
1732 }
1733
1734 return AttrBegin;
1735}
1736
1737std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1738CGHLSLRuntime::handleSemanticLoad(
1739 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1740 const clang::DeclaratorDecl *Decl,
1741 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1742 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1743 SemanticSignatures &Signature,
1744 llvm::hlsl::InterpolationModifier Modifiers) {
1745 assert(AttrBegin != AttrEnd);
1746 // Pass the enclosing declaration's mask down the traversal, replacing (not
1747 // merging) it when an inner field has its own interpolation modifiers.
1748 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
1749 Modifiers =
1750 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
1751 if (Type->isStructTy())
1752 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
1753 Signature, Modifiers);
1754
1755 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1756 ++AttrBegin;
1757 size_t PreviousSize = Signature.size();
1758 llvm::Value *Value =
1759 handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr, Signature);
1760 // Intrinsic-only system values and SPIR-V loads do not add DXIL signature
1761 // elements. Non-pixel inputs retain Undefined.
1762 if (Signature.size() != PreviousSize) {
1763 auto &Element = Signature.back();
1764 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
1765 Mode: llvm::hlsl::getInterpolationMode(Modifiers), CompType: Element.CompType,
1766 Kind: Element.SemanticKind, Stage: FD->getAttr<HLSLShaderAttr>()->getType(),
1767 IO: llvm::hlsl::IOType::In);
1768 }
1769 return std::make_pair(x&: Value, y&: AttrBegin);
1770}
1771
1772specific_attr_iterator<HLSLAppliedSemanticAttr>
1773CGHLSLRuntime::handleSemanticStore(
1774 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1775 const clang::DeclaratorDecl *Decl,
1776 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1777 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1778 SemanticSignatures &Signature,
1779 llvm::hlsl::InterpolationModifier Modifiers) {
1780 assert(AttrBegin != AttrEnd);
1781 // An inner field overrides the enclosing return declaration's modifiers.
1782 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
1783 Modifiers =
1784 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
1785 if (Source->getType()->isStructTy())
1786 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
1787 Signature, Modifiers);
1788
1789 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1790 ++AttrBegin;
1791 size_t PreviousSize = Signature.size();
1792 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr, Signature);
1793 if (Signature.size() != PreviousSize) {
1794 auto &Element = Signature.back();
1795 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
1796 Mode: llvm::hlsl::getInterpolationMode(Modifiers), CompType: Element.CompType,
1797 Kind: Element.SemanticKind, Stage: FD->getAttr<HLSLShaderAttr>()->getType(),
1798 IO: llvm::hlsl::IOType::Out);
1799 }
1800 return AttrBegin;
1801}
1802
1803void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
1804 llvm::Function *Fn) {
1805 SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
1806 SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
1807
1808 llvm::Module &M = CGM.getModule();
1809 llvm::LLVMContext &Ctx = M.getContext();
1810 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
1811 Function *EntryFn =
1812 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
1813
1814 // Copy function attributes over, we have no argument or return attributes
1815 // that can be valid on the real entry.
1816 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
1817 Attrs: Fn->getAttributes().getFnAttrs());
1818 EntryFn->setAttributes(NewAttrs);
1819 setHLSLEntryAttributes(FD, Fn: EntryFn);
1820
1821 // Set the called function as internal linkage.
1822 Fn->setLinkage(GlobalValue::InternalLinkage);
1823
1824 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
1825 IRBuilder<> B(BB);
1826 llvm::SmallVector<Value *> Args;
1827
1828 SmallVector<OperandBundleDef, 1> OB;
1829 if (CGM.shouldEmitConvergenceTokens()) {
1830 assert(EntryFn->isConvergent());
1831 llvm::Value *I =
1832 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
1833 llvm::Value *bundleArgs[] = {I};
1834 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1835 }
1836
1837 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
1838
1839 unsigned SRetOffset = 0;
1840 for (const auto &Param : Fn->args()) {
1841 if (Param.hasStructRetAttr()) {
1842 SRetOffset = 1;
1843 llvm::Type *VarType = Param.getParamStructRetType();
1844 llvm::Value *Var =
1845 CGM.getLangOpts().EmitLogicalPointer
1846 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
1847 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
1848 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
1849 Args.push_back(Elt: Var);
1850 continue;
1851 }
1852
1853 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
1854 llvm::Value *SemanticValue = nullptr;
1855 // FIXME: support inout/out parameters for semantics.
1856 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
1857 PD->getAttr<HLSLParamModifierAttr>()) {
1858 llvm_unreachable("Not handled yet");
1859 } else {
1860 llvm::Type *ParamType = nullptr;
1861 if (Param.hasByValAttr())
1862 ParamType = Param.getParamByValType();
1863 else if (PD->getType()->isRecordType())
1864 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
1865 else
1866 ParamType = Param.getType();
1867
1868 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1869 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
1870 auto Result = handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd,
1871 Signature&: InputSignature);
1872 SemanticValue = Result.first;
1873 if (!SemanticValue)
1874 return;
1875 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
1876 llvm::Value *Var =
1877 CGM.getLangOpts().EmitLogicalPointer
1878 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
1879 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
1880 B.CreateStore(Val: SemanticValue, Ptr: Var);
1881 SemanticValue = Var;
1882 }
1883 }
1884
1885 assert(SemanticValue);
1886 Args.push_back(Elt: SemanticValue);
1887 }
1888
1889 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
1890 CI->setCallingConv(Fn->getCallingConv());
1891
1892 if (Fn->getReturnType() != CGM.VoidTy)
1893 // Element type is unused, so set to dummy value (NULL).
1894 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
1895
1896 for (auto &SourcePair : OutputSemantic) {
1897 llvm::Value *Source = SourcePair.first;
1898 llvm::Type *ElementType = SourcePair.second;
1899 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
1900 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
1901
1902 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1903 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
1904 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd,
1905 Signature&: OutputSignature);
1906 }
1907
1908 B.CreateRetVoid();
1909
1910 // Add and identify root signature to function, if applicable
1911 for (const Attr *Attr : FD->getAttrs()) {
1912 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
1913 auto *RSDecl = RSAttr->getSignatureDecl();
1914 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
1915 Fn: EntryFn, M);
1916 }
1917 }
1918
1919 if (CGM.getTarget().getTriple().isDXIL()) {
1920 // Use the entry's shader stage, not the target environment, which may be
1921 // 'library' when compiling multiple entry points.
1922 auto Stage = FD->getAttr<HLSLShaderAttr>()->getType();
1923 using PackingMode = CodeGenOptions::HLSLSemanticSignaturePackingMode;
1924 auto Mode = CGM.getCodeGenOpts().getHLSLSemanticSignaturePacking();
1925 for (auto IOTy : {llvm::hlsl::IOType::In, llvm::hlsl::IOType::Out}) {
1926 bool IsOutput = IOTy == llvm::hlsl::IOType::Out;
1927 auto &Signature = IsOutput ? OutputSignature : InputSignature;
1928 auto Packed = packSemanticSignature(Elements: Signature, Stage, IOTy, Mode,
1929 UseNative16BitTypes: CGM.getLangOpts().NativeHalfType);
1930 if (!Packed) {
1931 CGM.getDiags().Report(Loc: FD->getLocation(),
1932 DiagID: diag::err_hlsl_signature_packing)
1933 << IsOutput << llvm::toString(E: Packed.takeError());
1934 return;
1935 }
1936 // Library entry points retain stage-specific packing without warnings.
1937 if (Mode != PackingMode::Default &&
1938 CGM.getTarget().getTriple().getEnvironment() !=
1939 llvm::Triple::Library &&
1940 ((Stage == llvm::Triple::Vertex && !IsOutput) ||
1941 (Stage == llvm::Triple::Pixel && IsOutput)))
1942 CGM.getDiags().Report(Loc: FD->getLocation(),
1943 DiagID: diag::warn_hlsl_signature_packing_ignored)
1944 << (Mode == PackingMode::Optimized ? "optimized" : "prefix-stable")
1945 << IsOutput;
1946 }
1947 }
1948
1949 addSemanticSignatureMD(InputElements: InputSignature, OutputElements: OutputSignature, Fn: EntryFn, M);
1950}
1951
1952static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
1953 bool CtorOrDtor) {
1954 const auto *GV =
1955 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
1956 if (!GV)
1957 return;
1958 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1959 if (!CA)
1960 return;
1961 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
1962 // HLSL neither supports priorities or COMDat values, so we will check those
1963 // in an assert but not handle them.
1964
1965 for (const auto &Ctor : CA->operands()) {
1966 if (isa<ConstantAggregateZero>(Val: Ctor))
1967 continue;
1968 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
1969
1970 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
1971 "HLSL doesn't support setting priority for global ctors.");
1972 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
1973 "HLSL doesn't support COMDat for global ctors.");
1974 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
1975 }
1976}
1977
1978void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
1979 llvm::Module &M = CGM.getModule();
1980 SmallVector<Function *> CtorFns;
1981 SmallVector<Function *> DtorFns;
1982 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
1983 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
1984
1985 // Insert a call to the global constructor at the beginning of the entry block
1986 // to externally exported functions. This is a bit of a hack, but HLSL allows
1987 // global constructors, but doesn't support driver initialization of globals.
1988 for (auto &F : M.functions()) {
1989 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
1990 continue;
1991 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
1992 Instruction *IP = &*F.getEntryBlock().begin();
1993 SmallVector<OperandBundleDef, 1> OB;
1994 if (Token) {
1995 llvm::Value *bundleArgs[] = {Token};
1996 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1997 IP = Token->getNextNode();
1998 }
1999 IRBuilder<> B(IP);
2000 for (auto *Fn : CtorFns) {
2001 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
2002 CI->setCallingConv(Fn->getCallingConv());
2003 }
2004
2005 // Insert global dtors before the terminator of the last instruction
2006 B.SetInsertPoint(F.back().getTerminator());
2007 for (auto *Fn : DtorFns) {
2008 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
2009 CI->setCallingConv(Fn->getCallingConv());
2010 }
2011 }
2012
2013 // No need to keep global ctors/dtors for non-lib profile after call to
2014 // ctors/dtors added for entry.
2015 Triple T(M.getTargetTriple());
2016 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
2017 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
2018 GV->eraseFromParent();
2019 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
2020 GV->eraseFromParent();
2021 }
2022}
2023
2024static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
2025 Intrinsic::ID IntrID,
2026 ArrayRef<llvm::Value *> Args) {
2027
2028 LLVMContext &Ctx = CGM.getLLVMContext();
2029 llvm::Function *InitResFunc =
2030 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
2031 Linkage: llvm::GlobalValue::InternalLinkage,
2032 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
2033 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
2034
2035 llvm::BasicBlock *EntryBB =
2036 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
2037 CGBuilderTy Builder(CGM, Ctx);
2038 const DataLayout &DL = CGM.getModule().getDataLayout();
2039 Builder.SetInsertPoint(EntryBB);
2040
2041 // Make sure the global variable is buffer resource handle
2042 llvm::Type *HandleTy = GV->getValueType();
2043 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
2044
2045 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
2046 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
2047 Name: Twine(GV->getName()).concat(Suffix: "_h"));
2048
2049 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
2050 Builder.CreateRetVoid();
2051
2052 CGM.AddCXXGlobalInit(F: InitResFunc);
2053}
2054
2055void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
2056 llvm::GlobalVariable *GV) {
2057 ResourceBindingAttrs Binding(BufDecl);
2058 assert(Binding.hasBinding() &&
2059 "cbuffer/tbuffer should always have resource binding attribute");
2060
2061 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2062 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
2063 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
2064 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
2065
2066 // buffer with explicit binding
2067 if (Binding.isExplicit()) {
2068 llvm::Intrinsic::ID IntrinsicID =
2069 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
2070 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
2071 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
2072 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2073 } else {
2074 // buffer with implicit binding
2075 llvm::Intrinsic::ID IntrinsicID =
2076 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
2077 auto *OrderID =
2078 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
2079 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
2080 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2081 }
2082}
2083
2084void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
2085 llvm::GlobalVariable *GV) {
2086 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
2087 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
2088 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
2089 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
2090}
2091
2092llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
2093 if (!CGM.shouldEmitConvergenceTokens())
2094 return nullptr;
2095
2096 auto E = BB.end();
2097 for (auto I = BB.begin(); I != E; ++I) {
2098 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
2099 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
2100 return II;
2101 }
2102 }
2103 llvm_unreachable("Convergence token should have been emitted.");
2104 return nullptr;
2105}
2106
2107class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
2108public:
2109 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
2110 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
2111 OpaqueValueVisitor() {}
2112
2113 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
2114 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
2115 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
2116 // traversal, the temporary containing the copy out will not have
2117 // been created yet.
2118 return false;
2119 }
2120
2121 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
2122 // Traverse the source expression first.
2123 if (E->getSourceExpr())
2124 TraverseStmt(S: E->getSourceExpr());
2125
2126 // Then add this OVE if we haven't seen it before.
2127 if (Visited.insert(Ptr: E).second)
2128 OVEs.push_back(Elt: E);
2129
2130 return true;
2131 }
2132};
2133
2134void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
2135 InitListExpr *E) {
2136
2137 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
2138 OpaqueValueVisitor Visitor;
2139 Visitor.TraverseStmt(S: E);
2140 for (auto *OVE : Visitor.OVEs) {
2141 if (CGF.isOpaqueValueEmitted(E: OVE))
2142 continue;
2143 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
2144 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
2145 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
2146 } else {
2147 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
2148 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
2149 }
2150 }
2151}
2152
2153std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
2154 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
2155 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
2156 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
2157 "expected resource array subscript expression");
2158
2159 // Let clang codegen handle local and static resource array subscripts,
2160 // or when the subscript references on opaque expression (as part of
2161 // ArrayInitLoopExpr AST node).
2162 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
2163 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
2164 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2165 ArrayDecl->getStorageClass() == SC_Static)
2166 return std::nullopt;
2167
2168 // get the resource array type
2169 ASTContext &AST = ArrayDecl->getASTContext();
2170 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
2171 assert(ResArrayTy->isHLSLResourceRecordArray() &&
2172 "expected array of resource classes");
2173
2174 // Iterate through all nested array subscript expressions to calculate
2175 // the index in the flattened resource array (if this is a multi-
2176 // dimensional array). The index is calculated as a sum of all indices
2177 // multiplied by the total size of the array at that level.
2178 Value *Index = nullptr;
2179 const ArraySubscriptExpr *ASE = ArraySubsExpr;
2180 while (ASE != nullptr) {
2181 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
2182 if (const auto *ArrayTy =
2183 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
2184 Value *Multiplier = llvm::ConstantInt::get(
2185 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
2186 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
2187 }
2188 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
2189 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
2190 }
2191
2192 // Find binding info for the resource array. For implicit binding
2193 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
2194 ResourceBindingAttrs Binding(ArrayDecl);
2195 assert(Binding.hasBinding() &&
2196 "resource array must have a binding attribute");
2197
2198 // Find the individual resource type.
2199 QualType ResultTy = ArraySubsExpr->getType();
2200 QualType ResourceTy =
2201 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
2202
2203 // Create a temporary variable for the result, which is either going
2204 // to be a single resource instance or a local array of resources (we need to
2205 // return an LValue).
2206 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
2207 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
2208 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
2209 kind: NormalEHLifetimeMarker, A: TmpVar);
2210
2211 AggValueSlot ValueSlot = AggValueSlot::forAddr(
2212 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
2213 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
2214 mayOverlap: AggValueSlot::DoesNotOverlap);
2215
2216 // Calculate total array size (= range size).
2217 llvm::Value *Range = llvm::ConstantInt::getSigned(
2218 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
2219
2220 // If the result of the subscript operation is a single resource, call the
2221 // constructor.
2222 if (ResultTy == ResourceTy) {
2223 CallArgList Args;
2224 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
2225 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
2226 Name: ArrayDecl->getName(), Binding, Args);
2227
2228 if (!CreateMethod) {
2229 // This can happen if someone creates an array of structs that looks like
2230 // an HLSL resource record array but it does not have the required static
2231 // create method. No binding will be generated for it.
2232 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
2233 "create method lookup should always succeed for built-in resource "
2234 "records");
2235 return std::nullopt;
2236 }
2237
2238 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
2239
2240 } else {
2241 // The result of the subscript operation is a local resource array which
2242 // needs to be initialized.
2243 const ConstantArrayType *ArrayTy =
2244 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
2245 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2246 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
2247 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
2248 if (!EndIndex)
2249 return std::nullopt;
2250 }
2251 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
2252}
2253
2254// Initialize all resources of a global resource array into provided slot.
2255bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
2256 const VarDecl *ArrayDecl,
2257 AggValueSlot &DestSlot) {
2258 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2259 ArrayDecl->hasGlobalStorage() &&
2260 ArrayDecl->getStorageClass() != SC_Static &&
2261 "expected global non-static resource array");
2262
2263 // Find binding info for the resource array. For implicit binding
2264 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
2265 ResourceBindingAttrs Binding(ArrayDecl);
2266 assert(Binding.hasBinding() &&
2267 "resource array must have a binding attribute");
2268
2269 // Find the individual resource type.
2270 ASTContext &AST = ArrayDecl->getASTContext();
2271 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
2272 const auto *ResArrayTy =
2273 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
2274
2275 // Create Value for index and total array size (= range size).
2276 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
2277 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2278 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
2279
2280 // Initialize individual resources in the array into DestSlot.
2281 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2282 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
2283 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
2284 return EndIndex.has_value();
2285}
2286
2287// If the expression is a global resource array, initialize all of its resources
2288// into Dest. Returns false if no initialization has been performed and the
2289// array copy should be handled by the default codegen.
2290bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
2291 AggValueSlot &DestSlot) {
2292 assert(E->getType()->isHLSLResourceRecordArray() &&
2293 "expected resource array");
2294
2295 // Find the array declaration for the expression. Fallback to the default
2296 // handling if it's not a global resource array.
2297 const VarDecl *ArrayDecl =
2298 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
2299 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2300 ArrayDecl->getStorageClass() == SC_Static)
2301 return false;
2302
2303 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
2304}
2305
2306// If the expression is a global resource array, create a temporary and
2307// initialize all of its resources, and return it as an LValue. Returns nullopt
2308// if no initialization has been performed and the handling should follow the
2309// default path.
2310std::optional<LValue>
2311CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
2312 const VarDecl *ArrayDecl) {
2313 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2314 "expected resource array declaration");
2315
2316 if (!ArrayDecl->hasGlobalStorage() ||
2317 ArrayDecl->getStorageClass() == SC_Static)
2318 return std::nullopt;
2319
2320 AggValueSlot TmpArraySlot =
2321 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
2322 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
2323 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
2324 Source: AlignmentSource::Decl);
2325 return std::nullopt;
2326}
2327
2328RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
2329 CodeGenFunction &CGF) {
2330
2331 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
2332 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
2333 "expected cbuffer matrix");
2334
2335 QualType MatQualTy = LV.getType();
2336 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
2337 Address SrcAddr = LV.getAddress();
2338
2339 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
2340 return SrcAddr;
2341
2342 RawAddress DestAlloca =
2343 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
2344 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
2345 return DestAlloca;
2346}
2347
2348std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
2349 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
2350 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
2351 // Find the element type to index by first padding the element type per HLSL
2352 // buffer rules, and then padding out to a 16-byte register boundary if
2353 // necessary.
2354 llvm::Type *LayoutTy =
2355 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
2356 uint64_t LayoutSizeInBits =
2357 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
2358 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
2359 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
2360 if (RowAlignedSize > ElementSize) {
2361 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
2362 CGM, NumBytes: RowAlignedSize - ElementSize);
2363 assert(Padding && "No padding type for target?");
2364 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
2365 /*isPacked=*/true);
2366 }
2367
2368 // If the layout type doesn't introduce any padding, we don't need to do
2369 // anything special.
2370 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
2371 if (LayoutTy == OrigTy)
2372 return std::nullopt;
2373
2374 LValueBaseInfo EltBaseInfo;
2375 TBAAAccessInfo EltTBAAInfo;
2376
2377 // Index into the object as-if we have an array of the padded element type,
2378 // and then dereference the element itself to avoid reading padding that may
2379 // be past the end of the in-memory object.
2380 SmallVector<llvm::Value *, 2> Indices;
2381 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
2382 Indices.push_back(Elt: Idx);
2383 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
2384
2385 if (CGF.getLangOpts().EmitLogicalPointer) {
2386 // The fact that we emit an array-to-pointer decay might be an oversight,
2387 // but for now, we simply ignore it (see #179951).
2388 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
2389 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
2390
2391 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
2392 Address Addr = LV.getAddress();
2393 LayoutTy = llvm::ArrayType::get(
2394 ElementType: LayoutTy,
2395 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
2396 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
2397 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
2398 Addr =
2399 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
2400 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2401 }
2402
2403 Address Addr =
2404 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
2405 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
2406 IdxList: Indices, Name: "cbufferidx");
2407 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
2408 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2409}
2410
2411std::optional<LValue>
2412CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
2413 const MemberExpr *ME) {
2414 assert((ME->getType()->isHLSLResourceRecord() ||
2415 ME->getType()->isHLSLResourceRecordArray()) &&
2416 "expected resource member expression");
2417
2418 const VarDecl *ResourceVD =
2419 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
2420 if (!ResourceVD)
2421 return std::nullopt;
2422
2423 // Handle member of resource array type.
2424 if (ResourceVD->getType()->isHLSLResourceRecordArray())
2425 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
2426
2427 GlobalVariable *ResGV =
2428 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
2429 const DataLayout &DL = CGM.getDataLayout();
2430 llvm::Type *Ty = ResGV->getValueType();
2431 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
2432 Address Addr = Address(ResGV, Ty, Align);
2433 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
2434 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2435 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
2436 return LV;
2437}
2438
2439bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
2440 const LValue &SrcLV,
2441 AggValueSlot &DestSlot) {
2442 assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
2443 "expected expression in HLSL constant address space");
2444 assert(!E->getType()->isHLSLResourceRecord() &&
2445 !E->getType()->isHLSLResourceRecordArray() &&
2446 "direct accesses to resource types should be handled separately");
2447
2448 if (DestSlot.isIgnored())
2449 return false;
2450
2451 QualType Ty = E->getType();
2452 Address DstPtr = DestSlot.getAddress();
2453 Address SrcPtr = SrcLV.getAddress();
2454
2455 // If there are no intangible types, we don't need to lookup associated
2456 // resources.
2457 if (!Ty->isHLSLIntangibleType())
2458 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty);
2459
2460 // Handle structs with intangible types by setting the resource fields
2461 // of the destination struct with the resources associated with the global
2462 // struct.
2463 EmbeddedResourceNameBuilder NameBuilder;
2464 const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
2465 AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
2466
2467 // Callback to fill in the associated resource.
2468 auto EmitResFn = [&](AggValueSlot &ResSlot) {
2469 const VarDecl *ResDecl = AssociatedResources.getNextResource();
2470 assert(ResDecl && "associated resource declaration not found");
2471
2472 // Check that the resource type of dest and src matches.
2473 [[maybe_unused]] llvm::Type *DestType =
2474 ResSlot.getAddress().getElementType();
2475 [[maybe_unused]] llvm::Type *SrcConvertedType =
2476 CGM.getTypes().ConvertTypeForMem(T: ResDecl->getType());
2477 assert(DestType == SrcConvertedType && "resource slot type mismatch");
2478
2479 if (ResDecl->getType()->isHLSLResourceRecord())
2480 copyGlobalResource(CGF, ResourceVD: ResDecl, DestSlot&: ResSlot);
2481 else
2482 initializeGlobalResourceArray(CGF, ArrayDecl: ResDecl, DestSlot&: ResSlot);
2483 };
2484
2485 auto Result =
2486 HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty, EmitResFn);
2487 assert(AssociatedResources.getNextResource() == nullptr &&
2488 "expected all associated resources to be processed");
2489 return Result;
2490}
2491
2492LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
2493 const MemberExpr *E) {
2494 LValue Base =
2495 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
2496 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
2497 assert(Field && "Unexpected access into HLSL buffer");
2498
2499 const RecordDecl *Rec = Field->getParent();
2500
2501 // Work out the buffer layout type to index into.
2502 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
2503 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
2504 // Since this is a member of an object in the buffer and not the buffer's
2505 // struct/class itself, we shouldn't have any offsets on the members we need
2506 // to contend with.
2507 CGHLSLOffsetInfo EmptyOffsets;
2508 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
2509 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
2510
2511 // Get the field index for the layout struct, accounting for padding.
2512 unsigned FieldIdx =
2513 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
2514 assert(FieldIdx < LayoutTy->getNumElements() &&
2515 "Layout struct is smaller than member struct");
2516 unsigned Skipped = 0;
2517 for (unsigned I = 0; I <= FieldIdx;) {
2518 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
2519 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
2520 ++Skipped;
2521 else
2522 ++I;
2523 }
2524 FieldIdx += Skipped;
2525 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
2526
2527 // Now index into the struct, making sure that the type we return is the
2528 // buffer layout type rather than the original type in the AST.
2529 QualType FieldType = Field->getType();
2530 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
2531 CharUnits Align = CharUnits::fromQuantity(
2532 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
2533
2534 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
2535 ? CGF.Builder.CreateStructuredGEP(
2536 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
2537 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
2538 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
2539 Idx: FieldIdx, Name: Field->getName());
2540 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
2541
2542 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
2543 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2544 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
2545 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
2546
2547 return LV;
2548}
2549