1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/MatrixUtils.h"
28#include "clang/AST/RecursiveASTVisitor.h"
29#include "clang/AST/Type.h"
30#include "clang/Basic/DiagnosticDriver.h"
31#include "clang/Basic/DiagnosticFrontend.h"
32#include "clang/Basic/SourceManager.h"
33#include "clang/Basic/TargetOptions.h"
34#include "llvm/ADT/DenseMap.h"
35#include "llvm/ADT/Enum.h"
36#include "llvm/ADT/STLExtras.h"
37#include "llvm/ADT/ScopeExit.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/SmallString.h"
40#include "llvm/ADT/SmallVector.h"
41#include "llvm/ADT/StringSwitch.h"
42#include "llvm/Frontend/HLSL/HLSLResource.h"
43#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
44#include "llvm/Frontend/HLSL/SemanticSignaturePacking.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/GlobalVariable.h"
48#include "llvm/IR/IntrinsicInst.h"
49#include "llvm/IR/LLVMContext.h"
50#include "llvm/IR/Metadata.h"
51#include "llvm/IR/Module.h"
52#include "llvm/IR/Type.h"
53#include "llvm/IR/Value.h"
54#include "llvm/Support/Alignment.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/FormatVariadic.h"
57#include "llvm/Support/Path.h"
58#include "llvm/Transforms/Utils/ModuleUtils.h"
59#include <array>
60#include <cstdint>
61#include <optional>
62
63using namespace clang;
64using namespace CodeGen;
65using namespace clang::hlsl;
66using namespace llvm;
67
68using llvm::hlsl::CBufferRowSizeInBytes;
69
70namespace {
71
72enum class SPIRVExecutionMode : uint32_t {
73 DerivativeGroupQuadsKHR = 5289,
74 DerivativeGroupLinearKHR = 5290
75};
76
77void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
78 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
79 // Assume ValVersionStr is legal here.
80 VersionTuple Version;
81 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
82 Version.getSubminor() || !Version.getMinor()) {
83 return;
84 }
85
86 uint64_t Major = Version.getMajor();
87 uint64_t Minor = *Version.getMinor();
88
89 auto &Ctx = M.getContext();
90 IRBuilder<> B(M);
91 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
92 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
93 StringRef DXILValKey = "dx.valver";
94 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
95 DXILValMD->addOperand(M: Val);
96}
97
98void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
99 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
100 llvm::Function *Fn, llvm::Module &M) {
101 auto &Ctx = M.getContext();
102
103 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
104 MDNode *RootSignature = RSBuilder.BuildRootSignature();
105
106 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
107 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
108 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
109 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
110
111 StringRef RootSignatureValKey = "dx.rootsignatures";
112 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
113 RootSignatureValMD->addOperand(M: MDVals);
114}
115
116Expected<unsigned> packSemanticSignature(
117 MutableArrayRef<llvm::hlsl::SemanticSignatureElement> Elements,
118 llvm::Triple::EnvironmentType Stage, llvm::hlsl::IOType IOTy,
119 CodeGenOptions::HLSLSemanticSignaturePackingMode Mode,
120 bool UseNative16BitTypes) {
121 using namespace llvm::hlsl;
122 // Vertex inputs and pixel outputs have fixed packing rules, independent of
123 // the packing mode used between programmable stages.
124 if (Stage == llvm::Triple::Vertex && IOTy == IOType::In)
125 return packSignatureStacked(Elements, ShaderStage: Stage, IOTy);
126 if (Stage == llvm::Triple::Pixel && IOTy == IOType::Out)
127 return packSignatureIndexed(Elements, ShaderStage: Stage, IOTy);
128 if (Mode == CodeGenOptions::HLSLSemanticSignaturePackingMode::Optimized)
129 return packSignatureOptimized(Elements, ShaderStage: Stage, IOTy, UseNative16BitTypes);
130 return packSignaturePrefixStable(Elements, ShaderStage: Stage, IOTy, UseNative16BitTypes);
131}
132
133MDNode *buildSemanticSignatureMD(
134 ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
135 if (Elements.empty())
136 return nullptr;
137
138 SmallVector<Metadata *> ElementMD;
139 for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
140 ElementMD.push_back(Elt: Element.toMetadata(Ctx));
141 return MDNode::get(Context&: Ctx, MDs: ElementMD);
142}
143
144void addSemanticSignatureMD(
145 ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
146 ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
147 llvm::Function *Fn, llvm::Module &M) {
148 if (InputElements.empty() && OutputElements.empty())
149 return;
150
151 LLVMContext &Ctx = M.getContext();
152 MDNode *InputSignature = buildSemanticSignatureMD(Elements: InputElements, Ctx);
153 MDNode *OutputSignature = buildSemanticSignatureMD(Elements: OutputElements, Ctx);
154 MDNode *MDVals = MDNode::get(
155 Context&: Ctx, MDs: {ValueAsMetadata::get(V: Fn), InputSignature, OutputSignature});
156
157 M.getOrInsertNamedMetadata(Name: "dx.semantic.signatures")->addOperand(M: MDVals);
158}
159
160static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
161 AggValueSlot &DestSlot) {
162 GlobalVariable *ResGV =
163 cast<GlobalVariable>(Val: CGF.CGM.GetAddrOfGlobalVar(D: ResourceVD));
164 assert(ResGV && "expected valid global variable");
165 CGF.Builder.CreateStore(Val: ResGV, Addr: DestSlot.getAddress());
166}
167
168// Given a MemberExpr of a resource or resource array type, find the parent
169// VarDecl of the struct or class instance that contains this resource and
170// build the full resource name based on the member access path.
171//
172// For example, for a member access like "myStructArray[0].memberA",
173// this function will find the VarDecl of "myStructArray" and use the
174// EmbeddedResourceNameBuilder to build the resource name
175// "myStructArray.0.memberA".
176//
177// This also works for a record type expression that has some embedded
178// resources. It finds the parent VarDecl of that record and builds a partial
179// name which is the prefix of the resource globals associated with the
180// declaration.
181static const VarDecl *findStructResourceParentDeclAndBuildName(
182 const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
183
184 SmallVector<const Expr *> WorkList;
185 const VarDecl *VD = nullptr;
186
187 for (;;) {
188 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
189 assert(isa<VarDecl>(DRE->getDecl()) &&
190 "member expr base is not a var decl");
191 VD = cast<VarDecl>(Val: DRE->getDecl());
192 NameBuilder.pushName(N: VD->getName());
193 break;
194 }
195
196 WorkList.push_back(Elt: E);
197 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
198 E = MExp->getBase();
199 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
200 E = ICE->getSubExpr();
201 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
202 E = ASE->getBase();
203 else if (isa<CXXThisExpr>(Val: E))
204 // Resource member access on "this" pointer not yet implemented
205 // (llvm/llvm-project#190299)
206 return nullptr;
207 else
208 llvm_unreachable("unexpected expr type in resource member access");
209
210 assert(E && "expected valid expression");
211 }
212
213 while (!WorkList.empty()) {
214 E = WorkList.pop_back_val();
215 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
216 NameBuilder.pushName(
217 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
218 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
219 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
220 CXXRecordDecl *DerivedRD =
221 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
222 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
223 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
224 }
225 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
226 const Expr *IdxExpr = ASE->getIdx();
227 std::optional<llvm::APSInt> Value =
228 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
229 assert(Value &&
230 "expected constant index in struct with resource array access");
231 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
232 } else {
233 llvm_unreachable("unexpected expr type in resource member access");
234 }
235 }
236 return VD;
237}
238
239// Given a MemberExpr of a resource or resource array type, find the
240// corresponding global resource declaration associated with the owning struct
241// or class instance via HLSLAssociatedResourceDeclAttr.
242static const VarDecl *
243findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
244
245 EmbeddedResourceNameBuilder NameBuilder;
246 const VarDecl *ParentVD =
247 findStructResourceParentDeclAndBuildName(E: ME, NameBuilder);
248 if (!ParentVD)
249 return nullptr;
250
251 if (!ParentVD->hasGlobalStorage())
252 return nullptr;
253
254 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
255 for (const Attr *A : ParentVD->getAttrs()) {
256 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
257 VarDecl *AssocResVD = ADA->getResDecl();
258 if (AssocResVD->getIdentifier() == II)
259 return AssocResVD;
260 }
261 }
262 return nullptr;
263}
264
265void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
266 auto &SM = CGM.getContext().getSourceManager();
267 auto &Macros = CGM.getPreprocessorOpts().Macros;
268 auto &CodeGenOpts = CGM.getCodeGenOpts();
269 auto &Ctx = M.getContext();
270
271 // Names and content of shader source code files.
272 llvm::NamedMDNode *DXContents =
273 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
274 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
275 llvm::MDTuple *FileInfo =
276 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
277 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
278 DXContents->addOperand(M: FileInfo);
279 };
280
281 bool Invalid = false;
282 const SrcMgr::SLocEntry *MainLocEntry =
283 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
284 assert(!Invalid && "Main file SLocEntry must not be invalid!");
285 const SrcMgr::ContentCache &MainCCEntry =
286 MainLocEntry->getFile().getContentCache();
287
288 SmallVector<std::pair<std::string, StringRef>> Files;
289 std::optional<SmallString<256>> MainFileName;
290 Files.reserve(N: SM.local_sloc_entry_size());
291 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
292 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
293 if (!LocEntry.isFile())
294 continue;
295
296 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
297 if (isSystem(CK: FInfo.getFileCharacteristic()))
298 continue;
299
300 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
301 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
302 if (!FEntry)
303 continue;
304
305 llvm::SmallString<256> Path = FEntry->getName();
306 llvm::sys::path::native(path&: Path);
307 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
308 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
309 if (!Buffer) {
310 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
311 << Path;
312 continue;
313 }
314
315 if (&MainCCEntry != &CCEntry) {
316 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
317 } else {
318 // Main file should be at first position.
319 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
320 MainFileName.emplace(args&: Path);
321 }
322 }
323 assert(MainFileName && "Main file not found.");
324
325 // Files other that main one should be sorted by name.
326 llvm::sort(C&: Files);
327#ifndef NDEBUG
328 for (unsigned I = 1; I < Files.size(); ++I)
329 assert((Files[I - 1].first != Files[I].first) &&
330 "duplicate files in dx.source.contents");
331#endif
332 llvm::for_each(Range&: Files, F: addFile);
333
334 SmallVector<llvm::Metadata *> Defines;
335 Defines.reserve(N: Macros.size());
336 for (const auto &Macro : Macros) {
337 // Ignore undefs.
338 if (!Macro.second)
339 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
340 }
341 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
342 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
343
344 if (!CodeGenOpts.MainFileName.empty())
345 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
346 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
347 ->addOperand(
348 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
349
350 SmallVector<llvm::Metadata *> Args;
351 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
352 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
353 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
354 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
355 M.getOrInsertNamedMetadata(Name: "dx.source.args")
356 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
357}
358
359// Find array variable declaration from DeclRef expression
360static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
361 E = E->IgnoreImpCasts();
362 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
363 return DRE->getDecl();
364 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
365 E = OVE->getSourceExpr()->IgnoreImpCasts();
366 if (isa<MemberExpr>(Val: E))
367 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
368 return nullptr;
369}
370
371// Find array variable declaration from nested array subscript AST nodes
372static const ValueDecl *getArrayDecl(ASTContext &AST,
373 const ArraySubscriptExpr *ASE) {
374 const Expr *E = nullptr;
375 while (ASE != nullptr) {
376 E = ASE->getBase()->IgnoreImpCasts();
377 if (!E)
378 return nullptr;
379 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
380 }
381 return getArrayDecl(AST, E);
382}
383
384// Get the total size of the array, or 0 if the array is unbounded.
385static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
386 Ty = Ty->getUnqualifiedDesugaredType();
387 assert(Ty->isArrayType() && "expected array type");
388 if (Ty->isIncompleteArrayType())
389 return 0;
390 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
391}
392
393static Value *buildNameForResource(llvm::StringRef BaseName,
394 CodeGenModule &CGM) {
395 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
396 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
397 .getPointer();
398}
399
400static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
401 StorageClass SC = SC_None) {
402 for (auto *Method : Record->methods()) {
403 if (Method->getStorageClass() == SC && Method->getName() == Name)
404 return Method;
405 }
406 return nullptr;
407}
408
409static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
410 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
411 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
412 CallArgList &Args) {
413 assert(Binding.hasBinding() && "at least one binding attribute expected");
414
415 ASTContext &AST = CGM.getContext();
416 CXXMethodDecl *CreateMethod = nullptr;
417 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
418 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
419
420 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
421 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
422 "resources with counter handle must have a binding with counter "
423 "implicit order ID");
424 if (Binding.isExplicit()) {
425 // explicit binding
426 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
427 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
428 const char *Name = Binding.hasCounterImplicitOrderID()
429 ? "__createFromBindingWithImplicitCounter"
430 : "__createFromBinding";
431 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
432 } else {
433 // implicit binding
434 auto *OrderID =
435 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
436 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
437 const char *Name = Binding.hasCounterImplicitOrderID()
438 ? "__createFromImplicitBindingWithImplicitCounter"
439 : "__createFromImplicitBinding";
440 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
441 }
442 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
443 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
444 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
445 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
446 if (HasCounter) {
447 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
448 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
449 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
450 }
451
452 return CreateMethod;
453}
454
455static void callResourceInitMethod(CodeGenFunction &CGF,
456 CXXMethodDecl *CreateMethod,
457 CallArgList &Args, Address ReturnAddress) {
458 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
459 const FunctionProtoType *Proto =
460 CreateMethod->getType()->getAs<FunctionProtoType>();
461 // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
462 // caller declaration is needed for x86 SysV ABI selection.
463 const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
464 Args, Ty: Proto, ChainCall: false, /*ABIInfoFD=*/nullptr);
465 ReturnValueSlot ReturnValue(ReturnAddress, false);
466 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
467 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
468}
469
470// Initializes local resource array variable with global resource array
471// elements. For multi-dimensional arrays it calls itself recursively to
472// initialize its sub-arrays. The Index used in the resource constructor calls
473// will begin at StartIndex and will be incremented for each array element. The
474// last used resource Index is returned to the caller. If the function returns
475// std::nullopt, it indicates an error.
476static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
477 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
478 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
479 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
480 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
481
482 ASTContext &AST = CGF.getContext();
483 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
484 llvm::Value *Index = StartIndex;
485 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
486 const uint64_t ArraySize = ArrayTy->getSExtSize();
487 QualType ElemType = ArrayTy->getElementType();
488 Address TmpArrayAddr = ValueSlot.getAddress();
489
490 // Add additional index to the getelementptr call indices.
491 // This index will be updated for each array element in the loops below.
492 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
493 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
494
495 // For array of arrays, recursively initialize the sub-arrays.
496 if (ElemType->isArrayType()) {
497 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
498 for (uint64_t I = 0; I < ArraySize; I++) {
499 if (I > 0) {
500 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
501 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
502 }
503 std::optional<llvm::Value *> MaybeIndex =
504 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
505 ValueSlot, Range, StartIndex: Index,
506 ResourceName, Binding, PrevGEPIndices: GEPIndices);
507 if (!MaybeIndex)
508 return std::nullopt;
509 Index = *MaybeIndex;
510 }
511 return Index;
512 }
513
514 // For array of resources, initialize each resource in the array.
515 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
516 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
517 CharUnits Align =
518 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
519
520 for (uint64_t I = 0; I < ArraySize; I++) {
521 if (I > 0) {
522 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
523 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
524 }
525 Address ReturnAddress =
526 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
527
528 CallArgList Args;
529 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
530 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
531
532 if (!CreateMethod)
533 // This can happen if someone creates an array of structs that looks like
534 // an HLSL resource record array but it does not have the required static
535 // create method. No binding will be generated for it.
536 return std::nullopt;
537
538 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
539 }
540 return Index;
541}
542
543/// Utility for emitting copies following the HLSL buffer layout rules (ie,
544/// copying out of a cbuffer).
545class HLSLBufferCopyEmitter {
546 CodeGenFunction &CGF;
547 Address DstPtr;
548 Address SrcPtr;
549 llvm::Type *LayoutTy = nullptr;
550
551 SmallVector<llvm::Value *> CurStoreIndices;
552 SmallVector<llvm::Value *> CurLoadIndices;
553
554 using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
555
556 // Creates & returns either a structured.gep or a ptradd/gep depending on
557 // langopts.
558 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
559 ArrayRef<llvm::Value *> Indices) {
560 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
561 if (EmitLogical)
562 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
563
564 llvm::SmallVector<llvm::Value *> GEPIndices;
565 GEPIndices.reserve(N: Indices.size() + 1);
566 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
567 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
568 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
569 }
570
571 bool isBufferLayoutArray(llvm::StructType *ST) {
572 // A buffer layout array is a struct with two elements: the padded array,
573 // and the last element. That is, is should look something like this:
574 //
575 // { [%n x { %type, %padding }], %type }
576 //
577 if (!ST || ST->getNumElements() != 2)
578 return false;
579
580 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
581 if (!PaddedEltsTy)
582 return false;
583
584 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
585 if (!PaddedTy || PaddedTy->getNumElements() != 2)
586 return false;
587
588 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
589 Ty: PaddedTy->getElementType(N: 1)))
590 return false;
591
592 llvm::Type *ElementTy = ST->getElementType(N: 1);
593 if (PaddedTy->getElementType(N: 0) != ElementTy)
594 return false;
595 return true;
596 }
597
598 // Returns true if the type is either a struct representing a resource record,
599 // or an array of structs that are resource records. This assumes a struct is
600 // a resource record if the first element is a target type (resource handle).
601 // This is the case for all target types used by HLSL except the padding type
602 // ("{dx|spirv.Padding"), but padding will never be the first element of a
603 // struct.
604 bool isResourceOrResourceArray(llvm::Type *Ty) {
605 while (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
606 Ty = AT->getElementType();
607
608 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
609 if (!ST || ST->getNumElements() < 1)
610 return false;
611
612 auto *TargetTy = dyn_cast<llvm::TargetExtType>(Val: ST->getElementType(N: 0));
613 return TargetTy != nullptr;
614 }
615
616 void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
617 EmitResourceFnTy EmitResFn) {
618 CharUnits DstAlign =
619 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
620 Address DstAddr(Dst, DstTy, DstAlign);
621 AggValueSlot Slot = AggValueSlot::forAddr(
622 addr: DstAddr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
623 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
624 mayOverlap: AggValueSlot::DoesNotOverlap);
625
626 EmitResFn(Slot);
627 }
628
629 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
630 llvm::ArrayType *DstTy,
631 EmitResourceFnTy EmitResFn) {
632 // Those assumptions are checked by isBufferLayoutArray.
633 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
634 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
635 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
636 ->getElementType(0) == SrcTy->getElementType(1));
637
638 auto *SrcDataTy = SrcTy->getElementType(N: 1);
639 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
640
641 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
642 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
643 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
644 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
645 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
646 EmitResFn);
647 }
648
649 auto *SrcElt =
650 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
651 auto *DstElt = emitAccessChain(
652 BaseTy: DstTy, Base: Dst,
653 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
654 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
655 EmitResFn);
656 }
657
658 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
659 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
660 assert(!isResourceOrResourceArray(DstTy) &&
661 "direct access to resources or resource arrays should be handled "
662 "separately");
663
664 if (isBufferLayoutArray(ST: SrcTy))
665 return emitBufferLayoutCopy(Src, SrcTy, Dst, DstTy: cast<llvm::ArrayType>(Val: DstTy),
666 EmitResFn);
667
668 unsigned SrcIndex = 0;
669 unsigned DstIndex = 0;
670
671 // DstTy layout is in default address space and can include resource types.
672 // SrcTy is in cbuffer layout where resources are filtered out, so the
673 // number of elements in SrcTy can be less than the number of elements in
674 // DstTy.
675 auto *DstST = cast<llvm::StructType>(Val: DstTy);
676 while (DstIndex < DstST->getNumElements()) {
677 llvm::Type *DstEltTy = DstST->getElementType(N: DstIndex);
678 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: DstEltTy)) {
679 DstIndex += 1;
680 continue;
681 }
682 if (isResourceOrResourceArray(Ty: DstEltTy)) {
683 auto *DstElt = emitAccessChain(
684 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
685 emitResourceOrResourceArray(Dst: DstElt, DstTy: DstEltTy, EmitResFn);
686 DstIndex += 1;
687 continue;
688 }
689
690 assert(SrcIndex < SrcTy->getNumElements());
691 llvm::Type *SrcEltTy = SrcTy->getElementType(N: SrcIndex);
692 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: SrcEltTy)) {
693 SrcIndex += 1;
694 continue;
695 }
696
697 auto *SrcElt = emitAccessChain(
698 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
699 auto *DstElt = emitAccessChain(
700 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
701 emitElementCopy(Src: SrcElt, SrcTy: SrcEltTy, Dst: DstElt, DstTy: DstEltTy, EmitResFn);
702 DstIndex += 1;
703 SrcIndex += 1;
704 }
705 }
706
707 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
708 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
709 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
710 auto *SrcElt =
711 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
712 auto *DstElt =
713 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
714 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
715 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType(),
716 EmitResFn);
717 }
718 }
719
720 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
721 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
722 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
723 return emitCopy(Src, SrcTy: AT, Dst, DstTy, EmitResFn);
724 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
725 return emitCopy(Src, SrcTy: ST, Dst, DstTy, EmitResFn);
726
727 // When we have a scalar or vector element we can emit the copy.
728 CharUnits SrcAlign =
729 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
730 CharUnits DstAlign =
731 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
732 Address SrcAddr(Src, SrcTy, SrcAlign);
733 Address DstAddr(Dst, DstTy, DstAlign);
734 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
735 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
736 }
737
738public:
739 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
740 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
741
742 bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
743 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
744
745 // TODO: We should be able to fall back to a regular memcpy if the layout
746 // type doesn't have any padding, but that runs into issues in the backend
747 // currently.
748 //
749 // See https://github.com/llvm/wg-hlsl/issues/351
750 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
751 DstTy: DstPtr.getElementType(), EmitResFn);
752 return true;
753 }
754};
755
756// Represents a list resources associated with a global struct whose name
757// starts with the specified prefix.
758// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
759// order of the depth-first traversal of the corresponding fields in the struct.
760// The resources are always returned in that order, which is the same order
761// we need when a struct is copied element-by-element.
762class AssociatedResourcesList {
763 // Iterator pointers for the associated resource attributes that match the
764 // prefix. Begin = begin of the range of attributes that match the prefix End
765 // = end of the range of attributes that match the prefix Next = the current
766 // attribute in the iteration to be returned by getNextResource
767 specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
768
769public:
770 AssociatedResourcesList(const VarDecl *StructVD,
771 StringRef ResourceNamePrefix) {
772 auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
773 auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
774
775 // Skip over associated resources that don't match the prefix.
776 while (I != E &&
777 !I->getResDecl()->getName().starts_with(Prefix: ResourceNamePrefix))
778 ++I;
779 assert(I != E && "expected associated resource not found");
780 Begin = End = I;
781
782 // Scan over associated resources that do match the prefix to find the end
783 // of the range.
784 while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
785 ->getResDecl()
786 ->getName()
787 .starts_with(Prefix: ResourceNamePrefix))
788 End = ++I;
789
790 Next = Begin;
791 }
792
793 const VarDecl *getNextResource() {
794 if (Next == End)
795 return nullptr;
796
797 const VarDecl *Res = Next->getResDecl();
798 ++Next;
799 return Res;
800 }
801};
802
803} // namespace
804
805llvm::Type *
806CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
807 const CGHLSLOffsetInfo &OffsetInfo) {
808 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
809
810 // Check if the target has a specific translation for this type first.
811 if (llvm::Type *TargetTy =
812 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
813 return TargetTy;
814
815 llvm_unreachable("Generic handling of HLSL types is not supported.");
816}
817
818llvm::Triple::ArchType CGHLSLRuntime::getArch() {
819 return CGM.getTarget().getTriple().getArch();
820}
821
822// Emits constant global variables for buffer constants declarations
823// and creates metadata linking the constant globals with the buffer global.
824void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
825 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
826 const CGHLSLOffsetInfo &OffsetInfo) {
827 LLVMContext &Ctx = CGM.getLLVMContext();
828
829 // get the layout struct from constant buffer target type
830 llvm::Type *BufType = BufGV->getValueType();
831 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
832 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
833
834 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
835 size_t OffsetIdx = 0;
836 for (Decl *D : BufDecl->buffer_decls()) {
837 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
838 // Nothing to do for this declaration.
839 continue;
840 if (isa<FunctionDecl>(Val: D)) {
841 // A function within an cbuffer is effectively a top-level function.
842 CGM.EmitTopLevelDecl(D);
843 continue;
844 }
845 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
846 if (!VD)
847 continue;
848
849 QualType VDTy = VD->getType();
850 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
851 if (VD->getStorageClass() == SC_Static ||
852 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
853 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
854 // Emit static and groupshared variables and resource classes inside
855 // cbuffer as regular globals
856 CGM.EmitGlobal(D: VD);
857 }
858 continue;
859 }
860
861 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
862 }
863
864 if (!OffsetInfo.empty())
865 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
866 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
867 });
868
869 // Associate the buffer global variable with its constants
870 SmallVector<llvm::Metadata *> BufGlobals;
871 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
872 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
873
874 auto ElemIt = LayoutStruct->element_begin();
875 for (auto &[VD, _] : DeclsWithOffset) {
876 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
877 ++ElemIt;
878
879 assert(ElemIt != LayoutStruct->element_end() &&
880 "number of elements in layout struct does not match");
881 llvm::Type *LayoutType = *ElemIt++;
882
883 GlobalVariable *ElemGV =
884 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
885 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
886 }
887 assert(ElemIt == LayoutStruct->element_end() &&
888 "number of elements in layout struct does not match");
889
890 // add buffer metadata to the module
891 CGM.getModule()
892 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
893 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
894}
895
896// Creates resource handle type for the HLSL buffer declaration
897static const clang::HLSLAttributedResourceType *
898createBufferHandleType(const HLSLBufferDecl *BufDecl) {
899 ASTContext &AST = BufDecl->getASTContext();
900 QualType QT = AST.getHLSLAttributedResourceType(
901 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
902 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
903 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
904}
905
906CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
907 CGHLSLOffsetInfo Result;
908
909 // If we don't have packoffset info, just return an empty result.
910 if (!BufDecl.hasValidPackoffset())
911 return Result;
912
913 for (Decl *D : BufDecl.buffer_decls()) {
914 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
915 continue;
916 }
917 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
918 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
919 continue;
920
921 if (!VD->hasAttrs()) {
922 Result.Offsets.push_back(Elt: Unspecified);
923 continue;
924 }
925
926 uint32_t Offset = Unspecified;
927 for (auto *Attr : VD->getAttrs()) {
928 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
929 Offset = POA->getOffsetInBytes();
930 break;
931 }
932 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
933 if (RBA &&
934 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
935 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
936 break;
937 }
938 }
939 Result.Offsets.push_back(Elt: Offset);
940 }
941 return Result;
942}
943
944// Codegen for HLSLBufferDecl
945void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
946
947 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
948
949 // create resource handle type for the buffer
950 const clang::HLSLAttributedResourceType *ResHandleTy =
951 createBufferHandleType(BufDecl);
952
953 // empty constant buffer is ignored
954 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
955 return;
956
957 // create global variable for the constant buffer
958 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
959 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
960 llvm::GlobalVariable *BufGV = new GlobalVariable(
961 LayoutTy, /*isConstant*/ false,
962 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
963 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
964 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
965 GlobalValue::NotThreadLocal);
966
967 llvm::Module &M = CGM.getModule();
968 M.insertGlobalVariable(GV: BufGV);
969
970 // Add the global variable to the compiler used list so it does not
971 // get optimized away by GlobalOptPass before it reaches
972 // {DXIL|SPIRV}CBufferAccess pass.
973 llvm::appendToCompilerUsed(M, Values: {BufGV});
974
975 // Add globals for constant buffer elements and create metadata nodes
976 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
977
978 // Initialize cbuffer from binding (implicit or explicit)
979 initializeBufferFromBinding(BufDecl, GV: BufGV);
980}
981
982void CGHLSLRuntime::addRootSignature(
983 const HLSLRootSignatureDecl *SignatureDecl) {
984 llvm::Module &M = CGM.getModule();
985 Triple T(M.getTargetTriple());
986
987 // Generated later with the function decl if not targeting root signature
988 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
989 return;
990
991 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
992 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
993}
994
995llvm::StructType *
996CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
997 const auto Entry = LayoutTypes.find(Val: StructType);
998 if (Entry != LayoutTypes.end())
999 return Entry->getSecond();
1000 return nullptr;
1001}
1002
1003void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
1004 llvm::StructType *LayoutTy) {
1005 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
1006 "layout type for this struct already exist");
1007 LayoutTypes[StructType] = LayoutTy;
1008}
1009
1010static std::array<uint32_t, 3> parseNumThreads(StringRef NumThreads) {
1011 SmallVector<StringRef, 3> Dimensions;
1012 NumThreads.split(A&: Dimensions, Separator: ',');
1013 assert(Dimensions.size() == 3 && "expected three numthreads dimensions");
1014 std::array<uint32_t, 3> Threads;
1015 for (auto [Dimension, Thread] : llvm::zip(t&: Dimensions, u&: Threads)) {
1016 [[maybe_unused]] bool Invalid = Dimension.getAsInteger(Radix: 10, Result&: Thread);
1017 assert(!Invalid && "expected an integer numthreads dimension");
1018 }
1019 return Threads;
1020}
1021
1022static void addSPIRVQuadAllExecutionModes(CodeGenModule &CGM) {
1023 llvm::Module &M = CGM.getModule();
1024 Function *QuadAll =
1025 Intrinsic::getDeclarationIfExists(M: &M, id: Intrinsic::spv_quad_all);
1026 if (!QuadAll)
1027 return;
1028
1029 SmallVector<Function *, 8> Worklist{QuadAll};
1030 SmallPtrSet<Function *, 16> Visited;
1031 IRBuilder<> B(M);
1032 while (!Worklist.empty()) {
1033 Function *F = Worklist.pop_back_val();
1034 if (!Visited.insert(Ptr: F).second)
1035 continue;
1036
1037 Attribute NumThreadsAttr = F->getFnAttribute(Kind: "hlsl.numthreads");
1038 if (NumThreadsAttr.isValid()) {
1039 std::array<uint32_t, 3> Threads =
1040 parseNumThreads(NumThreads: NumThreadsAttr.getValueAsString());
1041 bool IsLinear = Threads[1] == 1 && Threads[2] == 1;
1042 // QuadAll needs derivative groups for SPIR-V compute entry points.
1043 // Linear groups need complete sets of four threads. Quad groups need
1044 // complete 2-by-2 quads in the X and Y dimensions.
1045 bool InvalidThreads;
1046 SPIRVExecutionMode Mode;
1047 if (IsLinear) {
1048 InvalidThreads =
1049 (uint64_t(Threads[0]) * Threads[1] * Threads[2]) % 4 != 0;
1050 Mode = SPIRVExecutionMode::DerivativeGroupLinearKHR;
1051 } else {
1052 InvalidThreads = Threads[0] % 2 != 0 || Threads[1] % 2 != 0;
1053 Mode = SPIRVExecutionMode::DerivativeGroupQuadsKHR;
1054 }
1055 if (InvalidThreads) {
1056 CGM.getDiags().Report(DiagID: diag::err_hlsl_spirv_derivative_group_numthreads)
1057 << F->getName() << IsLinear;
1058 CGM.getDiags().Report(
1059 DiagID: diag::note_hlsl_spirv_derivative_group_requirement)
1060 << "QuadAll" << F->getName();
1061 continue;
1062 }
1063 MDNode *MD = MDNode::get(
1064 Context&: M.getContext(),
1065 MDs: {ValueAsMetadata::get(V: F),
1066 ConstantAsMetadata::get(C: B.getInt32(C: llvm::to_underlying(E: Mode)))});
1067 M.getOrInsertNamedMetadata(Name: "spirv.ExecutionMode")->addOperand(M: MD);
1068 }
1069
1070 for (User *U : F->users())
1071 if (auto *Call = dyn_cast<CallBase>(Val: U))
1072 Worklist.push_back(Elt: Call->getFunction());
1073 }
1074}
1075
1076void CGHLSLRuntime::finishCodeGen() {
1077 auto &TargetOpts = CGM.getTarget().getTargetOpts();
1078 auto &CodeGenOpts = CGM.getCodeGenOpts();
1079 auto &LangOpts = CGM.getLangOpts();
1080 llvm::Module &M = CGM.getModule();
1081 Triple T(M.getTargetTriple());
1082 if (T.getArch() == Triple::ArchType::dxil)
1083 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
1084 if (!CodeGenOpts.DisableDXSourceMetadata &&
1085 CodeGenOpts.getDebugInfo() >=
1086 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
1087 addSourceInfo(CGM, M);
1088 if (CodeGenOpts.ResMayAlias)
1089 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
1090 if (CodeGenOpts.AllResourcesBound)
1091 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
1092 Key: "dx.allresourcesbound", Val: 1);
1093 if (CodeGenOpts.OptimizationLevel == 0)
1094 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
1095 Key: "dx.disable_optimizations", Val: 1);
1096
1097 // NativeHalfType corresponds to the -fnative-half-type clang option which is
1098 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
1099 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
1100 if (LangOpts.NativeHalfType)
1101 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
1102 Val: 1);
1103
1104 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
1105 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
1106 const ASTContext &Ctx = CGM.getContext();
1107 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
1108 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
1109 SmallVector<GlobalValue *, 8> InterfaceVars;
1110 for (GlobalVariable &GV : M.globals()) {
1111 unsigned AS = GV.getAddressSpace();
1112 if (AS == InputAS || AS == OutputAS)
1113 InterfaceVars.push_back(Elt: &GV);
1114 }
1115 if (!InterfaceVars.empty())
1116 appendToCompilerUsed(M, Values: InterfaceVars);
1117 }
1118
1119 generateGlobalCtorDtorCalls();
1120 if (T.isSPIRV())
1121 addSPIRVQuadAllExecutionModes(CGM);
1122}
1123
1124void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
1125 const FunctionDecl *FD, llvm::Function *Fn) {
1126 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1127 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
1128 const StringRef ShaderAttrKindStr = "hlsl.shader";
1129 Fn->addFnAttr(Kind: ShaderAttrKindStr,
1130 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
1131 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
1132 const StringRef NumThreadsKindStr = "hlsl.numthreads";
1133 std::string NumThreadsStr =
1134 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
1135 Vals: NumThreadsAttr->getZ());
1136 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
1137 }
1138 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
1139 const StringRef WaveSizeKindStr = "hlsl.wavesize";
1140 std::string WaveSizeStr =
1141 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
1142 Vals: WaveSizeAttr->getPreferred());
1143 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
1144 }
1145 // HLSL entry functions are materialized for module functions with
1146 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
1147 // later in the compiler-flow for such module functions is not aware of and
1148 // hence not able to set attributes of the newly materialized entry functions.
1149 // So, set attributes of entry function here, as appropriate.
1150 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
1151
1152 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
1153 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
1154 }
1155}
1156
1157static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
1158 // Compute ID intrinsics return i32 components, but the semantic may use
1159 // 16-bit integers. Narrow each component before assembling the input.
1160 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1161 Value *Result = PoisonValue::get(T: Ty);
1162 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
1163 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
1164 Elt = B.CreateTrunc(V: Elt, DestTy: VT->getElementType());
1165 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
1166 }
1167 return Result;
1168 }
1169 return B.CreateTrunc(V: B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)}), DestTy: Ty);
1170}
1171
1172// A fragment shader input interface variable whose base type is an integer or
1173// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
1174// specification requires these variables to be decorated with Flat (see
1175// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
1176// their base scalar type.
1177static bool requiresFlatInterpolation(llvm::Type *Ty) {
1178 while (true) {
1179 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
1180 Ty = AT->getElementType();
1181 continue;
1182 }
1183 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: Ty)) {
1184 Ty = VT->getElementType();
1185 continue;
1186 }
1187 break;
1188 }
1189 return Ty->isIntegerTy() || Ty->isDoubleTy();
1190}
1191
1192static llvm::hlsl::InterpolationMode
1193getSPIRVInterpolationMode(llvm::hlsl::InterpolationMode InterpMode,
1194 llvm::Type *Ty, llvm::Triple::EnvironmentType Stage,
1195 llvm::hlsl::IOType IO) {
1196 using llvm::hlsl::InterpolationMode;
1197 if (!((Stage == llvm::Triple::Pixel && IO == llvm::hlsl::IOType::In) ||
1198 (Stage == llvm::Triple::Vertex && IO == llvm::hlsl::IOType::Out)))
1199 return InterpolationMode::Undefined;
1200 if (InterpMode == InterpolationMode::Undefined &&
1201 requiresFlatInterpolation(Ty))
1202 return InterpolationMode::Constant;
1203 return InterpMode;
1204}
1205
1206static void
1207addSPIRVInterfaceDecorations(llvm::GlobalVariable *GV,
1208 llvm::hlsl::SPIRVInterfaceDecoration Decoration,
1209 unsigned Operand,
1210 llvm::hlsl::InterpolationMode InterpMode =
1211 llvm::hlsl::InterpolationMode::Undefined) {
1212 using llvm::hlsl::InterpolationMode;
1213 using llvm::hlsl::SPIRVInterpolationMode;
1214
1215 LLVMContext &Ctx = GV->getContext();
1216 IRBuilder<> B(*GV->getParent());
1217 SmallVector<Metadata *, 3> Decorations;
1218 Decorations.push_back(Elt: MDNode::get(
1219 Context&: Ctx,
1220 MDs: {ConstantAsMetadata::get(C: B.getInt32(C: llvm::to_underlying(E: Decoration))),
1221 ConstantAsMetadata::get(C: B.getInt32(C: Operand))}));
1222 auto Add = [&](SPIRVInterpolationMode InterpolationDecoration) {
1223 Decorations.push_back(
1224 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(
1225 C: llvm::to_underlying(E: InterpolationDecoration)))}));
1226 };
1227 switch (InterpMode) {
1228 case InterpolationMode::Undefined:
1229 case InterpolationMode::Linear:
1230 break;
1231 case InterpolationMode::Constant:
1232 Add(SPIRVInterpolationMode::Flat);
1233 break;
1234 case InterpolationMode::LinearNoperspective:
1235 Add(SPIRVInterpolationMode::NoPerspective);
1236 break;
1237 case InterpolationMode::LinearNoperspectiveCentroid:
1238 Add(SPIRVInterpolationMode::NoPerspective);
1239 [[fallthrough]];
1240 case InterpolationMode::LinearCentroid:
1241 Add(SPIRVInterpolationMode::Centroid);
1242 break;
1243 case InterpolationMode::LinearNoperspectiveSample:
1244 Add(SPIRVInterpolationMode::NoPerspective);
1245 [[fallthrough]];
1246 case InterpolationMode::LinearSample:
1247 Add(SPIRVInterpolationMode::Sample);
1248 break;
1249 default:
1250 llvm_unreachable("invalid interpolation mode");
1251 }
1252 // The backend consumes one decoration list per global, including both the
1253 // Location or BuiltIn and any interpolation decorations.
1254 GV->addMetadata(Kind: "spirv.Decorations", MD&: *MDNode::get(Context&: Ctx, MDs: Decorations));
1255}
1256
1257static llvm::Value *createSPIRVInputLoad(
1258 IRBuilder<> &B, llvm::Module &M, llvm::Type *Ty, const Twine &Name,
1259 llvm::hlsl::SPIRVInterfaceDecoration Decoration, unsigned Operand,
1260 llvm::hlsl::InterpolationMode InterpMode) {
1261 // SPIR-V integer built-ins are 32-bit, so a narrower integer declaration
1262 // loads the 32-bit built-in and truncates it. Other types are loaded as is.
1263 // i1 is excluded: bool built-ins such as FrontFacing are OpTypeBool.
1264 bool Truncate = Decoration == llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn &&
1265 Ty->isIntegerTy() && Ty->getIntegerBitWidth() > 1 &&
1266 Ty->getIntegerBitWidth() < 32;
1267 llvm::Type *LoadTy = Truncate ? B.getInt32Ty() : Ty;
1268 auto *GV = new llvm::GlobalVariable(
1269 M, LoadTy, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1270 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1271 llvm::GlobalVariable::GeneralDynamicTLSModel,
1272 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1273 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1274 addSPIRVInterfaceDecorations(GV, Decoration, Operand, InterpMode);
1275 llvm::Value *Value = B.CreateLoad(Ty: LoadTy, Ptr: GV);
1276 return Truncate ? B.CreateTrunc(V: Value, DestTy: Ty) : Value;
1277}
1278
1279// For Vulkan builtins which begin counting from a driver-supplied base offset,
1280// where the corresponding HLSL system value is always zero-based within the
1281// current draw call.
1282static llvm::Value *
1283createSPIRVBuiltinDelta(IRBuilder<> &B, llvm::Module &M, llvm::Type *Ty,
1284 const Twine &Name, unsigned BuiltInID,
1285 unsigned BaseBuiltInID,
1286 llvm::hlsl::InterpolationMode InterpMode) {
1287 llvm::Value *Value = createSPIRVInputLoad(
1288 B, M, Ty, Name, Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Operand: BuiltInID,
1289 InterpMode);
1290 llvm::Value *Base = createSPIRVInputLoad(
1291 B, M, Ty, Name: Name + Twine(".base"),
1292 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Operand: BaseBuiltInID, InterpMode);
1293 return B.CreateSub(LHS: Value, RHS: Base);
1294}
1295
1296llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
1297 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1298 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1299 llvm::hlsl::InterpolationMode InterpMode) {
1300 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1301 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1302
1303 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
1304 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1305 Location = L->getLocation();
1306
1307 // DXC completely ignores the semantic/index pair. Location are assigned from
1308 // the first semantic to the last.
1309 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
1310 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1311 SPIRVLastAssignedInputSemanticLocation += ElementCount;
1312
1313 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type, Name: VariableName.str(),
1314 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::Location,
1315 Operand: Location, InterpMode);
1316}
1317
1318static void createSPIRVOutputStore(
1319 IRBuilder<> &B, llvm::Module &M, llvm::Value *Source, const Twine &Name,
1320 llvm::hlsl::SPIRVInterfaceDecoration Decoration, unsigned Operand,
1321 llvm::hlsl::InterpolationMode InterpMode) {
1322 auto *GV = new llvm::GlobalVariable(
1323 M, Source->getType(), /* isConstant= */ false,
1324 llvm::GlobalValue::ExternalLinkage,
1325 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1326 llvm::GlobalVariable::GeneralDynamicTLSModel,
1327 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1328 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1329 addSPIRVInterfaceDecorations(GV, Decoration, Operand, InterpMode);
1330 B.CreateStore(Val: Source, Ptr: GV);
1331}
1332
1333void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1334 llvm::IRBuilder<> &B, llvm::Value *Source,
1335 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1336 std::optional<unsigned> Index, llvm::hlsl::InterpolationMode InterpMode) {
1337 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1338 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1339
1340 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1341 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1342 Location = L->getLocation();
1343
1344 // DXC completely ignores the semantic/index pair. Location are assigned from
1345 // the first semantic to the last.
1346 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1347 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1348 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1349 createSPIRVOutputStore(B, M&: CGM.getModule(), Source, Name: VariableName.str(),
1350 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::Location,
1351 Operand: Location, InterpMode);
1352}
1353
1354namespace {
1355// Describes how a semantic leaf lowers to signature rows
1356struct SemanticShape {
1357 // Array length followed by matrix major-axis length.
1358 SmallVector<unsigned> Dimensions;
1359 unsigned Cols;
1360 QualType RowType;
1361 bool IsMatrix = false;
1362 bool IsRowMajor = false;
1363
1364 // Direct matrix arguments/results use column-major flattened vectors even
1365 // when their signature (and aggregate memory representation) is row-major.
1366 unsigned getMatrixElementIndex(unsigned Row, unsigned Col) const {
1367 assert(IsMatrix);
1368 unsigned MatrixRow = Row % Dimensions.back();
1369 return IsRowMajor ? Col * Dimensions.back() + MatrixRow
1370 : MatrixRow * Cols + Col;
1371 }
1372
1373 unsigned getNumRows() const {
1374 unsigned Rows = 1;
1375 for (unsigned Dimension : Dimensions)
1376 Rows *= Dimension;
1377 return Rows;
1378 }
1379
1380 SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
1381 assert(Row < getNumRows() && "row exceeds semantic shape");
1382
1383 SmallVector<unsigned> Indices(Dimensions.size());
1384 for (auto [Index, Dimension] :
1385 llvm::zip_equal(t: llvm::reverse(C&: Indices), u: llvm::reverse(C: Dimensions))) {
1386 Index = Row % Dimension;
1387 Row /= Dimension;
1388 }
1389 return Indices;
1390 }
1391};
1392} // namespace
1393
1394// Returns the QualType of a semantic leaf declarator. For a function the
1395// declared return type is used, otherwise the declared type.
1396static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
1397 if (const auto *FD = dyn_cast<clang::FunctionDecl>(Val: Decl))
1398 return FD->getDeclaredReturnType();
1399 return Decl->getType();
1400}
1401
1402// Collects array dimensions and, for a matrix leaf, its major-axis dimension.
1403// RowType is the scalar or vector transferred by each signature load/store.
1404static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
1405 SemanticShape Shape{.Dimensions: {}, .Cols: 1, .RowType: Ty};
1406 while (const ConstantArrayType *CAT =
1407 Ctx.getAsConstantArrayType(T: Shape.RowType)) {
1408 Shape.Dimensions.push_back(Elt: CAT->getSize().getZExtValue());
1409 Shape.RowType = CAT->getElementType();
1410 }
1411
1412 if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
1413 Shape.Cols = VT->getNumElements();
1414 } else if (const auto *MT =
1415 Shape.RowType->getAs<clang::ConstantMatrixType>()) {
1416 Shape.IsMatrix = true;
1417 Shape.IsRowMajor = isMatrixRowMajor(LangOpts: Ctx.getLangOpts(), T: Shape.RowType);
1418 Shape.Dimensions.push_back(Elt: Shape.IsRowMajor ? MT->getNumRows()
1419 : MT->getNumColumns());
1420 Shape.Cols = Shape.IsRowMajor ? MT->getNumColumns() : MT->getNumRows();
1421 Shape.RowType = Ctx.getExtVectorType(VectorType: MT->getElementType(), NumElts: Shape.Cols);
1422 }
1423
1424 return Shape;
1425}
1426
1427static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
1428 QualType Ty) {
1429 if (const auto *VT = Ty->getAs<clang::VectorType>())
1430 Ty = VT->getElementType();
1431
1432 llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
1433 bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
1434 return llvm::hlsl::getDXILElementType(Ty: IRTy, IsSigned);
1435}
1436
1437static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
1438 CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
1439 std::optional<unsigned> Index, const SemanticShape &Shape) {
1440 StringRef Name = Semantic->getAttrName()->getName();
1441
1442 // One semantic index per row, starting from the declared index.
1443 SmallVector<uint32_t> SemanticIndices;
1444 uint32_t FirstSemanticIndex = Index.value_or(u: 0);
1445 for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
1446 SemanticIndices.push_back(Elt: FirstSemanticIndex + I);
1447
1448 llvm::dxbc::PSV::SemanticKind Kind = llvm::hlsl::getSemanticKind(SemanticName: Name);
1449 llvm::dxil::ElementType CompType =
1450 getSignatureComponentType(CGM, Ty: Shape.RowType);
1451 // SV_SampleIndex and SV_PrimitiveID may be declared as int or uint, as in
1452 // DXC. Both are indices generated by the system, so they are never negative
1453 // and a signed declaration reads the same value. DXIL signatures require an
1454 // unsigned component type.
1455 if (Kind == llvm::dxbc::PSV::SemanticKind::SampleIndex ||
1456 Kind == llvm::dxbc::PSV::SemanticKind::PrimitiveID) {
1457 if (CompType == llvm::dxil::ElementType::I16)
1458 CompType = llvm::dxil::ElementType::U16;
1459 else if (CompType == llvm::dxil::ElementType::I32)
1460 CompType = llvm::dxil::ElementType::U32;
1461 }
1462
1463 // The remaining members keep their default value and will be filled at a
1464 // later stage, either during packing or analysis of usage. Interpolation
1465 // is set by the load/store traversal after visiting each leaf.
1466 return llvm::hlsl::SemanticSignatureElement(SigId, Name, CompType, Kind,
1467 SemanticIndices,
1468 static_cast<uint8_t>(Shape.Cols));
1469}
1470
1471llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
1472 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1473 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1474 SemanticSignatures &Signature) {
1475 StringRef Name = Semantic->getAttrName()->getName();
1476 SemanticShape Shape =
1477 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1478
1479 uint32_t SigId = Signature.size();
1480 Signature.push_back(
1481 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1482
1483 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1484
1485 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1486 M: B.getModule(), id: llvm::Intrinsic::dx_load_input, OverloadTys: {RowTy});
1487
1488 SmallVector<OperandBundleDef, 1> OB;
1489 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1490 llvm::Value *bundleArgs[] = {Token};
1491 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1492 }
1493
1494 bool IsMatrixValue = Shape.IsMatrix && Type->isVectorTy();
1495 llvm::Type *LeafTy = Shape.IsMatrix && !IsMatrixValue
1496 ? RowTy
1497 : CGM.getTypes().ConvertType(T: Shape.RowType);
1498 llvm::Value *Result = llvm::PoisonValue::get(T: Type);
1499
1500 const unsigned NumRows = Shape.getNumRows();
1501
1502 for (unsigned Row = 0; Row < NumRows; ++Row) {
1503 std::array<Value *, 4> Args{
1504 /*SigElementId=*/B.getInt32(C: SigId),
1505 /*RowIndex=*/B.getInt32(C: Row),
1506 /*ColIndex=*/B.getInt8(C: 0),
1507 /*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(T: B.getInt32Ty())};
1508 llvm::Value *Value =
1509 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: Twine(Name).concat(Suffix: Twine(Row)));
1510 // Booleans use their memory representation in DXIL signatures, but
1511 // function parameters use their value representation.
1512 if (Value->getType() != LeafTy) {
1513 assert(Shape.RowType->hasBooleanRepresentation() &&
1514 "unexpected semantic load type mismatch");
1515 Value = B.CreateICmpNE(
1516 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
1517 }
1518
1519 if (IsMatrixValue) {
1520 for (unsigned Col = 0; Col < Shape.Cols; ++Col)
1521 Result =
1522 B.CreateInsertElement(Vec: Result, NewElt: B.CreateExtractElement(Vec: Value, Idx: Col),
1523 Idx: Shape.getMatrixElementIndex(Row, Col));
1524 } else {
1525 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1526 Result =
1527 Indices.empty() ? Value : B.CreateInsertValue(Agg: Result, Val: Value, Idxs: Indices);
1528 }
1529 }
1530 return Result;
1531}
1532
1533void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1534 llvm::Value *Source,
1535 const clang::DeclaratorDecl *Decl,
1536 HLSLAppliedSemanticAttr *Semantic,
1537 std::optional<unsigned> Index,
1538 SemanticSignatures &Signature) {
1539 SemanticShape Shape =
1540 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1541
1542 uint32_t SigId = Signature.size();
1543 Signature.push_back(
1544 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1545
1546 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1547
1548 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1549 M: B.getModule(), id: llvm::Intrinsic::dx_store_output, OverloadTys: {RowTy});
1550
1551 SmallVector<OperandBundleDef, 1> OB;
1552 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1553 llvm::Value *bundleArgs[] = {Token};
1554 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1555 }
1556
1557 bool IsMatrixValue = Shape.IsMatrix && Source->getType()->isVectorTy();
1558 const unsigned NumRows = Shape.getNumRows();
1559 for (unsigned Row = 0; Row < NumRows; ++Row) {
1560 llvm::Value *Val;
1561 if (IsMatrixValue) {
1562 SmallVector<int, 4> Mask;
1563 for (unsigned Col = 0; Col < Shape.Cols; ++Col)
1564 Mask.push_back(Elt: Shape.getMatrixElementIndex(Row, Col));
1565 Val = B.CreateShuffleVector(V: Source, Mask);
1566 } else {
1567 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1568 Val = Indices.empty() ? Source : B.CreateExtractValue(Agg: Source, Idxs: Indices);
1569 }
1570
1571 // Booleans use their memory representation in DXIL signatures, but direct
1572 // function results use their value representation.
1573 if (Val->getType() != RowTy) {
1574 assert(Shape.RowType->hasBooleanRepresentation() &&
1575 "unexpected semantic store type mismatch");
1576 Val = B.CreateZExt(V: Val, DestTy: RowTy, Name: "storedv");
1577 }
1578
1579 std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(C: SigId),
1580 /*RowIndex=*/B.getInt32(C: Row),
1581 /*ColIndex=*/B.getInt8(C: 0), /*Value=*/Val};
1582 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1583 }
1584}
1585
1586llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1587 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1588 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1589 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
1590 if (CGM.getTarget().getTriple().isSPIRV())
1591 return emitSPIRVUserSemanticLoad(B, Type, Decl, Semantic, Index,
1592 InterpMode);
1593
1594 if (CGM.getTarget().getTriple().isDXIL())
1595 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
1596
1597 llvm_unreachable("Unsupported target for user-semantic load.");
1598}
1599
1600void CGHLSLRuntime::emitUserSemanticStore(
1601 IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
1602 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1603 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
1604 if (CGM.getTarget().getTriple().isSPIRV())
1605 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index,
1606 InterpMode);
1607
1608 if (CGM.getTarget().getTriple().isDXIL())
1609 return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
1610 Signature);
1611
1612 llvm_unreachable("Unsupported target for user-semantic load.");
1613}
1614
1615llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1616 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1617 HLSLAppliedSemanticAttr *Semantic,
1618 llvm::dxbc::PSV::SemanticKind SemanticKind,
1619 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1620 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
1621 switch (SemanticKind) {
1622 case llvm::dxbc::PSV::SemanticKind::Barycentrics:
1623 assert(Stage == llvm::Triple::Pixel &&
1624 "SV_Barycentrics is in an unavailable stage and should have been "
1625 "diagnosed by Sema");
1626 if (CGM.getTarget().getTriple().isSPIRV()) {
1627 using llvm::hlsl::InterpolationMode;
1628 // Perspective is encoded by the built-in; only sampling decorations
1629 // belong on the interface variable.
1630 bool NoPerspective = false;
1631 switch (InterpMode) {
1632 case InterpolationMode::LinearNoperspective:
1633 NoPerspective = true;
1634 InterpMode = InterpolationMode::Linear;
1635 break;
1636 case InterpolationMode::LinearNoperspectiveCentroid:
1637 NoPerspective = true;
1638 InterpMode = InterpolationMode::LinearCentroid;
1639 break;
1640 case InterpolationMode::LinearNoperspectiveSample:
1641 NoPerspective = true;
1642 InterpMode = InterpolationMode::LinearSample;
1643 break;
1644 default:
1645 break;
1646 }
1647 // SPIR-V barycentric built-ins are float3, even for half3 inputs.
1648 llvm::Type *BuiltinType = llvm::FixedVectorType::get(ElementType: CGM.FloatTy, NumElts: 3);
1649 llvm::Value *Barycentrics = createSPIRVInputLoad(
1650 B, M&: CGM.getModule(), Ty: BuiltinType, Name: Semantic->getSemanticName(),
1651 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1652 Operand: NoPerspective ? /* BuiltIn::BaryCoordNoPerspKHR */ 5287
1653 : /* BuiltIn::BaryCoordKHR */ 5286,
1654 InterpMode);
1655 if (Type->getScalarType()->isHalfTy())
1656 return B.CreateFPTrunc(V: Barycentrics, DestTy: Type);
1657 return Barycentrics;
1658 }
1659 if (CGM.getTarget().getTriple().isDXIL())
1660 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1661 Signature);
1662 break;
1663 case llvm::dxbc::PSV::SemanticKind::GroupIndex: {
1664 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1665 llvm::Triple::Amplification},
1666 Stage) &&
1667 "SV_GroupIndex is in an unavailable stage and should have been "
1668 "diagnosed by Sema");
1669 assert(Stage != llvm::Triple::Mesh &&
1670 Stage != llvm::Triple::Amplification &&
1671 "FIXME: SV_GroupIndex is not yet implemented for this shader "
1672 "stage");
1673 llvm::Function *GroupIndex =
1674 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1675 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1676 }
1677 case llvm::dxbc::PSV::SemanticKind::DispatchThreadID: {
1678 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1679 llvm::Triple::Amplification},
1680 Stage) &&
1681 "SV_DispatchThreadID is in an unavailable stage and should have "
1682 "been diagnosed by Sema");
1683 assert(Stage != llvm::Triple::Mesh &&
1684 Stage != llvm::Triple::Amplification &&
1685 "FIXME: SV_DispatchThreadID is not yet implemented for this "
1686 "shader stage");
1687 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1688 llvm::Function *ThreadIDIntrinsic =
1689 llvm::Intrinsic::isOverloaded(id: IntrinID)
1690 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1691 : CGM.getIntrinsic(IID: IntrinID);
1692 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1693 }
1694 case llvm::dxbc::PSV::SemanticKind::GroupThreadID: {
1695 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1696 llvm::Triple::Amplification},
1697 Stage) &&
1698 "SV_GroupThreadID is in an unavailable stage and should have been "
1699 "diagnosed by Sema");
1700 assert(Stage != llvm::Triple::Mesh &&
1701 Stage != llvm::Triple::Amplification &&
1702 "FIXME: SV_GroupThreadID is not yet implemented for this shader "
1703 "stage");
1704 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1705 llvm::Function *GroupThreadIDIntrinsic =
1706 llvm::Intrinsic::isOverloaded(id: IntrinID)
1707 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1708 : CGM.getIntrinsic(IID: IntrinID);
1709 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1710 }
1711 case llvm::dxbc::PSV::SemanticKind::GroupID: {
1712 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1713 llvm::Triple::Amplification},
1714 Stage) &&
1715 "SV_GroupID is in an unavailable stage and should have been "
1716 "diagnosed by Sema");
1717 assert(Stage != llvm::Triple::Mesh &&
1718 Stage != llvm::Triple::Amplification &&
1719 "FIXME: SV_GroupID is not yet implemented for this shader stage");
1720 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1721 llvm::Function *GroupIDIntrinsic =
1722 llvm::Intrinsic::isOverloaded(id: IntrinID)
1723 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1724 : CGM.getIntrinsic(IID: IntrinID);
1725 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1726 }
1727 case llvm::dxbc::PSV::SemanticKind::Position:
1728 assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
1729 llvm::Triple::Geometry, llvm::Triple::Pixel},
1730 Stage) &&
1731 "SV_Position is in an unavailable stage and should have been "
1732 "diagnosed by Sema");
1733 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1734 Stage != llvm::Triple::Geometry &&
1735 "FIXME: loading SV_Position is not yet implemented for this "
1736 "shader stage");
1737 if (CGM.getTarget().getTriple().isSPIRV())
1738 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1739 Name: Semantic->getAttrName()->getName(),
1740 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1741 /* FragCoord */ Operand: 15, InterpMode);
1742 if (CGM.getTarget().getTriple().isDXIL())
1743 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1744 Signature);
1745 break;
1746 case llvm::dxbc::PSV::SemanticKind::VertexID:
1747 assert(Stage == llvm::Triple::Vertex &&
1748 "SV_VertexID is in an unavailable stage and should have been "
1749 "diagnosed by Sema");
1750 if (CGM.getTarget().getTriple().isSPIRV())
1751 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1752 Name: Semantic->getAttrName()->getName(),
1753 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1754 /* VertexIndex */ Operand: 42, InterpMode);
1755 if (CGM.getTarget().getTriple().isDXIL())
1756 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1757 Signature);
1758 break;
1759 case llvm::dxbc::PSV::SemanticKind::InstanceID:
1760 assert(Stage == llvm::Triple::Vertex &&
1761 "SV_InstanceID is in an unavailable stage and should have been "
1762 "diagnosed by Sema");
1763 if (CGM.getTarget().getTriple().isSPIRV()) {
1764 // Matches DXC: only subtract BaseInstance when
1765 // -fvk-support-nonzero-base-instance is given.
1766 if (CGM.getLangOpts().HLSLSpvSupportNonzeroBaseInstance)
1767 return createSPIRVBuiltinDelta(
1768 B, M&: CGM.getModule(), Ty: Type, Name: Semantic->getAttrName()->getName(),
1769 /* InstanceIndex */ BuiltInID: 43, /* BaseInstance */ BaseBuiltInID: 4425, InterpMode);
1770 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1771 Name: Semantic->getAttrName()->getName(),
1772 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1773 /* InstanceIndex */ Operand: 43, InterpMode);
1774 }
1775 if (CGM.getTarget().getTriple().isDXIL())
1776 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1777 Signature);
1778 break;
1779 case llvm::dxbc::PSV::SemanticKind::IsFrontFace:
1780 assert(Stage == llvm::Triple::Pixel &&
1781 "SV_IsFrontFace is in an unavailable stage and should have been "
1782 "diagnosed by Sema");
1783 if (CGM.getTarget().getTriple().isSPIRV()) {
1784 // The FrontFacing BuiltIn is always a bool in SPIR-V, whatever type the
1785 // entry point declares. Load it as bool, then convert below.
1786 llvm::Type *BoolTy = B.getInt1Ty();
1787 // Integer pixel inputs cannot be interpolated and must be Flat.
1788 llvm::Value *FF = createSPIRVInputLoad(
1789 B, M&: CGM.getModule(), Ty: BoolTy, Name: Semantic->getAttrName()->getName(),
1790 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1791 /* FrontFacing */ Operand: 17, InterpMode: llvm::hlsl::InterpolationMode::Constant);
1792 // Convert to the declared parameter type (eg uint).
1793 return Type == BoolTy ? FF : B.CreateZExt(V: FF, DestTy: Type);
1794 }
1795 if (CGM.getTarget().getTriple().isDXIL())
1796 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1797 Signature);
1798 break;
1799 case llvm::dxbc::PSV::SemanticKind::ShadingRate:
1800 assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
1801 llvm::Triple::Geometry, llvm::Triple::Pixel},
1802 Stage) &&
1803 "SV_ShadingRate is in an unavailable stage and should have been "
1804 "diagnosed by Sema");
1805 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1806 Stage != llvm::Triple::Geometry &&
1807 "FIXME: loading SV_ShadingRate is not yet implemented for this "
1808 "shader stage");
1809 if (CGM.getTarget().getTriple().isSPIRV())
1810 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1811 Name: Semantic->getAttrName()->getName(),
1812 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1813 /* ShadingRateKHR */ Operand: 4444, InterpMode);
1814 if (CGM.getTarget().getTriple().isDXIL())
1815 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1816 Signature);
1817 break;
1818 case llvm::dxbc::PSV::SemanticKind::StartVertexLocation:
1819 assert(Stage == llvm::Triple::Vertex &&
1820 "SV_StartVertexLocation is in an unavailable stage and should have "
1821 "been diagnosed by Sema");
1822 if (CGM.getTarget().getTriple().isSPIRV())
1823 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1824 Name: Semantic->getAttrName()->getName(),
1825 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1826 /* BaseVertex */ Operand: 4424, InterpMode);
1827 if (CGM.getTarget().getTriple().isDXIL())
1828 return B.CreateCall(
1829 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::dx_start_vertex_location));
1830 break;
1831 case llvm::dxbc::PSV::SemanticKind::PrimitiveID:
1832 assert(Stage == llvm::Triple::Pixel &&
1833 "SV_PrimitiveID is in an unavailable stage and should have been "
1834 "diagnosed by Sema");
1835 if (CGM.getTarget().getTriple().isSPIRV())
1836 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1837 Name: Semantic->getAttrName()->getName(),
1838 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1839 /* PrimitiveId */ Operand: 7, InterpMode);
1840 // Pixel shaders read the primitive ID from the input signature. The
1841 // dx.op.primitiveID operation is only available to other stages.
1842 if (CGM.getTarget().getTriple().isDXIL())
1843 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1844 Signature);
1845 break;
1846 case llvm::dxbc::PSV::SemanticKind::SampleIndex:
1847 assert(Stage == llvm::Triple::Pixel &&
1848 "SV_SampleIndex is in an unavailable stage and should have been "
1849 "diagnosed by Sema");
1850 if (CGM.getTarget().getTriple().isSPIRV())
1851 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1852 Name: Semantic->getAttrName()->getName(),
1853 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1854 /* SampleId */ Operand: 18, InterpMode);
1855 if (CGM.getTarget().getTriple().isDXIL()) {
1856 assert(Type->isIntegerTy() && Type->getIntegerBitWidth() <= 32 &&
1857 "SV_SampleIndex must be a 16 or 32 bit scalar integer, as "
1858 "diagnosed by Sema");
1859 // The sample index is read with a dedicated operation, but an unpacked
1860 // element is still added to the input signature for compatibility.
1861 SemanticShape Shape =
1862 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1863 Signature.push_back(Elt: createSemanticSignatureElement(
1864 CGM, SigId: Signature.size(), Semantic, Index, Shape));
1865 llvm::Value *SampleIndex =
1866 B.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::dx_sample_index));
1867 // dx.op.sampleIndex only has an i32 overload, so a 16-bit declaration
1868 // narrows the result. The sample index always fits in 16 bits.
1869 if (Type->getIntegerBitWidth() < 32)
1870 SampleIndex = B.CreateTrunc(V: SampleIndex, DestTy: Type);
1871 return SampleIndex;
1872 }
1873 break;
1874 case llvm::dxbc::PSV::SemanticKind::ViewID:
1875 assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
1876 llvm::Triple::Domain, llvm::Triple::Geometry,
1877 llvm::Triple::Pixel, llvm::Triple::Mesh},
1878 Stage) &&
1879 "SV_ViewID is in an unavailable stage and should have been "
1880 "diagnosed by Sema");
1881 if (CGM.getTarget().getTriple().isSPIRV())
1882 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1883 Name: Semantic->getAttrName()->getName(),
1884 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1885 /* ViewIndex */ Operand: 4440, InterpMode);
1886 if (CGM.getTarget().getTriple().isDXIL())
1887 return B.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::dx_view_id));
1888 break;
1889 case llvm::dxbc::PSV::SemanticKind::StartInstanceLocation:
1890 assert(Stage == llvm::Triple::Vertex &&
1891 "SV_StartInstanceLocation is in an unavailable stage and should "
1892 "have been diagnosed by Sema");
1893 if (CGM.getTarget().getTriple().isSPIRV())
1894 return createSPIRVInputLoad(B, M&: CGM.getModule(), Ty: Type,
1895 Name: Semantic->getAttrName()->getName(),
1896 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1897 /* BaseInstance */ Operand: 4425, InterpMode);
1898 if (CGM.getTarget().getTriple().isDXIL())
1899 return B.CreateCall(
1900 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::dx_start_instance_location));
1901 break;
1902 default:
1903 break;
1904 }
1905
1906 llvm_unreachable(
1907 "Load hasn't been implemented yet for this system semantic. FIXME");
1908}
1909
1910// Maps a [[vk::builtin(...)]] name to its SPIR-V spec `BuiltIn` decoration.
1911// Deliberately not creating dependency on llvm::SPIRV::BuiltIn in front end.
1912// Sema's handleVkBuiltInAttr only allows names handled here to reach CodeGen.
1913static unsigned getSPIRVVkBuiltinID(StringRef BuiltIn) {
1914 unsigned ID = llvm::StringSwitch<unsigned>(BuiltIn)
1915 .Case(S: "PointSize", /* BuiltIn::PointSize */ Value: 1)
1916 .Default(Value: -1);
1917 assert(ID != static_cast<unsigned>(-1) &&
1918 "Sema should only allow builtin names implemented here");
1919 return ID;
1920}
1921
1922void CGHLSLRuntime::emitSystemSemanticStore(
1923 IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
1924 HLSLAppliedSemanticAttr *Semantic,
1925 llvm::dxbc::PSV::SemanticKind SemanticKind,
1926 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1927 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
1928 switch (SemanticKind) {
1929 case llvm::dxbc::PSV::SemanticKind::Position:
1930 assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
1931 llvm::Triple::Domain, llvm::Triple::Geometry,
1932 llvm::Triple::Mesh},
1933 Stage) &&
1934 "SV_Position is in an unavailable stage and should have been "
1935 "diagnosed by Sema");
1936 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1937 Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
1938 "FIXME: storing SV_Position is not yet implemented for this "
1939 "shader stage");
1940 if (CGM.getTarget().getTriple().isDXIL()) {
1941 emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1942 return;
1943 }
1944 if (CGM.getTarget().getTriple().isSPIRV()) {
1945 createSPIRVOutputStore(B, M&: CGM.getModule(), Source,
1946 Name: Semantic->getAttrName()->getName(),
1947 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1948 /* Position */ Operand: 0, InterpMode);
1949 return;
1950 }
1951 break;
1952 case llvm::dxbc::PSV::SemanticKind::Target:
1953 assert(Stage == llvm::Triple::Pixel &&
1954 "SV_Target is in an unavailable stage and should have been "
1955 "diagnosed by Sema");
1956 emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature,
1957 InterpMode);
1958 return;
1959 case llvm::dxbc::PSV::SemanticKind::ShadingRate:
1960 assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
1961 llvm::Triple::Domain, llvm::Triple::Geometry,
1962 llvm::Triple::Mesh},
1963 Stage) &&
1964 "SV_ShadingRate is in an unavailable stage and should have been "
1965 "diagnosed by Sema");
1966 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1967 Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
1968 "FIXME: storing SV_ShadingRate is not yet implemented for this "
1969 "shader stage");
1970 if (CGM.getTarget().getTriple().isDXIL()) {
1971 emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1972 return;
1973 }
1974 if (CGM.getTarget().getTriple().isSPIRV()) {
1975 // Matches DXC: the PrimitiveShadingRateKHR built-in output does not
1976 // carry the Flat decoration implied for integer vertex outputs.
1977 createSPIRVOutputStore(B, M&: CGM.getModule(), Source,
1978 Name: Semantic->getAttrName()->getName(),
1979 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
1980 /* PrimitiveShadingRateKHR */ Operand: 4432,
1981 InterpMode: llvm::hlsl::InterpolationMode::Undefined);
1982 return;
1983 }
1984 break;
1985 default:
1986 break;
1987 }
1988
1989 llvm_unreachable(
1990 "Store hasn't been implemented yet for this system semantic. FIXME");
1991}
1992
1993llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1994 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1995 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1996 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
1997
1998 // [[vk::builtin(...)]] takes precedence over the recognized-semantic
1999 // dispatch, so without this check they'd always be treated as an arbitrary
2000 // user semantic. The attribute is only ever attached when targeting SPIR-V.
2001 if (const auto *BuiltInAttr = Decl->getAttr<HLSLVkBuiltInAttr>())
2002 return createSPIRVInputLoad(
2003 B, M&: CGM.getModule(), Ty: Type, Name: Semantic->getAttrName()->getName(),
2004 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
2005 Operand: getSPIRVVkBuiltinID(BuiltIn: BuiltInAttr->getBuiltIn()), InterpMode);
2006
2007 std::optional<unsigned> Index = Semantic->getSemanticIndex();
2008 llvm::dxbc::PSV::SemanticKind SemanticKind =
2009 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
2010 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
2011 assert(ShaderAttr && "Entry point has no shader attribute");
2012 llvm::hlsl::SemanticInterpretation Interpretation =
2013 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
2014 IOTy: llvm::hlsl::IOType::In);
2015 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
2016 "invalid semantic should have been diagnosed by Sema");
2017 if (CGM.getTarget().getTriple().isSPIRV())
2018 InterpMode = getSPIRVInterpolationMode(
2019 InterpMode, Ty: Type, Stage: ShaderAttr->getType(), IO: llvm::hlsl::IOType::In);
2020 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
2021 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature,
2022 InterpMode);
2023 return emitSystemSemanticLoad(B, Type, Decl, Semantic, SemanticKind,
2024 Stage: ShaderAttr->getType(), Index, Signature,
2025 InterpMode);
2026}
2027
2028void CGHLSLRuntime::handleScalarSemanticStore(
2029 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
2030 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
2031 SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
2032 // See the matching comment in `handleScalarSemanticLoad`
2033 if (const auto *BuiltInAttr = Decl->getAttr<HLSLVkBuiltInAttr>()) {
2034 createSPIRVOutputStore(
2035 B, M&: CGM.getModule(), Source, Name: Semantic->getAttrName()->getName(),
2036 Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
2037 Operand: getSPIRVVkBuiltinID(BuiltIn: BuiltInAttr->getBuiltIn()), InterpMode);
2038 return;
2039 }
2040
2041 std::optional<unsigned> Index = Semantic->getSemanticIndex();
2042 llvm::dxbc::PSV::SemanticKind SemanticKind =
2043 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
2044 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
2045 assert(ShaderAttr && "Entry point has no shader attribute");
2046
2047 llvm::hlsl::SemanticInterpretation Interpretation =
2048 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
2049 IOTy: llvm::hlsl::IOType::Out);
2050 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
2051 "invalid semantic should have been diagnosed by Sema");
2052 if (CGM.getTarget().getTriple().isSPIRV())
2053 InterpMode = getSPIRVInterpolationMode(InterpMode, Ty: Source->getType(),
2054 Stage: ShaderAttr->getType(),
2055 IO: llvm::hlsl::IOType::Out);
2056
2057 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
2058 return emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature,
2059 InterpMode);
2060 emitSystemSemanticStore(B, Source, Decl, Semantic, SemanticKind,
2061 Stage: ShaderAttr->getType(), Index, Signature, InterpMode);
2062}
2063
2064std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
2065CGHLSLRuntime::handleStructSemanticLoad(
2066 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
2067 const clang::DeclaratorDecl *Decl,
2068 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
2069 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
2070 SemanticSignatures &Signature,
2071 llvm::hlsl::InterpolationModifier Modifiers) {
2072 const llvm::StructType *ST = cast<StructType>(Val: Type);
2073 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
2074
2075 assert(RD->getNumFields() == ST->getNumElements());
2076
2077 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
2078 auto FieldDecl = RD->field_begin();
2079 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
2080 auto [ChildValue, NextAttr] =
2081 handleSemanticLoad(B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin,
2082 end: AttrEnd, Signature, Modifiers);
2083 AttrBegin = NextAttr;
2084 assert(ChildValue);
2085 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
2086 ++FieldDecl;
2087 }
2088
2089 return std::make_pair(x&: Aggregate, y&: AttrBegin);
2090}
2091
2092specific_attr_iterator<HLSLAppliedSemanticAttr>
2093CGHLSLRuntime::handleStructSemanticStore(
2094 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
2095 const clang::DeclaratorDecl *Decl,
2096 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
2097 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
2098 SemanticSignatures &Signature,
2099 llvm::hlsl::InterpolationModifier Modifiers) {
2100
2101 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
2102
2103 const clang::RecordDecl *RD = nullptr;
2104 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
2105 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
2106 else
2107 RD = Decl->getType()->getAsRecordDecl();
2108 assert(RD);
2109
2110 assert(RD->getNumFields() == ST->getNumElements());
2111
2112 auto FieldDecl = RD->field_begin();
2113 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
2114 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
2115 AttrBegin = handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin,
2116 AttrEnd, Signature, Modifiers);
2117 }
2118
2119 return AttrBegin;
2120}
2121
2122std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
2123CGHLSLRuntime::handleSemanticLoad(
2124 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
2125 const clang::DeclaratorDecl *Decl,
2126 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
2127 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
2128 SemanticSignatures &Signature,
2129 llvm::hlsl::InterpolationModifier Modifiers) {
2130 assert(AttrBegin != AttrEnd);
2131 // Pass the enclosing declaration's mask down the traversal, replacing (not
2132 // merging) it when an inner field has its own interpolation modifiers.
2133 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
2134 Modifiers =
2135 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
2136 if (Type->isStructTy())
2137 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
2138 Signature, Modifiers);
2139
2140 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
2141 ++AttrBegin;
2142 size_t PreviousSize = Signature.size();
2143 auto InterpMode = llvm::hlsl::getInterpolationMode(Modifiers);
2144 llvm::Value *Value =
2145 handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr, Signature, InterpMode);
2146 // Intrinsic-only system values and SPIR-V loads do not add DXIL signature
2147 // elements. Non-pixel inputs retain Undefined.
2148 if (Signature.size() != PreviousSize) {
2149 auto &Element = Signature.back();
2150 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
2151 Mode: InterpMode, CompType: Element.CompType, Kind: Element.SemanticKind,
2152 Stage: FD->getAttr<HLSLShaderAttr>()->getType(), IO: llvm::hlsl::IOType::In);
2153 }
2154 return std::make_pair(x&: Value, y&: AttrBegin);
2155}
2156
2157specific_attr_iterator<HLSLAppliedSemanticAttr>
2158CGHLSLRuntime::handleSemanticStore(
2159 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
2160 const clang::DeclaratorDecl *Decl,
2161 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
2162 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
2163 SemanticSignatures &Signature,
2164 llvm::hlsl::InterpolationModifier Modifiers) {
2165 assert(AttrBegin != AttrEnd);
2166 // An inner field overrides the enclosing return declaration's modifiers.
2167 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
2168 Modifiers =
2169 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
2170 if (Source->getType()->isStructTy())
2171 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
2172 Signature, Modifiers);
2173
2174 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
2175 ++AttrBegin;
2176 size_t PreviousSize = Signature.size();
2177 auto InterpMode = llvm::hlsl::getInterpolationMode(Modifiers);
2178 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr, Signature, InterpMode);
2179 if (Signature.size() != PreviousSize) {
2180 auto &Element = Signature.back();
2181 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
2182 Mode: InterpMode, CompType: Element.CompType, Kind: Element.SemanticKind,
2183 Stage: FD->getAttr<HLSLShaderAttr>()->getType(), IO: llvm::hlsl::IOType::Out);
2184 }
2185 return AttrBegin;
2186}
2187
2188void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
2189 llvm::Function *Fn) {
2190 SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
2191 SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
2192
2193 llvm::Module &M = CGM.getModule();
2194 llvm::LLVMContext &Ctx = M.getContext();
2195 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
2196 Function *EntryFn =
2197 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
2198
2199 // Copy function attributes over, we have no argument or return attributes
2200 // that can be valid on the real entry.
2201 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
2202 Attrs: Fn->getAttributes().getFnAttrs());
2203 EntryFn->setAttributes(NewAttrs);
2204 setHLSLEntryAttributes(FD, Fn: EntryFn);
2205
2206 // Set the called function as internal linkage.
2207 Fn->setLinkage(GlobalValue::InternalLinkage);
2208
2209 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
2210 IRBuilder<> B(BB);
2211 llvm::SmallVector<Value *> Args;
2212
2213 SmallVector<OperandBundleDef, 1> OB;
2214 if (CGM.shouldEmitConvergenceTokens()) {
2215 assert(EntryFn->isConvergent());
2216 llvm::Value *I =
2217 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
2218 llvm::Value *bundleArgs[] = {I};
2219 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
2220 }
2221
2222 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
2223
2224 unsigned SRetOffset = 0;
2225 for (const auto &Param : Fn->args()) {
2226 if (Param.hasStructRetAttr()) {
2227 SRetOffset = 1;
2228 llvm::Type *VarType = Param.getParamStructRetType();
2229 llvm::Value *Var =
2230 CGM.getLangOpts().EmitLogicalPointer
2231 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
2232 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
2233 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
2234 Args.push_back(Elt: Var);
2235 continue;
2236 }
2237
2238 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
2239 llvm::Value *SemanticValue = nullptr;
2240 // FIXME: support inout/out parameters for semantics.
2241 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
2242 PD->getAttr<HLSLParamModifierAttr>()) {
2243 llvm_unreachable("Not handled yet");
2244 } else {
2245 llvm::Type *ParamType = nullptr;
2246 if (Param.hasByValAttr())
2247 ParamType = Param.getParamByValType();
2248 else if (PD->getType()->isRecordType())
2249 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
2250 else
2251 ParamType = Param.getType();
2252
2253 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
2254 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
2255 auto Result = handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd,
2256 Signature&: InputSignature);
2257 SemanticValue = Result.first;
2258 if (!SemanticValue)
2259 return;
2260 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
2261 llvm::Value *Var =
2262 CGM.getLangOpts().EmitLogicalPointer
2263 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
2264 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
2265 B.CreateStore(Val: SemanticValue, Ptr: Var);
2266 SemanticValue = Var;
2267 }
2268 }
2269
2270 assert(SemanticValue);
2271 Args.push_back(Elt: SemanticValue);
2272 }
2273
2274 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
2275 CI->setCallingConv(Fn->getCallingConv());
2276
2277 if (Fn->getReturnType() != CGM.VoidTy)
2278 // Element type is unused, so set to dummy value (NULL).
2279 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
2280
2281 for (auto &SourcePair : OutputSemantic) {
2282 llvm::Value *Source = SourcePair.first;
2283 llvm::Type *ElementType = SourcePair.second;
2284 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
2285 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
2286
2287 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
2288 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
2289 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd,
2290 Signature&: OutputSignature);
2291 }
2292
2293 B.CreateRetVoid();
2294
2295 // Add and identify root signature to function, if applicable
2296 for (const Attr *Attr : FD->getAttrs()) {
2297 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
2298 auto *RSDecl = RSAttr->getSignatureDecl();
2299 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
2300 Fn: EntryFn, M);
2301 }
2302 }
2303
2304 if (CGM.getTarget().getTriple().isDXIL()) {
2305 // Use the entry's shader stage, not the target environment, which may be
2306 // 'library' when compiling multiple entry points.
2307 auto Stage = FD->getAttr<HLSLShaderAttr>()->getType();
2308 using PackingMode = CodeGenOptions::HLSLSemanticSignaturePackingMode;
2309 auto Mode = CGM.getCodeGenOpts().getHLSLSemanticSignaturePacking();
2310 for (auto IOTy : {llvm::hlsl::IOType::In, llvm::hlsl::IOType::Out}) {
2311 bool IsOutput = IOTy == llvm::hlsl::IOType::Out;
2312 auto &Signature = IsOutput ? OutputSignature : InputSignature;
2313 auto Packed = packSemanticSignature(Elements: Signature, Stage, IOTy, Mode,
2314 UseNative16BitTypes: CGM.getLangOpts().NativeHalfType);
2315 if (!Packed) {
2316 CGM.getDiags().Report(Loc: FD->getLocation(),
2317 DiagID: diag::err_hlsl_signature_packing)
2318 << IsOutput << llvm::toString(E: Packed.takeError());
2319 return;
2320 }
2321 // Library entry points retain stage-specific packing without warnings.
2322 if (Mode != PackingMode::Default &&
2323 CGM.getTarget().getTriple().getEnvironment() !=
2324 llvm::Triple::Library &&
2325 ((Stage == llvm::Triple::Vertex && !IsOutput) ||
2326 (Stage == llvm::Triple::Pixel && IsOutput)))
2327 CGM.getDiags().Report(Loc: FD->getLocation(),
2328 DiagID: diag::warn_hlsl_signature_packing_ignored)
2329 << (Mode == PackingMode::Optimized ? "optimized" : "prefix-stable")
2330 << IsOutput;
2331 }
2332 }
2333
2334 addSemanticSignatureMD(InputElements: InputSignature, OutputElements: OutputSignature, Fn: EntryFn, M);
2335}
2336
2337static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
2338 bool CtorOrDtor) {
2339 const auto *GV =
2340 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
2341 if (!GV)
2342 return;
2343 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
2344 if (!CA)
2345 return;
2346 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
2347 // HLSL neither supports priorities or COMDat values, so we will check those
2348 // in an assert but not handle them.
2349
2350 for (const auto &Ctor : CA->operands()) {
2351 if (isa<ConstantAggregateZero>(Val: Ctor))
2352 continue;
2353 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
2354
2355 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
2356 "HLSL doesn't support setting priority for global ctors.");
2357 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
2358 "HLSL doesn't support COMDat for global ctors.");
2359 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
2360 }
2361}
2362
2363void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
2364 llvm::Module &M = CGM.getModule();
2365 SmallVector<Function *> CtorFns;
2366 SmallVector<Function *> DtorFns;
2367 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
2368 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
2369
2370 // Insert a call to the global constructor at the beginning of the entry block
2371 // to externally exported functions. This is a bit of a hack, but HLSL allows
2372 // global constructors, but doesn't support driver initialization of globals.
2373 for (auto &F : M.functions()) {
2374 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
2375 continue;
2376 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
2377 Instruction *IP = &*F.getEntryBlock().begin();
2378 SmallVector<OperandBundleDef, 1> OB;
2379 if (Token) {
2380 llvm::Value *bundleArgs[] = {Token};
2381 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
2382 IP = Token->getNextNode();
2383 }
2384 IRBuilder<> B(IP);
2385 for (auto *Fn : CtorFns) {
2386 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
2387 CI->setCallingConv(Fn->getCallingConv());
2388 }
2389
2390 // Insert global dtors before the terminator of the last instruction
2391 B.SetInsertPoint(F.back().getTerminator());
2392 for (auto *Fn : DtorFns) {
2393 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
2394 CI->setCallingConv(Fn->getCallingConv());
2395 }
2396 }
2397
2398 // No need to keep global ctors/dtors for non-lib profile after call to
2399 // ctors/dtors added for entry.
2400 Triple T(M.getTargetTriple());
2401 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
2402 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
2403 GV->eraseFromParent();
2404 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
2405 GV->eraseFromParent();
2406 }
2407}
2408
2409static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
2410 Intrinsic::ID IntrID,
2411 ArrayRef<llvm::Value *> Args) {
2412
2413 LLVMContext &Ctx = CGM.getLLVMContext();
2414 llvm::Function *InitResFunc =
2415 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
2416 Linkage: llvm::GlobalValue::InternalLinkage,
2417 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
2418 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
2419
2420 llvm::BasicBlock *EntryBB =
2421 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
2422 CGBuilderTy Builder(CGM, Ctx);
2423 const DataLayout &DL = CGM.getModule().getDataLayout();
2424 Builder.SetInsertPoint(EntryBB);
2425
2426 // Make sure the global variable is buffer resource handle
2427 llvm::Type *HandleTy = GV->getValueType();
2428 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
2429
2430 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
2431 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
2432 Name: Twine(GV->getName()).concat(Suffix: "_h"));
2433
2434 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
2435 Builder.CreateRetVoid();
2436
2437 CGM.AddCXXGlobalInit(F: InitResFunc);
2438}
2439
2440void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
2441 llvm::GlobalVariable *GV) {
2442 ResourceBindingAttrs Binding(BufDecl);
2443 assert(Binding.hasBinding() &&
2444 "cbuffer/tbuffer should always have resource binding attribute");
2445
2446 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2447 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
2448 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
2449 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
2450
2451 // buffer with explicit binding
2452 if (Binding.isExplicit()) {
2453 llvm::Intrinsic::ID IntrinsicID =
2454 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
2455 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
2456 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
2457 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2458 } else {
2459 // buffer with implicit binding
2460 llvm::Intrinsic::ID IntrinsicID =
2461 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
2462 auto *OrderID =
2463 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
2464 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
2465 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2466 }
2467}
2468
2469void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
2470 llvm::GlobalVariable *GV) {
2471 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
2472 addSPIRVInterfaceDecorations(
2473 GV, Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Operand: Attr->getBuiltIn());
2474 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
2475 addSPIRVInterfaceDecorations(
2476 GV, Decoration: llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Operand: Attr->getBuiltIn());
2477}
2478
2479llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
2480 if (!CGM.shouldEmitConvergenceTokens())
2481 return nullptr;
2482
2483 auto E = BB.end();
2484 for (auto I = BB.begin(); I != E; ++I) {
2485 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
2486 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
2487 return II;
2488 }
2489 }
2490 llvm_unreachable("Convergence token should have been emitted.");
2491 return nullptr;
2492}
2493
2494class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
2495public:
2496 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
2497 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
2498 OpaqueValueVisitor() {}
2499
2500 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
2501 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
2502 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
2503 // traversal, the temporary containing the copy out will not have
2504 // been created yet.
2505 return false;
2506 }
2507
2508 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
2509 // Traverse the source expression first.
2510 if (E->getSourceExpr())
2511 TraverseStmt(S: E->getSourceExpr());
2512
2513 // Then add this OVE if we haven't seen it before.
2514 if (Visited.insert(Ptr: E).second)
2515 OVEs.push_back(Elt: E);
2516
2517 return true;
2518 }
2519};
2520
2521void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
2522 InitListExpr *E) {
2523
2524 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
2525 OpaqueValueVisitor Visitor;
2526 Visitor.TraverseStmt(S: E);
2527 for (auto *OVE : Visitor.OVEs) {
2528 if (CGF.isOpaqueValueEmitted(E: OVE))
2529 continue;
2530 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
2531 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
2532 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
2533 } else {
2534 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
2535 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
2536 }
2537 }
2538}
2539
2540std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
2541 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
2542 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
2543 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
2544 "expected resource array subscript expression");
2545
2546 // Let clang codegen handle local and static resource array subscripts,
2547 // or when the subscript references on opaque expression (as part of
2548 // ArrayInitLoopExpr AST node).
2549 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
2550 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
2551 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2552 ArrayDecl->getStorageClass() == SC_Static)
2553 return std::nullopt;
2554
2555 // get the resource array type
2556 ASTContext &AST = ArrayDecl->getASTContext();
2557 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
2558 assert(ResArrayTy->isHLSLResourceRecordArray() &&
2559 "expected array of resource classes");
2560
2561 // Iterate through all nested array subscript expressions to calculate
2562 // the index in the flattened resource array (if this is a multi-
2563 // dimensional array). The index is calculated as a sum of all indices
2564 // multiplied by the total size of the array at that level.
2565 Value *Index = nullptr;
2566 const ArraySubscriptExpr *ASE = ArraySubsExpr;
2567 while (ASE != nullptr) {
2568 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
2569 if (const auto *ArrayTy =
2570 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
2571 Value *Multiplier = llvm::ConstantInt::get(
2572 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
2573 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
2574 }
2575 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
2576 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
2577 }
2578
2579 // Find binding info for the resource array. For implicit binding
2580 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
2581 ResourceBindingAttrs Binding(ArrayDecl);
2582 assert(Binding.hasBinding() &&
2583 "resource array must have a binding attribute");
2584
2585 // Find the individual resource type.
2586 QualType ResultTy = ArraySubsExpr->getType();
2587 QualType ResourceTy =
2588 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
2589
2590 // Create a temporary variable for the result, which is either going
2591 // to be a single resource instance or a local array of resources (we need to
2592 // return an LValue).
2593 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
2594 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
2595 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
2596 kind: NormalEHLifetimeMarker, A: TmpVar);
2597
2598 AggValueSlot ValueSlot = AggValueSlot::forAddr(
2599 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
2600 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
2601 mayOverlap: AggValueSlot::DoesNotOverlap);
2602
2603 // Calculate total array size (= range size).
2604 llvm::Value *Range = llvm::ConstantInt::getSigned(
2605 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
2606
2607 // If the result of the subscript operation is a single resource, call the
2608 // constructor.
2609 if (ResultTy == ResourceTy) {
2610 CallArgList Args;
2611 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
2612 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
2613 Name: ArrayDecl->getName(), Binding, Args);
2614
2615 if (!CreateMethod) {
2616 // This can happen if someone creates an array of structs that looks like
2617 // an HLSL resource record array but it does not have the required static
2618 // create method. No binding will be generated for it.
2619 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
2620 "create method lookup should always succeed for built-in resource "
2621 "records");
2622 return std::nullopt;
2623 }
2624
2625 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
2626
2627 } else {
2628 // The result of the subscript operation is a local resource array which
2629 // needs to be initialized.
2630 const ConstantArrayType *ArrayTy =
2631 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
2632 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2633 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
2634 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
2635 if (!EndIndex)
2636 return std::nullopt;
2637 }
2638 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
2639}
2640
2641// Initialize all resources of a global resource array into provided slot.
2642bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
2643 const VarDecl *ArrayDecl,
2644 AggValueSlot &DestSlot) {
2645 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2646 ArrayDecl->hasGlobalStorage() &&
2647 ArrayDecl->getStorageClass() != SC_Static &&
2648 "expected global non-static resource array");
2649
2650 // Find binding info for the resource array. For implicit binding
2651 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
2652 ResourceBindingAttrs Binding(ArrayDecl);
2653 assert(Binding.hasBinding() &&
2654 "resource array must have a binding attribute");
2655
2656 // Find the individual resource type.
2657 ASTContext &AST = ArrayDecl->getASTContext();
2658 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
2659 const auto *ResArrayTy =
2660 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
2661
2662 // Create Value for index and total array size (= range size).
2663 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
2664 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2665 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
2666
2667 // Initialize individual resources in the array into DestSlot.
2668 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2669 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
2670 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
2671 return EndIndex.has_value();
2672}
2673
2674// If the expression is a global resource array, initialize all of its resources
2675// into Dest. Returns false if no initialization has been performed and the
2676// array copy should be handled by the default codegen.
2677bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
2678 AggValueSlot &DestSlot) {
2679 assert(E->getType()->isHLSLResourceRecordArray() &&
2680 "expected resource array");
2681
2682 // Find the array declaration for the expression. Fallback to the default
2683 // handling if it's not a global resource array.
2684 const VarDecl *ArrayDecl =
2685 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
2686 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2687 ArrayDecl->getStorageClass() == SC_Static)
2688 return false;
2689
2690 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
2691}
2692
2693// If the expression is a global resource array, create a temporary and
2694// initialize all of its resources, and return it as an LValue. Returns nullopt
2695// if no initialization has been performed and the handling should follow the
2696// default path.
2697std::optional<LValue>
2698CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
2699 const VarDecl *ArrayDecl) {
2700 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2701 "expected resource array declaration");
2702
2703 if (!ArrayDecl->hasGlobalStorage() ||
2704 ArrayDecl->getStorageClass() == SC_Static)
2705 return std::nullopt;
2706
2707 AggValueSlot TmpArraySlot =
2708 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
2709 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
2710 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
2711 Source: AlignmentSource::Decl);
2712 return std::nullopt;
2713}
2714
2715RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
2716 CodeGenFunction &CGF) {
2717
2718 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
2719 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
2720 "expected cbuffer matrix");
2721
2722 QualType MatQualTy = LV.getType();
2723 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
2724 Address SrcAddr = LV.getAddress();
2725
2726 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
2727 return SrcAddr;
2728
2729 RawAddress DestAlloca =
2730 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
2731 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
2732 return DestAlloca;
2733}
2734
2735std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
2736 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
2737 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
2738 // Find the element type to index by first padding the element type per HLSL
2739 // buffer rules, and then padding out to a 16-byte register boundary if
2740 // necessary.
2741 llvm::Type *LayoutTy =
2742 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
2743 uint64_t LayoutSizeInBits =
2744 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
2745 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
2746 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
2747 if (RowAlignedSize > ElementSize) {
2748 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
2749 CGM, NumBytes: RowAlignedSize - ElementSize);
2750 assert(Padding && "No padding type for target?");
2751 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
2752 /*isPacked=*/true);
2753 }
2754
2755 // If the layout type doesn't introduce any padding, we don't need to do
2756 // anything special.
2757 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
2758 if (LayoutTy == OrigTy)
2759 return std::nullopt;
2760
2761 LValueBaseInfo EltBaseInfo;
2762 TBAAAccessInfo EltTBAAInfo;
2763
2764 // Index into the object as-if we have an array of the padded element type,
2765 // and then dereference the element itself to avoid reading padding that may
2766 // be past the end of the in-memory object.
2767 SmallVector<llvm::Value *, 2> Indices;
2768 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
2769 Indices.push_back(Elt: Idx);
2770 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
2771
2772 if (CGF.getLangOpts().EmitLogicalPointer) {
2773 // The fact that we emit an array-to-pointer decay might be an oversight,
2774 // but for now, we simply ignore it (see #179951).
2775 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
2776 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
2777
2778 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
2779 Address Addr = LV.getAddress();
2780 LayoutTy = llvm::ArrayType::get(
2781 ElementType: LayoutTy,
2782 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
2783 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
2784 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
2785 Addr =
2786 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
2787 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2788 }
2789
2790 Address Addr =
2791 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
2792 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
2793 IdxList: Indices, Name: "cbufferidx");
2794 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
2795 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2796}
2797
2798std::optional<LValue>
2799CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
2800 const MemberExpr *ME) {
2801 assert((ME->getType()->isHLSLResourceRecord() ||
2802 ME->getType()->isHLSLResourceRecordArray()) &&
2803 "expected resource member expression");
2804
2805 const VarDecl *ResourceVD =
2806 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
2807 if (!ResourceVD)
2808 return std::nullopt;
2809
2810 // Handle member of resource array type.
2811 if (ResourceVD->getType()->isHLSLResourceRecordArray())
2812 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
2813
2814 GlobalVariable *ResGV =
2815 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
2816 const DataLayout &DL = CGM.getDataLayout();
2817 llvm::Type *Ty = ResGV->getValueType();
2818 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
2819 Address Addr = Address(ResGV, Ty, Align);
2820 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
2821 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2822 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
2823 return LV;
2824}
2825
2826bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
2827 const LValue &SrcLV,
2828 AggValueSlot &DestSlot) {
2829 assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
2830 "expected expression in HLSL constant address space");
2831 assert(!E->getType()->isHLSLResourceRecord() &&
2832 !E->getType()->isHLSLResourceRecordArray() &&
2833 "direct accesses to resource types should be handled separately");
2834
2835 if (DestSlot.isIgnored())
2836 return false;
2837
2838 QualType Ty = E->getType();
2839 Address DstPtr = DestSlot.getAddress();
2840 Address SrcPtr = SrcLV.getAddress();
2841
2842 // If there are no intangible types, we don't need to lookup associated
2843 // resources.
2844 if (!Ty->isHLSLIntangibleType())
2845 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty);
2846
2847 // Handle structs with intangible types by setting the resource fields
2848 // of the destination struct with the resources associated with the global
2849 // struct.
2850 EmbeddedResourceNameBuilder NameBuilder;
2851 const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
2852 AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
2853
2854 // Callback to fill in the associated resource.
2855 auto EmitResFn = [&](AggValueSlot &ResSlot) {
2856 const VarDecl *ResDecl = AssociatedResources.getNextResource();
2857 assert(ResDecl && "associated resource declaration not found");
2858
2859 // Check that the resource type of dest and src matches.
2860 [[maybe_unused]] llvm::Type *DestType =
2861 ResSlot.getAddress().getElementType();
2862 [[maybe_unused]] llvm::Type *SrcConvertedType =
2863 CGM.getTypes().ConvertTypeForMem(T: ResDecl->getType());
2864 assert(DestType == SrcConvertedType && "resource slot type mismatch");
2865
2866 if (ResDecl->getType()->isHLSLResourceRecord())
2867 copyGlobalResource(CGF, ResourceVD: ResDecl, DestSlot&: ResSlot);
2868 else
2869 initializeGlobalResourceArray(CGF, ArrayDecl: ResDecl, DestSlot&: ResSlot);
2870 };
2871
2872 auto Result =
2873 HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty, EmitResFn);
2874 assert(AssociatedResources.getNextResource() == nullptr &&
2875 "expected all associated resources to be processed");
2876 return Result;
2877}
2878
2879LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
2880 const MemberExpr *E) {
2881 LValue Base =
2882 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
2883 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
2884 assert(Field && "Unexpected access into HLSL buffer");
2885
2886 const RecordDecl *Rec = Field->getParent();
2887
2888 // Work out the buffer layout type to index into.
2889 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
2890 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
2891 // Since this is a member of an object in the buffer and not the buffer's
2892 // struct/class itself, we shouldn't have any offsets on the members we need
2893 // to contend with.
2894 CGHLSLOffsetInfo EmptyOffsets;
2895 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
2896 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
2897
2898 // Get the field index for the layout struct, accounting for padding.
2899 unsigned FieldIdx =
2900 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
2901 assert(FieldIdx < LayoutTy->getNumElements() &&
2902 "Layout struct is smaller than member struct");
2903 unsigned Skipped = 0;
2904 for (unsigned I = 0; I <= FieldIdx;) {
2905 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
2906 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
2907 ++Skipped;
2908 else
2909 ++I;
2910 }
2911 FieldIdx += Skipped;
2912 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
2913
2914 // Now index into the struct, making sure that the type we return is the
2915 // buffer layout type rather than the original type in the AST.
2916 QualType FieldType = Field->getType();
2917 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
2918 CharUnits Align = CharUnits::fromQuantity(
2919 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
2920
2921 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
2922 ? CGF.Builder.CreateStructuredGEP(
2923 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
2924 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
2925 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
2926 Idx: FieldIdx, Name: Field->getName());
2927 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
2928
2929 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
2930 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2931 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
2932 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
2933
2934 return LV;
2935}
2936