1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/RecursiveASTVisitor.h"
28#include "clang/AST/Type.h"
29#include "clang/Basic/DiagnosticDriver.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetOptions.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/Enum.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/ScopeExit.h"
37#include "llvm/ADT/SmallString.h"
38#include "llvm/ADT/SmallVector.h"
39#include "llvm/Frontend/HLSL/HLSLResource.h"
40#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/GlobalVariable.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Alignment.h"
51#include "llvm/Support/ErrorHandling.h"
52#include "llvm/Support/FormatVariadic.h"
53#include "llvm/Support/Path.h"
54#include "llvm/Transforms/Utils/ModuleUtils.h"
55#include <array>
56#include <cstdint>
57#include <optional>
58
59using namespace clang;
60using namespace CodeGen;
61using namespace clang::hlsl;
62using namespace llvm;
63
64using llvm::hlsl::CBufferRowSizeInBytes;
65
66namespace {
67
68void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
69 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
70 // Assume ValVersionStr is legal here.
71 VersionTuple Version;
72 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
73 Version.getSubminor() || !Version.getMinor()) {
74 return;
75 }
76
77 uint64_t Major = Version.getMajor();
78 uint64_t Minor = *Version.getMinor();
79
80 auto &Ctx = M.getContext();
81 IRBuilder<> B(M);
82 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
83 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
84 StringRef DXILValKey = "dx.valver";
85 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
86 DXILValMD->addOperand(M: Val);
87}
88
89void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
90 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
91 llvm::Function *Fn, llvm::Module &M) {
92 auto &Ctx = M.getContext();
93
94 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
95 MDNode *RootSignature = RSBuilder.BuildRootSignature();
96
97 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
98 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
99 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
100 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
101
102 StringRef RootSignatureValKey = "dx.rootsignatures";
103 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
104 RootSignatureValMD->addOperand(M: MDVals);
105}
106
107MDNode *buildSemanticSignatureMD(
108 ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
109 if (Elements.empty())
110 return nullptr;
111
112 SmallVector<Metadata *> ElementMD;
113 for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
114 ElementMD.push_back(Elt: Element.toMetadata(Ctx));
115 return MDNode::get(Context&: Ctx, MDs: ElementMD);
116}
117
118void addSemanticSignatureMD(
119 ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
120 ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
121 llvm::Function *Fn, llvm::Module &M) {
122 if (InputElements.empty() && OutputElements.empty())
123 return;
124
125 LLVMContext &Ctx = M.getContext();
126 MDNode *InputSignature = buildSemanticSignatureMD(Elements: InputElements, Ctx);
127 MDNode *OutputSignature = buildSemanticSignatureMD(Elements: OutputElements, Ctx);
128 MDNode *MDVals = MDNode::get(
129 Context&: Ctx, MDs: {ValueAsMetadata::get(V: Fn), InputSignature, OutputSignature});
130
131 M.getOrInsertNamedMetadata(Name: "dx.semantic.signatures")->addOperand(M: MDVals);
132}
133
134static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
135 AggValueSlot &DestSlot) {
136 GlobalVariable *ResGV =
137 cast<GlobalVariable>(Val: CGF.CGM.GetAddrOfGlobalVar(D: ResourceVD));
138 assert(ResGV && "expected valid global variable");
139 CGF.Builder.CreateStore(Val: ResGV, Addr: DestSlot.getAddress());
140}
141
142// Given a MemberExpr of a resource or resource array type, find the parent
143// VarDecl of the struct or class instance that contains this resource and
144// build the full resource name based on the member access path.
145//
146// For example, for a member access like "myStructArray[0].memberA",
147// this function will find the VarDecl of "myStructArray" and use the
148// EmbeddedResourceNameBuilder to build the resource name
149// "myStructArray.0.memberA".
150//
151// This also works for a record type expression that has some embedded
152// resources. It finds the parent VarDecl of that record and builds a partial
153// name which is the prefix of the resource globals associated with the
154// declaration.
155static const VarDecl *findStructResourceParentDeclAndBuildName(
156 const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
157
158 SmallVector<const Expr *> WorkList;
159 const VarDecl *VD = nullptr;
160
161 for (;;) {
162 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
163 assert(isa<VarDecl>(DRE->getDecl()) &&
164 "member expr base is not a var decl");
165 VD = cast<VarDecl>(Val: DRE->getDecl());
166 NameBuilder.pushName(N: VD->getName());
167 break;
168 }
169
170 WorkList.push_back(Elt: E);
171 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
172 E = MExp->getBase();
173 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
174 E = ICE->getSubExpr();
175 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
176 E = ASE->getBase();
177 else if (isa<CXXThisExpr>(Val: E))
178 // Resource member access on "this" pointer not yet implemented
179 // (llvm/llvm-project#190299)
180 return nullptr;
181 else
182 llvm_unreachable("unexpected expr type in resource member access");
183
184 assert(E && "expected valid expression");
185 }
186
187 while (!WorkList.empty()) {
188 E = WorkList.pop_back_val();
189 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
190 NameBuilder.pushName(
191 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
192 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
193 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
194 CXXRecordDecl *DerivedRD =
195 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
196 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
197 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
198 }
199 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
200 const Expr *IdxExpr = ASE->getIdx();
201 std::optional<llvm::APSInt> Value =
202 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
203 assert(Value &&
204 "expected constant index in struct with resource array access");
205 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
206 } else {
207 llvm_unreachable("unexpected expr type in resource member access");
208 }
209 }
210 return VD;
211}
212
213// Given a MemberExpr of a resource or resource array type, find the
214// corresponding global resource declaration associated with the owning struct
215// or class instance via HLSLAssociatedResourceDeclAttr.
216static const VarDecl *
217findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
218
219 EmbeddedResourceNameBuilder NameBuilder;
220 const VarDecl *ParentVD =
221 findStructResourceParentDeclAndBuildName(E: ME, NameBuilder);
222 if (!ParentVD)
223 return nullptr;
224
225 if (!ParentVD->hasGlobalStorage())
226 return nullptr;
227
228 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
229 for (const Attr *A : ParentVD->getAttrs()) {
230 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
231 VarDecl *AssocResVD = ADA->getResDecl();
232 if (AssocResVD->getIdentifier() == II)
233 return AssocResVD;
234 }
235 }
236 return nullptr;
237}
238
239void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
240 auto &SM = CGM.getContext().getSourceManager();
241 auto &Macros = CGM.getPreprocessorOpts().Macros;
242 auto &CodeGenOpts = CGM.getCodeGenOpts();
243 auto &Ctx = M.getContext();
244
245 // Names and content of shader source code files.
246 llvm::NamedMDNode *DXContents =
247 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
248 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
249 llvm::MDTuple *FileInfo =
250 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
251 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
252 DXContents->addOperand(M: FileInfo);
253 };
254
255 bool Invalid = false;
256 const SrcMgr::SLocEntry *MainLocEntry =
257 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
258 assert(!Invalid && "Main file SLocEntry must not be invalid!");
259 const SrcMgr::ContentCache &MainCCEntry =
260 MainLocEntry->getFile().getContentCache();
261
262 SmallVector<std::pair<std::string, StringRef>> Files;
263 std::optional<SmallString<256>> MainFileName;
264 Files.reserve(N: SM.local_sloc_entry_size());
265 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
266 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
267 if (!LocEntry.isFile())
268 continue;
269
270 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
271 if (isSystem(CK: FInfo.getFileCharacteristic()))
272 continue;
273
274 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
275 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
276 if (!FEntry)
277 continue;
278
279 llvm::SmallString<256> Path = FEntry->getName();
280 llvm::sys::path::native(path&: Path);
281 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
282 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
283 if (!Buffer) {
284 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
285 << Path;
286 continue;
287 }
288
289 if (&MainCCEntry != &CCEntry) {
290 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
291 } else {
292 // Main file should be at first position.
293 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
294 MainFileName.emplace(args&: Path);
295 }
296 }
297 assert(MainFileName && "Main file not found.");
298
299 // Files other that main one should be sorted by name.
300 llvm::sort(C&: Files);
301#ifndef NDEBUG
302 for (unsigned I = 1; I < Files.size(); ++I)
303 assert((Files[I - 1].first != Files[I].first) &&
304 "duplicate files in dx.source.contents");
305#endif
306 llvm::for_each(Range&: Files, F: addFile);
307
308 SmallVector<llvm::Metadata *> Defines;
309 Defines.reserve(N: Macros.size());
310 for (const auto &Macro : Macros) {
311 // Ignore undefs.
312 if (!Macro.second)
313 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
314 }
315 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
316 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
317
318 if (!CodeGenOpts.MainFileName.empty())
319 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
320 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
321 ->addOperand(
322 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
323
324 SmallVector<llvm::Metadata *> Args;
325 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
326 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
327 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
328 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
329 M.getOrInsertNamedMetadata(Name: "dx.source.args")
330 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
331}
332
333// Find array variable declaration from DeclRef expression
334static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
335 E = E->IgnoreImpCasts();
336 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
337 return DRE->getDecl();
338 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
339 E = OVE->getSourceExpr()->IgnoreImpCasts();
340 if (isa<MemberExpr>(Val: E))
341 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
342 return nullptr;
343}
344
345// Find array variable declaration from nested array subscript AST nodes
346static const ValueDecl *getArrayDecl(ASTContext &AST,
347 const ArraySubscriptExpr *ASE) {
348 const Expr *E = nullptr;
349 while (ASE != nullptr) {
350 E = ASE->getBase()->IgnoreImpCasts();
351 if (!E)
352 return nullptr;
353 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
354 }
355 return getArrayDecl(AST, E);
356}
357
358// Get the total size of the array, or 0 if the array is unbounded.
359static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
360 Ty = Ty->getUnqualifiedDesugaredType();
361 assert(Ty->isArrayType() && "expected array type");
362 if (Ty->isIncompleteArrayType())
363 return 0;
364 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
365}
366
367static Value *buildNameForResource(llvm::StringRef BaseName,
368 CodeGenModule &CGM) {
369 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
370 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
371 .getPointer();
372}
373
374static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
375 StorageClass SC = SC_None) {
376 for (auto *Method : Record->methods()) {
377 if (Method->getStorageClass() == SC && Method->getName() == Name)
378 return Method;
379 }
380 return nullptr;
381}
382
383static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
384 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
385 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
386 CallArgList &Args) {
387 assert(Binding.hasBinding() && "at least one binding attribute expected");
388
389 ASTContext &AST = CGM.getContext();
390 CXXMethodDecl *CreateMethod = nullptr;
391 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
392 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
393
394 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
395 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
396 "resources with counter handle must have a binding with counter "
397 "implicit order ID");
398 if (Binding.isExplicit()) {
399 // explicit binding
400 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
401 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
402 const char *Name = Binding.hasCounterImplicitOrderID()
403 ? "__createFromBindingWithImplicitCounter"
404 : "__createFromBinding";
405 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
406 } else {
407 // implicit binding
408 auto *OrderID =
409 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
410 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
411 const char *Name = Binding.hasCounterImplicitOrderID()
412 ? "__createFromImplicitBindingWithImplicitCounter"
413 : "__createFromImplicitBinding";
414 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
415 }
416 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
417 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
418 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
419 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
420 if (HasCounter) {
421 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
422 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
423 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
424 }
425
426 return CreateMethod;
427}
428
429static void callResourceInitMethod(CodeGenFunction &CGF,
430 CXXMethodDecl *CreateMethod,
431 CallArgList &Args, Address ReturnAddress) {
432 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
433 const FunctionProtoType *Proto =
434 CreateMethod->getType()->getAs<FunctionProtoType>();
435 // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
436 // caller declaration is needed for x86 SysV ABI selection.
437 const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
438 Args, Ty: Proto, ChainCall: false, /*ABIInfoFD=*/nullptr);
439 ReturnValueSlot ReturnValue(ReturnAddress, false);
440 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
441 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
442}
443
444// Initializes local resource array variable with global resource array
445// elements. For multi-dimensional arrays it calls itself recursively to
446// initialize its sub-arrays. The Index used in the resource constructor calls
447// will begin at StartIndex and will be incremented for each array element. The
448// last used resource Index is returned to the caller. If the function returns
449// std::nullopt, it indicates an error.
450static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
451 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
452 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
453 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
454 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
455
456 ASTContext &AST = CGF.getContext();
457 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
458 llvm::Value *Index = StartIndex;
459 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
460 const uint64_t ArraySize = ArrayTy->getSExtSize();
461 QualType ElemType = ArrayTy->getElementType();
462 Address TmpArrayAddr = ValueSlot.getAddress();
463
464 // Add additional index to the getelementptr call indices.
465 // This index will be updated for each array element in the loops below.
466 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
467 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
468
469 // For array of arrays, recursively initialize the sub-arrays.
470 if (ElemType->isArrayType()) {
471 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
472 for (uint64_t I = 0; I < ArraySize; I++) {
473 if (I > 0) {
474 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
475 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
476 }
477 std::optional<llvm::Value *> MaybeIndex =
478 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
479 ValueSlot, Range, StartIndex: Index,
480 ResourceName, Binding, PrevGEPIndices: GEPIndices);
481 if (!MaybeIndex)
482 return std::nullopt;
483 Index = *MaybeIndex;
484 }
485 return Index;
486 }
487
488 // For array of resources, initialize each resource in the array.
489 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
490 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
491 CharUnits Align =
492 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
493
494 for (uint64_t I = 0; I < ArraySize; I++) {
495 if (I > 0) {
496 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
497 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
498 }
499 Address ReturnAddress =
500 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
501
502 CallArgList Args;
503 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
504 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
505
506 if (!CreateMethod)
507 // This can happen if someone creates an array of structs that looks like
508 // an HLSL resource record array but it does not have the required static
509 // create method. No binding will be generated for it.
510 return std::nullopt;
511
512 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
513 }
514 return Index;
515}
516
517/// Utility for emitting copies following the HLSL buffer layout rules (ie,
518/// copying out of a cbuffer).
519class HLSLBufferCopyEmitter {
520 CodeGenFunction &CGF;
521 Address DstPtr;
522 Address SrcPtr;
523 llvm::Type *LayoutTy = nullptr;
524
525 SmallVector<llvm::Value *> CurStoreIndices;
526 SmallVector<llvm::Value *> CurLoadIndices;
527
528 using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
529
530 // Creates & returns either a structured.gep or a ptradd/gep depending on
531 // langopts.
532 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
533 ArrayRef<llvm::Value *> Indices) {
534 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
535 if (EmitLogical)
536 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
537
538 llvm::SmallVector<llvm::Value *> GEPIndices;
539 GEPIndices.reserve(N: Indices.size() + 1);
540 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
541 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
542 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
543 }
544
545 bool isBufferLayoutArray(llvm::StructType *ST) {
546 // A buffer layout array is a struct with two elements: the padded array,
547 // and the last element. That is, is should look something like this:
548 //
549 // { [%n x { %type, %padding }], %type }
550 //
551 if (!ST || ST->getNumElements() != 2)
552 return false;
553
554 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
555 if (!PaddedEltsTy)
556 return false;
557
558 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
559 if (!PaddedTy || PaddedTy->getNumElements() != 2)
560 return false;
561
562 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
563 Ty: PaddedTy->getElementType(N: 1)))
564 return false;
565
566 llvm::Type *ElementTy = ST->getElementType(N: 1);
567 if (PaddedTy->getElementType(N: 0) != ElementTy)
568 return false;
569 return true;
570 }
571
572 // Returns true if the type is either a struct representing a resource record,
573 // or an array of structs that are resource records. This assumes a struct is
574 // a resource record if the first element is a target type (resource handle).
575 // This is the case for all target types used by HLSL except the padding type
576 // ("{dx|spirv.Padding"), but padding will never be the first element of a
577 // struct.
578 bool isResourceOrResourceArray(llvm::Type *Ty) {
579 while (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
580 Ty = AT->getElementType();
581
582 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
583 if (!ST || ST->getNumElements() < 1)
584 return false;
585
586 auto *TargetTy = dyn_cast<llvm::TargetExtType>(Val: ST->getElementType(N: 0));
587 return TargetTy != nullptr;
588 }
589
590 void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
591 EmitResourceFnTy EmitResFn) {
592 CharUnits DstAlign =
593 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
594 Address DstAddr(Dst, DstTy, DstAlign);
595 AggValueSlot Slot = AggValueSlot::forAddr(
596 addr: DstAddr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
597 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
598 mayOverlap: AggValueSlot::DoesNotOverlap);
599
600 EmitResFn(Slot);
601 }
602
603 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
604 llvm::ArrayType *DstTy,
605 EmitResourceFnTy EmitResFn) {
606 // Those assumptions are checked by isBufferLayoutArray.
607 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
608 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
609 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
610 ->getElementType(0) == SrcTy->getElementType(1));
611
612 auto *SrcDataTy = SrcTy->getElementType(N: 1);
613 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
614
615 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
616 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
617 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
618 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
619 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
620 EmitResFn);
621 }
622
623 auto *SrcElt =
624 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
625 auto *DstElt = emitAccessChain(
626 BaseTy: DstTy, Base: Dst,
627 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
628 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
629 EmitResFn);
630 }
631
632 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
633 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
634 assert(!isResourceOrResourceArray(DstTy) &&
635 "direct access to resources or resource arrays should be handled "
636 "separately");
637
638 if (isBufferLayoutArray(ST: SrcTy))
639 return emitBufferLayoutCopy(Src, SrcTy, Dst, DstTy: cast<llvm::ArrayType>(Val: DstTy),
640 EmitResFn);
641
642 unsigned SrcIndex = 0;
643 unsigned DstIndex = 0;
644
645 // DstTy layout is in default address space and can include resource types.
646 // SrcTy is in cbuffer layout where resources are filtered out, so the
647 // number of elements in SrcTy can be less than the number of elements in
648 // DstTy.
649 auto *DstST = cast<llvm::StructType>(Val: DstTy);
650 while (DstIndex < DstST->getNumElements()) {
651 llvm::Type *DstEltTy = DstST->getElementType(N: DstIndex);
652 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: DstEltTy)) {
653 DstIndex += 1;
654 continue;
655 }
656 if (isResourceOrResourceArray(Ty: DstEltTy)) {
657 auto *DstElt = emitAccessChain(
658 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
659 emitResourceOrResourceArray(Dst: DstElt, DstTy: DstEltTy, EmitResFn);
660 DstIndex += 1;
661 continue;
662 }
663
664 assert(SrcIndex < SrcTy->getNumElements());
665 llvm::Type *SrcEltTy = SrcTy->getElementType(N: SrcIndex);
666 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: SrcEltTy)) {
667 SrcIndex += 1;
668 continue;
669 }
670
671 auto *SrcElt = emitAccessChain(
672 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
673 auto *DstElt = emitAccessChain(
674 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
675 emitElementCopy(Src: SrcElt, SrcTy: SrcEltTy, Dst: DstElt, DstTy: DstEltTy, EmitResFn);
676 DstIndex += 1;
677 SrcIndex += 1;
678 }
679 }
680
681 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
682 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
683 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
684 auto *SrcElt =
685 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
686 auto *DstElt =
687 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
688 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
689 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType(),
690 EmitResFn);
691 }
692 }
693
694 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
695 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
696 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
697 return emitCopy(Src, SrcTy: AT, Dst, DstTy, EmitResFn);
698 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
699 return emitCopy(Src, SrcTy: ST, Dst, DstTy, EmitResFn);
700
701 // When we have a scalar or vector element we can emit the copy.
702 CharUnits SrcAlign =
703 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
704 CharUnits DstAlign =
705 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
706 Address SrcAddr(Src, SrcTy, SrcAlign);
707 Address DstAddr(Dst, DstTy, DstAlign);
708 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
709 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
710 }
711
712public:
713 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
714 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
715
716 bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
717 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
718
719 // TODO: We should be able to fall back to a regular memcpy if the layout
720 // type doesn't have any padding, but that runs into issues in the backend
721 // currently.
722 //
723 // See https://github.com/llvm/wg-hlsl/issues/351
724 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
725 DstTy: DstPtr.getElementType(), EmitResFn);
726 return true;
727 }
728};
729
730// Represents a list resources associated with a global struct whose name
731// starts with the specified prefix.
732// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
733// order of the depth-first traversal of the corresponding fields in the struct.
734// The resources are always returned in that order, which is the same order
735// we need when a struct is copied element-by-element.
736class AssociatedResourcesList {
737 // Iterator pointers for the associated resource attributes that match the
738 // prefix. Begin = begin of the range of attributes that match the prefix End
739 // = end of the range of attributes that match the prefix Next = the current
740 // attribute in the iteration to be returned by getNextResource
741 specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
742
743public:
744 AssociatedResourcesList(const VarDecl *StructVD,
745 StringRef ResourceNamePrefix) {
746 auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
747 auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
748
749 // Skip over associated resources that don't match the prefix.
750 while (I != E &&
751 !I->getResDecl()->getName().starts_with(Prefix: ResourceNamePrefix))
752 ++I;
753 assert(I != E && "expected associated resource not found");
754 Begin = End = I;
755
756 // Scan over associated resources that do match the prefix to find the end
757 // of the range.
758 while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
759 ->getResDecl()
760 ->getName()
761 .starts_with(Prefix: ResourceNamePrefix))
762 End = ++I;
763
764 Next = Begin;
765 }
766
767 const VarDecl *getNextResource() {
768 if (Next == End)
769 return nullptr;
770
771 const VarDecl *Res = Next->getResDecl();
772 ++Next;
773 return Res;
774 }
775};
776
777} // namespace
778
779llvm::Type *
780CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
781 const CGHLSLOffsetInfo &OffsetInfo) {
782 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
783
784 // Check if the target has a specific translation for this type first.
785 if (llvm::Type *TargetTy =
786 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
787 return TargetTy;
788
789 llvm_unreachable("Generic handling of HLSL types is not supported.");
790}
791
792llvm::Triple::ArchType CGHLSLRuntime::getArch() {
793 return CGM.getTarget().getTriple().getArch();
794}
795
796// Emits constant global variables for buffer constants declarations
797// and creates metadata linking the constant globals with the buffer global.
798void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
799 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
800 const CGHLSLOffsetInfo &OffsetInfo) {
801 LLVMContext &Ctx = CGM.getLLVMContext();
802
803 // get the layout struct from constant buffer target type
804 llvm::Type *BufType = BufGV->getValueType();
805 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
806 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
807
808 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
809 size_t OffsetIdx = 0;
810 for (Decl *D : BufDecl->buffer_decls()) {
811 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
812 // Nothing to do for this declaration.
813 continue;
814 if (isa<FunctionDecl>(Val: D)) {
815 // A function within an cbuffer is effectively a top-level function.
816 CGM.EmitTopLevelDecl(D);
817 continue;
818 }
819 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
820 if (!VD)
821 continue;
822
823 QualType VDTy = VD->getType();
824 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
825 if (VD->getStorageClass() == SC_Static ||
826 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
827 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
828 // Emit static and groupshared variables and resource classes inside
829 // cbuffer as regular globals
830 CGM.EmitGlobal(D: VD);
831 }
832 continue;
833 }
834
835 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
836 }
837
838 if (!OffsetInfo.empty())
839 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
840 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
841 });
842
843 // Associate the buffer global variable with its constants
844 SmallVector<llvm::Metadata *> BufGlobals;
845 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
846 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
847
848 auto ElemIt = LayoutStruct->element_begin();
849 for (auto &[VD, _] : DeclsWithOffset) {
850 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
851 ++ElemIt;
852
853 assert(ElemIt != LayoutStruct->element_end() &&
854 "number of elements in layout struct does not match");
855 llvm::Type *LayoutType = *ElemIt++;
856
857 GlobalVariable *ElemGV =
858 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
859 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
860 }
861 assert(ElemIt == LayoutStruct->element_end() &&
862 "number of elements in layout struct does not match");
863
864 // add buffer metadata to the module
865 CGM.getModule()
866 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
867 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
868}
869
870// Creates resource handle type for the HLSL buffer declaration
871static const clang::HLSLAttributedResourceType *
872createBufferHandleType(const HLSLBufferDecl *BufDecl) {
873 ASTContext &AST = BufDecl->getASTContext();
874 QualType QT = AST.getHLSLAttributedResourceType(
875 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
876 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
877 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
878}
879
880CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
881 CGHLSLOffsetInfo Result;
882
883 // If we don't have packoffset info, just return an empty result.
884 if (!BufDecl.hasValidPackoffset())
885 return Result;
886
887 for (Decl *D : BufDecl.buffer_decls()) {
888 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
889 continue;
890 }
891 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
892 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
893 continue;
894
895 if (!VD->hasAttrs()) {
896 Result.Offsets.push_back(Elt: Unspecified);
897 continue;
898 }
899
900 uint32_t Offset = Unspecified;
901 for (auto *Attr : VD->getAttrs()) {
902 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
903 Offset = POA->getOffsetInBytes();
904 break;
905 }
906 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
907 if (RBA &&
908 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
909 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
910 break;
911 }
912 }
913 Result.Offsets.push_back(Elt: Offset);
914 }
915 return Result;
916}
917
918// Codegen for HLSLBufferDecl
919void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
920
921 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
922
923 // create resource handle type for the buffer
924 const clang::HLSLAttributedResourceType *ResHandleTy =
925 createBufferHandleType(BufDecl);
926
927 // empty constant buffer is ignored
928 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
929 return;
930
931 // create global variable for the constant buffer
932 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
933 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
934 llvm::GlobalVariable *BufGV = new GlobalVariable(
935 LayoutTy, /*isConstant*/ false,
936 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
937 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
938 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
939 GlobalValue::NotThreadLocal);
940
941 llvm::Module &M = CGM.getModule();
942 M.insertGlobalVariable(GV: BufGV);
943
944 // Add the global variable to the compiler used list so it does not
945 // get optimized away by GlobalOptPass before it reaches
946 // {DXIL|SPIRV}CBufferAccess pass.
947 llvm::appendToCompilerUsed(M, Values: {BufGV});
948
949 // Add globals for constant buffer elements and create metadata nodes
950 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
951
952 // Initialize cbuffer from binding (implicit or explicit)
953 initializeBufferFromBinding(BufDecl, GV: BufGV);
954}
955
956void CGHLSLRuntime::addRootSignature(
957 const HLSLRootSignatureDecl *SignatureDecl) {
958 llvm::Module &M = CGM.getModule();
959 Triple T(M.getTargetTriple());
960
961 // Generated later with the function decl if not targeting root signature
962 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
963 return;
964
965 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
966 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
967}
968
969llvm::StructType *
970CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
971 const auto Entry = LayoutTypes.find(Val: StructType);
972 if (Entry != LayoutTypes.end())
973 return Entry->getSecond();
974 return nullptr;
975}
976
977void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
978 llvm::StructType *LayoutTy) {
979 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
980 "layout type for this struct already exist");
981 LayoutTypes[StructType] = LayoutTy;
982}
983
984void CGHLSLRuntime::finishCodeGen() {
985 auto &TargetOpts = CGM.getTarget().getTargetOpts();
986 auto &CodeGenOpts = CGM.getCodeGenOpts();
987 auto &LangOpts = CGM.getLangOpts();
988 llvm::Module &M = CGM.getModule();
989 Triple T(M.getTargetTriple());
990 if (T.getArch() == Triple::ArchType::dxil)
991 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
992 if (!CodeGenOpts.DisableDXSourceMetadata &&
993 CodeGenOpts.getDebugInfo() >=
994 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
995 addSourceInfo(CGM, M);
996 if (CodeGenOpts.ResMayAlias)
997 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
998 if (CodeGenOpts.AllResourcesBound)
999 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
1000 Key: "dx.allresourcesbound", Val: 1);
1001 if (CodeGenOpts.OptimizationLevel == 0)
1002 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
1003 Key: "dx.disable_optimizations", Val: 1);
1004
1005 // NativeHalfType corresponds to the -fnative-half-type clang option which is
1006 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
1007 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
1008 if (LangOpts.NativeHalfType)
1009 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
1010 Val: 1);
1011
1012 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
1013 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
1014 const ASTContext &Ctx = CGM.getContext();
1015 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
1016 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
1017 SmallVector<GlobalValue *, 8> InterfaceVars;
1018 for (GlobalVariable &GV : M.globals()) {
1019 unsigned AS = GV.getAddressSpace();
1020 if (AS == InputAS || AS == OutputAS)
1021 InterfaceVars.push_back(Elt: &GV);
1022 }
1023 if (!InterfaceVars.empty())
1024 appendToCompilerUsed(M, Values: InterfaceVars);
1025 }
1026
1027 generateGlobalCtorDtorCalls();
1028}
1029
1030void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
1031 const FunctionDecl *FD, llvm::Function *Fn) {
1032 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1033 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
1034 const StringRef ShaderAttrKindStr = "hlsl.shader";
1035 Fn->addFnAttr(Kind: ShaderAttrKindStr,
1036 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
1037 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
1038 const StringRef NumThreadsKindStr = "hlsl.numthreads";
1039 std::string NumThreadsStr =
1040 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
1041 Vals: NumThreadsAttr->getZ());
1042 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
1043 }
1044 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
1045 const StringRef WaveSizeKindStr = "hlsl.wavesize";
1046 std::string WaveSizeStr =
1047 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
1048 Vals: WaveSizeAttr->getPreferred());
1049 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
1050 }
1051 // HLSL entry functions are materialized for module functions with
1052 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
1053 // later in the compiler-flow for such module functions is not aware of and
1054 // hence not able to set attributes of the newly materialized entry functions.
1055 // So, set attributes of entry function here, as appropriate.
1056 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
1057
1058 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
1059 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
1060 }
1061}
1062
1063static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
1064 // Compute ID intrinsics return i32 components, but the semantic may use
1065 // 16-bit integers. Narrow each component before assembling the input.
1066 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1067 Value *Result = PoisonValue::get(T: Ty);
1068 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
1069 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
1070 Elt = B.CreateTrunc(V: Elt, DestTy: VT->getElementType());
1071 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
1072 }
1073 return Result;
1074 }
1075 return B.CreateTrunc(V: B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)}), DestTy: Ty);
1076}
1077
1078static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
1079 unsigned BuiltIn) {
1080 LLVMContext &Ctx = GV->getContext();
1081 IRBuilder<> B(*GV->getParent());
1082 MDNode *Operands = MDNode::get(
1083 Context&: Ctx,
1084 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
1085 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
1086 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1087 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1088}
1089
1090static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
1091 LLVMContext &Ctx = GV->getContext();
1092 IRBuilder<> B(*GV->getParent());
1093 MDNode *Operands =
1094 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Location */ C: 30)),
1095 ConstantAsMetadata::get(C: B.getInt32(C: Location))});
1096 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1097 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1098}
1099
1100// A fragment shader input interface variable whose base type is an integer or
1101// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
1102// specification requires these variables to be decorated with Flat (see
1103// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
1104// their base scalar type.
1105static bool inputRequiresFlatDecoration(llvm::Type *Ty) {
1106 while (true) {
1107 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
1108 Ty = AT->getElementType();
1109 continue;
1110 }
1111 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: Ty)) {
1112 Ty = VT->getElementType();
1113 continue;
1114 }
1115 break;
1116 }
1117 return Ty->isIntegerTy() || Ty->isDoubleTy();
1118}
1119
1120static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
1121 llvm::Type *Ty, const Twine &Name,
1122 unsigned BuiltInID) {
1123 auto *GV = new llvm::GlobalVariable(
1124 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1125 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1126 llvm::GlobalVariable::GeneralDynamicTLSModel,
1127 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1128 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1129 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1130 return B.CreateLoad(Ty, Ptr: GV);
1131}
1132
1133static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
1134 llvm::Type *Ty, unsigned Location,
1135 StringRef Name, bool NeedsFlat) {
1136 auto *GV = new llvm::GlobalVariable(
1137 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1138 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1139 llvm::GlobalVariable::GeneralDynamicTLSModel,
1140 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1141 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1142
1143 // Emit all decorations as a single `spirv.Decorations` node. Attaching
1144 // multiple `spirv.Decorations` metadata nodes to the same global is not
1145 // supported by the SPIR-V backend and results in all but one being dropped.
1146 LLVMContext &Ctx = GV->getContext();
1147 SmallVector<Metadata *, 2> Decorations;
1148 Decorations.push_back(
1149 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1150 C: B.getInt32(/* SPIRV::Decoration::Location */ C: 30)),
1151 ConstantAsMetadata::get(C: B.getInt32(C: Location))}));
1152 if (NeedsFlat)
1153 Decorations.push_back(
1154 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1155 C: B.getInt32(/* SPIRV::Decoration::Flat */ C: 14))}));
1156 GV->addMetadata(Kind: "spirv.Decorations", MD&: *MDNode::get(Context&: Ctx, MDs: Decorations));
1157
1158 return B.CreateLoad(Ty, Ptr: GV);
1159}
1160
1161llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
1162 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1163 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1164 std::optional<unsigned> Index) {
1165 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1166 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1167
1168 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
1169 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1170 Location = L->getLocation();
1171
1172 // DXC completely ignores the semantic/index pair. Location are assigned from
1173 // the first semantic to the last.
1174 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
1175 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1176 SPIRVLastAssignedInputSemanticLocation += ElementCount;
1177
1178 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1179 bool NeedsFlat =
1180 ShaderAttr &&
1181 ShaderAttr->getType() == llvm::Triple::EnvironmentType::Pixel &&
1182 inputRequiresFlatDecoration(Ty: Type);
1183
1184 return createSPIRVLocationLoad(B, M&: CGM.getModule(), Ty: Type, Location,
1185 Name: VariableName.str(), NeedsFlat);
1186}
1187
1188static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
1189 llvm::Value *Source, unsigned Location,
1190 StringRef Name) {
1191 auto *GV = new llvm::GlobalVariable(
1192 M, Source->getType(), /* isConstant= */ false,
1193 llvm::GlobalValue::ExternalLinkage,
1194 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1195 llvm::GlobalVariable::GeneralDynamicTLSModel,
1196 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1197 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1198 addLocationDecoration(GV, Location);
1199 B.CreateStore(Val: Source, Ptr: GV);
1200}
1201
1202void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1203 llvm::IRBuilder<> &B, llvm::Value *Source,
1204 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1205 std::optional<unsigned> Index) {
1206 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1207 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1208
1209 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1210 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1211 Location = L->getLocation();
1212
1213 // DXC completely ignores the semantic/index pair. Location are assigned from
1214 // the first semantic to the last.
1215 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1216 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1217 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1218 createSPIRVLocationStore(B, M&: CGM.getModule(), Source, Location,
1219 Name: VariableName.str());
1220}
1221
1222namespace {
1223// Describes how a semantic leaf lowers to signature rows
1224struct SemanticShape {
1225 SmallVector<unsigned> Dimensions; // Empty dims denotes a scalar
1226 unsigned Cols;
1227 QualType RowType;
1228
1229 unsigned getNumRows() const {
1230 unsigned Rows = 1;
1231 for (unsigned Dimension : Dimensions)
1232 Rows *= Dimension;
1233 return Rows;
1234 }
1235
1236 SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
1237 assert(Row < getNumRows() && "row exceeds semantic shape");
1238
1239 SmallVector<unsigned> Indices(Dimensions.size());
1240 for (auto [Index, Dimension] :
1241 llvm::zip_equal(t: llvm::reverse(C&: Indices), u: llvm::reverse(C: Dimensions))) {
1242 Index = Row % Dimension;
1243 Row /= Dimension;
1244 }
1245 return Indices;
1246 }
1247};
1248} // namespace
1249
1250// Returns the QualType of a semantic leaf declarator. For a function the
1251// declared return type is used, otherwise the declared type.
1252static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
1253 if (const auto *FD = dyn_cast<clang::FunctionDecl>(Val: Decl))
1254 return FD->getDeclaredReturnType();
1255 return Decl->getType();
1256}
1257
1258// Walks through the surrounding constant array types of \p Ty, collecting their
1259// dimensions until reaching a scalar, vector, or matrix leaf.
1260static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
1261 SemanticShape Shape{.Dimensions: {}, .Cols: 1, .RowType: Ty};
1262 while (const ConstantArrayType *CAT =
1263 Ctx.getAsConstantArrayType(T: Shape.RowType)) {
1264 Shape.Dimensions.push_back(Elt: CAT->getSize().getZExtValue());
1265 Shape.RowType = CAT->getElementType();
1266 }
1267
1268 if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
1269 Shape.Cols = VT->getNumElements();
1270 } else if (const auto *MT =
1271 Shape.RowType->getAs<clang::ConstantMatrixType>()) {
1272 // FIXME: a matrix leaf lowers to one row per matrix row but if column_major
1273 // is specified we transpose the num rows and num cols, this depends on
1274 // #211977 to resolve
1275 Shape.Cols = MT->getNumColumns();
1276 }
1277
1278 return Shape;
1279}
1280
1281static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
1282 QualType Ty) {
1283 if (const auto *VT = Ty->getAs<clang::VectorType>())
1284 Ty = VT->getElementType();
1285 else if (const auto *MT = Ty->getAs<clang::ConstantMatrixType>())
1286 Ty = MT->getElementType();
1287
1288 llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
1289 bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
1290 return llvm::hlsl::getDXILElementType(Ty: IRTy, IsSigned);
1291}
1292
1293static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
1294 CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
1295 std::optional<unsigned> Index, const SemanticShape &Shape) {
1296 StringRef Name = Semantic->getAttrName()->getName();
1297
1298 // One semantic index per row, starting from the declared index.
1299 SmallVector<uint32_t> SemanticIndices;
1300 uint32_t FirstSemanticIndex = Index.value_or(u: 0);
1301 for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
1302 SemanticIndices.push_back(Elt: FirstSemanticIndex + I);
1303
1304 // The remaining members keep their default value and will be filled at a
1305 // later stage, either during packing or analysis of usage. Interpolation
1306 // is set by the load/store traversal after visiting each leaf.
1307 return llvm::hlsl::SemanticSignatureElement(
1308 SigId, Name, getSignatureComponentType(CGM, Ty: Shape.RowType),
1309 llvm::hlsl::getSemanticKind(SemanticName: Name), SemanticIndices,
1310 static_cast<uint8_t>(Shape.Cols));
1311}
1312
1313llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
1314 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1315 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1316 SemanticSignatures &Signature) {
1317 StringRef Name = Semantic->getAttrName()->getName();
1318 SemanticShape Shape =
1319 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1320
1321 uint32_t SigId = Signature.size();
1322 Signature.push_back(
1323 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1324
1325 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1326
1327 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1328 M: B.getModule(), id: llvm::Intrinsic::dx_load_input, OverloadTys: {RowTy});
1329
1330 SmallVector<OperandBundleDef, 1> OB;
1331 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1332 llvm::Value *bundleArgs[] = {Token};
1333 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1334 }
1335
1336 llvm::Type *LeafTy = CGM.getTypes().ConvertType(T: Shape.RowType);
1337 llvm::Value *Result = llvm::PoisonValue::get(T: Type);
1338
1339 const unsigned NumRows = Shape.getNumRows();
1340
1341 for (unsigned Row = 0; Row < NumRows; ++Row) {
1342 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1343 std::array<Value *, 4> Args{
1344 /*SigElementId=*/B.getInt32(C: SigId),
1345 /*RowIndex=*/B.getInt32(C: Row),
1346 /*ColIndex=*/B.getInt8(C: 0),
1347 /*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(T: B.getInt32Ty())};
1348 llvm::Value *Value =
1349 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: Twine(Name).concat(Suffix: Twine(Row)));
1350 // Booleans use their memory representation in DXIL signatures, but
1351 // function parameters use their value representation.
1352 if (Value->getType() != LeafTy) {
1353 assert(Shape.RowType->hasBooleanRepresentation() &&
1354 "unexpected semantic load type mismatch");
1355 Value = B.CreateICmpNE(
1356 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
1357 }
1358
1359 Result =
1360 Indices.empty() ? Value : B.CreateInsertValue(Agg: Result, Val: Value, Idxs: Indices);
1361 }
1362 return Result;
1363}
1364
1365void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1366 llvm::Value *Source,
1367 const clang::DeclaratorDecl *Decl,
1368 HLSLAppliedSemanticAttr *Semantic,
1369 std::optional<unsigned> Index,
1370 SemanticSignatures &Signature) {
1371 SemanticShape Shape =
1372 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1373
1374 uint32_t SigId = Signature.size();
1375 Signature.push_back(
1376 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1377
1378 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1379
1380 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1381 M: B.getModule(), id: llvm::Intrinsic::dx_store_output, OverloadTys: {RowTy});
1382
1383 SmallVector<OperandBundleDef, 1> OB;
1384 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1385 llvm::Value *bundleArgs[] = {Token};
1386 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1387 }
1388
1389 const unsigned NumRows = Shape.getNumRows();
1390 for (unsigned Row = 0; Row < NumRows; ++Row) {
1391 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1392 llvm::Value *Val =
1393 Indices.empty() ? Source : B.CreateExtractValue(Agg: Source, Idxs: Indices);
1394
1395 // Booleans use their memory representation in DXIL signatures, but direct
1396 // function results use their value representation.
1397 if (Val->getType() != RowTy) {
1398 assert(Shape.RowType->hasBooleanRepresentation() &&
1399 "unexpected semantic store type mismatch");
1400 Val = B.CreateZExt(V: Val, DestTy: RowTy, Name: "storedv");
1401 }
1402
1403 std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(C: SigId),
1404 /*RowIndex=*/B.getInt32(C: Row),
1405 /*ColIndex=*/B.getInt8(C: 0), /*Value=*/Val};
1406 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1407 }
1408}
1409
1410llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1411 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1412 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1413 std::optional<unsigned> Index, SemanticSignatures &Signature) {
1414 if (CGM.getTarget().getTriple().isSPIRV())
1415 return emitSPIRVUserSemanticLoad(B, FD, Type, Decl, Semantic, Index);
1416
1417 if (CGM.getTarget().getTriple().isDXIL())
1418 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
1419
1420 llvm_unreachable("Unsupported target for user-semantic load.");
1421}
1422
1423void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1424 const clang::DeclaratorDecl *Decl,
1425 HLSLAppliedSemanticAttr *Semantic,
1426 std::optional<unsigned> Index,
1427 SemanticSignatures &Signature) {
1428 if (CGM.getTarget().getTriple().isSPIRV())
1429 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
1430
1431 if (CGM.getTarget().getTriple().isDXIL())
1432 return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
1433 Signature);
1434
1435 llvm_unreachable("Unsupported target for user-semantic load.");
1436}
1437
1438llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1439 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1440 HLSLAppliedSemanticAttr *Semantic,
1441 llvm::dxbc::PSV::SemanticKind SemanticKind,
1442 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1443 SemanticSignatures &Signature) {
1444 switch (SemanticKind) {
1445 case llvm::dxbc::PSV::SemanticKind::GroupIndex: {
1446 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1447 llvm::Triple::Amplification},
1448 Stage) &&
1449 "SV_GroupIndex is in an unavailable stage and should have been "
1450 "diagnosed by Sema");
1451 assert(Stage != llvm::Triple::Mesh &&
1452 Stage != llvm::Triple::Amplification &&
1453 "FIXME: SV_GroupIndex is not yet implemented for this shader "
1454 "stage");
1455 llvm::Function *GroupIndex =
1456 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1457 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1458 }
1459 case llvm::dxbc::PSV::SemanticKind::DispatchThreadID: {
1460 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1461 llvm::Triple::Amplification},
1462 Stage) &&
1463 "SV_DispatchThreadID is in an unavailable stage and should have "
1464 "been diagnosed by Sema");
1465 assert(Stage != llvm::Triple::Mesh &&
1466 Stage != llvm::Triple::Amplification &&
1467 "FIXME: SV_DispatchThreadID is not yet implemented for this "
1468 "shader stage");
1469 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1470 llvm::Function *ThreadIDIntrinsic =
1471 llvm::Intrinsic::isOverloaded(id: IntrinID)
1472 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1473 : CGM.getIntrinsic(IID: IntrinID);
1474 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1475 }
1476 case llvm::dxbc::PSV::SemanticKind::GroupThreadID: {
1477 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1478 llvm::Triple::Amplification},
1479 Stage) &&
1480 "SV_GroupThreadID is in an unavailable stage and should have been "
1481 "diagnosed by Sema");
1482 assert(Stage != llvm::Triple::Mesh &&
1483 Stage != llvm::Triple::Amplification &&
1484 "FIXME: SV_GroupThreadID is not yet implemented for this shader "
1485 "stage");
1486 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1487 llvm::Function *GroupThreadIDIntrinsic =
1488 llvm::Intrinsic::isOverloaded(id: IntrinID)
1489 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1490 : CGM.getIntrinsic(IID: IntrinID);
1491 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1492 }
1493 case llvm::dxbc::PSV::SemanticKind::GroupID: {
1494 assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
1495 llvm::Triple::Amplification},
1496 Stage) &&
1497 "SV_GroupID is in an unavailable stage and should have been "
1498 "diagnosed by Sema");
1499 assert(Stage != llvm::Triple::Mesh &&
1500 Stage != llvm::Triple::Amplification &&
1501 "FIXME: SV_GroupID is not yet implemented for this shader stage");
1502 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1503 llvm::Function *GroupIDIntrinsic =
1504 llvm::Intrinsic::isOverloaded(id: IntrinID)
1505 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1506 : CGM.getIntrinsic(IID: IntrinID);
1507 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1508 }
1509 case llvm::dxbc::PSV::SemanticKind::Position:
1510 assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
1511 llvm::Triple::Geometry, llvm::Triple::Pixel},
1512 Stage) &&
1513 "SV_Position is in an unavailable stage and should have been "
1514 "diagnosed by Sema");
1515 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1516 Stage != llvm::Triple::Geometry &&
1517 "FIXME: loading SV_Position is not yet implemented for this "
1518 "shader stage");
1519 if (CGM.getTarget().getTriple().isSPIRV())
1520 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1521 Name: Semantic->getAttrName()->getName(),
1522 /* BuiltIn::FragCoord */ BuiltInID: 15);
1523 if (CGM.getTarget().getTriple().isDXIL())
1524 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1525 Signature);
1526 break;
1527 case llvm::dxbc::PSV::SemanticKind::VertexID:
1528 assert(Stage == llvm::Triple::Vertex &&
1529 "SV_VertexID is in an unavailable stage and should have been "
1530 "diagnosed by Sema");
1531 if (CGM.getTarget().getTriple().isSPIRV())
1532 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1533 Name: Semantic->getAttrName()->getName(),
1534 /* BuiltIn::VertexIndex */ BuiltInID: 42);
1535 if (CGM.getTarget().getTriple().isDXIL())
1536 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1537 Signature);
1538 break;
1539 case llvm::dxbc::PSV::SemanticKind::InstanceID:
1540 assert(Stage == llvm::Triple::Vertex &&
1541 "SV_InstanceID is in an unavailable stage and should have been "
1542 "diagnosed by Sema");
1543 if (CGM.getTarget().getTriple().isSPIRV())
1544 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1545 Name: Semantic->getAttrName()->getName(),
1546 /* BuiltIn::InstanceIndex */ BuiltInID: 43);
1547 if (CGM.getTarget().getTriple().isDXIL())
1548 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1549 Signature);
1550 break;
1551 default:
1552 break;
1553 }
1554
1555 llvm_unreachable(
1556 "Load hasn't been implemented yet for this system semantic. FIXME");
1557}
1558
1559static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
1560 llvm::Value *Source, const Twine &Name,
1561 unsigned BuiltInID) {
1562 auto *GV = new llvm::GlobalVariable(
1563 M, Source->getType(), /* isConstant= */ false,
1564 llvm::GlobalValue::ExternalLinkage,
1565 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1566 llvm::GlobalVariable::GeneralDynamicTLSModel,
1567 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1568 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1569 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1570 B.CreateStore(Val: Source, Ptr: GV);
1571}
1572
1573void CGHLSLRuntime::emitSystemSemanticStore(
1574 IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
1575 HLSLAppliedSemanticAttr *Semantic,
1576 llvm::dxbc::PSV::SemanticKind SemanticKind,
1577 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
1578 SemanticSignatures &Signature) {
1579 switch (SemanticKind) {
1580 case llvm::dxbc::PSV::SemanticKind::Position:
1581 assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
1582 llvm::Triple::Domain, llvm::Triple::Geometry,
1583 llvm::Triple::Mesh},
1584 Stage) &&
1585 "SV_Position is in an unavailable stage and should have been "
1586 "diagnosed by Sema");
1587 assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
1588 Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
1589 "FIXME: storing SV_Position is not yet implemented for this "
1590 "shader stage");
1591 if (CGM.getTarget().getTriple().isDXIL()) {
1592 emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1593 return;
1594 }
1595 if (CGM.getTarget().getTriple().isSPIRV()) {
1596 createSPIRVBuiltinStore(B, M&: CGM.getModule(), Source,
1597 Name: Semantic->getAttrName()->getName(),
1598 /* BuiltIn::Position */ BuiltInID: 0);
1599 return;
1600 }
1601 break;
1602 case llvm::dxbc::PSV::SemanticKind::Target:
1603 assert(Stage == llvm::Triple::Pixel &&
1604 "SV_Target is in an unavailable stage and should have been "
1605 "diagnosed by Sema");
1606 emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1607 return;
1608 default:
1609 break;
1610 }
1611
1612 llvm_unreachable(
1613 "Store hasn't been implemented yet for this system semantic. FIXME");
1614}
1615
1616llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1617 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1618 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1619 SemanticSignatures &Signature) {
1620
1621 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1622 llvm::dxbc::PSV::SemanticKind SemanticKind =
1623 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
1624 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1625 assert(ShaderAttr && "Entry point has no shader attribute");
1626 llvm::hlsl::SemanticInterpretation Interpretation =
1627 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
1628 IOTy: llvm::hlsl::IOType::In);
1629 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
1630 "invalid semantic should have been diagnosed by Sema");
1631 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
1632 return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index, Signature);
1633 return emitSystemSemanticLoad(B, Type, Decl, Semantic, SemanticKind,
1634 Stage: ShaderAttr->getType(), Index, Signature);
1635}
1636
1637void CGHLSLRuntime::handleScalarSemanticStore(IRBuilder<> &B,
1638 const FunctionDecl *FD,
1639 llvm::Value *Source,
1640 const clang::DeclaratorDecl *Decl,
1641 HLSLAppliedSemanticAttr *Semantic,
1642 SemanticSignatures &Signature) {
1643 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1644 llvm::dxbc::PSV::SemanticKind SemanticKind =
1645 llvm::hlsl::getSemanticKind(SemanticName: Semantic->getAttrName()->getName());
1646 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1647 assert(ShaderAttr && "Entry point has no shader attribute");
1648
1649 llvm::hlsl::SemanticInterpretation Interpretation =
1650 llvm::hlsl::getInterpretationKind(SemanticKind, ShaderStage: ShaderAttr->getType(),
1651 IOTy: llvm::hlsl::IOType::Out);
1652 assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
1653 "invalid semantic should have been diagnosed by Sema");
1654
1655 if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
1656 return emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1657 emitSystemSemanticStore(B, Source, Decl, Semantic, SemanticKind,
1658 Stage: ShaderAttr->getType(), Index, Signature);
1659}
1660
1661std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1662CGHLSLRuntime::handleStructSemanticLoad(
1663 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1664 const clang::DeclaratorDecl *Decl,
1665 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1666 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1667 SemanticSignatures &Signature,
1668 llvm::hlsl::InterpolationModifier Modifiers) {
1669 const llvm::StructType *ST = cast<StructType>(Val: Type);
1670 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
1671
1672 assert(RD->getNumFields() == ST->getNumElements());
1673
1674 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
1675 auto FieldDecl = RD->field_begin();
1676 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
1677 auto [ChildValue, NextAttr] =
1678 handleSemanticLoad(B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin,
1679 end: AttrEnd, Signature, Modifiers);
1680 AttrBegin = NextAttr;
1681 assert(ChildValue);
1682 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
1683 ++FieldDecl;
1684 }
1685
1686 return std::make_pair(x&: Aggregate, y&: AttrBegin);
1687}
1688
1689specific_attr_iterator<HLSLAppliedSemanticAttr>
1690CGHLSLRuntime::handleStructSemanticStore(
1691 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1692 const clang::DeclaratorDecl *Decl,
1693 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1694 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1695 SemanticSignatures &Signature,
1696 llvm::hlsl::InterpolationModifier Modifiers) {
1697
1698 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
1699
1700 const clang::RecordDecl *RD = nullptr;
1701 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
1702 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
1703 else
1704 RD = Decl->getType()->getAsRecordDecl();
1705 assert(RD);
1706
1707 assert(RD->getNumFields() == ST->getNumElements());
1708
1709 auto FieldDecl = RD->field_begin();
1710 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
1711 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
1712 AttrBegin = handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin,
1713 AttrEnd, Signature, Modifiers);
1714 }
1715
1716 return AttrBegin;
1717}
1718
1719std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1720CGHLSLRuntime::handleSemanticLoad(
1721 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1722 const clang::DeclaratorDecl *Decl,
1723 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1724 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1725 SemanticSignatures &Signature,
1726 llvm::hlsl::InterpolationModifier Modifiers) {
1727 assert(AttrBegin != AttrEnd);
1728 // Pass the enclosing declaration's mask down the traversal, replacing (not
1729 // merging) it when an inner field has its own interpolation modifiers.
1730 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
1731 Modifiers =
1732 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
1733 if (Type->isStructTy())
1734 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
1735 Signature, Modifiers);
1736
1737 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1738 ++AttrBegin;
1739 size_t PreviousSize = Signature.size();
1740 llvm::Value *Value =
1741 handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr, Signature);
1742 // Intrinsic-only system values and SPIR-V loads do not add DXIL signature
1743 // elements. Non-pixel inputs retain Undefined.
1744 if (Signature.size() != PreviousSize) {
1745 auto &Element = Signature.back();
1746 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
1747 Mode: llvm::hlsl::getInterpolationMode(Modifiers), CompType: Element.CompType,
1748 Kind: Element.SemanticKind, Stage: FD->getAttr<HLSLShaderAttr>()->getType(),
1749 IO: llvm::hlsl::IOType::In);
1750 }
1751 return std::make_pair(x&: Value, y&: AttrBegin);
1752}
1753
1754specific_attr_iterator<HLSLAppliedSemanticAttr>
1755CGHLSLRuntime::handleSemanticStore(
1756 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1757 const clang::DeclaratorDecl *Decl,
1758 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1759 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1760 SemanticSignatures &Signature,
1761 llvm::hlsl::InterpolationModifier Modifiers) {
1762 assert(AttrBegin != AttrEnd);
1763 // An inner field overrides the enclosing return declaration's modifiers.
1764 if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
1765 Modifiers =
1766 static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
1767 if (Source->getType()->isStructTy())
1768 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
1769 Signature, Modifiers);
1770
1771 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1772 ++AttrBegin;
1773 size_t PreviousSize = Signature.size();
1774 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr, Signature);
1775 if (Signature.size() != PreviousSize) {
1776 auto &Element = Signature.back();
1777 Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
1778 Mode: llvm::hlsl::getInterpolationMode(Modifiers), CompType: Element.CompType,
1779 Kind: Element.SemanticKind, Stage: FD->getAttr<HLSLShaderAttr>()->getType(),
1780 IO: llvm::hlsl::IOType::Out);
1781 }
1782 return AttrBegin;
1783}
1784
1785void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
1786 llvm::Function *Fn) {
1787 SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
1788 SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
1789
1790 llvm::Module &M = CGM.getModule();
1791 llvm::LLVMContext &Ctx = M.getContext();
1792 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
1793 Function *EntryFn =
1794 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
1795
1796 // Copy function attributes over, we have no argument or return attributes
1797 // that can be valid on the real entry.
1798 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
1799 Attrs: Fn->getAttributes().getFnAttrs());
1800 EntryFn->setAttributes(NewAttrs);
1801 setHLSLEntryAttributes(FD, Fn: EntryFn);
1802
1803 // Set the called function as internal linkage.
1804 Fn->setLinkage(GlobalValue::InternalLinkage);
1805
1806 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
1807 IRBuilder<> B(BB);
1808 llvm::SmallVector<Value *> Args;
1809
1810 SmallVector<OperandBundleDef, 1> OB;
1811 if (CGM.shouldEmitConvergenceTokens()) {
1812 assert(EntryFn->isConvergent());
1813 llvm::Value *I =
1814 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
1815 llvm::Value *bundleArgs[] = {I};
1816 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1817 }
1818
1819 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
1820
1821 unsigned SRetOffset = 0;
1822 for (const auto &Param : Fn->args()) {
1823 if (Param.hasStructRetAttr()) {
1824 SRetOffset = 1;
1825 llvm::Type *VarType = Param.getParamStructRetType();
1826 llvm::Value *Var =
1827 CGM.getLangOpts().EmitLogicalPointer
1828 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
1829 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
1830 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
1831 Args.push_back(Elt: Var);
1832 continue;
1833 }
1834
1835 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
1836 llvm::Value *SemanticValue = nullptr;
1837 // FIXME: support inout/out parameters for semantics.
1838 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
1839 PD->getAttr<HLSLParamModifierAttr>()) {
1840 llvm_unreachable("Not handled yet");
1841 } else {
1842 llvm::Type *ParamType = nullptr;
1843 if (Param.hasByValAttr())
1844 ParamType = Param.getParamByValType();
1845 else if (PD->getType()->isRecordType())
1846 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
1847 else
1848 ParamType = Param.getType();
1849
1850 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1851 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
1852 auto Result = handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd,
1853 Signature&: InputSignature);
1854 SemanticValue = Result.first;
1855 if (!SemanticValue)
1856 return;
1857 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
1858 llvm::Value *Var =
1859 CGM.getLangOpts().EmitLogicalPointer
1860 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
1861 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
1862 B.CreateStore(Val: SemanticValue, Ptr: Var);
1863 SemanticValue = Var;
1864 }
1865 }
1866
1867 assert(SemanticValue);
1868 Args.push_back(Elt: SemanticValue);
1869 }
1870
1871 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
1872 CI->setCallingConv(Fn->getCallingConv());
1873
1874 if (Fn->getReturnType() != CGM.VoidTy)
1875 // Element type is unused, so set to dummy value (NULL).
1876 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
1877
1878 for (auto &SourcePair : OutputSemantic) {
1879 llvm::Value *Source = SourcePair.first;
1880 llvm::Type *ElementType = SourcePair.second;
1881 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
1882 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
1883
1884 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1885 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
1886 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd,
1887 Signature&: OutputSignature);
1888 }
1889
1890 B.CreateRetVoid();
1891
1892 // Add and identify root signature to function, if applicable
1893 for (const Attr *Attr : FD->getAttrs()) {
1894 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
1895 auto *RSDecl = RSAttr->getSignatureDecl();
1896 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
1897 Fn: EntryFn, M);
1898 }
1899 }
1900
1901 addSemanticSignatureMD(InputElements: InputSignature, OutputElements: OutputSignature, Fn: EntryFn, M);
1902}
1903
1904static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
1905 bool CtorOrDtor) {
1906 const auto *GV =
1907 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
1908 if (!GV)
1909 return;
1910 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1911 if (!CA)
1912 return;
1913 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
1914 // HLSL neither supports priorities or COMDat values, so we will check those
1915 // in an assert but not handle them.
1916
1917 for (const auto &Ctor : CA->operands()) {
1918 if (isa<ConstantAggregateZero>(Val: Ctor))
1919 continue;
1920 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
1921
1922 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
1923 "HLSL doesn't support setting priority for global ctors.");
1924 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
1925 "HLSL doesn't support COMDat for global ctors.");
1926 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
1927 }
1928}
1929
1930void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
1931 llvm::Module &M = CGM.getModule();
1932 SmallVector<Function *> CtorFns;
1933 SmallVector<Function *> DtorFns;
1934 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
1935 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
1936
1937 // Insert a call to the global constructor at the beginning of the entry block
1938 // to externally exported functions. This is a bit of a hack, but HLSL allows
1939 // global constructors, but doesn't support driver initialization of globals.
1940 for (auto &F : M.functions()) {
1941 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
1942 continue;
1943 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
1944 Instruction *IP = &*F.getEntryBlock().begin();
1945 SmallVector<OperandBundleDef, 1> OB;
1946 if (Token) {
1947 llvm::Value *bundleArgs[] = {Token};
1948 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1949 IP = Token->getNextNode();
1950 }
1951 IRBuilder<> B(IP);
1952 for (auto *Fn : CtorFns) {
1953 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1954 CI->setCallingConv(Fn->getCallingConv());
1955 }
1956
1957 // Insert global dtors before the terminator of the last instruction
1958 B.SetInsertPoint(F.back().getTerminator());
1959 for (auto *Fn : DtorFns) {
1960 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1961 CI->setCallingConv(Fn->getCallingConv());
1962 }
1963 }
1964
1965 // No need to keep global ctors/dtors for non-lib profile after call to
1966 // ctors/dtors added for entry.
1967 Triple T(M.getTargetTriple());
1968 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
1969 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
1970 GV->eraseFromParent();
1971 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
1972 GV->eraseFromParent();
1973 }
1974}
1975
1976static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
1977 Intrinsic::ID IntrID,
1978 ArrayRef<llvm::Value *> Args) {
1979
1980 LLVMContext &Ctx = CGM.getLLVMContext();
1981 llvm::Function *InitResFunc =
1982 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
1983 Linkage: llvm::GlobalValue::InternalLinkage,
1984 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
1985 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1986
1987 llvm::BasicBlock *EntryBB =
1988 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
1989 CGBuilderTy Builder(CGM, Ctx);
1990 const DataLayout &DL = CGM.getModule().getDataLayout();
1991 Builder.SetInsertPoint(EntryBB);
1992
1993 // Make sure the global variable is buffer resource handle
1994 llvm::Type *HandleTy = GV->getValueType();
1995 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
1996
1997 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
1998 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
1999 Name: Twine(GV->getName()).concat(Suffix: "_h"));
2000
2001 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
2002 Builder.CreateRetVoid();
2003
2004 CGM.AddCXXGlobalInit(F: InitResFunc);
2005}
2006
2007void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
2008 llvm::GlobalVariable *GV) {
2009 ResourceBindingAttrs Binding(BufDecl);
2010 assert(Binding.hasBinding() &&
2011 "cbuffer/tbuffer should always have resource binding attribute");
2012
2013 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2014 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
2015 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
2016 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
2017
2018 // buffer with explicit binding
2019 if (Binding.isExplicit()) {
2020 llvm::Intrinsic::ID IntrinsicID =
2021 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
2022 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
2023 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
2024 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2025 } else {
2026 // buffer with implicit binding
2027 llvm::Intrinsic::ID IntrinsicID =
2028 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
2029 auto *OrderID =
2030 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
2031 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
2032 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
2033 }
2034}
2035
2036void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
2037 llvm::GlobalVariable *GV) {
2038 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
2039 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
2040 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
2041 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
2042}
2043
2044llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
2045 if (!CGM.shouldEmitConvergenceTokens())
2046 return nullptr;
2047
2048 auto E = BB.end();
2049 for (auto I = BB.begin(); I != E; ++I) {
2050 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
2051 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
2052 return II;
2053 }
2054 }
2055 llvm_unreachable("Convergence token should have been emitted.");
2056 return nullptr;
2057}
2058
2059class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
2060public:
2061 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
2062 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
2063 OpaqueValueVisitor() {}
2064
2065 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
2066 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
2067 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
2068 // traversal, the temporary containing the copy out will not have
2069 // been created yet.
2070 return false;
2071 }
2072
2073 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
2074 // Traverse the source expression first.
2075 if (E->getSourceExpr())
2076 TraverseStmt(S: E->getSourceExpr());
2077
2078 // Then add this OVE if we haven't seen it before.
2079 if (Visited.insert(Ptr: E).second)
2080 OVEs.push_back(Elt: E);
2081
2082 return true;
2083 }
2084};
2085
2086void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
2087 InitListExpr *E) {
2088
2089 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
2090 OpaqueValueVisitor Visitor;
2091 Visitor.TraverseStmt(S: E);
2092 for (auto *OVE : Visitor.OVEs) {
2093 if (CGF.isOpaqueValueEmitted(E: OVE))
2094 continue;
2095 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
2096 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
2097 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
2098 } else {
2099 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
2100 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
2101 }
2102 }
2103}
2104
2105std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
2106 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
2107 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
2108 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
2109 "expected resource array subscript expression");
2110
2111 // Let clang codegen handle local and static resource array subscripts,
2112 // or when the subscript references on opaque expression (as part of
2113 // ArrayInitLoopExpr AST node).
2114 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
2115 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
2116 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2117 ArrayDecl->getStorageClass() == SC_Static)
2118 return std::nullopt;
2119
2120 // get the resource array type
2121 ASTContext &AST = ArrayDecl->getASTContext();
2122 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
2123 assert(ResArrayTy->isHLSLResourceRecordArray() &&
2124 "expected array of resource classes");
2125
2126 // Iterate through all nested array subscript expressions to calculate
2127 // the index in the flattened resource array (if this is a multi-
2128 // dimensional array). The index is calculated as a sum of all indices
2129 // multiplied by the total size of the array at that level.
2130 Value *Index = nullptr;
2131 const ArraySubscriptExpr *ASE = ArraySubsExpr;
2132 while (ASE != nullptr) {
2133 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
2134 if (const auto *ArrayTy =
2135 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
2136 Value *Multiplier = llvm::ConstantInt::get(
2137 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
2138 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
2139 }
2140 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
2141 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
2142 }
2143
2144 // Find binding info for the resource array. For implicit binding
2145 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
2146 ResourceBindingAttrs Binding(ArrayDecl);
2147 assert(Binding.hasBinding() &&
2148 "resource array must have a binding attribute");
2149
2150 // Find the individual resource type.
2151 QualType ResultTy = ArraySubsExpr->getType();
2152 QualType ResourceTy =
2153 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
2154
2155 // Create a temporary variable for the result, which is either going
2156 // to be a single resource instance or a local array of resources (we need to
2157 // return an LValue).
2158 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
2159 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
2160 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
2161 kind: NormalEHLifetimeMarker, A: TmpVar);
2162
2163 AggValueSlot ValueSlot = AggValueSlot::forAddr(
2164 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
2165 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
2166 mayOverlap: AggValueSlot::DoesNotOverlap);
2167
2168 // Calculate total array size (= range size).
2169 llvm::Value *Range = llvm::ConstantInt::getSigned(
2170 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
2171
2172 // If the result of the subscript operation is a single resource, call the
2173 // constructor.
2174 if (ResultTy == ResourceTy) {
2175 CallArgList Args;
2176 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
2177 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
2178 Name: ArrayDecl->getName(), Binding, Args);
2179
2180 if (!CreateMethod) {
2181 // This can happen if someone creates an array of structs that looks like
2182 // an HLSL resource record array but it does not have the required static
2183 // create method. No binding will be generated for it.
2184 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
2185 "create method lookup should always succeed for built-in resource "
2186 "records");
2187 return std::nullopt;
2188 }
2189
2190 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
2191
2192 } else {
2193 // The result of the subscript operation is a local resource array which
2194 // needs to be initialized.
2195 const ConstantArrayType *ArrayTy =
2196 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
2197 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2198 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
2199 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
2200 if (!EndIndex)
2201 return std::nullopt;
2202 }
2203 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
2204}
2205
2206// Initialize all resources of a global resource array into provided slot.
2207bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
2208 const VarDecl *ArrayDecl,
2209 AggValueSlot &DestSlot) {
2210 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2211 ArrayDecl->hasGlobalStorage() &&
2212 ArrayDecl->getStorageClass() != SC_Static &&
2213 "expected global non-static resource array");
2214
2215 // Find binding info for the resource array. For implicit binding
2216 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
2217 ResourceBindingAttrs Binding(ArrayDecl);
2218 assert(Binding.hasBinding() &&
2219 "resource array must have a binding attribute");
2220
2221 // Find the individual resource type.
2222 ASTContext &AST = ArrayDecl->getASTContext();
2223 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
2224 const auto *ResArrayTy =
2225 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
2226
2227 // Create Value for index and total array size (= range size).
2228 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
2229 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2230 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
2231
2232 // Initialize individual resources in the array into DestSlot.
2233 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2234 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
2235 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
2236 return EndIndex.has_value();
2237}
2238
2239// If the expression is a global resource array, initialize all of its resources
2240// into Dest. Returns false if no initialization has been performed and the
2241// array copy should be handled by the default codegen.
2242bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
2243 AggValueSlot &DestSlot) {
2244 assert(E->getType()->isHLSLResourceRecordArray() &&
2245 "expected resource array");
2246
2247 // Find the array declaration for the expression. Fallback to the default
2248 // handling if it's not a global resource array.
2249 const VarDecl *ArrayDecl =
2250 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
2251 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2252 ArrayDecl->getStorageClass() == SC_Static)
2253 return false;
2254
2255 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
2256}
2257
2258// If the expression is a global resource array, create a temporary and
2259// initialize all of its resources, and return it as an LValue. Returns nullopt
2260// if no initialization has been performed and the handling should follow the
2261// default path.
2262std::optional<LValue>
2263CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
2264 const VarDecl *ArrayDecl) {
2265 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2266 "expected resource array declaration");
2267
2268 if (!ArrayDecl->hasGlobalStorage() ||
2269 ArrayDecl->getStorageClass() == SC_Static)
2270 return std::nullopt;
2271
2272 AggValueSlot TmpArraySlot =
2273 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
2274 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
2275 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
2276 Source: AlignmentSource::Decl);
2277 return std::nullopt;
2278}
2279
2280RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
2281 CodeGenFunction &CGF) {
2282
2283 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
2284 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
2285 "expected cbuffer matrix");
2286
2287 QualType MatQualTy = LV.getType();
2288 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
2289 Address SrcAddr = LV.getAddress();
2290
2291 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
2292 return SrcAddr;
2293
2294 RawAddress DestAlloca =
2295 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
2296 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
2297 return DestAlloca;
2298}
2299
2300std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
2301 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
2302 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
2303 // Find the element type to index by first padding the element type per HLSL
2304 // buffer rules, and then padding out to a 16-byte register boundary if
2305 // necessary.
2306 llvm::Type *LayoutTy =
2307 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
2308 uint64_t LayoutSizeInBits =
2309 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
2310 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
2311 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
2312 if (RowAlignedSize > ElementSize) {
2313 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
2314 CGM, NumBytes: RowAlignedSize - ElementSize);
2315 assert(Padding && "No padding type for target?");
2316 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
2317 /*isPacked=*/true);
2318 }
2319
2320 // If the layout type doesn't introduce any padding, we don't need to do
2321 // anything special.
2322 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
2323 if (LayoutTy == OrigTy)
2324 return std::nullopt;
2325
2326 LValueBaseInfo EltBaseInfo;
2327 TBAAAccessInfo EltTBAAInfo;
2328
2329 // Index into the object as-if we have an array of the padded element type,
2330 // and then dereference the element itself to avoid reading padding that may
2331 // be past the end of the in-memory object.
2332 SmallVector<llvm::Value *, 2> Indices;
2333 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
2334 Indices.push_back(Elt: Idx);
2335 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
2336
2337 if (CGF.getLangOpts().EmitLogicalPointer) {
2338 // The fact that we emit an array-to-pointer decay might be an oversight,
2339 // but for now, we simply ignore it (see #179951).
2340 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
2341 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
2342
2343 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
2344 Address Addr = LV.getAddress();
2345 LayoutTy = llvm::ArrayType::get(
2346 ElementType: LayoutTy,
2347 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
2348 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
2349 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
2350 Addr =
2351 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
2352 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2353 }
2354
2355 Address Addr =
2356 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
2357 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
2358 IdxList: Indices, Name: "cbufferidx");
2359 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
2360 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2361}
2362
2363std::optional<LValue>
2364CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
2365 const MemberExpr *ME) {
2366 assert((ME->getType()->isHLSLResourceRecord() ||
2367 ME->getType()->isHLSLResourceRecordArray()) &&
2368 "expected resource member expression");
2369
2370 const VarDecl *ResourceVD =
2371 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
2372 if (!ResourceVD)
2373 return std::nullopt;
2374
2375 // Handle member of resource array type.
2376 if (ResourceVD->getType()->isHLSLResourceRecordArray())
2377 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
2378
2379 GlobalVariable *ResGV =
2380 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
2381 const DataLayout &DL = CGM.getDataLayout();
2382 llvm::Type *Ty = ResGV->getValueType();
2383 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
2384 Address Addr = Address(ResGV, Ty, Align);
2385 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
2386 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2387 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
2388 return LV;
2389}
2390
2391bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
2392 const LValue &SrcLV,
2393 AggValueSlot &DestSlot) {
2394 assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
2395 "expected expression in HLSL constant address space");
2396 assert(!E->getType()->isHLSLResourceRecord() &&
2397 !E->getType()->isHLSLResourceRecordArray() &&
2398 "direct accesses to resource types should be handled separately");
2399
2400 if (DestSlot.isIgnored())
2401 return false;
2402
2403 QualType Ty = E->getType();
2404 Address DstPtr = DestSlot.getAddress();
2405 Address SrcPtr = SrcLV.getAddress();
2406
2407 // If there are no intangible types, we don't need to lookup associated
2408 // resources.
2409 if (!Ty->isHLSLIntangibleType())
2410 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty);
2411
2412 // Handle structs with intangible types by setting the resource fields
2413 // of the destination struct with the resources associated with the global
2414 // struct.
2415 EmbeddedResourceNameBuilder NameBuilder;
2416 const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
2417 AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
2418
2419 // Callback to fill in the associated resource.
2420 auto EmitResFn = [&](AggValueSlot &ResSlot) {
2421 const VarDecl *ResDecl = AssociatedResources.getNextResource();
2422 assert(ResDecl && "associated resource declaration not found");
2423
2424 // Check that the resource type of dest and src matches.
2425 [[maybe_unused]] llvm::Type *DestType =
2426 ResSlot.getAddress().getElementType();
2427 [[maybe_unused]] llvm::Type *SrcConvertedType =
2428 CGM.getTypes().ConvertTypeForMem(T: ResDecl->getType());
2429 assert(DestType == SrcConvertedType && "resource slot type mismatch");
2430
2431 if (ResDecl->getType()->isHLSLResourceRecord())
2432 copyGlobalResource(CGF, ResourceVD: ResDecl, DestSlot&: ResSlot);
2433 else
2434 initializeGlobalResourceArray(CGF, ArrayDecl: ResDecl, DestSlot&: ResSlot);
2435 };
2436
2437 auto Result =
2438 HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty, EmitResFn);
2439 assert(AssociatedResources.getNextResource() == nullptr &&
2440 "expected all associated resources to be processed");
2441 return Result;
2442}
2443
2444LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
2445 const MemberExpr *E) {
2446 LValue Base =
2447 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
2448 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
2449 assert(Field && "Unexpected access into HLSL buffer");
2450
2451 const RecordDecl *Rec = Field->getParent();
2452
2453 // Work out the buffer layout type to index into.
2454 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
2455 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
2456 // Since this is a member of an object in the buffer and not the buffer's
2457 // struct/class itself, we shouldn't have any offsets on the members we need
2458 // to contend with.
2459 CGHLSLOffsetInfo EmptyOffsets;
2460 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
2461 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
2462
2463 // Get the field index for the layout struct, accounting for padding.
2464 unsigned FieldIdx =
2465 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
2466 assert(FieldIdx < LayoutTy->getNumElements() &&
2467 "Layout struct is smaller than member struct");
2468 unsigned Skipped = 0;
2469 for (unsigned I = 0; I <= FieldIdx;) {
2470 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
2471 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
2472 ++Skipped;
2473 else
2474 ++I;
2475 }
2476 FieldIdx += Skipped;
2477 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
2478
2479 // Now index into the struct, making sure that the type we return is the
2480 // buffer layout type rather than the original type in the AST.
2481 QualType FieldType = Field->getType();
2482 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
2483 CharUnits Align = CharUnits::fromQuantity(
2484 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
2485
2486 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
2487 ? CGF.Builder.CreateStructuredGEP(
2488 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
2489 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
2490 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
2491 Idx: FieldIdx, Name: Field->getName());
2492 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
2493
2494 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
2495 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2496 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
2497 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
2498
2499 return LV;
2500}
2501