1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/RecursiveASTVisitor.h"
28#include "clang/AST/Type.h"
29#include "clang/Basic/DiagnosticDriver.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetOptions.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/Enum.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/ScopeExit.h"
37#include "llvm/ADT/SmallString.h"
38#include "llvm/ADT/SmallVector.h"
39#include "llvm/Frontend/HLSL/HLSLResource.h"
40#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/GlobalVariable.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Alignment.h"
51#include "llvm/Support/ErrorHandling.h"
52#include "llvm/Support/FormatVariadic.h"
53#include "llvm/Support/Path.h"
54#include "llvm/Transforms/Utils/ModuleUtils.h"
55#include <array>
56#include <cstdint>
57#include <optional>
58
59using namespace clang;
60using namespace CodeGen;
61using namespace clang::hlsl;
62using namespace llvm;
63
64using llvm::hlsl::CBufferRowSizeInBytes;
65
66namespace {
67
68void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
69 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
70 // Assume ValVersionStr is legal here.
71 VersionTuple Version;
72 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
73 Version.getSubminor() || !Version.getMinor()) {
74 return;
75 }
76
77 uint64_t Major = Version.getMajor();
78 uint64_t Minor = *Version.getMinor();
79
80 auto &Ctx = M.getContext();
81 IRBuilder<> B(M.getContext());
82 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
83 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
84 StringRef DXILValKey = "dx.valver";
85 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
86 DXILValMD->addOperand(M: Val);
87}
88
89void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
90 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
91 llvm::Function *Fn, llvm::Module &M) {
92 auto &Ctx = M.getContext();
93
94 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
95 MDNode *RootSignature = RSBuilder.BuildRootSignature();
96
97 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
98 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
99 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
100 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
101
102 StringRef RootSignatureValKey = "dx.rootsignatures";
103 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
104 RootSignatureValMD->addOperand(M: MDVals);
105}
106
107MDNode *buildSemanticSignatureMD(
108 ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
109 if (Elements.empty())
110 return nullptr;
111
112 SmallVector<Metadata *> ElementMD;
113 for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
114 ElementMD.push_back(Elt: Element.toMetadata(Ctx));
115 return MDNode::get(Context&: Ctx, MDs: ElementMD);
116}
117
118void addSemanticSignatureMD(
119 ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
120 ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
121 llvm::Function *Fn, llvm::Module &M) {
122 if (InputElements.empty() && OutputElements.empty())
123 return;
124
125 LLVMContext &Ctx = M.getContext();
126 MDNode *InputSignature = buildSemanticSignatureMD(Elements: InputElements, Ctx);
127 MDNode *OutputSignature = buildSemanticSignatureMD(Elements: OutputElements, Ctx);
128 MDNode *MDVals = MDNode::get(
129 Context&: Ctx, MDs: {ValueAsMetadata::get(V: Fn), InputSignature, OutputSignature});
130
131 M.getOrInsertNamedMetadata(Name: "dx.semantic.signatures")->addOperand(M: MDVals);
132}
133
134static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
135 AggValueSlot &DestSlot) {
136 GlobalVariable *ResGV =
137 cast<GlobalVariable>(Val: CGF.CGM.GetAddrOfGlobalVar(D: ResourceVD));
138 assert(ResGV && "expected valid global variable");
139 CGF.Builder.CreateStore(Val: ResGV, Addr: DestSlot.getAddress());
140}
141
142// Given a MemberExpr of a resource or resource array type, find the parent
143// VarDecl of the struct or class instance that contains this resource and
144// build the full resource name based on the member access path.
145//
146// For example, for a member access like "myStructArray[0].memberA",
147// this function will find the VarDecl of "myStructArray" and use the
148// EmbeddedResourceNameBuilder to build the resource name
149// "myStructArray.0.memberA".
150//
151// This also works for a record type expression that has some embedded
152// resources. It finds the parent VarDecl of that record and builds a partial
153// name which is the prefix of the resource globals associated with the
154// declaration.
155static const VarDecl *findStructResourceParentDeclAndBuildName(
156 const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
157
158 SmallVector<const Expr *> WorkList;
159 const VarDecl *VD = nullptr;
160
161 for (;;) {
162 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
163 assert(isa<VarDecl>(DRE->getDecl()) &&
164 "member expr base is not a var decl");
165 VD = cast<VarDecl>(Val: DRE->getDecl());
166 NameBuilder.pushName(N: VD->getName());
167 break;
168 }
169
170 WorkList.push_back(Elt: E);
171 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
172 E = MExp->getBase();
173 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
174 E = ICE->getSubExpr();
175 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
176 E = ASE->getBase();
177 else if (isa<CXXThisExpr>(Val: E))
178 // Resource member access on "this" pointer not yet implemented
179 // (llvm/llvm-project#190299)
180 return nullptr;
181 else
182 llvm_unreachable("unexpected expr type in resource member access");
183
184 assert(E && "expected valid expression");
185 }
186
187 while (!WorkList.empty()) {
188 E = WorkList.pop_back_val();
189 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
190 NameBuilder.pushName(
191 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
192 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
193 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
194 CXXRecordDecl *DerivedRD =
195 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
196 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
197 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
198 }
199 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
200 const Expr *IdxExpr = ASE->getIdx();
201 std::optional<llvm::APSInt> Value =
202 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
203 assert(Value &&
204 "expected constant index in struct with resource array access");
205 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
206 } else {
207 llvm_unreachable("unexpected expr type in resource member access");
208 }
209 }
210 return VD;
211}
212
213// Given a MemberExpr of a resource or resource array type, find the
214// corresponding global resource declaration associated with the owning struct
215// or class instance via HLSLAssociatedResourceDeclAttr.
216static const VarDecl *
217findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
218
219 EmbeddedResourceNameBuilder NameBuilder;
220 const VarDecl *ParentVD =
221 findStructResourceParentDeclAndBuildName(E: ME, NameBuilder);
222 if (!ParentVD)
223 return nullptr;
224
225 if (!ParentVD->hasGlobalStorage())
226 return nullptr;
227
228 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
229 for (const Attr *A : ParentVD->getAttrs()) {
230 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
231 VarDecl *AssocResVD = ADA->getResDecl();
232 if (AssocResVD->getIdentifier() == II)
233 return AssocResVD;
234 }
235 }
236 return nullptr;
237}
238
239void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
240 auto &SM = CGM.getContext().getSourceManager();
241 auto &Macros = CGM.getPreprocessorOpts().Macros;
242 auto &CodeGenOpts = CGM.getCodeGenOpts();
243 auto &Ctx = M.getContext();
244
245 // Names and content of shader source code files.
246 llvm::NamedMDNode *DXContents =
247 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
248 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
249 llvm::MDTuple *FileInfo =
250 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
251 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
252 DXContents->addOperand(M: FileInfo);
253 };
254
255 bool Invalid = false;
256 const SrcMgr::SLocEntry *MainLocEntry =
257 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
258 assert(!Invalid && "Main file SLocEntry must not be invalid!");
259 const SrcMgr::ContentCache &MainCCEntry =
260 MainLocEntry->getFile().getContentCache();
261
262 SmallVector<std::pair<std::string, StringRef>> Files;
263 std::optional<SmallString<256>> MainFileName;
264 Files.reserve(N: SM.local_sloc_entry_size());
265 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
266 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
267 if (!LocEntry.isFile())
268 continue;
269
270 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
271 if (isSystem(CK: FInfo.getFileCharacteristic()))
272 continue;
273
274 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
275 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
276 if (!FEntry)
277 continue;
278
279 llvm::SmallString<256> Path = FEntry->getName();
280 llvm::sys::path::native(path&: Path);
281 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
282 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
283 if (!Buffer) {
284 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
285 << Path;
286 continue;
287 }
288
289 if (&MainCCEntry != &CCEntry) {
290 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
291 } else {
292 // Main file should be at first position.
293 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
294 MainFileName.emplace(args&: Path);
295 }
296 }
297 assert(MainFileName && "Main file not found.");
298
299 // Files other that main one should be sorted by name.
300 llvm::sort(C&: Files);
301#ifndef NDEBUG
302 for (unsigned I = 1; I < Files.size(); ++I)
303 assert((Files[I - 1].first != Files[I].first) &&
304 "duplicate files in dx.source.contents");
305#endif
306 llvm::for_each(Range&: Files, F: addFile);
307
308 SmallVector<llvm::Metadata *> Defines;
309 Defines.reserve(N: Macros.size());
310 for (const auto &Macro : Macros) {
311 // Ignore undefs.
312 if (!Macro.second)
313 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
314 }
315 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
316 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
317
318 if (!CodeGenOpts.MainFileName.empty())
319 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
320 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
321 ->addOperand(
322 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
323
324 SmallVector<llvm::Metadata *> Args;
325 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
326 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
327 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
328 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
329 M.getOrInsertNamedMetadata(Name: "dx.source.args")
330 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
331}
332
333// Find array variable declaration from DeclRef expression
334static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
335 E = E->IgnoreImpCasts();
336 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
337 return DRE->getDecl();
338 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
339 E = OVE->getSourceExpr()->IgnoreImpCasts();
340 if (isa<MemberExpr>(Val: E))
341 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
342 return nullptr;
343}
344
345// Find array variable declaration from nested array subscript AST nodes
346static const ValueDecl *getArrayDecl(ASTContext &AST,
347 const ArraySubscriptExpr *ASE) {
348 const Expr *E = nullptr;
349 while (ASE != nullptr) {
350 E = ASE->getBase()->IgnoreImpCasts();
351 if (!E)
352 return nullptr;
353 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
354 }
355 return getArrayDecl(AST, E);
356}
357
358// Get the total size of the array, or 0 if the array is unbounded.
359static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
360 Ty = Ty->getUnqualifiedDesugaredType();
361 assert(Ty->isArrayType() && "expected array type");
362 if (Ty->isIncompleteArrayType())
363 return 0;
364 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
365}
366
367static Value *buildNameForResource(llvm::StringRef BaseName,
368 CodeGenModule &CGM) {
369 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
370 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
371 .getPointer();
372}
373
374static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
375 StorageClass SC = SC_None) {
376 for (auto *Method : Record->methods()) {
377 if (Method->getStorageClass() == SC && Method->getName() == Name)
378 return Method;
379 }
380 return nullptr;
381}
382
383static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
384 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
385 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
386 CallArgList &Args) {
387 assert(Binding.hasBinding() && "at least one binding attribute expected");
388
389 ASTContext &AST = CGM.getContext();
390 CXXMethodDecl *CreateMethod = nullptr;
391 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
392 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
393
394 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
395 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
396 "resources with counter handle must have a binding with counter "
397 "implicit order ID");
398 if (Binding.isExplicit()) {
399 // explicit binding
400 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
401 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
402 const char *Name = Binding.hasCounterImplicitOrderID()
403 ? "__createFromBindingWithImplicitCounter"
404 : "__createFromBinding";
405 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
406 } else {
407 // implicit binding
408 auto *OrderID =
409 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
410 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
411 const char *Name = Binding.hasCounterImplicitOrderID()
412 ? "__createFromImplicitBindingWithImplicitCounter"
413 : "__createFromImplicitBinding";
414 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
415 }
416 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
417 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
418 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
419 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
420 if (HasCounter) {
421 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
422 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
423 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
424 }
425
426 return CreateMethod;
427}
428
429static void callResourceInitMethod(CodeGenFunction &CGF,
430 CXXMethodDecl *CreateMethod,
431 CallArgList &Args, Address ReturnAddress) {
432 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
433 const FunctionProtoType *Proto =
434 CreateMethod->getType()->getAs<FunctionProtoType>();
435 // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
436 // caller declaration is needed for x86 SysV ABI selection.
437 const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
438 Args, Ty: Proto, ChainCall: false, /*ABIInfoFD=*/nullptr);
439 ReturnValueSlot ReturnValue(ReturnAddress, false);
440 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
441 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
442}
443
444// Initializes local resource array variable with global resource array
445// elements. For multi-dimensional arrays it calls itself recursively to
446// initialize its sub-arrays. The Index used in the resource constructor calls
447// will begin at StartIndex and will be incremented for each array element. The
448// last used resource Index is returned to the caller. If the function returns
449// std::nullopt, it indicates an error.
450static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
451 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
452 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
453 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
454 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
455
456 ASTContext &AST = CGF.getContext();
457 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
458 llvm::Value *Index = StartIndex;
459 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
460 const uint64_t ArraySize = ArrayTy->getSExtSize();
461 QualType ElemType = ArrayTy->getElementType();
462 Address TmpArrayAddr = ValueSlot.getAddress();
463
464 // Add additional index to the getelementptr call indices.
465 // This index will be updated for each array element in the loops below.
466 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
467 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
468
469 // For array of arrays, recursively initialize the sub-arrays.
470 if (ElemType->isArrayType()) {
471 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
472 for (uint64_t I = 0; I < ArraySize; I++) {
473 if (I > 0) {
474 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
475 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
476 }
477 std::optional<llvm::Value *> MaybeIndex =
478 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
479 ValueSlot, Range, StartIndex: Index,
480 ResourceName, Binding, PrevGEPIndices: GEPIndices);
481 if (!MaybeIndex)
482 return std::nullopt;
483 Index = *MaybeIndex;
484 }
485 return Index;
486 }
487
488 // For array of resources, initialize each resource in the array.
489 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
490 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
491 CharUnits Align =
492 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
493
494 for (uint64_t I = 0; I < ArraySize; I++) {
495 if (I > 0) {
496 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
497 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
498 }
499 Address ReturnAddress =
500 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
501
502 CallArgList Args;
503 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
504 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
505
506 if (!CreateMethod)
507 // This can happen if someone creates an array of structs that looks like
508 // an HLSL resource record array but it does not have the required static
509 // create method. No binding will be generated for it.
510 return std::nullopt;
511
512 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
513 }
514 return Index;
515}
516
517/// Utility for emitting copies following the HLSL buffer layout rules (ie,
518/// copying out of a cbuffer).
519class HLSLBufferCopyEmitter {
520 CodeGenFunction &CGF;
521 Address DstPtr;
522 Address SrcPtr;
523 llvm::Type *LayoutTy = nullptr;
524
525 SmallVector<llvm::Value *> CurStoreIndices;
526 SmallVector<llvm::Value *> CurLoadIndices;
527
528 using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
529
530 // Creates & returns either a structured.gep or a ptradd/gep depending on
531 // langopts.
532 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
533 ArrayRef<llvm::Value *> Indices) {
534 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
535 if (EmitLogical)
536 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
537
538 llvm::SmallVector<llvm::Value *> GEPIndices;
539 GEPIndices.reserve(N: Indices.size() + 1);
540 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
541 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
542 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
543 }
544
545 bool isBufferLayoutArray(llvm::StructType *ST) {
546 // A buffer layout array is a struct with two elements: the padded array,
547 // and the last element. That is, is should look something like this:
548 //
549 // { [%n x { %type, %padding }], %type }
550 //
551 if (!ST || ST->getNumElements() != 2)
552 return false;
553
554 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
555 if (!PaddedEltsTy)
556 return false;
557
558 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
559 if (!PaddedTy || PaddedTy->getNumElements() != 2)
560 return false;
561
562 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
563 Ty: PaddedTy->getElementType(N: 1)))
564 return false;
565
566 llvm::Type *ElementTy = ST->getElementType(N: 1);
567 if (PaddedTy->getElementType(N: 0) != ElementTy)
568 return false;
569 return true;
570 }
571
572 // Returns true if the type is either a struct representing a resource record,
573 // or an array of structs that are resource records. This assumes a struct is
574 // a resource record if the first element is a target type (resource handle).
575 // This is the case for all target types used by HLSL except the padding type
576 // ("{dx|spirv.Padding"), but padding will never be the first element of a
577 // struct.
578 bool isResourceOrResourceArray(llvm::Type *Ty) {
579 while (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
580 Ty = AT->getElementType();
581
582 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
583 if (!ST || ST->getNumElements() < 1)
584 return false;
585
586 auto *TargetTy = dyn_cast<llvm::TargetExtType>(Val: ST->getElementType(N: 0));
587 return TargetTy != nullptr;
588 }
589
590 void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
591 EmitResourceFnTy EmitResFn) {
592 CharUnits DstAlign =
593 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
594 Address DstAddr(Dst, DstTy, DstAlign);
595 AggValueSlot Slot = AggValueSlot::forAddr(
596 addr: DstAddr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
597 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
598 mayOverlap: AggValueSlot::DoesNotOverlap);
599
600 EmitResFn(Slot);
601 }
602
603 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
604 llvm::ArrayType *DstTy,
605 EmitResourceFnTy EmitResFn) {
606 // Those assumptions are checked by isBufferLayoutArray.
607 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
608 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
609 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
610 ->getElementType(0) == SrcTy->getElementType(1));
611
612 auto *SrcDataTy = SrcTy->getElementType(N: 1);
613 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
614
615 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
616 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
617 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
618 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
619 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
620 EmitResFn);
621 }
622
623 auto *SrcElt =
624 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
625 auto *DstElt = emitAccessChain(
626 BaseTy: DstTy, Base: Dst,
627 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
628 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType(),
629 EmitResFn);
630 }
631
632 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
633 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
634 assert(!isResourceOrResourceArray(DstTy) &&
635 "direct access to resources or resource arrays should be handled "
636 "separately");
637
638 if (isBufferLayoutArray(ST: SrcTy))
639 return emitBufferLayoutCopy(Src, SrcTy, Dst, DstTy: cast<llvm::ArrayType>(Val: DstTy),
640 EmitResFn);
641
642 unsigned SrcIndex = 0;
643 unsigned DstIndex = 0;
644
645 // DstTy layout is in default address space and can include resource types.
646 // SrcTy is in cbuffer layout where resources are filtered out, so the
647 // number of elements in SrcTy can be less than the number of elements in
648 // DstTy.
649 auto *DstST = cast<llvm::StructType>(Val: DstTy);
650 while (DstIndex < DstST->getNumElements()) {
651 llvm::Type *DstEltTy = DstST->getElementType(N: DstIndex);
652 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: DstEltTy)) {
653 DstIndex += 1;
654 continue;
655 }
656 if (isResourceOrResourceArray(Ty: DstEltTy)) {
657 auto *DstElt = emitAccessChain(
658 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
659 emitResourceOrResourceArray(Dst: DstElt, DstTy: DstEltTy, EmitResFn);
660 DstIndex += 1;
661 continue;
662 }
663
664 assert(SrcIndex < SrcTy->getNumElements());
665 llvm::Type *SrcEltTy = SrcTy->getElementType(N: SrcIndex);
666 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: SrcEltTy)) {
667 SrcIndex += 1;
668 continue;
669 }
670
671 auto *SrcElt = emitAccessChain(
672 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
673 auto *DstElt = emitAccessChain(
674 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
675 emitElementCopy(Src: SrcElt, SrcTy: SrcEltTy, Dst: DstElt, DstTy: DstEltTy, EmitResFn);
676 DstIndex += 1;
677 SrcIndex += 1;
678 }
679 }
680
681 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
682 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
683 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
684 auto *SrcElt =
685 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
686 auto *DstElt =
687 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
688 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
689 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType(),
690 EmitResFn);
691 }
692 }
693
694 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
695 llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
696 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
697 return emitCopy(Src, SrcTy: AT, Dst, DstTy, EmitResFn);
698 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
699 return emitCopy(Src, SrcTy: ST, Dst, DstTy, EmitResFn);
700
701 // When we have a scalar or vector element we can emit the copy.
702 CharUnits SrcAlign =
703 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
704 CharUnits DstAlign =
705 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
706 Address SrcAddr(Src, SrcTy, SrcAlign);
707 Address DstAddr(Dst, DstTy, DstAlign);
708 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
709 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
710 }
711
712public:
713 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
714 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
715
716 bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
717 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
718
719 // TODO: We should be able to fall back to a regular memcpy if the layout
720 // type doesn't have any padding, but that runs into issues in the backend
721 // currently.
722 //
723 // See https://github.com/llvm/wg-hlsl/issues/351
724 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
725 DstTy: DstPtr.getElementType(), EmitResFn);
726 return true;
727 }
728};
729
730// Represents a list resources associated with a global struct whose name
731// starts with the specified prefix.
732// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
733// order of the depth-first traversal of the corresponding fields in the struct.
734// The resources are always returned in that order, which is the same order
735// we need when a struct is copied element-by-element.
736class AssociatedResourcesList {
737 // Iterator pointers for the associated resource attributes that match the
738 // prefix. Begin = begin of the range of attributes that match the prefix End
739 // = end of the range of attributes that match the prefix Next = the current
740 // attribute in the iteration to be returned by getNextResource
741 specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
742
743public:
744 AssociatedResourcesList(const VarDecl *StructVD,
745 StringRef ResourceNamePrefix) {
746 auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
747 auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
748
749 // Skip over associated resources that don't match the prefix.
750 while (I != E &&
751 !I->getResDecl()->getName().starts_with(Prefix: ResourceNamePrefix))
752 ++I;
753 assert(I != E && "expected associated resource not found");
754 Begin = End = I;
755
756 // Scan over associated resources that do match the prefix to find the end
757 // of the range.
758 while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
759 ->getResDecl()
760 ->getName()
761 .starts_with(Prefix: ResourceNamePrefix))
762 End = ++I;
763
764 Next = Begin;
765 }
766
767 const VarDecl *getNextResource() {
768 if (Next == End)
769 return nullptr;
770
771 const VarDecl *Res = Next->getResDecl();
772 ++Next;
773 return Res;
774 }
775};
776
777} // namespace
778
779llvm::Type *
780CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
781 const CGHLSLOffsetInfo &OffsetInfo) {
782 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
783
784 // Check if the target has a specific translation for this type first.
785 if (llvm::Type *TargetTy =
786 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
787 return TargetTy;
788
789 llvm_unreachable("Generic handling of HLSL types is not supported.");
790}
791
792llvm::Triple::ArchType CGHLSLRuntime::getArch() {
793 return CGM.getTarget().getTriple().getArch();
794}
795
796// Emits constant global variables for buffer constants declarations
797// and creates metadata linking the constant globals with the buffer global.
798void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
799 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
800 const CGHLSLOffsetInfo &OffsetInfo) {
801 LLVMContext &Ctx = CGM.getLLVMContext();
802
803 // get the layout struct from constant buffer target type
804 llvm::Type *BufType = BufGV->getValueType();
805 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
806 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
807
808 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
809 size_t OffsetIdx = 0;
810 for (Decl *D : BufDecl->buffer_decls()) {
811 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
812 // Nothing to do for this declaration.
813 continue;
814 if (isa<FunctionDecl>(Val: D)) {
815 // A function within an cbuffer is effectively a top-level function.
816 CGM.EmitTopLevelDecl(D);
817 continue;
818 }
819 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
820 if (!VD)
821 continue;
822
823 QualType VDTy = VD->getType();
824 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
825 if (VD->getStorageClass() == SC_Static ||
826 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
827 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
828 // Emit static and groupshared variables and resource classes inside
829 // cbuffer as regular globals
830 CGM.EmitGlobal(D: VD);
831 }
832 continue;
833 }
834
835 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
836 }
837
838 if (!OffsetInfo.empty())
839 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
840 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
841 });
842
843 // Associate the buffer global variable with its constants
844 SmallVector<llvm::Metadata *> BufGlobals;
845 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
846 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
847
848 auto ElemIt = LayoutStruct->element_begin();
849 for (auto &[VD, _] : DeclsWithOffset) {
850 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
851 ++ElemIt;
852
853 assert(ElemIt != LayoutStruct->element_end() &&
854 "number of elements in layout struct does not match");
855 llvm::Type *LayoutType = *ElemIt++;
856
857 GlobalVariable *ElemGV =
858 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
859 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
860 }
861 assert(ElemIt == LayoutStruct->element_end() &&
862 "number of elements in layout struct does not match");
863
864 // add buffer metadata to the module
865 CGM.getModule()
866 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
867 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
868}
869
870// Creates resource handle type for the HLSL buffer declaration
871static const clang::HLSLAttributedResourceType *
872createBufferHandleType(const HLSLBufferDecl *BufDecl) {
873 ASTContext &AST = BufDecl->getASTContext();
874 QualType QT = AST.getHLSLAttributedResourceType(
875 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
876 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
877 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
878}
879
880CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
881 CGHLSLOffsetInfo Result;
882
883 // If we don't have packoffset info, just return an empty result.
884 if (!BufDecl.hasValidPackoffset())
885 return Result;
886
887 for (Decl *D : BufDecl.buffer_decls()) {
888 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
889 continue;
890 }
891 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
892 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
893 continue;
894
895 if (!VD->hasAttrs()) {
896 Result.Offsets.push_back(Elt: Unspecified);
897 continue;
898 }
899
900 uint32_t Offset = Unspecified;
901 for (auto *Attr : VD->getAttrs()) {
902 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
903 Offset = POA->getOffsetInBytes();
904 break;
905 }
906 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
907 if (RBA &&
908 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
909 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
910 break;
911 }
912 }
913 Result.Offsets.push_back(Elt: Offset);
914 }
915 return Result;
916}
917
918// Codegen for HLSLBufferDecl
919void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
920
921 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
922
923 // create resource handle type for the buffer
924 const clang::HLSLAttributedResourceType *ResHandleTy =
925 createBufferHandleType(BufDecl);
926
927 // empty constant buffer is ignored
928 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
929 return;
930
931 // create global variable for the constant buffer
932 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
933 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
934 llvm::GlobalVariable *BufGV = new GlobalVariable(
935 LayoutTy, /*isConstant*/ false,
936 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
937 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
938 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
939 GlobalValue::NotThreadLocal);
940
941 llvm::Module &M = CGM.getModule();
942 M.insertGlobalVariable(GV: BufGV);
943
944 // Add the global variable to the compiler used list so it does not
945 // get optimized away by GlobalOptPass before it reaches
946 // {DXIL|SPIRV}CBufferAccess pass.
947 llvm::appendToCompilerUsed(M, Values: {BufGV});
948
949 // Add globals for constant buffer elements and create metadata nodes
950 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
951
952 // Initialize cbuffer from binding (implicit or explicit)
953 initializeBufferFromBinding(BufDecl, GV: BufGV);
954}
955
956void CGHLSLRuntime::addRootSignature(
957 const HLSLRootSignatureDecl *SignatureDecl) {
958 llvm::Module &M = CGM.getModule();
959 Triple T(M.getTargetTriple());
960
961 // Generated later with the function decl if not targeting root signature
962 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
963 return;
964
965 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
966 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
967}
968
969llvm::StructType *
970CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
971 const auto Entry = LayoutTypes.find(Val: StructType);
972 if (Entry != LayoutTypes.end())
973 return Entry->getSecond();
974 return nullptr;
975}
976
977void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
978 llvm::StructType *LayoutTy) {
979 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
980 "layout type for this struct already exist");
981 LayoutTypes[StructType] = LayoutTy;
982}
983
984void CGHLSLRuntime::finishCodeGen() {
985 auto &TargetOpts = CGM.getTarget().getTargetOpts();
986 auto &CodeGenOpts = CGM.getCodeGenOpts();
987 auto &LangOpts = CGM.getLangOpts();
988 llvm::Module &M = CGM.getModule();
989 Triple T(M.getTargetTriple());
990 if (T.getArch() == Triple::ArchType::dxil)
991 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
992 if (!CodeGenOpts.DisableDXSourceMetadata &&
993 CodeGenOpts.getDebugInfo() >=
994 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
995 addSourceInfo(CGM, M);
996 if (CodeGenOpts.ResMayAlias)
997 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
998 if (CodeGenOpts.AllResourcesBound)
999 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
1000 Key: "dx.allresourcesbound", Val: 1);
1001 if (CodeGenOpts.OptimizationLevel == 0)
1002 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
1003 Key: "dx.disable_optimizations", Val: 1);
1004
1005 // NativeHalfType corresponds to the -fnative-half-type clang option which is
1006 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
1007 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
1008 if (LangOpts.NativeHalfType)
1009 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
1010 Val: 1);
1011
1012 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
1013 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
1014 const ASTContext &Ctx = CGM.getContext();
1015 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
1016 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
1017 SmallVector<GlobalValue *, 8> InterfaceVars;
1018 for (GlobalVariable &GV : M.globals()) {
1019 unsigned AS = GV.getAddressSpace();
1020 if (AS == InputAS || AS == OutputAS)
1021 InterfaceVars.push_back(Elt: &GV);
1022 }
1023 if (!InterfaceVars.empty())
1024 appendToCompilerUsed(M, Values: InterfaceVars);
1025 }
1026
1027 generateGlobalCtorDtorCalls();
1028}
1029
1030void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
1031 const FunctionDecl *FD, llvm::Function *Fn) {
1032 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1033 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
1034 const StringRef ShaderAttrKindStr = "hlsl.shader";
1035 Fn->addFnAttr(Kind: ShaderAttrKindStr,
1036 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
1037 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
1038 const StringRef NumThreadsKindStr = "hlsl.numthreads";
1039 std::string NumThreadsStr =
1040 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
1041 Vals: NumThreadsAttr->getZ());
1042 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
1043 }
1044 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
1045 const StringRef WaveSizeKindStr = "hlsl.wavesize";
1046 std::string WaveSizeStr =
1047 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
1048 Vals: WaveSizeAttr->getPreferred());
1049 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
1050 }
1051 // HLSL entry functions are materialized for module functions with
1052 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
1053 // later in the compiler-flow for such module functions is not aware of and
1054 // hence not able to set attributes of the newly materialized entry functions.
1055 // So, set attributes of entry function here, as appropriate.
1056 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
1057
1058 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
1059 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
1060 }
1061}
1062
1063static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
1064 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1065 Value *Result = PoisonValue::get(T: Ty);
1066 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
1067 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
1068 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
1069 }
1070 return Result;
1071 }
1072 return B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)});
1073}
1074
1075static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
1076 unsigned BuiltIn) {
1077 LLVMContext &Ctx = GV->getContext();
1078 IRBuilder<> B(GV->getContext());
1079 MDNode *Operands = MDNode::get(
1080 Context&: Ctx,
1081 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
1082 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
1083 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1084 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1085}
1086
1087static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
1088 LLVMContext &Ctx = GV->getContext();
1089 IRBuilder<> B(GV->getContext());
1090 MDNode *Operands =
1091 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Location */ C: 30)),
1092 ConstantAsMetadata::get(C: B.getInt32(C: Location))});
1093 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
1094 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
1095}
1096
1097// A fragment shader input interface variable whose base type is an integer or
1098// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
1099// specification requires these variables to be decorated with Flat (see
1100// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
1101// their base scalar type.
1102static bool inputRequiresFlatDecoration(llvm::Type *Ty) {
1103 while (true) {
1104 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
1105 Ty = AT->getElementType();
1106 continue;
1107 }
1108 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: Ty)) {
1109 Ty = VT->getElementType();
1110 continue;
1111 }
1112 break;
1113 }
1114 return Ty->isIntegerTy() || Ty->isDoubleTy();
1115}
1116
1117static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
1118 llvm::Type *Ty, const Twine &Name,
1119 unsigned BuiltInID) {
1120 auto *GV = new llvm::GlobalVariable(
1121 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1122 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1123 llvm::GlobalVariable::GeneralDynamicTLSModel,
1124 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1125 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1126 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1127 return B.CreateLoad(Ty, Ptr: GV);
1128}
1129
1130static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
1131 llvm::Type *Ty, unsigned Location,
1132 StringRef Name, bool NeedsFlat) {
1133 auto *GV = new llvm::GlobalVariable(
1134 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
1135 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1136 llvm::GlobalVariable::GeneralDynamicTLSModel,
1137 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
1138 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1139
1140 // Emit all decorations as a single `spirv.Decorations` node. Attaching
1141 // multiple `spirv.Decorations` metadata nodes to the same global is not
1142 // supported by the SPIR-V backend and results in all but one being dropped.
1143 LLVMContext &Ctx = GV->getContext();
1144 SmallVector<Metadata *, 2> Decorations;
1145 Decorations.push_back(
1146 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1147 C: B.getInt32(/* SPIRV::Decoration::Location */ C: 30)),
1148 ConstantAsMetadata::get(C: B.getInt32(C: Location))}));
1149 if (NeedsFlat)
1150 Decorations.push_back(
1151 Elt: MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(
1152 C: B.getInt32(/* SPIRV::Decoration::Flat */ C: 14))}));
1153 GV->addMetadata(Kind: "spirv.Decorations", MD&: *MDNode::get(Context&: Ctx, MDs: Decorations));
1154
1155 return B.CreateLoad(Ty, Ptr: GV);
1156}
1157
1158llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
1159 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1160 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1161 std::optional<unsigned> Index) {
1162 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1163 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1164
1165 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
1166 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1167 Location = L->getLocation();
1168
1169 // DXC completely ignores the semantic/index pair. Location are assigned from
1170 // the first semantic to the last.
1171 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
1172 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1173 SPIRVLastAssignedInputSemanticLocation += ElementCount;
1174
1175 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1176 bool NeedsFlat =
1177 ShaderAttr &&
1178 ShaderAttr->getType() == llvm::Triple::EnvironmentType::Pixel &&
1179 inputRequiresFlatDecoration(Ty: Type);
1180
1181 return createSPIRVLocationLoad(B, M&: CGM.getModule(), Ty: Type, Location,
1182 Name: VariableName.str(), NeedsFlat);
1183}
1184
1185static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
1186 llvm::Value *Source, unsigned Location,
1187 StringRef Name) {
1188 auto *GV = new llvm::GlobalVariable(
1189 M, Source->getType(), /* isConstant= */ false,
1190 llvm::GlobalValue::ExternalLinkage,
1191 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1192 llvm::GlobalVariable::GeneralDynamicTLSModel,
1193 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1194 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1195 addLocationDecoration(GV, Location);
1196 B.CreateStore(Val: Source, Ptr: GV);
1197}
1198
1199void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1200 llvm::IRBuilder<> &B, llvm::Value *Source,
1201 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1202 std::optional<unsigned> Index) {
1203 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1204 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1205
1206 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1207 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1208 Location = L->getLocation();
1209
1210 // DXC completely ignores the semantic/index pair. Location are assigned from
1211 // the first semantic to the last.
1212 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1213 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1214 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1215 createSPIRVLocationStore(B, M&: CGM.getModule(), Source, Location,
1216 Name: VariableName.str());
1217}
1218
1219namespace {
1220// Describes how a semantic leaf lowers to signature rows
1221struct SemanticShape {
1222 SmallVector<unsigned> Dimensions; // Empty dims denotes a scalar
1223 unsigned Cols;
1224 QualType RowType;
1225
1226 unsigned getNumRows() const {
1227 unsigned Rows = 1;
1228 for (unsigned Dimension : Dimensions)
1229 Rows *= Dimension;
1230 return Rows;
1231 }
1232
1233 SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
1234 assert(Row < getNumRows() && "row exceeds semantic shape");
1235
1236 SmallVector<unsigned> Indices(Dimensions.size());
1237 for (auto [Index, Dimension] :
1238 llvm::zip_equal(t: llvm::reverse(C&: Indices), u: llvm::reverse(C: Dimensions))) {
1239 Index = Row % Dimension;
1240 Row /= Dimension;
1241 }
1242 return Indices;
1243 }
1244};
1245} // namespace
1246
1247// Returns the QualType of a semantic leaf declarator. For a function the
1248// declared return type is used, otherwise the declared type.
1249static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
1250 if (const auto *FD = dyn_cast<clang::FunctionDecl>(Val: Decl))
1251 return FD->getDeclaredReturnType();
1252 return Decl->getType();
1253}
1254
1255// Walks through the surrounding constant array types of \p Ty, collecting their
1256// dimensions until reaching a scalar, vector, or matrix leaf.
1257static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
1258 SemanticShape Shape{.Dimensions: {}, .Cols: 1, .RowType: Ty};
1259 while (const ConstantArrayType *CAT =
1260 Ctx.getAsConstantArrayType(T: Shape.RowType)) {
1261 Shape.Dimensions.push_back(Elt: CAT->getSize().getZExtValue());
1262 Shape.RowType = CAT->getElementType();
1263 }
1264
1265 if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
1266 Shape.Cols = VT->getNumElements();
1267 } else if (const auto *MT =
1268 Shape.RowType->getAs<clang::ConstantMatrixType>()) {
1269 // FIXME: a matrix leaf lowers to one row per matrix row but if column_major
1270 // is specified we transpose the num rows and num cols, this depends on
1271 // #211977 to resolve
1272 Shape.Cols = MT->getNumColumns();
1273 }
1274
1275 return Shape;
1276}
1277
1278static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
1279 QualType Ty) {
1280 if (const auto *VT = Ty->getAs<clang::VectorType>())
1281 Ty = VT->getElementType();
1282 else if (const auto *MT = Ty->getAs<clang::ConstantMatrixType>())
1283 Ty = MT->getElementType();
1284
1285 llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
1286 bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
1287 return llvm::hlsl::getDXILElementType(Ty: IRTy, IsSigned);
1288}
1289
1290static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
1291 CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
1292 std::optional<unsigned> Index, const SemanticShape &Shape) {
1293 StringRef Name = Semantic->getAttrName()->getName();
1294
1295 // One semantic index per row, starting from the declared index.
1296 SmallVector<uint32_t> SemanticIndices;
1297 uint32_t FirstSemanticIndex = Index.value_or(u: 0);
1298 for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
1299 SemanticIndices.push_back(Elt: FirstSemanticIndex + I);
1300
1301 // The remaining members keep their default value and will be filled at a
1302 // later stage, either during packing or analysis of usage
1303 //
1304 // FIXME #189762: Element.InterpMode is to be set
1305 return llvm::hlsl::SemanticSignatureElement(
1306 SigId, Name, getSignatureComponentType(CGM, Ty: Shape.RowType),
1307 llvm::hlsl::getSemanticKind(SemanticName: Name), SemanticIndices,
1308 static_cast<uint8_t>(Shape.Cols));
1309}
1310
1311llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
1312 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1313 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
1314 SemanticSignatures &Signature) {
1315 StringRef Name = Semantic->getAttrName()->getName();
1316 SemanticShape Shape =
1317 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1318
1319 uint32_t SigId = Signature.size();
1320 Signature.push_back(
1321 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1322
1323 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1324
1325 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1326 M: B.GetInsertBlock()->getModule(), id: llvm::Intrinsic::dx_load_input, OverloadTys: {RowTy});
1327
1328 SmallVector<OperandBundleDef, 1> OB;
1329 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1330 llvm::Value *bundleArgs[] = {Token};
1331 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1332 }
1333
1334 llvm::Type *LeafTy = CGM.getTypes().ConvertType(T: Shape.RowType);
1335 llvm::Value *Result = llvm::PoisonValue::get(T: Type);
1336
1337 const unsigned NumRows = Shape.getNumRows();
1338
1339 for (unsigned Row = 0; Row < NumRows; ++Row) {
1340 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1341 std::array<Value *, 4> Args{
1342 /*SigElementId=*/B.getInt32(C: SigId),
1343 /*RowIndex=*/B.getInt32(C: Row),
1344 /*ColIndex=*/B.getInt8(C: 0),
1345 /*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(T: B.getInt32Ty())};
1346 llvm::Value *Value =
1347 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: Twine(Name).concat(Suffix: Twine(Row)));
1348 // Booleans use their memory representation in DXIL signatures, but
1349 // function parameters use their value representation.
1350 if (Value->getType() != LeafTy) {
1351 assert(Shape.RowType->hasBooleanRepresentation() &&
1352 "unexpected semantic load type mismatch");
1353 Value = B.CreateICmpNE(
1354 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
1355 }
1356
1357 Result =
1358 Indices.empty() ? Value : B.CreateInsertValue(Agg: Result, Val: Value, Idxs: Indices);
1359 }
1360 return Result;
1361}
1362
1363void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1364 llvm::Value *Source,
1365 const clang::DeclaratorDecl *Decl,
1366 HLSLAppliedSemanticAttr *Semantic,
1367 std::optional<unsigned> Index,
1368 SemanticSignatures &Signature) {
1369 SemanticShape Shape =
1370 getSemanticShape(Ctx&: CGM.getContext(), Ty: getSemanticLeafType(Decl));
1371
1372 uint32_t SigId = Signature.size();
1373 Signature.push_back(
1374 Elt: createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
1375
1376 llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(T: Shape.RowType);
1377
1378 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1379 M: B.GetInsertBlock()->getModule(), id: llvm::Intrinsic::dx_store_output,
1380 OverloadTys: {RowTy});
1381
1382 SmallVector<OperandBundleDef, 1> OB;
1383 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1384 llvm::Value *bundleArgs[] = {Token};
1385 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1386 }
1387
1388 const unsigned NumRows = Shape.getNumRows();
1389 for (unsigned Row = 0; Row < NumRows; ++Row) {
1390 SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
1391 llvm::Value *Val =
1392 Indices.empty() ? Source : B.CreateExtractValue(Agg: Source, Idxs: Indices);
1393
1394 // Booleans use their memory representation in DXIL signatures, but direct
1395 // function results use their value representation.
1396 if (Val->getType() != RowTy) {
1397 assert(Shape.RowType->hasBooleanRepresentation() &&
1398 "unexpected semantic store type mismatch");
1399 Val = B.CreateZExt(V: Val, DestTy: RowTy, Name: "storedv");
1400 }
1401
1402 std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(C: SigId),
1403 /*RowIndex=*/B.getInt32(C: Row),
1404 /*ColIndex=*/B.getInt8(C: 0), /*Value=*/Val};
1405 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1406 }
1407}
1408
1409llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1410 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1411 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1412 std::optional<unsigned> Index, SemanticSignatures &Signature) {
1413 if (CGM.getTarget().getTriple().isSPIRV())
1414 return emitSPIRVUserSemanticLoad(B, FD, Type, Decl, Semantic, Index);
1415
1416 if (CGM.getTarget().getTriple().isDXIL())
1417 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
1418
1419 llvm_unreachable("Unsupported target for user-semantic load.");
1420}
1421
1422void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1423 const clang::DeclaratorDecl *Decl,
1424 HLSLAppliedSemanticAttr *Semantic,
1425 std::optional<unsigned> Index,
1426 SemanticSignatures &Signature) {
1427 if (CGM.getTarget().getTriple().isSPIRV())
1428 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
1429
1430 if (CGM.getTarget().getTriple().isDXIL())
1431 return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
1432 Signature);
1433
1434 llvm_unreachable("Unsupported target for user-semantic load.");
1435}
1436
1437llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1438 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1439 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1440 std::optional<unsigned> Index, SemanticSignatures &Signature) {
1441
1442 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1443 if (SemanticName == "SV_GROUPINDEX") {
1444 llvm::Function *GroupIndex =
1445 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1446 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1447 }
1448
1449 if (SemanticName == "SV_DISPATCHTHREADID") {
1450 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1451 llvm::Function *ThreadIDIntrinsic =
1452 llvm::Intrinsic::isOverloaded(id: IntrinID)
1453 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1454 : CGM.getIntrinsic(IID: IntrinID);
1455 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1456 }
1457
1458 if (SemanticName == "SV_GROUPTHREADID") {
1459 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1460 llvm::Function *GroupThreadIDIntrinsic =
1461 llvm::Intrinsic::isOverloaded(id: IntrinID)
1462 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1463 : CGM.getIntrinsic(IID: IntrinID);
1464 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1465 }
1466
1467 if (SemanticName == "SV_GROUPID") {
1468 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1469 llvm::Function *GroupIDIntrinsic =
1470 llvm::Intrinsic::isOverloaded(id: IntrinID)
1471 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1472 : CGM.getIntrinsic(IID: IntrinID);
1473 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1474 }
1475
1476 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1477 assert(ShaderAttr && "Entry point has no shader attribute");
1478 llvm::Triple::EnvironmentType ST = ShaderAttr->getType();
1479
1480 if (SemanticName == "SV_POSITION") {
1481 if (ST == Triple::EnvironmentType::Pixel) {
1482 if (CGM.getTarget().getTriple().isSPIRV())
1483 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1484 Name: Semantic->getAttrName()->getName(),
1485 /* BuiltIn::FragCoord */ BuiltInID: 15);
1486 if (CGM.getTarget().getTriple().isDXIL())
1487 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1488 Signature);
1489 }
1490
1491 if (ST == Triple::EnvironmentType::Vertex) {
1492 return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index,
1493 Signature);
1494 }
1495 }
1496
1497 if (SemanticName == "SV_VERTEXID") {
1498 if (ST == Triple::EnvironmentType::Vertex) {
1499 if (CGM.getTarget().getTriple().isSPIRV())
1500 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1501 Name: Semantic->getAttrName()->getName(),
1502 /* BuiltIn::VertexIndex */ BuiltInID: 42);
1503 else
1504 return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
1505 Signature);
1506 }
1507 }
1508
1509 llvm_unreachable(
1510 "Load hasn't been implemented yet for this system semantic. FIXME");
1511}
1512
1513static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
1514 llvm::Value *Source, const Twine &Name,
1515 unsigned BuiltInID) {
1516 auto *GV = new llvm::GlobalVariable(
1517 M, Source->getType(), /* isConstant= */ false,
1518 llvm::GlobalValue::ExternalLinkage,
1519 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1520 llvm::GlobalVariable::GeneralDynamicTLSModel,
1521 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1522 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1523 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1524 B.CreateStore(Val: Source, Ptr: GV);
1525}
1526
1527void CGHLSLRuntime::emitSystemSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1528 const clang::DeclaratorDecl *Decl,
1529 HLSLAppliedSemanticAttr *Semantic,
1530 std::optional<unsigned> Index,
1531 SemanticSignatures &Signature) {
1532
1533 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1534 if (SemanticName == "SV_POSITION") {
1535 if (CGM.getTarget().getTriple().isDXIL()) {
1536 emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1537 return;
1538 }
1539
1540 if (CGM.getTarget().getTriple().isSPIRV()) {
1541 createSPIRVBuiltinStore(B, M&: CGM.getModule(), Source,
1542 Name: Semantic->getAttrName()->getName(),
1543 /* BuiltIn::Position */ BuiltInID: 0);
1544 return;
1545 }
1546 }
1547
1548 if (SemanticName == "SV_TARGET") {
1549 emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1550 return;
1551 }
1552
1553 llvm_unreachable(
1554 "Store hasn't been implemented yet for this system semantic. FIXME");
1555}
1556
1557llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1558 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1559 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1560 SemanticSignatures &Signature) {
1561
1562 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1563 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1564 return emitSystemSemanticLoad(B, FD, Type, Decl, Semantic, Index,
1565 Signature);
1566 return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index, Signature);
1567}
1568
1569void CGHLSLRuntime::handleScalarSemanticStore(IRBuilder<> &B,
1570 const FunctionDecl *FD,
1571 llvm::Value *Source,
1572 const clang::DeclaratorDecl *Decl,
1573 HLSLAppliedSemanticAttr *Semantic,
1574 SemanticSignatures &Signature) {
1575 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1576 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1577 emitSystemSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1578 else
1579 emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
1580}
1581
1582std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1583CGHLSLRuntime::handleStructSemanticLoad(
1584 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1585 const clang::DeclaratorDecl *Decl,
1586 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1587 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1588 SemanticSignatures &Signature) {
1589 const llvm::StructType *ST = cast<StructType>(Val: Type);
1590 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
1591
1592 assert(RD->getNumFields() == ST->getNumElements());
1593
1594 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
1595 auto FieldDecl = RD->field_begin();
1596 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
1597 auto [ChildValue, NextAttr] =
1598 handleSemanticLoad(B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin,
1599 end: AttrEnd, Signature);
1600 AttrBegin = NextAttr;
1601 assert(ChildValue);
1602 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
1603 ++FieldDecl;
1604 }
1605
1606 return std::make_pair(x&: Aggregate, y&: AttrBegin);
1607}
1608
1609specific_attr_iterator<HLSLAppliedSemanticAttr>
1610CGHLSLRuntime::handleStructSemanticStore(
1611 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1612 const clang::DeclaratorDecl *Decl,
1613 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1614 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1615 SemanticSignatures &Signature) {
1616
1617 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
1618
1619 const clang::RecordDecl *RD = nullptr;
1620 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
1621 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
1622 else
1623 RD = Decl->getType()->getAsRecordDecl();
1624 assert(RD);
1625
1626 assert(RD->getNumFields() == ST->getNumElements());
1627
1628 auto FieldDecl = RD->field_begin();
1629 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
1630 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
1631 AttrBegin = handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin,
1632 AttrEnd, Signature);
1633 }
1634
1635 return AttrBegin;
1636}
1637
1638std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1639CGHLSLRuntime::handleSemanticLoad(
1640 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1641 const clang::DeclaratorDecl *Decl,
1642 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1643 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1644 SemanticSignatures &Signature) {
1645 assert(AttrBegin != AttrEnd);
1646 if (Type->isStructTy())
1647 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
1648 Signature);
1649
1650 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1651 ++AttrBegin;
1652 return std::make_pair(
1653 x: handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr, Signature), y&: AttrBegin);
1654}
1655
1656specific_attr_iterator<HLSLAppliedSemanticAttr>
1657CGHLSLRuntime::handleSemanticStore(
1658 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1659 const clang::DeclaratorDecl *Decl,
1660 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1661 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
1662 SemanticSignatures &Signature) {
1663 assert(AttrBegin != AttrEnd);
1664 if (Source->getType()->isStructTy())
1665 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
1666 Signature);
1667
1668 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1669 ++AttrBegin;
1670 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr, Signature);
1671 return AttrBegin;
1672}
1673
1674void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
1675 llvm::Function *Fn) {
1676 SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
1677 SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
1678
1679 llvm::Module &M = CGM.getModule();
1680 llvm::LLVMContext &Ctx = M.getContext();
1681 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
1682 Function *EntryFn =
1683 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
1684
1685 // Copy function attributes over, we have no argument or return attributes
1686 // that can be valid on the real entry.
1687 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
1688 Attrs: Fn->getAttributes().getFnAttrs());
1689 EntryFn->setAttributes(NewAttrs);
1690 setHLSLEntryAttributes(FD, Fn: EntryFn);
1691
1692 // Set the called function as internal linkage.
1693 Fn->setLinkage(GlobalValue::InternalLinkage);
1694
1695 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
1696 IRBuilder<> B(BB);
1697 llvm::SmallVector<Value *> Args;
1698
1699 SmallVector<OperandBundleDef, 1> OB;
1700 if (CGM.shouldEmitConvergenceTokens()) {
1701 assert(EntryFn->isConvergent());
1702 llvm::Value *I =
1703 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
1704 llvm::Value *bundleArgs[] = {I};
1705 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1706 }
1707
1708 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
1709
1710 unsigned SRetOffset = 0;
1711 for (const auto &Param : Fn->args()) {
1712 if (Param.hasStructRetAttr()) {
1713 SRetOffset = 1;
1714 llvm::Type *VarType = Param.getParamStructRetType();
1715 llvm::Value *Var =
1716 CGM.getLangOpts().EmitLogicalPointer
1717 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
1718 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
1719 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
1720 Args.push_back(Elt: Var);
1721 continue;
1722 }
1723
1724 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
1725 llvm::Value *SemanticValue = nullptr;
1726 // FIXME: support inout/out parameters for semantics.
1727 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
1728 PD->getAttr<HLSLParamModifierAttr>()) {
1729 llvm_unreachable("Not handled yet");
1730 } else {
1731 llvm::Type *ParamType = nullptr;
1732 if (Param.hasByValAttr())
1733 ParamType = Param.getParamByValType();
1734 else if (PD->getType()->isRecordType())
1735 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
1736 else
1737 ParamType = Param.getType();
1738
1739 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1740 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
1741 auto Result = handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd,
1742 Signature&: InputSignature);
1743 SemanticValue = Result.first;
1744 if (!SemanticValue)
1745 return;
1746 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
1747 llvm::Value *Var =
1748 CGM.getLangOpts().EmitLogicalPointer
1749 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
1750 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
1751 B.CreateStore(Val: SemanticValue, Ptr: Var);
1752 SemanticValue = Var;
1753 }
1754 }
1755
1756 assert(SemanticValue);
1757 Args.push_back(Elt: SemanticValue);
1758 }
1759
1760 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
1761 CI->setCallingConv(Fn->getCallingConv());
1762
1763 if (Fn->getReturnType() != CGM.VoidTy)
1764 // Element type is unused, so set to dummy value (NULL).
1765 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
1766
1767 for (auto &SourcePair : OutputSemantic) {
1768 llvm::Value *Source = SourcePair.first;
1769 llvm::Type *ElementType = SourcePair.second;
1770 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
1771 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
1772
1773 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1774 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
1775 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd,
1776 Signature&: OutputSignature);
1777 }
1778
1779 B.CreateRetVoid();
1780
1781 // Add and identify root signature to function, if applicable
1782 for (const Attr *Attr : FD->getAttrs()) {
1783 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
1784 auto *RSDecl = RSAttr->getSignatureDecl();
1785 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
1786 Fn: EntryFn, M);
1787 }
1788 }
1789
1790 addSemanticSignatureMD(InputElements: InputSignature, OutputElements: OutputSignature, Fn: EntryFn, M);
1791}
1792
1793static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
1794 bool CtorOrDtor) {
1795 const auto *GV =
1796 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
1797 if (!GV)
1798 return;
1799 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1800 if (!CA)
1801 return;
1802 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
1803 // HLSL neither supports priorities or COMDat values, so we will check those
1804 // in an assert but not handle them.
1805
1806 for (const auto &Ctor : CA->operands()) {
1807 if (isa<ConstantAggregateZero>(Val: Ctor))
1808 continue;
1809 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
1810
1811 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
1812 "HLSL doesn't support setting priority for global ctors.");
1813 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
1814 "HLSL doesn't support COMDat for global ctors.");
1815 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
1816 }
1817}
1818
1819void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
1820 llvm::Module &M = CGM.getModule();
1821 SmallVector<Function *> CtorFns;
1822 SmallVector<Function *> DtorFns;
1823 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
1824 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
1825
1826 // Insert a call to the global constructor at the beginning of the entry block
1827 // to externally exported functions. This is a bit of a hack, but HLSL allows
1828 // global constructors, but doesn't support driver initialization of globals.
1829 for (auto &F : M.functions()) {
1830 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
1831 continue;
1832 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
1833 Instruction *IP = &*F.getEntryBlock().begin();
1834 SmallVector<OperandBundleDef, 1> OB;
1835 if (Token) {
1836 llvm::Value *bundleArgs[] = {Token};
1837 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1838 IP = Token->getNextNode();
1839 }
1840 IRBuilder<> B(IP);
1841 for (auto *Fn : CtorFns) {
1842 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1843 CI->setCallingConv(Fn->getCallingConv());
1844 }
1845
1846 // Insert global dtors before the terminator of the last instruction
1847 B.SetInsertPoint(F.back().getTerminator());
1848 for (auto *Fn : DtorFns) {
1849 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1850 CI->setCallingConv(Fn->getCallingConv());
1851 }
1852 }
1853
1854 // No need to keep global ctors/dtors for non-lib profile after call to
1855 // ctors/dtors added for entry.
1856 Triple T(M.getTargetTriple());
1857 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
1858 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
1859 GV->eraseFromParent();
1860 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
1861 GV->eraseFromParent();
1862 }
1863}
1864
1865static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
1866 Intrinsic::ID IntrID,
1867 ArrayRef<llvm::Value *> Args) {
1868
1869 LLVMContext &Ctx = CGM.getLLVMContext();
1870 llvm::Function *InitResFunc =
1871 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
1872 Linkage: llvm::GlobalValue::InternalLinkage,
1873 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
1874 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1875
1876 llvm::BasicBlock *EntryBB =
1877 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
1878 CGBuilderTy Builder(CGM, Ctx);
1879 const DataLayout &DL = CGM.getModule().getDataLayout();
1880 Builder.SetInsertPoint(EntryBB);
1881
1882 // Make sure the global variable is buffer resource handle
1883 llvm::Type *HandleTy = GV->getValueType();
1884 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
1885
1886 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
1887 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
1888 Name: Twine(GV->getName()).concat(Suffix: "_h"));
1889
1890 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
1891 Builder.CreateRetVoid();
1892
1893 CGM.AddCXXGlobalInit(F: InitResFunc);
1894}
1895
1896void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
1897 llvm::GlobalVariable *GV) {
1898 ResourceBindingAttrs Binding(BufDecl);
1899 assert(Binding.hasBinding() &&
1900 "cbuffer/tbuffer should always have resource binding attribute");
1901
1902 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
1903 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
1904 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
1905 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
1906
1907 // buffer with explicit binding
1908 if (Binding.isExplicit()) {
1909 llvm::Intrinsic::ID IntrinsicID =
1910 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1911 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
1912 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
1913 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1914 } else {
1915 // buffer with implicit binding
1916 llvm::Intrinsic::ID IntrinsicID =
1917 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1918 auto *OrderID =
1919 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
1920 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
1921 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1922 }
1923}
1924
1925void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
1926 llvm::GlobalVariable *GV) {
1927 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
1928 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1929 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
1930 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1931}
1932
1933llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
1934 if (!CGM.shouldEmitConvergenceTokens())
1935 return nullptr;
1936
1937 auto E = BB.end();
1938 for (auto I = BB.begin(); I != E; ++I) {
1939 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
1940 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
1941 return II;
1942 }
1943 }
1944 llvm_unreachable("Convergence token should have been emitted.");
1945 return nullptr;
1946}
1947
1948class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
1949public:
1950 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
1951 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
1952 OpaqueValueVisitor() {}
1953
1954 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
1955 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
1956 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
1957 // traversal, the temporary containing the copy out will not have
1958 // been created yet.
1959 return false;
1960 }
1961
1962 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
1963 // Traverse the source expression first.
1964 if (E->getSourceExpr())
1965 TraverseStmt(S: E->getSourceExpr());
1966
1967 // Then add this OVE if we haven't seen it before.
1968 if (Visited.insert(Ptr: E).second)
1969 OVEs.push_back(Elt: E);
1970
1971 return true;
1972 }
1973};
1974
1975void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
1976 InitListExpr *E) {
1977
1978 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
1979 OpaqueValueVisitor Visitor;
1980 Visitor.TraverseStmt(S: E);
1981 for (auto *OVE : Visitor.OVEs) {
1982 if (CGF.isOpaqueValueEmitted(E: OVE))
1983 continue;
1984 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
1985 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
1986 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
1987 } else {
1988 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
1989 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
1990 }
1991 }
1992}
1993
1994std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
1995 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
1996 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
1997 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
1998 "expected resource array subscript expression");
1999
2000 // Let clang codegen handle local and static resource array subscripts,
2001 // or when the subscript references on opaque expression (as part of
2002 // ArrayInitLoopExpr AST node).
2003 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
2004 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
2005 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2006 ArrayDecl->getStorageClass() == SC_Static)
2007 return std::nullopt;
2008
2009 // get the resource array type
2010 ASTContext &AST = ArrayDecl->getASTContext();
2011 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
2012 assert(ResArrayTy->isHLSLResourceRecordArray() &&
2013 "expected array of resource classes");
2014
2015 // Iterate through all nested array subscript expressions to calculate
2016 // the index in the flattened resource array (if this is a multi-
2017 // dimensional array). The index is calculated as a sum of all indices
2018 // multiplied by the total size of the array at that level.
2019 Value *Index = nullptr;
2020 const ArraySubscriptExpr *ASE = ArraySubsExpr;
2021 while (ASE != nullptr) {
2022 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
2023 if (const auto *ArrayTy =
2024 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
2025 Value *Multiplier = llvm::ConstantInt::get(
2026 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
2027 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
2028 }
2029 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
2030 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
2031 }
2032
2033 // Find binding info for the resource array. For implicit binding
2034 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
2035 ResourceBindingAttrs Binding(ArrayDecl);
2036 assert(Binding.hasBinding() &&
2037 "resource array must have a binding attribute");
2038
2039 // Find the individual resource type.
2040 QualType ResultTy = ArraySubsExpr->getType();
2041 QualType ResourceTy =
2042 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
2043
2044 // Create a temporary variable for the result, which is either going
2045 // to be a single resource instance or a local array of resources (we need to
2046 // return an LValue).
2047 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
2048 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
2049 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
2050 kind: NormalEHLifetimeMarker, A: TmpVar);
2051
2052 AggValueSlot ValueSlot = AggValueSlot::forAddr(
2053 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
2054 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
2055 mayOverlap: AggValueSlot::DoesNotOverlap);
2056
2057 // Calculate total array size (= range size).
2058 llvm::Value *Range = llvm::ConstantInt::getSigned(
2059 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
2060
2061 // If the result of the subscript operation is a single resource, call the
2062 // constructor.
2063 if (ResultTy == ResourceTy) {
2064 CallArgList Args;
2065 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
2066 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
2067 Name: ArrayDecl->getName(), Binding, Args);
2068
2069 if (!CreateMethod) {
2070 // This can happen if someone creates an array of structs that looks like
2071 // an HLSL resource record array but it does not have the required static
2072 // create method. No binding will be generated for it.
2073 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
2074 "create method lookup should always succeed for built-in resource "
2075 "records");
2076 return std::nullopt;
2077 }
2078
2079 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
2080
2081 } else {
2082 // The result of the subscript operation is a local resource array which
2083 // needs to be initialized.
2084 const ConstantArrayType *ArrayTy =
2085 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
2086 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2087 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
2088 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
2089 if (!EndIndex)
2090 return std::nullopt;
2091 }
2092 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
2093}
2094
2095// Initialize all resources of a global resource array into provided slot.
2096bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
2097 const VarDecl *ArrayDecl,
2098 AggValueSlot &DestSlot) {
2099 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2100 ArrayDecl->hasGlobalStorage() &&
2101 ArrayDecl->getStorageClass() != SC_Static &&
2102 "expected global non-static resource array");
2103
2104 // Find binding info for the resource array. For implicit binding
2105 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
2106 ResourceBindingAttrs Binding(ArrayDecl);
2107 assert(Binding.hasBinding() &&
2108 "resource array must have a binding attribute");
2109
2110 // Find the individual resource type.
2111 ASTContext &AST = ArrayDecl->getASTContext();
2112 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
2113 const auto *ResArrayTy =
2114 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
2115
2116 // Create Value for index and total array size (= range size).
2117 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
2118 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
2119 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
2120
2121 // Initialize individual resources in the array into DestSlot.
2122 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
2123 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
2124 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
2125 return EndIndex.has_value();
2126}
2127
2128// If the expression is a global resource array, initialize all of its resources
2129// into Dest. Returns false if no initialization has been performed and the
2130// array copy should be handled by the default codegen.
2131bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
2132 AggValueSlot &DestSlot) {
2133 assert(E->getType()->isHLSLResourceRecordArray() &&
2134 "expected resource array");
2135
2136 // Find the array declaration for the expression. Fallback to the default
2137 // handling if it's not a global resource array.
2138 const VarDecl *ArrayDecl =
2139 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
2140 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
2141 ArrayDecl->getStorageClass() == SC_Static)
2142 return false;
2143
2144 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
2145}
2146
2147// If the expression is a global resource array, create a temporary and
2148// initialize all of its resources, and return it as an LValue. Returns nullopt
2149// if no initialization has been performed and the handling should follow the
2150// default path.
2151std::optional<LValue>
2152CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
2153 const VarDecl *ArrayDecl) {
2154 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
2155 "expected resource array declaration");
2156
2157 if (!ArrayDecl->hasGlobalStorage() ||
2158 ArrayDecl->getStorageClass() == SC_Static)
2159 return std::nullopt;
2160
2161 AggValueSlot TmpArraySlot =
2162 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
2163 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
2164 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
2165 Source: AlignmentSource::Decl);
2166 return std::nullopt;
2167}
2168
2169RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
2170 CodeGenFunction &CGF) {
2171
2172 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
2173 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
2174 "expected cbuffer matrix");
2175
2176 QualType MatQualTy = LV.getType();
2177 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
2178 Address SrcAddr = LV.getAddress();
2179
2180 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
2181 return SrcAddr;
2182
2183 RawAddress DestAlloca =
2184 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
2185 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
2186 return DestAlloca;
2187}
2188
2189std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
2190 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
2191 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
2192 // Find the element type to index by first padding the element type per HLSL
2193 // buffer rules, and then padding out to a 16-byte register boundary if
2194 // necessary.
2195 llvm::Type *LayoutTy =
2196 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
2197 uint64_t LayoutSizeInBits =
2198 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
2199 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
2200 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
2201 if (RowAlignedSize > ElementSize) {
2202 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
2203 CGM, NumBytes: RowAlignedSize - ElementSize);
2204 assert(Padding && "No padding type for target?");
2205 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
2206 /*isPacked=*/true);
2207 }
2208
2209 // If the layout type doesn't introduce any padding, we don't need to do
2210 // anything special.
2211 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
2212 if (LayoutTy == OrigTy)
2213 return std::nullopt;
2214
2215 LValueBaseInfo EltBaseInfo;
2216 TBAAAccessInfo EltTBAAInfo;
2217
2218 // Index into the object as-if we have an array of the padded element type,
2219 // and then dereference the element itself to avoid reading padding that may
2220 // be past the end of the in-memory object.
2221 SmallVector<llvm::Value *, 2> Indices;
2222 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
2223 Indices.push_back(Elt: Idx);
2224 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
2225
2226 if (CGF.getLangOpts().EmitLogicalPointer) {
2227 // The fact that we emit an array-to-pointer decay might be an oversight,
2228 // but for now, we simply ignore it (see #179951).
2229 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
2230 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
2231
2232 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
2233 Address Addr = LV.getAddress();
2234 LayoutTy = llvm::ArrayType::get(
2235 ElementType: LayoutTy,
2236 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
2237 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
2238 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
2239 Addr =
2240 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
2241 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2242 }
2243
2244 Address Addr =
2245 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
2246 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
2247 IdxList: Indices, Name: "cbufferidx");
2248 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
2249 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
2250}
2251
2252std::optional<LValue>
2253CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
2254 const MemberExpr *ME) {
2255 assert((ME->getType()->isHLSLResourceRecord() ||
2256 ME->getType()->isHLSLResourceRecordArray()) &&
2257 "expected resource member expression");
2258
2259 const VarDecl *ResourceVD =
2260 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
2261 if (!ResourceVD)
2262 return std::nullopt;
2263
2264 // Handle member of resource array type.
2265 if (ResourceVD->getType()->isHLSLResourceRecordArray())
2266 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
2267
2268 GlobalVariable *ResGV =
2269 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
2270 const DataLayout &DL = CGM.getDataLayout();
2271 llvm::Type *Ty = ResGV->getValueType();
2272 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
2273 Address Addr = Address(ResGV, Ty, Align);
2274 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
2275 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2276 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
2277 return LV;
2278}
2279
2280bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
2281 const LValue &SrcLV,
2282 AggValueSlot &DestSlot) {
2283 assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
2284 "expected expression in HLSL constant address space");
2285 assert(!E->getType()->isHLSLResourceRecord() &&
2286 !E->getType()->isHLSLResourceRecordArray() &&
2287 "direct accesses to resource types should be handled separately");
2288
2289 if (DestSlot.isIgnored())
2290 return false;
2291
2292 QualType Ty = E->getType();
2293 Address DstPtr = DestSlot.getAddress();
2294 Address SrcPtr = SrcLV.getAddress();
2295
2296 // If there are no intangible types, we don't need to lookup associated
2297 // resources.
2298 if (!Ty->isHLSLIntangibleType())
2299 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty);
2300
2301 // Handle structs with intangible types by setting the resource fields
2302 // of the destination struct with the resources associated with the global
2303 // struct.
2304 EmbeddedResourceNameBuilder NameBuilder;
2305 const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
2306 AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
2307
2308 // Callback to fill in the associated resource.
2309 auto EmitResFn = [&](AggValueSlot &ResSlot) {
2310 const VarDecl *ResDecl = AssociatedResources.getNextResource();
2311 assert(ResDecl && "associated resource declaration not found");
2312
2313 // Check that the resource type of dest and src matches.
2314 [[maybe_unused]] llvm::Type *DestType =
2315 ResSlot.getAddress().getElementType();
2316 [[maybe_unused]] llvm::Type *SrcConvertedType =
2317 CGM.getTypes().ConvertTypeForMem(T: ResDecl->getType());
2318 assert(DestType == SrcConvertedType && "resource slot type mismatch");
2319
2320 if (ResDecl->getType()->isHLSLResourceRecord())
2321 copyGlobalResource(CGF, ResourceVD: ResDecl, DestSlot&: ResSlot);
2322 else
2323 initializeGlobalResourceArray(CGF, ArrayDecl: ResDecl, DestSlot&: ResSlot);
2324 };
2325
2326 auto Result =
2327 HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType: Ty, EmitResFn);
2328 assert(AssociatedResources.getNextResource() == nullptr &&
2329 "expected all associated resources to be processed");
2330 return Result;
2331}
2332
2333LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
2334 const MemberExpr *E) {
2335 LValue Base =
2336 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
2337 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
2338 assert(Field && "Unexpected access into HLSL buffer");
2339
2340 const RecordDecl *Rec = Field->getParent();
2341
2342 // Work out the buffer layout type to index into.
2343 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
2344 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
2345 // Since this is a member of an object in the buffer and not the buffer's
2346 // struct/class itself, we shouldn't have any offsets on the members we need
2347 // to contend with.
2348 CGHLSLOffsetInfo EmptyOffsets;
2349 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
2350 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
2351
2352 // Get the field index for the layout struct, accounting for padding.
2353 unsigned FieldIdx =
2354 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
2355 assert(FieldIdx < LayoutTy->getNumElements() &&
2356 "Layout struct is smaller than member struct");
2357 unsigned Skipped = 0;
2358 for (unsigned I = 0; I <= FieldIdx;) {
2359 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
2360 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
2361 ++Skipped;
2362 else
2363 ++I;
2364 }
2365 FieldIdx += Skipped;
2366 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
2367
2368 // Now index into the struct, making sure that the type we return is the
2369 // buffer layout type rather than the original type in the AST.
2370 QualType FieldType = Field->getType();
2371 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
2372 CharUnits Align = CharUnits::fromQuantity(
2373 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
2374
2375 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
2376 ? CGF.Builder.CreateStructuredGEP(
2377 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
2378 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
2379 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
2380 Idx: FieldIdx, Name: Field->getName());
2381 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
2382
2383 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
2384 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
2385 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
2386 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
2387
2388 return LV;
2389}
2390