1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGHLSLRuntime.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "HLSLBufferLayoutBuilder.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/Decl.h"
25#include "clang/AST/Expr.h"
26#include "clang/AST/HLSLResource.h"
27#include "clang/AST/RecursiveASTVisitor.h"
28#include "clang/AST/Type.h"
29#include "clang/Basic/DiagnosticDriver.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetOptions.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/STLExtras.h"
35#include "llvm/ADT/ScopeExit.h"
36#include "llvm/ADT/SmallString.h"
37#include "llvm/ADT/SmallVector.h"
38#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/Type.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/Alignment.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Support/FormatVariadic.h"
51#include "llvm/Support/Path.h"
52#include "llvm/Transforms/Utils/ModuleUtils.h"
53#include <cstdint>
54#include <optional>
55
56using namespace clang;
57using namespace CodeGen;
58using namespace clang::hlsl;
59using namespace llvm;
60
61using llvm::hlsl::CBufferRowSizeInBytes;
62
63namespace {
64
65void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
66 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
67 // Assume ValVersionStr is legal here.
68 VersionTuple Version;
69 if (Version.tryParse(string: ValVersionStr) || Version.getBuild() ||
70 Version.getSubminor() || !Version.getMinor()) {
71 return;
72 }
73
74 uint64_t Major = Version.getMajor();
75 uint64_t Minor = *Version.getMinor();
76
77 auto &Ctx = M.getContext();
78 IRBuilder<> B(M.getContext());
79 MDNode *Val = MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(C: Major)),
80 ConstantAsMetadata::get(C: B.getInt32(C: Minor))});
81 StringRef DXILValKey = "dx.valver";
82 auto *DXILValMD = M.getOrInsertNamedMetadata(Name: DXILValKey);
83 DXILValMD->addOperand(M: Val);
84}
85
86void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
87 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
88 llvm::Function *Fn, llvm::Module &M) {
89 auto &Ctx = M.getContext();
90
91 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
92 MDNode *RootSignature = RSBuilder.BuildRootSignature();
93
94 ConstantAsMetadata *Version = ConstantAsMetadata::get(C: ConstantInt::get(
95 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: llvm::to_underlying(E: RootSigVer)));
96 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(V: Fn) : nullptr;
97 MDNode *MDVals = MDNode::get(Context&: Ctx, MDs: {EntryFunc, RootSignature, Version});
98
99 StringRef RootSignatureValKey = "dx.rootsignatures";
100 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(Name: RootSignatureValKey);
101 RootSignatureValMD->addOperand(M: MDVals);
102}
103
104// Given a MemberExpr of a resource or resource array type, find the parent
105// VarDecl of the struct or class instance that contains this resource and
106// build the full resource name based on the member access path.
107//
108// For example, for a member access like "myStructArray[0].memberA",
109// this function will find the VarDecl of "myStructArray" and use the
110// EmbeddedResourceNameBuilder to build the resource name
111// "myStructArray.0.memberA".
112static const VarDecl *findStructResourceParentDeclAndBuildName(
113 const MemberExpr *ME, EmbeddedResourceNameBuilder &NameBuilder) {
114
115 SmallVector<const Expr *> WorkList;
116 const VarDecl *VD = nullptr;
117 const Expr *E = ME;
118
119 for (;;) {
120 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
121 assert(isa<VarDecl>(DRE->getDecl()) &&
122 "member expr base is not a var decl");
123 VD = cast<VarDecl>(Val: DRE->getDecl());
124 NameBuilder.pushName(N: VD->getName());
125 break;
126 }
127
128 WorkList.push_back(Elt: E);
129 if (const auto *MExp = dyn_cast<MemberExpr>(Val: E))
130 E = MExp->getBase();
131 else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E))
132 E = ICE->getSubExpr();
133 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
134 E = ASE->getBase();
135 else if (isa<CXXThisExpr>(Val: E))
136 // Resource member access on "this" pointer not yet implemented
137 // (llvm/llvm-project#190299)
138 return nullptr;
139 else
140 llvm_unreachable("unexpected expr type in resource member access");
141
142 assert(E && "expected valid expression");
143 }
144
145 while (!WorkList.empty()) {
146 E = WorkList.pop_back_val();
147 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
148 NameBuilder.pushName(
149 N: ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
150 } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
151 if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
152 CXXRecordDecl *DerivedRD =
153 ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
154 CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
155 NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
156 }
157 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E)) {
158 const Expr *IdxExpr = ASE->getIdx();
159 std::optional<llvm::APSInt> Value =
160 IdxExpr->getIntegerConstantExpr(Ctx: VD->getASTContext());
161 assert(Value &&
162 "expected constant index in struct with resource array access");
163 NameBuilder.pushArrayIndex(Index: Value->getZExtValue());
164 } else {
165 llvm_unreachable("unexpected expr type in resource member access");
166 }
167 }
168 return VD;
169}
170
171// Given a MemberExpr of a resource or resource array type, find the
172// corresponding global resource declaration associated with the owning struct
173// or class instance via HLSLAssociatedResourceDeclAttr.
174static const VarDecl *
175findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
176
177 EmbeddedResourceNameBuilder NameBuilder;
178 const VarDecl *ParentVD =
179 findStructResourceParentDeclAndBuildName(ME, NameBuilder);
180 if (!ParentVD)
181 return nullptr;
182
183 if (!ParentVD->hasGlobalStorage())
184 return nullptr;
185
186 IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
187 for (const Attr *A : ParentVD->getAttrs()) {
188 if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(Val: A)) {
189 VarDecl *AssocResVD = ADA->getResDecl();
190 if (AssocResVD->getIdentifier() == II)
191 return AssocResVD;
192 }
193 }
194 return nullptr;
195}
196
197void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
198 auto &SM = CGM.getContext().getSourceManager();
199 auto &Macros = CGM.getPreprocessorOpts().Macros;
200 auto &CodeGenOpts = CGM.getCodeGenOpts();
201 auto &Ctx = M.getContext();
202
203 // Names and content of shader source code files.
204 llvm::NamedMDNode *DXContents =
205 M.getOrInsertNamedMetadata(Name: "dx.source.contents");
206 auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
207 llvm::MDTuple *FileInfo =
208 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: NameContent.first),
209 llvm::MDString::get(Context&: Ctx, Str: NameContent.second)});
210 DXContents->addOperand(M: FileInfo);
211 };
212
213 bool Invalid = false;
214 const SrcMgr::SLocEntry *MainLocEntry =
215 &SM.getSLocEntry(FID: SM.getMainFileID(), Invalid: &Invalid);
216 assert(!Invalid && "Main file SLocEntry must not be invalid!");
217 const SrcMgr::ContentCache &MainCCEntry =
218 MainLocEntry->getFile().getContentCache();
219
220 SmallVector<std::pair<std::string, StringRef>> Files;
221 std::optional<SmallString<256>> MainFileName;
222 Files.reserve(N: SM.local_sloc_entry_size());
223 for (unsigned I : llvm::seq(Size: SM.local_sloc_entry_size())) {
224 const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(Index: I);
225 if (!LocEntry.isFile())
226 continue;
227
228 const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
229 if (isSystem(CK: FInfo.getFileCharacteristic()))
230 continue;
231
232 const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
233 OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
234 if (!FEntry)
235 continue;
236
237 llvm::SmallString<256> Path = FEntry->getName();
238 llvm::sys::path::native(path&: Path);
239 std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
240 Diag&: SM.getDiagnostics(), FM&: SM.getFileManager(), Loc: SourceLocation());
241 if (!Buffer) {
242 SM.getDiagnostics().Report(DiagID: diag::warn_hlsl_failed_to_embed_source)
243 << Path;
244 continue;
245 }
246
247 if (&MainCCEntry != &CCEntry) {
248 Files.emplace_back(Args&: Path, Args: Buffer->getBuffer());
249 } else {
250 // Main file should be at first position.
251 addFile(std::make_pair(x&: Path, y: Buffer->getBuffer()));
252 MainFileName.emplace(args&: Path);
253 }
254 }
255 assert(MainFileName && "Main file not found.");
256
257 // Files other that main one should be sorted by name.
258 llvm::sort(C&: Files);
259#ifndef NDEBUG
260 for (unsigned I = 1; I < Files.size(); ++I)
261 assert((Files[I - 1].first != Files[I].first) &&
262 "duplicate files in dx.source.contents");
263#endif
264 llvm::for_each(Range&: Files, F: addFile);
265
266 SmallVector<llvm::Metadata *> Defines;
267 Defines.reserve(N: Macros.size());
268 for (const auto &Macro : Macros) {
269 // Ignore undefs.
270 if (!Macro.second)
271 Defines.emplace_back(Args: llvm::MDString::get(Context&: Ctx, Str: Macro.first));
272 }
273 M.getOrInsertNamedMetadata(Name: "dx.source.defines")
274 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Defines));
275
276 if (!CodeGenOpts.MainFileName.empty())
277 llvm::sys::path::native(path: CodeGenOpts.MainFileName, result&: *MainFileName);
278 M.getOrInsertNamedMetadata(Name: "dx.source.mainFileName")
279 ->addOperand(
280 M: llvm::MDNode::get(Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: *MainFileName)));
281
282 SmallVector<llvm::Metadata *> Args;
283 Args.reserve(N: CodeGenOpts.HLSLParsedCommandLine.size());
284 if (!CodeGenOpts.HLSLParsedCommandLine.empty())
285 for (const auto &Arg : llvm::drop_begin(RangeOrContainer: CodeGenOpts.HLSLParsedCommandLine))
286 Args.push_back(Elt: llvm::MDString::get(Context&: Ctx, Str: Arg));
287 M.getOrInsertNamedMetadata(Name: "dx.source.args")
288 ->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Args));
289}
290
291// Find array variable declaration from DeclRef expression
292static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
293 E = E->IgnoreImpCasts();
294 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: E))
295 return DRE->getDecl();
296 if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
297 E = OVE->getSourceExpr()->IgnoreImpCasts();
298 if (isa<MemberExpr>(Val: E))
299 return findAssociatedResourceDeclForStruct(AST, ME: cast<MemberExpr>(Val: E));
300 return nullptr;
301}
302
303// Find array variable declaration from nested array subscript AST nodes
304static const ValueDecl *getArrayDecl(ASTContext &AST,
305 const ArraySubscriptExpr *ASE) {
306 const Expr *E = nullptr;
307 while (ASE != nullptr) {
308 E = ASE->getBase()->IgnoreImpCasts();
309 if (!E)
310 return nullptr;
311 ASE = dyn_cast<ArraySubscriptExpr>(Val: E);
312 }
313 return getArrayDecl(AST, E);
314}
315
316// Get the total size of the array, or 0 if the array is unbounded.
317static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
318 Ty = Ty->getUnqualifiedDesugaredType();
319 assert(Ty->isArrayType() && "expected array type");
320 if (Ty->isIncompleteArrayType())
321 return 0;
322 return AST.getConstantArrayElementCount(CA: cast<ConstantArrayType>(Val: Ty));
323}
324
325static Value *buildNameForResource(llvm::StringRef BaseName,
326 CodeGenModule &CGM) {
327 llvm::SmallString<64> GlobalName = {BaseName, ".str"};
328 return CGM.GetAddrOfConstantCString(Str: BaseName.str(), GlobalName: GlobalName.c_str())
329 .getPointer();
330}
331
332static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
333 StorageClass SC = SC_None) {
334 for (auto *Method : Record->methods()) {
335 if (Method->getStorageClass() == SC && Method->getName() == Name)
336 return Method;
337 }
338 return nullptr;
339}
340
341static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
342 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
343 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
344 CallArgList &Args) {
345 assert(Binding.hasBinding() && "at least one binding attribute expected");
346
347 ASTContext &AST = CGM.getContext();
348 CXXMethodDecl *CreateMethod = nullptr;
349 Value *NameStr = buildNameForResource(BaseName: Name, CGM);
350 Value *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
351
352 bool HasCounter = hasCounterHandle(RD: ResourceDecl);
353 assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
354 "resources with counter handle must have a binding with counter "
355 "implicit order ID");
356 if (Binding.isExplicit()) {
357 // explicit binding
358 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
359 Args.add(rvalue: RValue::get(V: RegSlot), type: AST.UnsignedIntTy);
360 const char *Name = Binding.hasCounterImplicitOrderID()
361 ? "__createFromBindingWithImplicitCounter"
362 : "__createFromBinding";
363 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
364 } else {
365 // implicit binding
366 auto *OrderID =
367 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
368 Args.add(rvalue: RValue::get(V: OrderID), type: AST.UnsignedIntTy);
369 const char *Name = Binding.hasCounterImplicitOrderID()
370 ? "__createFromImplicitBindingWithImplicitCounter"
371 : "__createFromImplicitBinding";
372 CreateMethod = lookupMethod(Record: ResourceDecl, Name, SC: SC_Static);
373 }
374 Args.add(rvalue: RValue::get(V: Space), type: AST.UnsignedIntTy);
375 Args.add(rvalue: RValue::get(V: Range), type: AST.IntTy);
376 Args.add(rvalue: RValue::get(V: Index), type: AST.UnsignedIntTy);
377 Args.add(rvalue: RValue::get(V: NameStr), type: AST.getPointerType(T: AST.CharTy.withConst()));
378 if (HasCounter) {
379 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
380 auto *CounterOrderID = llvm::ConstantInt::get(Ty: CGM.IntTy, V: CounterBinding);
381 Args.add(rvalue: RValue::get(V: CounterOrderID), type: AST.UnsignedIntTy);
382 }
383
384 return CreateMethod;
385}
386
387static void callResourceInitMethod(CodeGenFunction &CGF,
388 CXXMethodDecl *CreateMethod,
389 CallArgList &Args, Address ReturnAddress) {
390 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(GD: CreateMethod);
391 const FunctionProtoType *Proto =
392 CreateMethod->getType()->getAs<FunctionProtoType>();
393 const CGFunctionInfo &FnInfo =
394 CGF.CGM.getTypes().arrangeFreeFunctionCall(Args, Ty: Proto, ChainCall: false);
395 ReturnValueSlot ReturnValue(ReturnAddress, false);
396 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
397 CGF.EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: nullptr);
398}
399
400// Initializes local resource array variable with global resource array
401// elements. For multi-dimensional arrays it calls itself recursively to
402// initialize its sub-arrays. The Index used in the resource constructor calls
403// will begin at StartIndex and will be incremented for each array element. The
404// last used resource Index is returned to the caller. If the function returns
405// std::nullopt, it indicates an error.
406static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
407 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
408 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
409 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
410 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
411
412 ASTContext &AST = CGF.getContext();
413 llvm::IntegerType *IntTy = CGF.CGM.IntTy;
414 llvm::Value *Index = StartIndex;
415 llvm::Value *One = llvm::ConstantInt::get(Ty: IntTy, V: 1);
416 const uint64_t ArraySize = ArrayTy->getSExtSize();
417 QualType ElemType = ArrayTy->getElementType();
418 Address TmpArrayAddr = ValueSlot.getAddress();
419
420 // Add additional index to the getelementptr call indices.
421 // This index will be updated for each array element in the loops below.
422 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
423 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: IntTy, V: 0));
424
425 // For array of arrays, recursively initialize the sub-arrays.
426 if (ElemType->isArrayType()) {
427 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(Val&: ElemType);
428 for (uint64_t I = 0; I < ArraySize; I++) {
429 if (I > 0) {
430 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
431 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
432 }
433 std::optional<llvm::Value *> MaybeIndex =
434 initializeResourceArrayFromGlobal(CGF, ResourceDecl, ArrayTy: SubArrayTy,
435 ValueSlot, Range, StartIndex: Index,
436 ResourceName, Binding, PrevGEPIndices: GEPIndices);
437 if (!MaybeIndex)
438 return std::nullopt;
439 Index = *MaybeIndex;
440 }
441 return Index;
442 }
443
444 // For array of resources, initialize each resource in the array.
445 llvm::Type *Ty = CGF.ConvertTypeForMem(T: ElemType);
446 CharUnits ElemSize = AST.getTypeSizeInChars(T: ElemType);
447 CharUnits Align =
448 TmpArrayAddr.getAlignment().alignmentOfArrayElement(elementSize: ElemSize);
449
450 for (uint64_t I = 0; I < ArraySize; I++) {
451 if (I > 0) {
452 Index = CGF.Builder.CreateAdd(LHS: Index, RHS: One);
453 GEPIndices.back() = llvm::ConstantInt::get(Ty: IntTy, V: I);
454 }
455 Address ReturnAddress =
456 CGF.Builder.CreateGEP(Addr: TmpArrayAddr, IdxList: GEPIndices, ElementType: Ty, Align);
457
458 CallArgList Args;
459 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
460 CGM&: CGF.CGM, ResourceDecl, Range, Index, Name: ResourceName, Binding, Args);
461
462 if (!CreateMethod)
463 // This can happen if someone creates an array of structs that looks like
464 // an HLSL resource record array but it does not have the required static
465 // create method. No binding will be generated for it.
466 return std::nullopt;
467
468 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
469 }
470 return Index;
471}
472
473/// Utility for emitting copies following the HLSL buffer layout rules (ie,
474/// copying out of a cbuffer).
475class HLSLBufferCopyEmitter {
476 CodeGenFunction &CGF;
477 Address DstPtr;
478 Address SrcPtr;
479 llvm::Type *LayoutTy = nullptr;
480
481 SmallVector<llvm::Value *> CurStoreIndices;
482 SmallVector<llvm::Value *> CurLoadIndices;
483
484 // Creates & returns either a structured.gep or a ptradd/gep depending on
485 // langopts.
486 llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
487 ArrayRef<llvm::Value *> Indices) {
488 bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
489 if (EmitLogical)
490 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: Indices);
491
492 llvm::SmallVector<llvm::Value *> GEPIndices;
493 GEPIndices.reserve(N: Indices.size() + 1);
494 GEPIndices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0));
495 GEPIndices.append(in_start: Indices.begin(), in_end: Indices.end());
496 return CGF.Builder.CreateAccessChain(Logical: EmitLogical, BaseType: BaseTy, PtrBase: Base, IdxList: GEPIndices);
497 }
498
499 bool isBufferLayoutArray(llvm::StructType *ST) {
500 // A buffer layout array is a struct with two elements: the padded array,
501 // and the last element. That is, is should look something like this:
502 //
503 // { [%n x { %type, %padding }], %type }
504 //
505 if (!ST || ST->getNumElements() != 2)
506 return false;
507
508 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
509 if (!PaddedEltsTy)
510 return false;
511
512 auto *PaddedTy = dyn_cast<llvm::StructType>(Val: PaddedEltsTy->getElementType());
513 if (!PaddedTy || PaddedTy->getNumElements() != 2)
514 return false;
515
516 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
517 Ty: PaddedTy->getElementType(N: 1)))
518 return false;
519
520 llvm::Type *ElementTy = ST->getElementType(N: 1);
521 if (PaddedTy->getElementType(N: 0) != ElementTy)
522 return false;
523 return true;
524 }
525
526 void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
527 llvm::ArrayType *DstTy) {
528 // Those assumptions are checked by isBufferLayoutArray.
529 auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(Val: SrcTy->getElementType(N: 0));
530 assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
531 assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
532 ->getElementType(0) == SrcTy->getElementType(1));
533
534 auto *SrcDataTy = SrcTy->getElementType(N: 1);
535 auto Zero = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
536
537 for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
538 auto Index = llvm::ConstantInt::get(Ty: CGF.IntTy, V: I);
539 auto *SrcElt = emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {Zero, Index, Zero});
540 auto *DstElt = emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {Index});
541 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType());
542 }
543
544 auto *SrcElt =
545 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1)});
546 auto *DstElt = emitAccessChain(
547 BaseTy: DstTy, Base: Dst,
548 Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstTy->getNumElements() - 1)});
549 emitElementCopy(Src: SrcElt, SrcTy: SrcDataTy, Dst: DstElt, DstTy: DstTy->getElementType());
550 }
551
552 void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
553 llvm::Type *DstTy) {
554 if (isBufferLayoutArray(ST: SrcTy))
555 return emitBufferLayoutCopy(Src, SrcTy, Dst,
556 DstTy: cast<llvm::ArrayType>(Val: DstTy));
557
558 unsigned SrcIndex = 0;
559 unsigned DstIndex = 0;
560
561 auto *DstST = cast<llvm::StructType>(Val: DstTy);
562 while (SrcIndex < SrcTy->getNumElements() &&
563 DstIndex < DstST->getNumElements()) {
564 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
565 Ty: SrcTy->getElementType(N: SrcIndex))) {
566 SrcIndex += 1;
567 continue;
568 }
569
570 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
571 Ty: DstST->getElementType(N: DstIndex))) {
572 DstIndex += 1;
573 continue;
574 }
575
576 auto *SrcElt = emitAccessChain(
577 BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: SrcIndex)});
578 auto *DstElt = emitAccessChain(
579 BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: DstIndex)});
580 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(N: SrcIndex), Dst: DstElt,
581 DstTy: DstST->getElementType(N: DstIndex));
582 DstIndex += 1;
583 SrcIndex += 1;
584 }
585 }
586
587 void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
588 llvm::Type *DstTy) {
589 for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
590 auto *SrcElt =
591 emitAccessChain(BaseTy: SrcTy, Base: Src, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
592 auto *DstElt =
593 emitAccessChain(BaseTy: DstTy, Base: Dst, Indices: {llvm::ConstantInt::get(Ty: CGF.IntTy, V: I)});
594 emitElementCopy(Src: SrcElt, SrcTy: SrcTy->getElementType(), Dst: DstElt,
595 DstTy: cast<llvm::ArrayType>(Val: DstTy)->getElementType());
596 }
597 }
598
599 void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
600 llvm::Type *DstTy) {
601 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: SrcTy))
602 return emitCopy(Src, SrcTy: AT, Dst, DstTy);
603 if (auto *ST = dyn_cast<llvm::StructType>(Val: SrcTy))
604 return emitCopy(Src, SrcTy: ST, Dst, DstTy);
605
606 // When we have a scalar or vector element we can emit the copy.
607 CharUnits SrcAlign =
608 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: SrcTy));
609 CharUnits DstAlign =
610 CharUnits::fromQuantity(Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: DstTy));
611 Address SrcAddr(Src, SrcTy, SrcAlign);
612 Address DstAddr(Dst, DstTy, DstAlign);
613 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: SrcAddr, Name: "cbuf.load");
614 CGF.Builder.CreateStore(Val: Load, Addr: DstAddr);
615 }
616
617public:
618 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
619 : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
620
621 bool emitCopy(QualType CType) {
622 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: CType);
623
624 // TODO: We should be able to fall back to a regular memcpy if the layout
625 // type doesn't have any padding, but that runs into issues in the backend
626 // currently.
627 //
628 // See https://github.com/llvm/wg-hlsl/issues/351
629 emitElementCopy(Src: SrcPtr.getBasePointer(), SrcTy: LayoutTy, Dst: DstPtr.getBasePointer(),
630 DstTy: DstPtr.getElementType());
631 return true;
632 }
633};
634
635} // namespace
636
637llvm::Type *
638CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
639 const CGHLSLOffsetInfo &OffsetInfo) {
640 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
641
642 // Check if the target has a specific translation for this type first.
643 if (llvm::Type *TargetTy =
644 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
645 return TargetTy;
646
647 llvm_unreachable("Generic handling of HLSL types is not supported.");
648}
649
650llvm::Triple::ArchType CGHLSLRuntime::getArch() {
651 return CGM.getTarget().getTriple().getArch();
652}
653
654// Emits constant global variables for buffer constants declarations
655// and creates metadata linking the constant globals with the buffer global.
656void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
657 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
658 const CGHLSLOffsetInfo &OffsetInfo) {
659 LLVMContext &Ctx = CGM.getLLVMContext();
660
661 // get the layout struct from constant buffer target type
662 llvm::Type *BufType = BufGV->getValueType();
663 llvm::StructType *LayoutStruct = cast<llvm::StructType>(
664 Val: cast<llvm::TargetExtType>(Val: BufType)->getTypeParameter(i: 0));
665
666 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
667 size_t OffsetIdx = 0;
668 for (Decl *D : BufDecl->buffer_decls()) {
669 if (isa<CXXRecordDecl, EmptyDecl>(Val: D))
670 // Nothing to do for this declaration.
671 continue;
672 if (isa<FunctionDecl>(Val: D)) {
673 // A function within an cbuffer is effectively a top-level function.
674 CGM.EmitTopLevelDecl(D);
675 continue;
676 }
677 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
678 if (!VD)
679 continue;
680
681 QualType VDTy = VD->getType();
682 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
683 if (VD->getStorageClass() == SC_Static ||
684 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
685 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
686 // Emit static and groupshared variables and resource classes inside
687 // cbuffer as regular globals
688 CGM.EmitGlobal(D: VD);
689 }
690 continue;
691 }
692
693 DeclsWithOffset.emplace_back(Args&: VD, Args: OffsetInfo[OffsetIdx++]);
694 }
695
696 if (!OffsetInfo.empty())
697 llvm::stable_sort(Range&: DeclsWithOffset, C: [](const auto &LHS, const auto &RHS) {
698 return CGHLSLOffsetInfo::compareOffsets(LHS: LHS.second, RHS: RHS.second);
699 });
700
701 // Associate the buffer global variable with its constants
702 SmallVector<llvm::Metadata *> BufGlobals;
703 BufGlobals.reserve(N: DeclsWithOffset.size() + 1);
704 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: BufGV));
705
706 auto ElemIt = LayoutStruct->element_begin();
707 for (auto &[VD, _] : DeclsWithOffset) {
708 if (CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: *ElemIt))
709 ++ElemIt;
710
711 assert(ElemIt != LayoutStruct->element_end() &&
712 "number of elements in layout struct does not match");
713 llvm::Type *LayoutType = *ElemIt++;
714
715 GlobalVariable *ElemGV =
716 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: VD, Ty: LayoutType));
717 BufGlobals.push_back(Elt: ValueAsMetadata::get(V: ElemGV));
718 }
719 assert(ElemIt == LayoutStruct->element_end() &&
720 "number of elements in layout struct does not match");
721
722 // add buffer metadata to the module
723 CGM.getModule()
724 .getOrInsertNamedMetadata(Name: "hlsl.cbs")
725 ->addOperand(M: MDNode::get(Context&: Ctx, MDs: BufGlobals));
726}
727
728// Creates resource handle type for the HLSL buffer declaration
729static const clang::HLSLAttributedResourceType *
730createBufferHandleType(const HLSLBufferDecl *BufDecl) {
731 ASTContext &AST = BufDecl->getASTContext();
732 QualType QT = AST.getHLSLAttributedResourceType(
733 Wrapped: AST.HLSLResourceTy, Contained: AST.getCanonicalTagType(TD: BufDecl->getLayoutStruct()),
734 Attrs: HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
735 return cast<HLSLAttributedResourceType>(Val: QT.getTypePtr());
736}
737
738CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
739 CGHLSLOffsetInfo Result;
740
741 // If we don't have packoffset info, just return an empty result.
742 if (!BufDecl.hasValidPackoffset())
743 return Result;
744
745 for (Decl *D : BufDecl.buffer_decls()) {
746 if (isa<CXXRecordDecl, EmptyDecl>(Val: D) || isa<FunctionDecl>(Val: D)) {
747 continue;
748 }
749 VarDecl *VD = dyn_cast<VarDecl>(Val: D);
750 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
751 continue;
752
753 if (!VD->hasAttrs()) {
754 Result.Offsets.push_back(Elt: Unspecified);
755 continue;
756 }
757
758 uint32_t Offset = Unspecified;
759 for (auto *Attr : VD->getAttrs()) {
760 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Val: Attr)) {
761 Offset = POA->getOffsetInBytes();
762 break;
763 }
764 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Val: Attr);
765 if (RBA &&
766 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
767 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
768 break;
769 }
770 }
771 Result.Offsets.push_back(Elt: Offset);
772 }
773 return Result;
774}
775
776// Codegen for HLSLBufferDecl
777void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
778
779 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
780
781 // create resource handle type for the buffer
782 const clang::HLSLAttributedResourceType *ResHandleTy =
783 createBufferHandleType(BufDecl);
784
785 // empty constant buffer is ignored
786 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
787 return;
788
789 // create global variable for the constant buffer
790 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(BufDecl: *BufDecl);
791 llvm::Type *LayoutTy = convertHLSLSpecificType(T: ResHandleTy, OffsetInfo);
792 llvm::GlobalVariable *BufGV = new GlobalVariable(
793 LayoutTy, /*isConstant*/ false,
794 GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(T: LayoutTy),
795 llvm::formatv(Fmt: "{0}{1}", Vals: BufDecl->getName(),
796 Vals: BufDecl->isCBuffer() ? ".cb" : ".tb"),
797 GlobalValue::NotThreadLocal);
798
799 llvm::Module &M = CGM.getModule();
800 M.insertGlobalVariable(GV: BufGV);
801
802 // Add the global variable to the compiler used list so it does not
803 // get optimized away by GlobalOptPass before it reaches
804 // {DXIL|SPIRV}CBufferAccess pass.
805 llvm::appendToCompilerUsed(M, Values: {BufGV});
806
807 // Add globals for constant buffer elements and create metadata nodes
808 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
809
810 // Initialize cbuffer from binding (implicit or explicit)
811 initializeBufferFromBinding(BufDecl, GV: BufGV);
812}
813
814void CGHLSLRuntime::addRootSignature(
815 const HLSLRootSignatureDecl *SignatureDecl) {
816 llvm::Module &M = CGM.getModule();
817 Triple T(M.getTargetTriple());
818
819 // Generated later with the function decl if not targeting root signature
820 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
821 return;
822
823 addRootSignatureMD(RootSigVer: SignatureDecl->getVersion(),
824 Elements: SignatureDecl->getRootElements(), Fn: nullptr, M);
825}
826
827llvm::StructType *
828CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
829 const auto Entry = LayoutTypes.find(Val: StructType);
830 if (Entry != LayoutTypes.end())
831 return Entry->getSecond();
832 return nullptr;
833}
834
835void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
836 llvm::StructType *LayoutTy) {
837 assert(getHLSLBufferLayoutType(StructType) == nullptr &&
838 "layout type for this struct already exist");
839 LayoutTypes[StructType] = LayoutTy;
840}
841
842void CGHLSLRuntime::finishCodeGen() {
843 auto &TargetOpts = CGM.getTarget().getTargetOpts();
844 auto &CodeGenOpts = CGM.getCodeGenOpts();
845 auto &LangOpts = CGM.getLangOpts();
846 llvm::Module &M = CGM.getModule();
847 Triple T(M.getTargetTriple());
848 if (T.getArch() == Triple::ArchType::dxil)
849 addDxilValVersion(ValVersionStr: TargetOpts.DxilValidatorVersion, M);
850 if (!CodeGenOpts.DisableDXSourceMetadata &&
851 CodeGenOpts.getDebugInfo() >=
852 llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
853 addSourceInfo(CGM, M);
854 if (CodeGenOpts.ResMayAlias)
855 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.resmayalias", Val: 1);
856 if (CodeGenOpts.AllResourcesBound)
857 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error,
858 Key: "dx.allresourcesbound", Val: 1);
859 if (CodeGenOpts.OptimizationLevel == 0)
860 M.addModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Override,
861 Key: "dx.disable_optimizations", Val: 1);
862
863 // NativeHalfType corresponds to the -fnative-half-type clang option which is
864 // aliased by clang-dxc's -enable-16bit-types option. This option is used to
865 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
866 if (LangOpts.NativeHalfType)
867 M.setModuleFlag(Behavior: llvm::Module::ModFlagBehavior::Error, Key: "dx.nativelowprec",
868 Val: 1);
869
870 if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
871 // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
872 const ASTContext &Ctx = CGM.getContext();
873 unsigned InputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_input);
874 unsigned OutputAS = Ctx.getTargetAddressSpace(AS: LangAS::hlsl_output);
875 SmallVector<GlobalValue *, 8> InterfaceVars;
876 for (GlobalVariable &GV : M.globals()) {
877 unsigned AS = GV.getAddressSpace();
878 if (AS == InputAS || AS == OutputAS)
879 InterfaceVars.push_back(Elt: &GV);
880 }
881 if (!InterfaceVars.empty())
882 appendToCompilerUsed(M, Values: InterfaceVars);
883 }
884
885 generateGlobalCtorDtorCalls();
886}
887
888void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
889 const FunctionDecl *FD, llvm::Function *Fn) {
890 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
891 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
892 const StringRef ShaderAttrKindStr = "hlsl.shader";
893 Fn->addFnAttr(Kind: ShaderAttrKindStr,
894 Val: llvm::Triple::getEnvironmentTypeName(Kind: ShaderAttr->getType()));
895 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
896 const StringRef NumThreadsKindStr = "hlsl.numthreads";
897 std::string NumThreadsStr =
898 formatv(Fmt: "{0},{1},{2}", Vals: NumThreadsAttr->getX(), Vals: NumThreadsAttr->getY(),
899 Vals: NumThreadsAttr->getZ());
900 Fn->addFnAttr(Kind: NumThreadsKindStr, Val: NumThreadsStr);
901 }
902 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
903 const StringRef WaveSizeKindStr = "hlsl.wavesize";
904 std::string WaveSizeStr =
905 formatv(Fmt: "{0},{1},{2}", Vals: WaveSizeAttr->getMin(), Vals: WaveSizeAttr->getMax(),
906 Vals: WaveSizeAttr->getPreferred());
907 Fn->addFnAttr(Kind: WaveSizeKindStr, Val: WaveSizeStr);
908 }
909 // HLSL entry functions are materialized for module functions with
910 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
911 // later in the compiler-flow for such module functions is not aware of and
912 // hence not able to set attributes of the newly materialized entry functions.
913 // So, set attributes of entry function here, as appropriate.
914 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
915
916 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
917 Fn->addFnAttr(Kind: "enable-maximal-reconvergence", Val: "true");
918 }
919}
920
921static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
922 if (const auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
923 Value *Result = PoisonValue::get(T: Ty);
924 for (unsigned I = 0; I < VT->getNumElements(); ++I) {
925 Value *Elt = B.CreateCall(Callee: F, Args: {B.getInt32(C: I)});
926 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: I);
927 }
928 return Result;
929 }
930 return B.CreateCall(Callee: F, Args: {B.getInt32(C: 0)});
931}
932
933static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
934 unsigned BuiltIn) {
935 LLVMContext &Ctx = GV->getContext();
936 IRBuilder<> B(GV->getContext());
937 MDNode *Operands = MDNode::get(
938 Context&: Ctx,
939 MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Spirv::Decoration::BuiltIn */ C: 11)),
940 ConstantAsMetadata::get(C: B.getInt32(C: BuiltIn))});
941 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
942 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
943}
944
945static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
946 LLVMContext &Ctx = GV->getContext();
947 IRBuilder<> B(GV->getContext());
948 MDNode *Operands =
949 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: B.getInt32(/* Location */ C: 30)),
950 ConstantAsMetadata::get(C: B.getInt32(C: Location))});
951 MDNode *Decoration = MDNode::get(Context&: Ctx, MDs: {Operands});
952 GV->addMetadata(Kind: "spirv.Decorations", MD&: *Decoration);
953}
954
955static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
956 llvm::Type *Ty, const Twine &Name,
957 unsigned BuiltInID) {
958 auto *GV = new llvm::GlobalVariable(
959 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
960 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
961 llvm::GlobalVariable::GeneralDynamicTLSModel,
962 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
963 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
964 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
965 return B.CreateLoad(Ty, Ptr: GV);
966}
967
968static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
969 llvm::Type *Ty, unsigned Location,
970 StringRef Name) {
971 auto *GV = new llvm::GlobalVariable(
972 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
973 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
974 llvm::GlobalVariable::GeneralDynamicTLSModel,
975 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
976 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
977 addLocationDecoration(GV, Location);
978 return B.CreateLoad(Ty, Ptr: GV);
979}
980
981llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
982 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
983 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {
984 Twine BaseName = Twine(Semantic->getAttrName()->getName());
985 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
986
987 unsigned Location = SPIRVLastAssignedInputSemanticLocation;
988 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
989 Location = L->getLocation();
990
991 // DXC completely ignores the semantic/index pair. Location are assigned from
992 // the first semantic to the last.
993 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Type);
994 unsigned ElementCount = AT ? AT->getNumElements() : 1;
995 SPIRVLastAssignedInputSemanticLocation += ElementCount;
996
997 return createSPIRVLocationLoad(B, M&: CGM.getModule(), Ty: Type, Location,
998 Name: VariableName.str());
999}
1000
1001static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
1002 llvm::Value *Source, unsigned Location,
1003 StringRef Name) {
1004 auto *GV = new llvm::GlobalVariable(
1005 M, Source->getType(), /* isConstant= */ false,
1006 llvm::GlobalValue::ExternalLinkage,
1007 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
1008 llvm::GlobalVariable::GeneralDynamicTLSModel,
1009 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1010 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1011 addLocationDecoration(GV, Location);
1012 B.CreateStore(Val: Source, Ptr: GV);
1013}
1014
1015void CGHLSLRuntime::emitSPIRVUserSemanticStore(
1016 llvm::IRBuilder<> &B, llvm::Value *Source,
1017 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1018 std::optional<unsigned> Index) {
1019 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1020 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1021
1022 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
1023 if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
1024 Location = L->getLocation();
1025
1026 // DXC completely ignores the semantic/index pair. Location are assigned from
1027 // the first semantic to the last.
1028 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Val: Source->getType());
1029 unsigned ElementCount = AT ? AT->getNumElements() : 1;
1030 SPIRVLastAssignedOutputSemanticLocation += ElementCount;
1031 createSPIRVLocationStore(B, M&: CGM.getModule(), Source, Location,
1032 Name: VariableName.str());
1033}
1034
1035llvm::Value *
1036CGHLSLRuntime::emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
1037 HLSLAppliedSemanticAttr *Semantic,
1038 std::optional<unsigned> Index) {
1039 Twine BaseName = Twine(Semantic->getAttrName()->getName());
1040 Twine VariableName = BaseName.concat(Suffix: Twine(Index.value_or(u: 0)));
1041
1042 // DXIL packing rules etc shall be handled here.
1043 // FIXME: generate proper sigpoint, index, col, row values.
1044 // FIXME: also DXIL loads vectors element by element.
1045 SmallVector<Value *> Args{B.getInt32(C: 4), B.getInt32(C: 0), B.getInt32(C: 0),
1046 B.getInt8(C: 0),
1047 llvm::PoisonValue::get(T: B.getInt32Ty())};
1048
1049 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_load_input;
1050
1051 SmallVector<OperandBundleDef, 1> OB;
1052 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1053 llvm::Value *bundleArgs[] = {Token};
1054 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1055 }
1056
1057 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1058 M: B.GetInsertBlock()->getModule(), id: IntrinsicID, OverloadTys: {Type});
1059 llvm::Value *Value = B.CreateCall(Callee: IntrFn, Args, OpBundles: OB, Name: VariableName);
1060 return Value;
1061}
1062
1063void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
1064 llvm::Value *Source,
1065 HLSLAppliedSemanticAttr *Semantic,
1066 std::optional<unsigned> Index) {
1067 // DXIL packing rules etc shall be handled here.
1068 // FIXME: generate proper sigpoint, index, col, row values.
1069 SmallVector<Value *> Args{B.getInt32(C: 4),
1070 B.getInt32(C: 0),
1071 B.getInt32(C: 0),
1072 B.getInt8(C: 0),
1073 llvm::PoisonValue::get(T: B.getInt32Ty()),
1074 Source};
1075
1076 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_store_output;
1077
1078 SmallVector<OperandBundleDef, 1> OB;
1079 if (auto *Token = getConvergenceToken(BB&: *B.GetInsertBlock())) {
1080 llvm::Value *bundleArgs[] = {Token};
1081 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1082 }
1083
1084 llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
1085 M: B.GetInsertBlock()->getModule(), id: IntrinsicID, OverloadTys: {Source->getType()});
1086 B.CreateCall(Callee: IntrFn, Args, OpBundles: OB);
1087}
1088
1089llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
1090 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
1091 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {
1092 if (CGM.getTarget().getTriple().isSPIRV())
1093 return emitSPIRVUserSemanticLoad(B, Type, Decl, Semantic, Index);
1094
1095 if (CGM.getTarget().getTriple().isDXIL())
1096 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1097
1098 llvm_unreachable("Unsupported target for user-semantic load.");
1099}
1100
1101void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1102 const clang::DeclaratorDecl *Decl,
1103 HLSLAppliedSemanticAttr *Semantic,
1104 std::optional<unsigned> Index) {
1105 if (CGM.getTarget().getTriple().isSPIRV())
1106 return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
1107
1108 if (CGM.getTarget().getTriple().isDXIL())
1109 return emitDXILUserSemanticStore(B, Source, Semantic, Index);
1110
1111 llvm_unreachable("Unsupported target for user-semantic load.");
1112}
1113
1114llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
1115 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1116 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
1117 std::optional<unsigned> Index) {
1118
1119 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1120 if (SemanticName == "SV_GROUPINDEX") {
1121 llvm::Function *GroupIndex =
1122 CGM.getIntrinsic(IID: getFlattenedThreadIdInGroupIntrinsic());
1123 return B.CreateCall(Callee: FunctionCallee(GroupIndex));
1124 }
1125
1126 if (SemanticName == "SV_DISPATCHTHREADID") {
1127 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
1128 llvm::Function *ThreadIDIntrinsic =
1129 llvm::Intrinsic::isOverloaded(id: IntrinID)
1130 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1131 : CGM.getIntrinsic(IID: IntrinID);
1132 return buildVectorInput(B, F: ThreadIDIntrinsic, Ty: Type);
1133 }
1134
1135 if (SemanticName == "SV_GROUPTHREADID") {
1136 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
1137 llvm::Function *GroupThreadIDIntrinsic =
1138 llvm::Intrinsic::isOverloaded(id: IntrinID)
1139 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1140 : CGM.getIntrinsic(IID: IntrinID);
1141 return buildVectorInput(B, F: GroupThreadIDIntrinsic, Ty: Type);
1142 }
1143
1144 if (SemanticName == "SV_GROUPID") {
1145 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
1146 llvm::Function *GroupIDIntrinsic =
1147 llvm::Intrinsic::isOverloaded(id: IntrinID)
1148 ? CGM.getIntrinsic(IID: IntrinID, Tys: {CGM.Int32Ty})
1149 : CGM.getIntrinsic(IID: IntrinID);
1150 return buildVectorInput(B, F: GroupIDIntrinsic, Ty: Type);
1151 }
1152
1153 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
1154 assert(ShaderAttr && "Entry point has no shader attribute");
1155 llvm::Triple::EnvironmentType ST = ShaderAttr->getType();
1156
1157 if (SemanticName == "SV_POSITION") {
1158 if (ST == Triple::EnvironmentType::Pixel) {
1159 if (CGM.getTarget().getTriple().isSPIRV())
1160 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1161 Name: Semantic->getAttrName()->getName(),
1162 /* BuiltIn::FragCoord */ BuiltInID: 15);
1163 if (CGM.getTarget().getTriple().isDXIL())
1164 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1165 }
1166
1167 if (ST == Triple::EnvironmentType::Vertex) {
1168 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);
1169 }
1170 }
1171
1172 if (SemanticName == "SV_VERTEXID") {
1173 if (ST == Triple::EnvironmentType::Vertex) {
1174 if (CGM.getTarget().getTriple().isSPIRV())
1175 return createSPIRVBuiltinLoad(B, M&: CGM.getModule(), Ty: Type,
1176 Name: Semantic->getAttrName()->getName(),
1177 /* BuiltIn::VertexIndex */ BuiltInID: 42);
1178 else
1179 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);
1180 }
1181 }
1182
1183 llvm_unreachable(
1184 "Load hasn't been implemented yet for this system semantic. FIXME");
1185}
1186
1187static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
1188 llvm::Value *Source, const Twine &Name,
1189 unsigned BuiltInID) {
1190 auto *GV = new llvm::GlobalVariable(
1191 M, Source->getType(), /* isConstant= */ false,
1192 llvm::GlobalValue::ExternalLinkage,
1193 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
1194 llvm::GlobalVariable::GeneralDynamicTLSModel,
1195 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
1196 addSPIRVBuiltinDecoration(GV, BuiltIn: BuiltInID);
1197 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1198 B.CreateStore(Val: Source, Ptr: GV);
1199}
1200
1201void CGHLSLRuntime::emitSystemSemanticStore(IRBuilder<> &B, llvm::Value *Source,
1202 const clang::DeclaratorDecl *Decl,
1203 HLSLAppliedSemanticAttr *Semantic,
1204 std::optional<unsigned> Index) {
1205
1206 std::string SemanticName = Semantic->getAttrName()->getName().upper();
1207 if (SemanticName == "SV_POSITION") {
1208 if (CGM.getTarget().getTriple().isDXIL()) {
1209 emitDXILUserSemanticStore(B, Source, Semantic, Index);
1210 return;
1211 }
1212
1213 if (CGM.getTarget().getTriple().isSPIRV()) {
1214 createSPIRVBuiltinStore(B, M&: CGM.getModule(), Source,
1215 Name: Semantic->getAttrName()->getName(),
1216 /* BuiltIn::Position */ BuiltInID: 0);
1217 return;
1218 }
1219 }
1220
1221 if (SemanticName == "SV_TARGET") {
1222 emitUserSemanticStore(B, Source, Decl, Semantic, Index);
1223 return;
1224 }
1225
1226 llvm_unreachable(
1227 "Store hasn't been implemented yet for this system semantic. FIXME");
1228}
1229
1230llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
1231 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1232 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {
1233
1234 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1235 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1236 return emitSystemSemanticLoad(B, FD, Type, Decl, Semantic, Index);
1237 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);
1238}
1239
1240void CGHLSLRuntime::handleScalarSemanticStore(
1241 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1242 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {
1243 std::optional<unsigned> Index = Semantic->getSemanticIndex();
1244 if (Semantic->getAttrName()->getName().starts_with_insensitive(Prefix: "SV_"))
1245 emitSystemSemanticStore(B, Source, Decl, Semantic, Index);
1246 else
1247 emitUserSemanticStore(B, Source, Decl, Semantic, Index);
1248}
1249
1250std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1251CGHLSLRuntime::handleStructSemanticLoad(
1252 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1253 const clang::DeclaratorDecl *Decl,
1254 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1255 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1256 const llvm::StructType *ST = cast<StructType>(Val: Type);
1257 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
1258
1259 assert(RD->getNumFields() == ST->getNumElements());
1260
1261 llvm::Value *Aggregate = llvm::PoisonValue::get(T: Type);
1262 auto FieldDecl = RD->field_begin();
1263 for (unsigned I = 0; I < ST->getNumElements(); ++I) {
1264 auto [ChildValue, NextAttr] = handleSemanticLoad(
1265 B, FD, Type: ST->getElementType(N: I), Decl: *FieldDecl, begin: AttrBegin, end: AttrEnd);
1266 AttrBegin = NextAttr;
1267 assert(ChildValue);
1268 Aggregate = B.CreateInsertValue(Agg: Aggregate, Val: ChildValue, Idxs: I);
1269 ++FieldDecl;
1270 }
1271
1272 return std::make_pair(x&: Aggregate, y&: AttrBegin);
1273}
1274
1275specific_attr_iterator<HLSLAppliedSemanticAttr>
1276CGHLSLRuntime::handleStructSemanticStore(
1277 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1278 const clang::DeclaratorDecl *Decl,
1279 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1280 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1281
1282 const llvm::StructType *ST = cast<StructType>(Val: Source->getType());
1283
1284 const clang::RecordDecl *RD = nullptr;
1285 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: Decl))
1286 RD = FD->getDeclaredReturnType()->getAsRecordDecl();
1287 else
1288 RD = Decl->getType()->getAsRecordDecl();
1289 assert(RD);
1290
1291 assert(RD->getNumFields() == ST->getNumElements());
1292
1293 auto FieldDecl = RD->field_begin();
1294 for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
1295 llvm::Value *Extract = B.CreateExtractValue(Agg: Source, Idxs: I);
1296 AttrBegin =
1297 handleSemanticStore(B, FD, Source: Extract, Decl: *FieldDecl, AttrBegin, AttrEnd);
1298 }
1299
1300 return AttrBegin;
1301}
1302
1303std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
1304CGHLSLRuntime::handleSemanticLoad(
1305 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
1306 const clang::DeclaratorDecl *Decl,
1307 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1308 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1309 assert(AttrBegin != AttrEnd);
1310 if (Type->isStructTy())
1311 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd);
1312
1313 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1314 ++AttrBegin;
1315 return std::make_pair(x: handleScalarSemanticLoad(B, FD, Type, Decl, Semantic: Attr),
1316 y&: AttrBegin);
1317}
1318
1319specific_attr_iterator<HLSLAppliedSemanticAttr>
1320CGHLSLRuntime::handleSemanticStore(
1321 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
1322 const clang::DeclaratorDecl *Decl,
1323 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
1324 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {
1325 assert(AttrBegin != AttrEnd);
1326 if (Source->getType()->isStructTy())
1327 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd);
1328
1329 HLSLAppliedSemanticAttr *Attr = *AttrBegin;
1330 ++AttrBegin;
1331 handleScalarSemanticStore(B, FD, Source, Decl, Semantic: Attr);
1332 return AttrBegin;
1333}
1334
1335void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
1336 llvm::Function *Fn) {
1337 llvm::Module &M = CGM.getModule();
1338 llvm::LLVMContext &Ctx = M.getContext();
1339 auto *EntryTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx), isVarArg: false);
1340 Function *EntryFn =
1341 Function::Create(Ty: EntryTy, Linkage: Function::ExternalLinkage, N: FD->getName(), M: &M);
1342
1343 // Copy function attributes over, we have no argument or return attributes
1344 // that can be valid on the real entry.
1345 AttributeList NewAttrs = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
1346 Attrs: Fn->getAttributes().getFnAttrs());
1347 EntryFn->setAttributes(NewAttrs);
1348 setHLSLEntryAttributes(FD, Fn: EntryFn);
1349
1350 // Set the called function as internal linkage.
1351 Fn->setLinkage(GlobalValue::InternalLinkage);
1352
1353 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: EntryFn);
1354 IRBuilder<> B(BB);
1355 llvm::SmallVector<Value *> Args;
1356
1357 SmallVector<OperandBundleDef, 1> OB;
1358 if (CGM.shouldEmitConvergenceTokens()) {
1359 assert(EntryFn->isConvergent());
1360 llvm::Value *I =
1361 B.CreateIntrinsic(ID: llvm::Intrinsic::experimental_convergence_entry, Args: {});
1362 llvm::Value *bundleArgs[] = {I};
1363 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1364 }
1365
1366 SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
1367
1368 unsigned SRetOffset = 0;
1369 for (const auto &Param : Fn->args()) {
1370 if (Param.hasStructRetAttr()) {
1371 SRetOffset = 1;
1372 llvm::Type *VarType = Param.getParamStructRetType();
1373 llvm::Value *Var =
1374 CGM.getLangOpts().EmitLogicalPointer
1375 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: VarType))
1376 : cast<Instruction>(Val: B.CreateAlloca(Ty: VarType));
1377 OutputSemantic.push_back(Elt: std::make_pair(x&: Var, y&: VarType));
1378 Args.push_back(Elt: Var);
1379 continue;
1380 }
1381
1382 const ParmVarDecl *PD = FD->getParamDecl(i: Param.getArgNo() - SRetOffset);
1383 llvm::Value *SemanticValue = nullptr;
1384 // FIXME: support inout/out parameters for semantics.
1385 if ([[maybe_unused]] HLSLParamModifierAttr *MA =
1386 PD->getAttr<HLSLParamModifierAttr>()) {
1387 llvm_unreachable("Not handled yet");
1388 } else {
1389 llvm::Type *ParamType = nullptr;
1390 if (Param.hasByValAttr())
1391 ParamType = Param.getParamByValType();
1392 else if (PD->getType()->isRecordType())
1393 ParamType = CGM.getTypes().ConvertType(T: PD->getType());
1394 else
1395 ParamType = Param.getType();
1396
1397 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1398 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
1399 auto Result =
1400 handleSemanticLoad(B, FD, Type: ParamType, Decl: PD, AttrBegin, AttrEnd);
1401 SemanticValue = Result.first;
1402 if (!SemanticValue)
1403 return;
1404 if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
1405 llvm::Value *Var =
1406 CGM.getLangOpts().EmitLogicalPointer
1407 ? cast<Instruction>(Val: B.CreateStructuredAlloca(BaseType: ParamType))
1408 : cast<Instruction>(Val: B.CreateAlloca(Ty: ParamType));
1409 B.CreateStore(Val: SemanticValue, Ptr: Var);
1410 SemanticValue = Var;
1411 }
1412 }
1413
1414 assert(SemanticValue);
1415 Args.push_back(Elt: SemanticValue);
1416 }
1417
1418 CallInst *CI = B.CreateCall(Callee: FunctionCallee(Fn), Args, OpBundles: OB);
1419 CI->setCallingConv(Fn->getCallingConv());
1420
1421 if (Fn->getReturnType() != CGM.VoidTy)
1422 // Element type is unused, so set to dummy value (NULL).
1423 OutputSemantic.push_back(Elt: std::make_pair(x&: CI, y: nullptr));
1424
1425 for (auto &SourcePair : OutputSemantic) {
1426 llvm::Value *Source = SourcePair.first;
1427 llvm::Type *ElementType = SourcePair.second;
1428 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Source);
1429 llvm::Value *SourceValue = AI ? B.CreateLoad(Ty: ElementType, Ptr: Source) : Source;
1430
1431 auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
1432 auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
1433 handleSemanticStore(B, FD, Source: SourceValue, Decl: FD, AttrBegin, AttrEnd);
1434 }
1435
1436 B.CreateRetVoid();
1437
1438 // Add and identify root signature to function, if applicable
1439 for (const Attr *Attr : FD->getAttrs()) {
1440 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Val: Attr)) {
1441 auto *RSDecl = RSAttr->getSignatureDecl();
1442 addRootSignatureMD(RootSigVer: RSDecl->getVersion(), Elements: RSDecl->getRootElements(),
1443 Fn: EntryFn, M);
1444 }
1445 }
1446}
1447
1448static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
1449 bool CtorOrDtor) {
1450 const auto *GV =
1451 M.getNamedGlobal(Name: CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
1452 if (!GV)
1453 return;
1454 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1455 if (!CA)
1456 return;
1457 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
1458 // HLSL neither supports priorities or COMDat values, so we will check those
1459 // in an assert but not handle them.
1460
1461 for (const auto &Ctor : CA->operands()) {
1462 if (isa<ConstantAggregateZero>(Val: Ctor))
1463 continue;
1464 ConstantStruct *CS = cast<ConstantStruct>(Val: Ctor);
1465
1466 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
1467 "HLSL doesn't support setting priority for global ctors.");
1468 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
1469 "HLSL doesn't support COMDat for global ctors.");
1470 Fns.push_back(Elt: cast<Function>(Val: CS->getOperand(i_nocapture: 1)));
1471 }
1472}
1473
1474void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
1475 llvm::Module &M = CGM.getModule();
1476 SmallVector<Function *> CtorFns;
1477 SmallVector<Function *> DtorFns;
1478 gatherFunctions(Fns&: CtorFns, M, CtorOrDtor: true);
1479 gatherFunctions(Fns&: DtorFns, M, CtorOrDtor: false);
1480
1481 // Insert a call to the global constructor at the beginning of the entry block
1482 // to externally exported functions. This is a bit of a hack, but HLSL allows
1483 // global constructors, but doesn't support driver initialization of globals.
1484 for (auto &F : M.functions()) {
1485 if (!F.hasFnAttribute(Kind: "hlsl.shader"))
1486 continue;
1487 auto *Token = getConvergenceToken(BB&: F.getEntryBlock());
1488 Instruction *IP = &*F.getEntryBlock().begin();
1489 SmallVector<OperandBundleDef, 1> OB;
1490 if (Token) {
1491 llvm::Value *bundleArgs[] = {Token};
1492 OB.emplace_back(Args: "convergencectrl", Args&: bundleArgs);
1493 IP = Token->getNextNode();
1494 }
1495 IRBuilder<> B(IP);
1496 for (auto *Fn : CtorFns) {
1497 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1498 CI->setCallingConv(Fn->getCallingConv());
1499 }
1500
1501 // Insert global dtors before the terminator of the last instruction
1502 B.SetInsertPoint(F.back().getTerminator());
1503 for (auto *Fn : DtorFns) {
1504 auto CI = B.CreateCall(Callee: FunctionCallee(Fn), Args: {}, OpBundles: OB);
1505 CI->setCallingConv(Fn->getCallingConv());
1506 }
1507 }
1508
1509 // No need to keep global ctors/dtors for non-lib profile after call to
1510 // ctors/dtors added for entry.
1511 Triple T(M.getTargetTriple());
1512 if (T.getEnvironment() != Triple::EnvironmentType::Library) {
1513 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_ctors"))
1514 GV->eraseFromParent();
1515 if (auto *GV = M.getNamedGlobal(Name: "llvm.global_dtors"))
1516 GV->eraseFromParent();
1517 }
1518}
1519
1520static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
1521 Intrinsic::ID IntrID,
1522 ArrayRef<llvm::Value *> Args) {
1523
1524 LLVMContext &Ctx = CGM.getLLVMContext();
1525 llvm::Function *InitResFunc =
1526 llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false),
1527 Linkage: llvm::GlobalValue::InternalLinkage,
1528 N: "_init_buffer_" + GV->getName(), M&: CGM.getModule());
1529 InitResFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1530
1531 llvm::BasicBlock *EntryBB =
1532 llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitResFunc);
1533 CGBuilderTy Builder(CGM, Ctx);
1534 const DataLayout &DL = CGM.getModule().getDataLayout();
1535 Builder.SetInsertPoint(EntryBB);
1536
1537 // Make sure the global variable is buffer resource handle
1538 llvm::Type *HandleTy = GV->getValueType();
1539 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
1540
1541 llvm::Value *CreateHandle = Builder.CreateIntrinsic(
1542 /*ReturnType=*/RetTy: HandleTy, ID: IntrID, Args, FMFSource: nullptr,
1543 Name: Twine(GV->getName()).concat(Suffix: "_h"));
1544
1545 Builder.CreateAlignedStore(Val: CreateHandle, Ptr: GV, Align: GV->getPointerAlignment(DL));
1546 Builder.CreateRetVoid();
1547
1548 CGM.AddCXXGlobalInit(F: InitResFunc);
1549}
1550
1551void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
1552 llvm::GlobalVariable *GV) {
1553 ResourceBindingAttrs Binding(BufDecl);
1554 assert(Binding.hasBinding() &&
1555 "cbuffer/tbuffer should always have resource binding attribute");
1556
1557 auto *Index = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
1558 auto *RangeSize = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 1);
1559 auto *Space = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSpace());
1560 Value *Name = buildNameForResource(BaseName: BufDecl->getName(), CGM);
1561
1562 // buffer with explicit binding
1563 if (Binding.isExplicit()) {
1564 llvm::Intrinsic::ID IntrinsicID =
1565 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1566 auto *RegSlot = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getSlot());
1567 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
1568 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1569 } else {
1570 // buffer with implicit binding
1571 llvm::Intrinsic::ID IntrinsicID =
1572 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1573 auto *OrderID =
1574 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Binding.getImplicitOrderID());
1575 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
1576 initializeBuffer(CGM, GV, IntrID: IntrinsicID, Args);
1577 }
1578}
1579
1580void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
1581 llvm::GlobalVariable *GV) {
1582 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
1583 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1584 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
1585 addSPIRVBuiltinDecoration(GV, BuiltIn: Attr->getBuiltIn());
1586}
1587
1588llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
1589 if (!CGM.shouldEmitConvergenceTokens())
1590 return nullptr;
1591
1592 auto E = BB.end();
1593 for (auto I = BB.begin(); I != E; ++I) {
1594 auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &*I);
1595 if (II && llvm::isConvergenceControlIntrinsic(IntrinsicID: II->getIntrinsicID())) {
1596 return II;
1597 }
1598 }
1599 llvm_unreachable("Convergence token should have been emitted.");
1600 return nullptr;
1601}
1602
1603class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
1604public:
1605 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
1606 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
1607 OpaqueValueVisitor() {}
1608
1609 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
1610 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
1611 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
1612 // traversal, the temporary containing the copy out will not have
1613 // been created yet.
1614 return false;
1615 }
1616
1617 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
1618 // Traverse the source expression first.
1619 if (E->getSourceExpr())
1620 TraverseStmt(S: E->getSourceExpr());
1621
1622 // Then add this OVE if we haven't seen it before.
1623 if (Visited.insert(Ptr: E).second)
1624 OVEs.push_back(Elt: E);
1625
1626 return true;
1627 }
1628};
1629
1630void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
1631 InitListExpr *E) {
1632
1633 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
1634 OpaqueValueVisitor Visitor;
1635 Visitor.TraverseStmt(S: E);
1636 for (auto *OVE : Visitor.OVEs) {
1637 if (CGF.isOpaqueValueEmitted(E: OVE))
1638 continue;
1639 if (OpaqueValueMappingData::shouldBindAsLValue(expr: OVE)) {
1640 LValue LV = CGF.EmitLValue(E: OVE->getSourceExpr());
1641 OpaqueValueMappingData::bind(CGF, ov: OVE, lv: LV);
1642 } else {
1643 RValue RV = CGF.EmitAnyExpr(E: OVE->getSourceExpr());
1644 OpaqueValueMappingData::bind(CGF, ov: OVE, rv: RV);
1645 }
1646 }
1647}
1648
1649std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
1650 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
1651 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
1652 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
1653 "expected resource array subscript expression");
1654
1655 // Let clang codegen handle local and static resource array subscripts,
1656 // or when the subscript references on opaque expression (as part of
1657 // ArrayInitLoopExpr AST node).
1658 const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
1659 Val: getArrayDecl(AST&: CGF.CGM.getContext(), ASE: ArraySubsExpr));
1660 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
1661 ArrayDecl->getStorageClass() == SC_Static)
1662 return std::nullopt;
1663
1664 // get the resource array type
1665 ASTContext &AST = ArrayDecl->getASTContext();
1666 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
1667 assert(ResArrayTy->isHLSLResourceRecordArray() &&
1668 "expected array of resource classes");
1669
1670 // Iterate through all nested array subscript expressions to calculate
1671 // the index in the flattened resource array (if this is a multi-
1672 // dimensional array). The index is calculated as a sum of all indices
1673 // multiplied by the total size of the array at that level.
1674 Value *Index = nullptr;
1675 const ArraySubscriptExpr *ASE = ArraySubsExpr;
1676 while (ASE != nullptr) {
1677 Value *SubIndex = CGF.EmitScalarExpr(E: ASE->getIdx());
1678 if (const auto *ArrayTy =
1679 dyn_cast<ConstantArrayType>(Val: ASE->getType().getTypePtr())) {
1680 Value *Multiplier = llvm::ConstantInt::get(
1681 Ty: CGM.IntTy, V: AST.getConstantArrayElementCount(CA: ArrayTy));
1682 SubIndex = CGF.Builder.CreateMul(LHS: SubIndex, RHS: Multiplier);
1683 }
1684 Index = Index ? CGF.Builder.CreateAdd(LHS: Index, RHS: SubIndex) : SubIndex;
1685 ASE = dyn_cast<ArraySubscriptExpr>(Val: ASE->getBase()->IgnoreParenImpCasts());
1686 }
1687
1688 // Find binding info for the resource array. For implicit binding
1689 // an HLSLResourceBindingAttr should have been added by SemaHLSL.
1690 ResourceBindingAttrs Binding(ArrayDecl);
1691 assert(Binding.hasBinding() &&
1692 "resource array must have a binding attribute");
1693
1694 // Find the individual resource type.
1695 QualType ResultTy = ArraySubsExpr->getType();
1696 QualType ResourceTy =
1697 ResultTy->isArrayType() ? AST.getBaseElementType(QT: ResultTy) : ResultTy;
1698
1699 // Create a temporary variable for the result, which is either going
1700 // to be a single resource instance or a local array of resources (we need to
1701 // return an LValue).
1702 RawAddress TmpVar = CGF.CreateMemTempWithoutCast(T: ResultTy);
1703 if (CGF.EmitLifetimeStart(Addr: TmpVar.getPointer()))
1704 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
1705 kind: NormalEHLifetimeMarker, A: TmpVar);
1706
1707 AggValueSlot ValueSlot = AggValueSlot::forAddr(
1708 addr: TmpVar, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
1709 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
1710 mayOverlap: AggValueSlot::DoesNotOverlap);
1711
1712 // Calculate total array size (= range size).
1713 llvm::Value *Range = llvm::ConstantInt::getSigned(
1714 Ty: CGM.IntTy, V: getTotalArraySize(AST, Ty: ResArrayTy));
1715
1716 // If the result of the subscript operation is a single resource, call the
1717 // constructor.
1718 if (ResultTy == ResourceTy) {
1719 CallArgList Args;
1720 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
1721 CGM&: CGF.CGM, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), Range, Index,
1722 Name: ArrayDecl->getName(), Binding, Args);
1723
1724 if (!CreateMethod) {
1725 // This can happen if someone creates an array of structs that looks like
1726 // an HLSL resource record array but it does not have the required static
1727 // create method. No binding will be generated for it.
1728 assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
1729 "create method lookup should always succeed for built-in resource "
1730 "records");
1731 return std::nullopt;
1732 }
1733
1734 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress: ValueSlot.getAddress());
1735
1736 } else {
1737 // The result of the subscript operation is a local resource array which
1738 // needs to be initialized.
1739 const ConstantArrayType *ArrayTy =
1740 cast<ConstantArrayType>(Val: ResultTy.getTypePtr());
1741 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
1742 CGF, ResourceDecl: ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, StartIndex: Index,
1743 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)});
1744 if (!EndIndex)
1745 return std::nullopt;
1746 }
1747 return CGF.MakeAddrLValue(Addr: TmpVar, T: ResultTy, Source: AlignmentSource::Decl);
1748}
1749
1750// Initialize all resources of a global resource array into provided slot.
1751bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
1752 const VarDecl *ArrayDecl,
1753 AggValueSlot &DestSlot) {
1754 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
1755 ArrayDecl->hasGlobalStorage() &&
1756 ArrayDecl->getStorageClass() != SC_Static &&
1757 "expected global non-static resource array");
1758
1759 // Find binding info for the resource array. For implicit binding
1760 // the HLSLResourceBindingAttr should have been added by SemaHLSL.
1761 ResourceBindingAttrs Binding(ArrayDecl);
1762 assert(Binding.hasBinding() &&
1763 "resource array must have a binding attribute");
1764
1765 // Find the individual resource type.
1766 ASTContext &AST = ArrayDecl->getASTContext();
1767 QualType ResTy = AST.getBaseElementType(QT: ArrayDecl->getType());
1768 const auto *ResArrayTy =
1769 cast<ConstantArrayType>(Val: ArrayDecl->getType().getTypePtr());
1770
1771 // Create Value for index and total array size (= range size).
1772 int Size = getTotalArraySize(AST, Ty: ResArrayTy);
1773 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
1774 llvm::Value *Range = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size);
1775
1776 // Initialize individual resources in the array into DestSlot.
1777 std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
1778 CGF, ResourceDecl: ResTy->getAsCXXRecordDecl(), ArrayTy: ResArrayTy, ValueSlot&: DestSlot, Range, StartIndex: Zero,
1779 ResourceName: ArrayDecl->getName(), Binding, PrevGEPIndices: {Zero});
1780 return EndIndex.has_value();
1781}
1782
1783// If the expression is a global resource array, initialize all of its resources
1784// into Dest. Returns false if no initialization has been performed and the
1785// array copy should be handled by the default codegen.
1786bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
1787 AggValueSlot &DestSlot) {
1788 assert(E->getType()->isHLSLResourceRecordArray() &&
1789 "expected resource array");
1790
1791 // Find the array declaration for the expression. Fallback to the default
1792 // handling if it's not a global resource array.
1793 const VarDecl *ArrayDecl =
1794 dyn_cast_or_null<VarDecl>(Val: getArrayDecl(AST&: CGF.CGM.getContext(), E));
1795 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
1796 ArrayDecl->getStorageClass() == SC_Static)
1797 return false;
1798
1799 return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
1800}
1801
1802// If the expression is a global resource array, create a temporary and
1803// initialize all of its resources, and return it as an LValue. Returns nullopt
1804// if no initialization has been performed and the handling should follow the
1805// default path.
1806std::optional<LValue>
1807CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
1808 const VarDecl *ArrayDecl) {
1809 assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
1810 "expected resource array declaration");
1811
1812 if (!ArrayDecl->hasGlobalStorage() ||
1813 ArrayDecl->getStorageClass() == SC_Static)
1814 return std::nullopt;
1815
1816 AggValueSlot TmpArraySlot =
1817 CGF.CreateAggTemp(T: ArrayDecl->getType(), Name: "tmpResArray");
1818 if (initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot&: TmpArraySlot))
1819 return CGF.MakeAddrLValue(Addr: TmpArraySlot.getAddress(), T: ArrayDecl->getType(),
1820 Source: AlignmentSource::Decl);
1821 return std::nullopt;
1822}
1823
1824RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
1825 CodeGenFunction &CGF) {
1826
1827 assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
1828 assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
1829 "expected cbuffer matrix");
1830
1831 QualType MatQualTy = LV.getType();
1832 llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: MatQualTy);
1833 Address SrcAddr = LV.getAddress();
1834
1835 if (LayoutTy == CGF.ConvertTypeForMem(T: MatQualTy))
1836 return SrcAddr;
1837
1838 RawAddress DestAlloca =
1839 CGF.CreateMemTempWithoutCast(T: MatQualTy, Name: "matrix.buf.copy");
1840 HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(CType: MatQualTy);
1841 return DestAlloca;
1842}
1843
1844std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
1845 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
1846 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
1847 // Find the element type to index by first padding the element type per HLSL
1848 // buffer rules, and then padding out to a 16-byte register boundary if
1849 // necessary.
1850 llvm::Type *LayoutTy =
1851 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(Type: E->getType());
1852 uint64_t LayoutSizeInBits =
1853 CGM.getDataLayout().getTypeSizeInBits(Ty: LayoutTy).getFixedValue();
1854 CharUnits ElementSize = CharUnits::fromQuantity(Quantity: LayoutSizeInBits / 8);
1855 CharUnits RowAlignedSize = ElementSize.alignTo(Align: CharUnits::fromQuantity(Quantity: 16));
1856 if (RowAlignedSize > ElementSize) {
1857 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
1858 CGM, NumBytes: RowAlignedSize - ElementSize);
1859 assert(Padding && "No padding type for target?");
1860 LayoutTy = llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {LayoutTy, Padding},
1861 /*isPacked=*/true);
1862 }
1863
1864 // If the layout type doesn't introduce any padding, we don't need to do
1865 // anything special.
1866 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(T: E->getType());
1867 if (LayoutTy == OrigTy)
1868 return std::nullopt;
1869
1870 LValueBaseInfo EltBaseInfo;
1871 TBAAAccessInfo EltTBAAInfo;
1872
1873 // Index into the object as-if we have an array of the padded element type,
1874 // and then dereference the element itself to avoid reading padding that may
1875 // be past the end of the in-memory object.
1876 SmallVector<llvm::Value *, 2> Indices;
1877 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
1878 Indices.push_back(Elt: Idx);
1879 Indices.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1880
1881 if (CGF.getLangOpts().EmitLogicalPointer) {
1882 // The fact that we emit an array-to-pointer decay might be an oversight,
1883 // but for now, we simply ignore it (see #179951).
1884 const CastExpr *CE = cast<CastExpr>(Val: E->getBase());
1885 assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
1886
1887 LValue LV = CGF.EmitLValue(E: CE->getSubExpr());
1888 Address Addr = LV.getAddress();
1889 LayoutTy = llvm::ArrayType::get(
1890 ElementType: LayoutTy,
1891 NumElements: cast<llvm::ArrayType>(Val: Addr.getElementType())->getNumElements());
1892 auto *GEP = cast<StructuredGEPInst>(Val: CGF.Builder.CreateStructuredGEP(
1893 BaseType: LayoutTy, PtrBase: Addr.emitRawPointer(CGF), Indices, Name: "cbufferidx"));
1894 Addr =
1895 Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
1896 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
1897 }
1898
1899 Address Addr =
1900 CGF.EmitPointerWithAlignment(Addr: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
1901 llvm::Value *GEP = CGF.Builder.CreateGEP(Ty: LayoutTy, Ptr: Addr.emitRawPointer(CGF),
1902 IdxList: Indices, Name: "cbufferidx");
1903 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
1904 return CGF.MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
1905}
1906
1907std::optional<LValue>
1908CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
1909 const MemberExpr *ME) {
1910 assert((ME->getType()->isHLSLResourceRecord() ||
1911 ME->getType()->isHLSLResourceRecordArray()) &&
1912 "expected resource member expression");
1913
1914 const VarDecl *ResourceVD =
1915 findAssociatedResourceDeclForStruct(AST&: CGF.CGM.getContext(), ME);
1916 if (!ResourceVD)
1917 return std::nullopt;
1918
1919 // Handle member of resource array type.
1920 if (ResourceVD->getType()->isHLSLResourceRecordArray())
1921 return emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: ResourceVD);
1922
1923 GlobalVariable *ResGV =
1924 cast<GlobalVariable>(Val: CGM.GetAddrOfGlobalVar(D: ResourceVD));
1925 const DataLayout &DL = CGM.getDataLayout();
1926 llvm::Type *Ty = ResGV->getValueType();
1927 CharUnits Align = CharUnits::fromQuantity(Quantity: DL.getABITypeAlign(Ty));
1928 Address Addr = Address(ResGV, Ty, Align);
1929 LValue LV = LValue::MakeAddr(Addr, type: ME->getType(), Context&: CGM.getContext(),
1930 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
1931 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: ME->getType()));
1932 return LV;
1933}
1934
1935bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, Address DstPtr,
1936 Address SrcPtr, QualType CType) {
1937 return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(CType);
1938}
1939
1940LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
1941 const MemberExpr *E) {
1942 LValue Base =
1943 CGF.EmitCheckedLValue(E: E->getBase(), TCK: CodeGenFunction::TCK_MemberAccess);
1944 auto *Field = dyn_cast<FieldDecl>(Val: E->getMemberDecl());
1945 assert(Field && "Unexpected access into HLSL buffer");
1946
1947 const RecordDecl *Rec = Field->getParent();
1948
1949 // Work out the buffer layout type to index into.
1950 QualType RecType = CGM.getContext().getCanonicalTagType(TD: Rec);
1951 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
1952 // Since this is a member of an object in the buffer and not the buffer's
1953 // struct/class itself, we shouldn't have any offsets on the members we need
1954 // to contend with.
1955 CGHLSLOffsetInfo EmptyOffsets;
1956 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
1957 StructType: RecType->getAsCanonical<RecordType>(), OffsetInfo: EmptyOffsets);
1958
1959 // Get the field index for the layout struct, accounting for padding.
1960 unsigned FieldIdx =
1961 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(FD: Field);
1962 assert(FieldIdx < LayoutTy->getNumElements() &&
1963 "Layout struct is smaller than member struct");
1964 unsigned Skipped = 0;
1965 for (unsigned I = 0; I <= FieldIdx;) {
1966 llvm::Type *ElementTy = LayoutTy->getElementType(N: I + Skipped);
1967 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(Ty: ElementTy))
1968 ++Skipped;
1969 else
1970 ++I;
1971 }
1972 FieldIdx += Skipped;
1973 assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
1974
1975 // Now index into the struct, making sure that the type we return is the
1976 // buffer layout type rather than the original type in the AST.
1977 QualType FieldType = Field->getType();
1978 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(T: FieldType);
1979 CharUnits Align = CharUnits::fromQuantity(
1980 Quantity: CGF.CGM.getDataLayout().getABITypeAlign(Ty: FieldLLVMTy));
1981
1982 Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
1983 ? CGF.Builder.CreateStructuredGEP(
1984 BaseType: LayoutTy, PtrBase: Base.getPointer(CGF),
1985 Indices: llvm::ConstantInt::get(Ty: CGM.IntTy, V: FieldIdx))
1986 : CGF.Builder.CreateStructGEP(Ty: LayoutTy, Ptr: Base.getPointer(CGF),
1987 Idx: FieldIdx, Name: Field->getName());
1988 Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
1989
1990 LValue LV = LValue::MakeAddr(Addr, type: FieldType, Context&: CGM.getContext(),
1991 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
1992 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: FieldType));
1993 LV.getQuals().addCVRQualifiers(mask: Base.getVRQualifiers());
1994
1995 return LV;
1996}
1997