1//===----- CGHLSLRuntime.h - Interface to HLSL Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
16#define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
17
18#include "Address.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Decl.h"
21#include "clang/Basic/Builtins.h"
22#include "clang/Basic/HLSLRuntime.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Frontend/HLSL/HLSLResource.h"
27#include "llvm/Frontend/HLSL/SemanticSignatures.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/IntrinsicsDirectX.h"
31#include "llvm/IR/IntrinsicsSPIRV.h"
32
33#include <optional>
34#include <vector>
35
36// A function generator macro for picking the right intrinsic
37// for the target backend
38#define GENERATE_HLSL_INTRINSIC_FUNCTION(FunctionName, IntrinsicPostfix) \
39 llvm::Intrinsic::ID get##FunctionName##Intrinsic() { \
40 llvm::Triple::ArchType Arch = getArch(); \
41 switch (Arch) { \
42 case llvm::Triple::dxil: \
43 return llvm::Intrinsic::dx_##IntrinsicPostfix; \
44 case llvm::Triple::spirv: \
45 return llvm::Intrinsic::spv_##IntrinsicPostfix; \
46 default: \
47 llvm_unreachable("Intrinsic " #IntrinsicPostfix \
48 " not supported by target architecture"); \
49 } \
50 }
51
52using ResourceClass = llvm::dxil::ResourceClass;
53
54namespace llvm {
55class GlobalVariable;
56class Function;
57class StructType;
58class Metadata;
59} // namespace llvm
60
61namespace clang {
62class NamedDecl;
63class VarDecl;
64class ParmVarDecl;
65class InitListExpr;
66class HLSLBufferDecl;
67class HLSLRootSignatureDecl;
68class HLSLVkBindingAttr;
69class HLSLResourceBindingAttr;
70class Type;
71class RecordType;
72class DeclContext;
73class HLSLPackOffsetAttr;
74class ArraySubscriptExpr;
75
76class FunctionDecl;
77
78namespace CodeGen {
79
80class CodeGenModule;
81class CodeGenFunction;
82class LValue;
83class AggValueSlot;
84
85class CGHLSLOffsetInfo {
86 SmallVector<uint32_t> Offsets;
87
88public:
89 static const uint32_t Unspecified = ~0U;
90
91 /// Iterates over all declarations in the HLSL buffer and based on the
92 /// packoffset or register(c#) annotations it fills outs the Offsets vector
93 /// with the user-specified layout offsets. The buffer offsets can be
94 /// specified 2 ways: 1. declarations in cbuffer {} block can have a
95 /// packoffset annotation (translates to HLSLPackOffsetAttr) 2. default
96 /// constant buffer declarations at global scope can have register(c#)
97 /// annotations (translates to HLSLResourceBindingAttr with RegisterType::C)
98 /// It is not guaranteed that all declarations in a buffer have an annotation.
99 /// For those where it is not specified a `~0U` value is added to the Offsets
100 /// vector. In the final layout these declarations will be placed at the end
101 /// of the HLSL buffer after all of the elements with specified offset.
102 static CGHLSLOffsetInfo fromDecl(const HLSLBufferDecl &BufDecl);
103
104 /// Comparison function for offsets received from `operator[]` suitable for
105 /// use in a `stable_sort`. This will order implicit bindings after explicit
106 /// offsets.
107 static bool compareOffsets(uint32_t LHS, uint32_t RHS) { return LHS < RHS; }
108
109 /// Get the given offset, or `~0U` if there is no offset for the member.
110 uint32_t operator[](size_t I) const {
111 if (Offsets.empty())
112 return Unspecified;
113 return Offsets[I];
114 }
115
116 bool empty() const { return Offsets.empty(); }
117};
118
119class CGHLSLRuntime {
120public:
121 //===----------------------------------------------------------------------===//
122 // Start of reserved area for HLSL intrinsic getters.
123 //===----------------------------------------------------------------------===//
124
125 GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
126 GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
127 GENERATE_HLSL_INTRINSIC_FUNCTION(Frac, frac)
128 GENERATE_HLSL_INTRINSIC_FUNCTION(FlattenedThreadIdInGroup,
129 flattened_thread_id_in_group)
130 GENERATE_HLSL_INTRINSIC_FUNCTION(IsInf, isinf)
131 GENERATE_HLSL_INTRINSIC_FUNCTION(IsNaN, isnan)
132 GENERATE_HLSL_INTRINSIC_FUNCTION(Rsqrt, rsqrt)
133 GENERATE_HLSL_INTRINSIC_FUNCTION(Saturate, saturate)
134 GENERATE_HLSL_INTRINSIC_FUNCTION(Sign, sign)
135 GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
136 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupThreadId, thread_id_in_group)
137 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupId, group_id)
138 GENERATE_HLSL_INTRINSIC_FUNCTION(FDot, fdot)
139 GENERATE_HLSL_INTRINSIC_FUNCTION(SDot, sdot)
140 GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
141 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
142 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
143 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllEqual, wave_all_equal)
144 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
145 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
146 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
147 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
148 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
149 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
150 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
151 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
152 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMin, wave_reduce_umin)
153 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
154 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
155 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveGetLaneCount, wave_get_lane_count)
156 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneAt, wave_readlane)
157 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossX, quad_read_across_x)
158 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossY, quad_read_across_y)
159 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossDiagonal,
160 quad_read_across_diagonal)
161 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitUHigh, firstbituhigh)
162 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitSHigh, firstbitshigh)
163 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitLow, firstbitlow)
164 GENERATE_HLSL_INTRINSIC_FUNCTION(NClamp, nclamp)
165 GENERATE_HLSL_INTRINSIC_FUNCTION(SClamp, sclamp)
166 GENERATE_HLSL_INTRINSIC_FUNCTION(UClamp, uclamp)
167
168 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetBasePointer,
169 resource_getbasepointer)
170 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetPointer,
171 resource_getpointer)
172 GENERATE_HLSL_INTRINSIC_FUNCTION(Sample, resource_sample)
173 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleClamp, resource_sample_clamp)
174 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBias, resource_samplebias)
175 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBiasClamp, resource_samplebias_clamp)
176 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGrad, resource_samplegrad)
177 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGradClamp, resource_samplegrad_clamp)
178 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleLevel, resource_samplelevel)
179 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmp, resource_samplecmp)
180 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpClamp, resource_samplecmp_clamp)
181 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpLevelZero,
182 resource_samplecmplevelzero)
183 GENERATE_HLSL_INTRINSIC_FUNCTION(Gather, resource_gather)
184 GENERATE_HLSL_INTRINSIC_FUNCTION(GatherCmp, resource_gather_cmp)
185 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding,
186 resource_handlefrombinding)
187 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromImplicitBinding,
188 resource_handlefromimplicitbinding)
189 GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex,
190 resource_nonuniformindex)
191 GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
192 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier)
193 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync,
194 all_memory_barrier_with_group_sync)
195 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier)
196 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync,
197 device_memory_barrier_with_group_sync)
198 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier)
199 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
200 group_memory_barrier_with_group_sync)
201 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsX, resource_getdimensions_x)
202 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsXY, resource_getdimensions_xy)
203 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsLevelsXY,
204 resource_getdimensions_levels_xy)
205 GENERATE_HLSL_INTRINSIC_FUNCTION(LoadLevel, resource_load_level)
206 GENERATE_HLSL_INTRINSIC_FUNCTION(LoadMS, resource_load_ms)
207 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLod, resource_calculate_lod)
208 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLodUnclamped,
209 resource_calculate_lod_unclamped)
210 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxCoarse, ddx_coarse)
211 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyCoarse, ddy_coarse)
212 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxFine, ddx_fine)
213 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyFine, ddy_fine)
214
215 //===----------------------------------------------------------------------===//
216 // End of reserved area for HLSL intrinsic getters.
217 //===----------------------------------------------------------------------===//
218
219protected:
220 using SemanticSignatures =
221 llvm::SmallVectorImpl<llvm::hlsl::SemanticSignatureElement>;
222
223 CodeGenModule &CGM;
224
225 llvm::Value *emitSystemSemanticLoad(
226 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
227 HLSLAppliedSemanticAttr *Semantic,
228 llvm::dxbc::PSV::SemanticKind SemanticKind,
229 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
230 SemanticSignatures &Signature);
231
232 void emitSystemSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
233 const clang::DeclaratorDecl *Decl,
234 HLSLAppliedSemanticAttr *Semantic,
235 llvm::dxbc::PSV::SemanticKind SemanticKind,
236 llvm::Triple::EnvironmentType Stage,
237 std::optional<unsigned> Index,
238 SemanticSignatures &Signature);
239
240 llvm::Value *handleScalarSemanticLoad(llvm::IRBuilder<> &B,
241 const FunctionDecl *FD,
242 llvm::Type *Type,
243 const clang::DeclaratorDecl *Decl,
244 HLSLAppliedSemanticAttr *Semantic,
245 SemanticSignatures &Signature);
246
247 void handleScalarSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
248 llvm::Value *Source,
249 const clang::DeclaratorDecl *Decl,
250 HLSLAppliedSemanticAttr *Semantic,
251 SemanticSignatures &Signature);
252
253 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
254 handleStructSemanticLoad(
255 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
256 const clang::DeclaratorDecl *Decl,
257 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
258 specific_attr_iterator<HLSLAppliedSemanticAttr> end,
259 SemanticSignatures &Signature);
260
261 specific_attr_iterator<HLSLAppliedSemanticAttr> handleStructSemanticStore(
262 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
263 const clang::DeclaratorDecl *Decl,
264 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
265 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
266 SemanticSignatures &Signature);
267
268 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
269 handleSemanticLoad(llvm::IRBuilder<> &B, const FunctionDecl *FD,
270 llvm::Type *Type, const clang::DeclaratorDecl *Decl,
271 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
272 specific_attr_iterator<HLSLAppliedSemanticAttr> end,
273 SemanticSignatures &Signature);
274
275 specific_attr_iterator<HLSLAppliedSemanticAttr>
276 handleSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
277 llvm::Value *Source, const clang::DeclaratorDecl *Decl,
278 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
279 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
280 SemanticSignatures &Signature);
281
282public:
283 CGHLSLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
284 virtual ~CGHLSLRuntime() {}
285
286 llvm::Type *convertHLSLSpecificType(const Type *T,
287 const CGHLSLOffsetInfo &OffsetInfo);
288 llvm::Type *convertHLSLSpecificType(const Type *T) {
289 return convertHLSLSpecificType(T, OffsetInfo: CGHLSLOffsetInfo());
290 }
291
292 void generateGlobalCtorDtorCalls();
293
294 void addBuffer(const HLSLBufferDecl *D);
295 void addRootSignature(const HLSLRootSignatureDecl *D);
296 void finishCodeGen();
297
298 void setHLSLEntryAttributes(const FunctionDecl *FD, llvm::Function *Fn);
299
300 void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
301 void setHLSLFunctionAttributes(const FunctionDecl *FD, llvm::Function *Fn);
302 void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
303
304 llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
305
306 llvm::StructType *getHLSLBufferLayoutType(const RecordType *LayoutStructTy);
307 void addHLSLBufferLayoutType(const RecordType *LayoutStructTy,
308 llvm::StructType *LayoutTy);
309 void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E);
310
311 std::optional<LValue>
312 emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
313 CodeGenFunction &CGF);
314
315 bool emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
316 AggValueSlot &DestSlot);
317 std::optional<LValue>
318 emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
319 const VarDecl *ArrayDecl);
320
321 std::optional<LValue> emitBufferArraySubscriptExpr(
322 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
323 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase);
324
325 RawAddress createBufferMatrixTempAddress(const LValue &LV,
326 CodeGenFunction &CGF);
327
328 bool emitBufferCopy(CodeGenFunction &CGF, const Expr *E, const LValue &SrcLV,
329 AggValueSlot &DestSlot);
330
331 LValue emitBufferMemberExpr(CodeGenFunction &CGF, const MemberExpr *E);
332 std::optional<LValue> emitResourceMemberExpr(CodeGenFunction &CGF,
333 const MemberExpr *E);
334
335private:
336 void emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
337 llvm::GlobalVariable *BufGV,
338 const CGHLSLOffsetInfo &OffsetInfo);
339 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
340 llvm::GlobalVariable *GV);
341 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
342 llvm::GlobalVariable *GV,
343 HLSLResourceBindingAttr *RBA);
344
345 llvm::Value *emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B,
346 const FunctionDecl *FD,
347 llvm::Type *Type,
348 const clang::DeclaratorDecl *Decl,
349 HLSLAppliedSemanticAttr *Semantic,
350 std::optional<unsigned> Index);
351 llvm::Value *emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
352 const clang::DeclaratorDecl *Decl,
353 HLSLAppliedSemanticAttr *Semantic,
354 std::optional<unsigned> Index,
355 SemanticSignatures &Signature);
356 llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B,
357 const FunctionDecl *FD, llvm::Type *Type,
358 const clang::DeclaratorDecl *Decl,
359 HLSLAppliedSemanticAttr *Semantic,
360 std::optional<unsigned> Index,
361 SemanticSignatures &Signature);
362
363 void emitSPIRVUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
364 const clang::DeclaratorDecl *Decl,
365 HLSLAppliedSemanticAttr *Semantic,
366 std::optional<unsigned> Index);
367 void emitDXILUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
368 const clang::DeclaratorDecl *Decl,
369 HLSLAppliedSemanticAttr *Semantic,
370 std::optional<unsigned> Index,
371 SemanticSignatures &Signature);
372 void emitUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
373 const clang::DeclaratorDecl *Decl,
374 HLSLAppliedSemanticAttr *Semantic,
375 std::optional<unsigned> Index,
376 SemanticSignatures &Signature);
377
378 bool initializeGlobalResourceArray(CodeGenFunction &CGF,
379 const VarDecl *ArrayDecl,
380 AggValueSlot &DestSlot);
381
382 llvm::Triple::ArchType getArch();
383
384 llvm::DenseMap<const clang::RecordType *, llvm::StructType *> LayoutTypes;
385 unsigned SPIRVLastAssignedInputSemanticLocation = 0;
386 unsigned SPIRVLastAssignedOutputSemanticLocation = 0;
387};
388
389} // namespace CodeGen
390} // namespace clang
391
392#endif
393