1//===----- CGHLSLRuntime.h - Interface to HLSL Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
16#define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
17
18#include "Address.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Decl.h"
21#include "clang/Basic/Builtins.h"
22#include "clang/Basic/HLSLRuntime.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Frontend/HLSL/HLSLResource.h"
27#include "llvm/Frontend/HLSL/SemanticSignatures.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/IntrinsicsDirectX.h"
31#include "llvm/IR/IntrinsicsSPIRV.h"
32
33#include <optional>
34#include <vector>
35
36// Function generator macros for picking the right intrinsic for the target.
37#define GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(FunctionName, DxilPostfix, \
38 SpirvPostfix) \
39 llvm::Intrinsic::ID get##FunctionName##Intrinsic() { \
40 llvm::Triple::ArchType Arch = getArch(); \
41 switch (Arch) { \
42 case llvm::Triple::dxil: \
43 return llvm::Intrinsic::dx_##DxilPostfix; \
44 case llvm::Triple::spirv: \
45 return llvm::Intrinsic::spv_##SpirvPostfix; \
46 default: \
47 llvm_unreachable(#FunctionName " not supported by target architecture"); \
48 } \
49 }
50
51#define GENERATE_HLSL_INTRINSIC_FUNCTION(FunctionName, IntrinsicPostfix) \
52 GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(FunctionName, IntrinsicPostfix, \
53 IntrinsicPostfix)
54
55using ResourceClass = llvm::dxil::ResourceClass;
56
57namespace llvm {
58class GlobalVariable;
59class Function;
60class StructType;
61class Metadata;
62} // namespace llvm
63
64namespace clang {
65class NamedDecl;
66class VarDecl;
67class ParmVarDecl;
68class InitListExpr;
69class HLSLBufferDecl;
70class HLSLRootSignatureDecl;
71class HLSLVkBindingAttr;
72class HLSLResourceBindingAttr;
73class Type;
74class RecordType;
75class DeclContext;
76class HLSLPackOffsetAttr;
77class ArraySubscriptExpr;
78
79class FunctionDecl;
80
81namespace CodeGen {
82
83class CodeGenModule;
84class CodeGenFunction;
85class LValue;
86class AggValueSlot;
87
88class CGHLSLOffsetInfo {
89 SmallVector<uint32_t> Offsets;
90
91public:
92 static const uint32_t Unspecified = ~0U;
93
94 /// Iterates over all declarations in the HLSL buffer and based on the
95 /// packoffset or register(c#) annotations it fills outs the Offsets vector
96 /// with the user-specified layout offsets. The buffer offsets can be
97 /// specified 2 ways: 1. declarations in cbuffer {} block can have a
98 /// packoffset annotation (translates to HLSLPackOffsetAttr) 2. default
99 /// constant buffer declarations at global scope can have register(c#)
100 /// annotations (translates to HLSLResourceBindingAttr with RegisterType::C)
101 /// It is not guaranteed that all declarations in a buffer have an annotation.
102 /// For those where it is not specified a `~0U` value is added to the Offsets
103 /// vector. In the final layout these declarations will be placed at the end
104 /// of the HLSL buffer after all of the elements with specified offset.
105 static CGHLSLOffsetInfo fromDecl(const HLSLBufferDecl &BufDecl);
106
107 /// Comparison function for offsets received from `operator[]` suitable for
108 /// use in a `stable_sort`. This will order implicit bindings after explicit
109 /// offsets.
110 static bool compareOffsets(uint32_t LHS, uint32_t RHS) { return LHS < RHS; }
111
112 /// Get the given offset, or `~0U` if there is no offset for the member.
113 uint32_t operator[](size_t I) const {
114 if (Offsets.empty())
115 return Unspecified;
116 return Offsets[I];
117 }
118
119 bool empty() const { return Offsets.empty(); }
120};
121
122class CGHLSLRuntime {
123public:
124 //===----------------------------------------------------------------------===//
125 // Start of reserved area for HLSL intrinsic getters.
126 //===----------------------------------------------------------------------===//
127
128 GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
129 GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
130 GENERATE_HLSL_INTRINSIC_FUNCTION(Frac, frac)
131 GENERATE_HLSL_INTRINSIC_FUNCTION(FlattenedThreadIdInGroup,
132 flattened_thread_id_in_group)
133 GENERATE_HLSL_INTRINSIC_FUNCTION(IsFinite, isfinite)
134 GENERATE_HLSL_INTRINSIC_FUNCTION(IsInf, isinf)
135 GENERATE_HLSL_INTRINSIC_FUNCTION(IsNaN, isnan)
136 GENERATE_HLSL_INTRINSIC_FUNCTION(Rsqrt, rsqrt)
137 GENERATE_HLSL_INTRINSIC_FUNCTION(Saturate, saturate)
138 GENERATE_HLSL_INTRINSIC_FUNCTION(Sign, sign)
139 GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
140 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupThreadId, thread_id_in_group)
141 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupId, group_id)
142 GENERATE_HLSL_INTRINSIC_FUNCTION(FDot, fdot)
143 GENERATE_HLSL_INTRINSIC_FUNCTION(SDot, sdot)
144 GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
145 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
146 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
147 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllEqual, wave_all_equal)
148 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
149 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
150 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
151 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
152 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
153 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
154 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
155 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
156 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMin, wave_reduce_umin)
157 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
158 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
159 GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveGetLaneCount, wave_get_lane_count,
160 subgroup_size)
161 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneAt, wave_readlane)
162 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneFirst, wave_readlane_first)
163 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossX, quad_read_across_x)
164 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossY, quad_read_across_y)
165 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossDiagonal,
166 quad_read_across_diagonal)
167 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitUHigh, firstbituhigh)
168 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitSHigh, firstbitshigh)
169 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitLow, firstbitlow)
170 GENERATE_HLSL_INTRINSIC_FUNCTION(NClamp, nclamp)
171 GENERATE_HLSL_INTRINSIC_FUNCTION(SClamp, sclamp)
172 GENERATE_HLSL_INTRINSIC_FUNCTION(UClamp, uclamp)
173
174 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetBasePointer,
175 resource_getbasepointer)
176 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetPointer,
177 resource_getpointer)
178 GENERATE_HLSL_INTRINSIC_FUNCTION(Sample, resource_sample)
179 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleClamp, resource_sample_clamp)
180 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBias, resource_samplebias)
181 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBiasClamp, resource_samplebias_clamp)
182 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGrad, resource_samplegrad)
183 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGradClamp, resource_samplegrad_clamp)
184 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleLevel, resource_samplelevel)
185 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmp, resource_samplecmp)
186 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpClamp, resource_samplecmp_clamp)
187 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpLevelZero,
188 resource_samplecmplevelzero)
189 GENERATE_HLSL_INTRINSIC_FUNCTION(Gather, resource_gather)
190 GENERATE_HLSL_INTRINSIC_FUNCTION(GatherCmp, resource_gather_cmp)
191 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding,
192 resource_handlefrombinding)
193 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromImplicitBinding,
194 resource_handlefromimplicitbinding)
195 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromHeap,
196 resource_handlefromheap)
197 GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex,
198 resource_nonuniformindex)
199 GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
200 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier)
201 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync,
202 all_memory_barrier_with_group_sync)
203 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier)
204 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync,
205 device_memory_barrier_with_group_sync)
206 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier)
207 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
208 group_memory_barrier_with_group_sync)
209 llvm::Intrinsic::ID getBarrierByMemoryTypeIntrinsic() {
210 assert(getArch() == llvm::Triple::dxil &&
211 "Barrier by memory type is only supported for DXIL");
212 return llvm::Intrinsic::dx_barrier_by_memory_type;
213 }
214 llvm::Intrinsic::ID getBarrierByMemoryHandleIntrinsic() {
215 assert(getArch() == llvm::Triple::dxil &&
216 "Barrier by memory handle is only supported for DXIL");
217 return llvm::Intrinsic::dx_barrier_by_memory_handle;
218 }
219 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsX, resource_getdimensions_x)
220 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsXY, resource_getdimensions_xy)
221 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsLevelsXY,
222 resource_getdimensions_levels_xy)
223 GENERATE_HLSL_INTRINSIC_FUNCTION(LoadLevel, resource_load_level)
224 GENERATE_HLSL_INTRINSIC_FUNCTION(LoadMS, resource_load_ms)
225 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLod, resource_calculate_lod)
226 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLodUnclamped,
227 resource_calculate_lod_unclamped)
228 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxCoarse, ddx_coarse)
229 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyCoarse, ddy_coarse)
230 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxFine, ddx_fine)
231 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyFine, ddy_fine)
232
233 //===----------------------------------------------------------------------===//
234 // End of reserved area for HLSL intrinsic getters.
235 //===----------------------------------------------------------------------===//
236
237protected:
238 using SemanticSignatures =
239 llvm::SmallVectorImpl<llvm::hlsl::SemanticSignatureElement>;
240
241 CodeGenModule &CGM;
242
243 llvm::Value *emitSystemSemanticLoad(
244 llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
245 HLSLAppliedSemanticAttr *Semantic,
246 llvm::dxbc::PSV::SemanticKind SemanticKind,
247 llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
248 SemanticSignatures &Signature);
249
250 void emitSystemSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
251 const clang::DeclaratorDecl *Decl,
252 HLSLAppliedSemanticAttr *Semantic,
253 llvm::dxbc::PSV::SemanticKind SemanticKind,
254 llvm::Triple::EnvironmentType Stage,
255 std::optional<unsigned> Index,
256 SemanticSignatures &Signature);
257
258 llvm::Value *handleScalarSemanticLoad(llvm::IRBuilder<> &B,
259 const FunctionDecl *FD,
260 llvm::Type *Type,
261 const clang::DeclaratorDecl *Decl,
262 HLSLAppliedSemanticAttr *Semantic,
263 SemanticSignatures &Signature);
264
265 void handleScalarSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
266 llvm::Value *Source,
267 const clang::DeclaratorDecl *Decl,
268 HLSLAppliedSemanticAttr *Semantic,
269 SemanticSignatures &Signature);
270
271 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
272 handleStructSemanticLoad(
273 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
274 const clang::DeclaratorDecl *Decl,
275 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
276 specific_attr_iterator<HLSLAppliedSemanticAttr> end,
277 SemanticSignatures &Signature,
278 llvm::hlsl::InterpolationModifier Modifiers);
279
280 specific_attr_iterator<HLSLAppliedSemanticAttr> handleStructSemanticStore(
281 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
282 const clang::DeclaratorDecl *Decl,
283 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
284 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
285 SemanticSignatures &Signature,
286 llvm::hlsl::InterpolationModifier Modifiers);
287
288 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
289 handleSemanticLoad(llvm::IRBuilder<> &B, const FunctionDecl *FD,
290 llvm::Type *Type, const clang::DeclaratorDecl *Decl,
291 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
292 specific_attr_iterator<HLSLAppliedSemanticAttr> end,
293 SemanticSignatures &Signature,
294 llvm::hlsl::InterpolationModifier Modifiers =
295 llvm::hlsl::InterpolationModifier::None);
296
297 specific_attr_iterator<HLSLAppliedSemanticAttr>
298 handleSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
299 llvm::Value *Source, const clang::DeclaratorDecl *Decl,
300 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
301 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
302 SemanticSignatures &Signature,
303 llvm::hlsl::InterpolationModifier Modifiers =
304 llvm::hlsl::InterpolationModifier::None);
305
306public:
307 CGHLSLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
308 virtual ~CGHLSLRuntime() {}
309
310 llvm::Type *convertHLSLSpecificType(const Type *T,
311 const CGHLSLOffsetInfo &OffsetInfo);
312 llvm::Type *convertHLSLSpecificType(const Type *T) {
313 return convertHLSLSpecificType(T, OffsetInfo: CGHLSLOffsetInfo());
314 }
315
316 void generateGlobalCtorDtorCalls();
317
318 void addBuffer(const HLSLBufferDecl *D);
319 void addRootSignature(const HLSLRootSignatureDecl *D);
320 void finishCodeGen();
321
322 void setHLSLEntryAttributes(const FunctionDecl *FD, llvm::Function *Fn);
323
324 void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
325 void setHLSLFunctionAttributes(const FunctionDecl *FD, llvm::Function *Fn);
326 void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
327
328 llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
329
330 llvm::StructType *getHLSLBufferLayoutType(const RecordType *LayoutStructTy);
331 void addHLSLBufferLayoutType(const RecordType *LayoutStructTy,
332 llvm::StructType *LayoutTy);
333 void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E);
334
335 std::optional<LValue>
336 emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
337 CodeGenFunction &CGF);
338
339 bool emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
340 AggValueSlot &DestSlot);
341 std::optional<LValue>
342 emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
343 const VarDecl *ArrayDecl);
344
345 std::optional<LValue> emitBufferArraySubscriptExpr(
346 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
347 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase);
348
349 RawAddress createBufferMatrixTempAddress(const LValue &LV,
350 CodeGenFunction &CGF);
351
352 bool emitBufferCopy(CodeGenFunction &CGF, const Expr *E, const LValue &SrcLV,
353 AggValueSlot &DestSlot);
354
355 LValue emitBufferMemberExpr(CodeGenFunction &CGF, const MemberExpr *E);
356 std::optional<LValue> emitResourceMemberExpr(CodeGenFunction &CGF,
357 const MemberExpr *E);
358
359private:
360 void emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
361 llvm::GlobalVariable *BufGV,
362 const CGHLSLOffsetInfo &OffsetInfo);
363 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
364 llvm::GlobalVariable *GV);
365 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
366 llvm::GlobalVariable *GV,
367 HLSLResourceBindingAttr *RBA);
368
369 llvm::Value *emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B,
370 const FunctionDecl *FD,
371 llvm::Type *Type,
372 const clang::DeclaratorDecl *Decl,
373 HLSLAppliedSemanticAttr *Semantic,
374 std::optional<unsigned> Index);
375 llvm::Value *emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
376 const clang::DeclaratorDecl *Decl,
377 HLSLAppliedSemanticAttr *Semantic,
378 std::optional<unsigned> Index,
379 SemanticSignatures &Signature);
380 llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B,
381 const FunctionDecl *FD, llvm::Type *Type,
382 const clang::DeclaratorDecl *Decl,
383 HLSLAppliedSemanticAttr *Semantic,
384 std::optional<unsigned> Index,
385 SemanticSignatures &Signature);
386
387 void emitSPIRVUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
388 const clang::DeclaratorDecl *Decl,
389 HLSLAppliedSemanticAttr *Semantic,
390 std::optional<unsigned> Index);
391 void emitDXILUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
392 const clang::DeclaratorDecl *Decl,
393 HLSLAppliedSemanticAttr *Semantic,
394 std::optional<unsigned> Index,
395 SemanticSignatures &Signature);
396 void emitUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
397 const clang::DeclaratorDecl *Decl,
398 HLSLAppliedSemanticAttr *Semantic,
399 std::optional<unsigned> Index,
400 SemanticSignatures &Signature);
401
402 bool initializeGlobalResourceArray(CodeGenFunction &CGF,
403 const VarDecl *ArrayDecl,
404 AggValueSlot &DestSlot);
405
406 llvm::Triple::ArchType getArch();
407
408 llvm::DenseMap<const clang::RecordType *, llvm::StructType *> LayoutTypes;
409 unsigned SPIRVLastAssignedInputSemanticLocation = 0;
410 unsigned SPIRVLastAssignedOutputSemanticLocation = 0;
411};
412
413} // namespace CodeGen
414} // namespace clang
415
416#endif
417