1//===----- CGHLSLRuntime.h - Interface to HLSL Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides an abstract class for HLSL code generation. Concrete
10// subclasses of this implement code generation for specific HLSL
11// runtime libraries.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
16#define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
17
18#include "Address.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Decl.h"
21#include "clang/Basic/Builtins.h"
22#include "clang/Basic/HLSLRuntime.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Frontend/HLSL/HLSLResource.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsDirectX.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31
32#include <optional>
33#include <vector>
34
35// A function generator macro for picking the right intrinsic
36// for the target backend
37#define GENERATE_HLSL_INTRINSIC_FUNCTION(FunctionName, IntrinsicPostfix) \
38 llvm::Intrinsic::ID get##FunctionName##Intrinsic() { \
39 llvm::Triple::ArchType Arch = getArch(); \
40 switch (Arch) { \
41 case llvm::Triple::dxil: \
42 return llvm::Intrinsic::dx_##IntrinsicPostfix; \
43 case llvm::Triple::spirv: \
44 return llvm::Intrinsic::spv_##IntrinsicPostfix; \
45 default: \
46 llvm_unreachable("Intrinsic " #IntrinsicPostfix \
47 " not supported by target architecture"); \
48 } \
49 }
50
51using ResourceClass = llvm::dxil::ResourceClass;
52
53namespace llvm {
54class GlobalVariable;
55class Function;
56class StructType;
57class Metadata;
58} // namespace llvm
59
60namespace clang {
61class NamedDecl;
62class VarDecl;
63class ParmVarDecl;
64class InitListExpr;
65class HLSLBufferDecl;
66class HLSLRootSignatureDecl;
67class HLSLVkBindingAttr;
68class HLSLResourceBindingAttr;
69class Type;
70class RecordType;
71class DeclContext;
72class HLSLPackOffsetAttr;
73class ArraySubscriptExpr;
74
75class FunctionDecl;
76
77namespace CodeGen {
78
79class CodeGenModule;
80class CodeGenFunction;
81class LValue;
82class AggValueSlot;
83
84class CGHLSLOffsetInfo {
85 SmallVector<uint32_t> Offsets;
86
87public:
88 static const uint32_t Unspecified = ~0U;
89
90 /// Iterates over all declarations in the HLSL buffer and based on the
91 /// packoffset or register(c#) annotations it fills outs the Offsets vector
92 /// with the user-specified layout offsets. The buffer offsets can be
93 /// specified 2 ways: 1. declarations in cbuffer {} block can have a
94 /// packoffset annotation (translates to HLSLPackOffsetAttr) 2. default
95 /// constant buffer declarations at global scope can have register(c#)
96 /// annotations (translates to HLSLResourceBindingAttr with RegisterType::C)
97 /// It is not guaranteed that all declarations in a buffer have an annotation.
98 /// For those where it is not specified a `~0U` value is added to the Offsets
99 /// vector. In the final layout these declarations will be placed at the end
100 /// of the HLSL buffer after all of the elements with specified offset.
101 static CGHLSLOffsetInfo fromDecl(const HLSLBufferDecl &BufDecl);
102
103 /// Comparison function for offsets received from `operator[]` suitable for
104 /// use in a `stable_sort`. This will order implicit bindings after explicit
105 /// offsets.
106 static bool compareOffsets(uint32_t LHS, uint32_t RHS) { return LHS < RHS; }
107
108 /// Get the given offset, or `~0U` if there is no offset for the member.
109 uint32_t operator[](size_t I) const {
110 if (Offsets.empty())
111 return Unspecified;
112 return Offsets[I];
113 }
114
115 bool empty() const { return Offsets.empty(); }
116};
117
118class CGHLSLRuntime {
119public:
120 //===----------------------------------------------------------------------===//
121 // Start of reserved area for HLSL intrinsic getters.
122 //===----------------------------------------------------------------------===//
123
124 GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
125 GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
126 GENERATE_HLSL_INTRINSIC_FUNCTION(Cross, cross)
127 GENERATE_HLSL_INTRINSIC_FUNCTION(Degrees, degrees)
128 GENERATE_HLSL_INTRINSIC_FUNCTION(Frac, frac)
129 GENERATE_HLSL_INTRINSIC_FUNCTION(FlattenedThreadIdInGroup,
130 flattened_thread_id_in_group)
131 GENERATE_HLSL_INTRINSIC_FUNCTION(IsInf, isinf)
132 GENERATE_HLSL_INTRINSIC_FUNCTION(IsNaN, isnan)
133 GENERATE_HLSL_INTRINSIC_FUNCTION(Lerp, lerp)
134 GENERATE_HLSL_INTRINSIC_FUNCTION(Normalize, normalize)
135 GENERATE_HLSL_INTRINSIC_FUNCTION(Rsqrt, rsqrt)
136 GENERATE_HLSL_INTRINSIC_FUNCTION(Saturate, saturate)
137 GENERATE_HLSL_INTRINSIC_FUNCTION(Sign, sign)
138 GENERATE_HLSL_INTRINSIC_FUNCTION(Step, step)
139 GENERATE_HLSL_INTRINSIC_FUNCTION(Radians, radians)
140 GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
141 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupThreadId, thread_id_in_group)
142 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupId, group_id)
143 GENERATE_HLSL_INTRINSIC_FUNCTION(FDot, fdot)
144 GENERATE_HLSL_INTRINSIC_FUNCTION(SDot, sdot)
145 GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
146 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
147 GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
148 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllEqual, wave_all_equal)
149 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
150 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
151 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
152 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
153 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
154 GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedAdd, interlocked_add)
155 GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedOr, interlocked_or)
156 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
157 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
158 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
159 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMin, wave_reduce_umin)
160 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
161 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
162 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveGetLaneCount, wave_get_lane_count)
163 GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneAt, wave_readlane)
164 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossX, quad_read_across_x)
165 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossY, quad_read_across_y)
166 GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossDiagonal,
167 quad_read_across_diagonal)
168 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitUHigh, firstbituhigh)
169 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitSHigh, firstbitshigh)
170 GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitLow, firstbitlow)
171 GENERATE_HLSL_INTRINSIC_FUNCTION(NClamp, nclamp)
172 GENERATE_HLSL_INTRINSIC_FUNCTION(SClamp, sclamp)
173 GENERATE_HLSL_INTRINSIC_FUNCTION(UClamp, uclamp)
174
175 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetBasePointer,
176 resource_getbasepointer)
177 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetPointer,
178 resource_getpointer)
179 GENERATE_HLSL_INTRINSIC_FUNCTION(Sample, resource_sample)
180 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleClamp, resource_sample_clamp)
181 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBias, resource_samplebias)
182 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBiasClamp, resource_samplebias_clamp)
183 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGrad, resource_samplegrad)
184 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGradClamp, resource_samplegrad_clamp)
185 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleLevel, resource_samplelevel)
186 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmp, resource_samplecmp)
187 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpClamp, resource_samplecmp_clamp)
188 GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpLevelZero,
189 resource_samplecmplevelzero)
190 GENERATE_HLSL_INTRINSIC_FUNCTION(Gather, resource_gather)
191 GENERATE_HLSL_INTRINSIC_FUNCTION(GatherCmp, resource_gather_cmp)
192 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding,
193 resource_handlefrombinding)
194 GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromImplicitBinding,
195 resource_handlefromimplicitbinding)
196 GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex,
197 resource_nonuniformindex)
198 GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
199 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier)
200 GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync,
201 all_memory_barrier_with_group_sync)
202 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier)
203 GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync,
204 device_memory_barrier_with_group_sync)
205 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier)
206 GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
207 group_memory_barrier_with_group_sync)
208 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsX, resource_getdimensions_x)
209 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsXY, resource_getdimensions_xy)
210 GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsLevelsXY,
211 resource_getdimensions_levels_xy)
212 GENERATE_HLSL_INTRINSIC_FUNCTION(LoadLevel, resource_load_level)
213 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLod, resource_calculate_lod)
214 GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLodUnclamped,
215 resource_calculate_lod_unclamped)
216 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxCoarse, ddx_coarse)
217 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyCoarse, ddy_coarse)
218 GENERATE_HLSL_INTRINSIC_FUNCTION(DdxFine, ddx_fine)
219 GENERATE_HLSL_INTRINSIC_FUNCTION(DdyFine, ddy_fine)
220
221 //===----------------------------------------------------------------------===//
222 // End of reserved area for HLSL intrinsic getters.
223 //===----------------------------------------------------------------------===//
224
225protected:
226 CodeGenModule &CGM;
227
228 llvm::Value *emitSystemSemanticLoad(llvm::IRBuilder<> &B,
229 const FunctionDecl *FD, llvm::Type *Type,
230 const clang::DeclaratorDecl *Decl,
231 HLSLAppliedSemanticAttr *Semantic,
232 std::optional<unsigned> Index);
233
234 void emitSystemSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
235 const clang::DeclaratorDecl *Decl,
236 HLSLAppliedSemanticAttr *Semantic,
237 std::optional<unsigned> Index);
238
239 llvm::Value *handleScalarSemanticLoad(llvm::IRBuilder<> &B,
240 const FunctionDecl *FD,
241 llvm::Type *Type,
242 const clang::DeclaratorDecl *Decl,
243 HLSLAppliedSemanticAttr *Semantic);
244
245 void handleScalarSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
246 llvm::Value *Source,
247 const clang::DeclaratorDecl *Decl,
248 HLSLAppliedSemanticAttr *Semantic);
249
250 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
251 handleStructSemanticLoad(
252 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
253 const clang::DeclaratorDecl *Decl,
254 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
255 specific_attr_iterator<HLSLAppliedSemanticAttr> end);
256
257 specific_attr_iterator<HLSLAppliedSemanticAttr> handleStructSemanticStore(
258 llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
259 const clang::DeclaratorDecl *Decl,
260 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
261 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd);
262
263 std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
264 handleSemanticLoad(llvm::IRBuilder<> &B, const FunctionDecl *FD,
265 llvm::Type *Type, const clang::DeclaratorDecl *Decl,
266 specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
267 specific_attr_iterator<HLSLAppliedSemanticAttr> end);
268
269 specific_attr_iterator<HLSLAppliedSemanticAttr>
270 handleSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
271 llvm::Value *Source, const clang::DeclaratorDecl *Decl,
272 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
273 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd);
274
275public:
276 CGHLSLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
277 virtual ~CGHLSLRuntime() {}
278
279 llvm::Type *convertHLSLSpecificType(const Type *T,
280 const CGHLSLOffsetInfo &OffsetInfo);
281 llvm::Type *convertHLSLSpecificType(const Type *T) {
282 return convertHLSLSpecificType(T, OffsetInfo: CGHLSLOffsetInfo());
283 }
284
285 void generateGlobalCtorDtorCalls();
286
287 void addBuffer(const HLSLBufferDecl *D);
288 void addRootSignature(const HLSLRootSignatureDecl *D);
289 void finishCodeGen();
290
291 void setHLSLEntryAttributes(const FunctionDecl *FD, llvm::Function *Fn);
292
293 void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
294 void setHLSLFunctionAttributes(const FunctionDecl *FD, llvm::Function *Fn);
295 void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
296
297 llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
298
299 llvm::StructType *getHLSLBufferLayoutType(const RecordType *LayoutStructTy);
300 void addHLSLBufferLayoutType(const RecordType *LayoutStructTy,
301 llvm::StructType *LayoutTy);
302 void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E);
303
304 std::optional<LValue>
305 emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
306 CodeGenFunction &CGF);
307
308 bool emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
309 AggValueSlot &DestSlot);
310 std::optional<LValue>
311 emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
312 const VarDecl *ArrayDecl);
313
314 std::optional<LValue> emitBufferArraySubscriptExpr(
315 const ArraySubscriptExpr *E, CodeGenFunction &CGF,
316 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase);
317
318 RawAddress createBufferMatrixTempAddress(const LValue &LV,
319 CodeGenFunction &CGF);
320
321 bool emitBufferCopy(CodeGenFunction &CGF, const Expr *E, const LValue &SrcLV,
322 AggValueSlot &DestSlot);
323
324 LValue emitBufferMemberExpr(CodeGenFunction &CGF, const MemberExpr *E);
325 std::optional<LValue> emitResourceMemberExpr(CodeGenFunction &CGF,
326 const MemberExpr *E);
327
328private:
329 void emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
330 llvm::GlobalVariable *BufGV,
331 const CGHLSLOffsetInfo &OffsetInfo);
332 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
333 llvm::GlobalVariable *GV);
334 void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
335 llvm::GlobalVariable *GV,
336 HLSLResourceBindingAttr *RBA);
337
338 llvm::Value *emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
339 const clang::DeclaratorDecl *Decl,
340 HLSLAppliedSemanticAttr *Semantic,
341 std::optional<unsigned> Index);
342 llvm::Value *emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
343 HLSLAppliedSemanticAttr *Semantic,
344 std::optional<unsigned> Index);
345 llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
346 const clang::DeclaratorDecl *Decl,
347 HLSLAppliedSemanticAttr *Semantic,
348 std::optional<unsigned> Index);
349
350 void emitSPIRVUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
351 const clang::DeclaratorDecl *Decl,
352 HLSLAppliedSemanticAttr *Semantic,
353 std::optional<unsigned> Index);
354 void emitDXILUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
355 HLSLAppliedSemanticAttr *Semantic,
356 std::optional<unsigned> Index);
357 void emitUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
358 const clang::DeclaratorDecl *Decl,
359 HLSLAppliedSemanticAttr *Semantic,
360 std::optional<unsigned> Index);
361
362 bool initializeGlobalResourceArray(CodeGenFunction &CGF,
363 const VarDecl *ArrayDecl,
364 AggValueSlot &DestSlot);
365
366 llvm::Triple::ArchType getArch();
367
368 llvm::DenseMap<const clang::RecordType *, llvm::StructType *> LayoutTypes;
369 unsigned SPIRVLastAssignedInputSemanticLocation = 0;
370 unsigned SPIRVLastAssignedOutputSemanticLocation = 0;
371};
372
373} // namespace CodeGen
374} // namespace clang
375
376#endif
377