1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/IR/MatrixBuilder.h"
20#include "llvm/Support/DXILABI.h"
21
22using namespace clang;
23using namespace CodeGen;
24using namespace llvm;
25
26static uint64_t
27getEffectiveBarrierMemoryFlags(uint64_t MemoryFlags,
28 llvm::Triple::EnvironmentType Stage) {
29 constexpr uint64_t AllMemory =
30 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::ValidMask);
31 if (MemoryFlags != AllMemory || Stage == llvm::Triple::Library ||
32 Stage == llvm::Triple::UnknownEnvironment)
33 return MemoryFlags;
34
35 constexpr uint64_t GroupSharedMemory =
36 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::GroupSharedMemory);
37 constexpr uint64_t NodeMemory =
38 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::NodeMemory);
39 const bool HasVisibleGroup = Stage == llvm::Triple::Compute ||
40 Stage == llvm::Triple::Mesh ||
41 Stage == llvm::Triple::Amplification;
42 if (!HasVisibleGroup)
43 MemoryFlags &= ~GroupSharedMemory;
44 // Clang has no node shader stage yet, so no known stage exposes node memory.
45 return MemoryFlags & ~NodeMemory;
46}
47
48static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
49 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
50 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
51 "asdouble operands types mismatch");
52 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
53 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
54
55 llvm::Type *ResultType = CGF.DoubleTy;
56 int N = 1;
57 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
58 N = VTy->getNumElements();
59 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
60 }
61
62 if (CGF.CGM.getTarget().getTriple().isDXIL())
63 return CGF.Builder.CreateIntrinsic(
64 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
65 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
66
67 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
68 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
69 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
70 }
71
72 llvm::SmallVector<int> Mask;
73 for (int i = 0; i < N; i++) {
74 Mask.push_back(Elt: i);
75 Mask.push_back(Elt: i + N);
76 }
77
78 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
79
80 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
81}
82
83static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
84 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
85
86 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
87 Value *CMP;
88 Value *LastInstr;
89
90 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
91 FZeroConst = ConstantVector::getSplat(
92 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
93 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
94 CMP = CGF->Builder.CreateIntrinsic(
95 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
96 Args: {FCompInst});
97 } else {
98 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
99 }
100
101 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
102 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
103 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
104 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
105 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
106
107 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
108
109 CGF->Builder.SetInsertPoint(LT0);
110
111 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
112
113 LastInstr = CGF->Builder.CreateBr(Dest: End);
114 CGF->Builder.SetInsertPoint(End);
115 } else {
116 llvm_unreachable("Backend Codegen not supported.");
117 }
118
119 return LastInstr;
120}
121
122static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
123 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
124 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
125 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
126
127 CallArgList Args;
128 LValue Op1TmpLValue =
129 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
130 LValue Op2TmpLValue =
131 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
132
133 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
134 Args.reverseWritebacks();
135
136 Value *LowBits = nullptr;
137 Value *HighBits = nullptr;
138
139 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
140 llvm::Type *RetElementTy = CGF->Int32Ty;
141 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
142 RetElementTy = llvm::VectorType::get(
143 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
144 else if (auto *Op0MatTy =
145 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
146 RetElementTy = llvm::VectorType::get(
147 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
148 Op0MatTy->getNumColumns()));
149
150 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
151
152 Value *CI = CGF->Builder.CreateIntrinsic(
153 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
154
155 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
156 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
157 } else {
158 // For Non DXIL targets we generate the instructions.
159
160 if (!Op0->getType()->isVectorTy()) {
161 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
162 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
163
164 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
165 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
166 } else {
167 int NumElements = 1;
168 if (const auto *VecTy =
169 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
170 NumElements = VecTy->getNumElements();
171 else if (const auto *MatTy =
172 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
173 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
174
175 FixedVectorType *Uint32VecTy =
176 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
177 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
178 if (NumElements == 1) {
179 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
180 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
181 } else {
182 SmallVector<int> EvenMask, OddMask;
183 for (int I = 0, E = NumElements; I != E; ++I) {
184 EvenMask.push_back(Elt: I * 2);
185 OddMask.push_back(Elt: I * 2 + 1);
186 }
187 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
188 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
189 }
190 }
191 }
192 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
193 auto *LastInst =
194 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
195 CGF->EmitWritebacks(Args);
196 return LastInst;
197}
198
199static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
200 const CallExpr *E) {
201 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
202 llvm::Type *I32 = CGF.Int32Ty;
203
204 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
205 [[maybe_unused]] llvm::StructType *Struct4I32 =
206 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
207
208 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
209 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
210 Value *StructVal =
211 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
212 assert(StructVal->getType() == Struct4I32 &&
213 "dx.wave.ballot must return {i32,i32,i32,i32}");
214
215 // Reassemble struct to <4 x i32>
216 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
217 for (unsigned I = 0; I < 4; ++I) {
218 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
219 VecVal =
220 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
221 }
222
223 return VecVal;
224 }
225
226 if (CGF.CGM.getTarget().getTriple().isSPIRV())
227 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
228
229 llvm_unreachable(
230 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
231}
232
233static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
234 const CallExpr *E) {
235 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
236 QualType Op0Ty = E->getArg(Arg: 0)->getType();
237 llvm::Type *ResType = CGF.FloatTy;
238 uint64_t NumElements = 0;
239 if (Op0->getType()->isVectorTy()) {
240 NumElements =
241 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
242 ResType =
243 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
244 }
245 if (!Op0Ty->hasUnsignedIntegerRepresentation())
246 llvm_unreachable(
247 "f16tof32 operand must have an unsigned int representation");
248
249 if (CGF.CGM.getTriple().isDXIL())
250 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
251 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
252 Name: "hlsl.f16tof32");
253
254 if (CGF.CGM.getTriple().isSPIRV()) {
255 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
256 // Int16 and Float16 capabilities
257 auto *UnpackType =
258 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
259
260 if (NumElements == 0) {
261 // a scalar input - simply extract the first element of the unpacked
262 // vector
263 Value *Unpack = CGF.Builder.CreateIntrinsic(
264 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
265 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
266 }
267
268 // a vector input - build a congruent output vector by iterating through
269 // the input vector calling unpackhalf2x16 for each element
270 Value *Result = PoisonValue::get(T: ResType);
271 for (uint64_t I = 0; I < NumElements; I++) {
272 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
273 Value *Unpack = CGF.Builder.CreateIntrinsic(
274 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
275 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
276 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
277 }
278 return Result;
279 }
280
281 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
282}
283
284static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
285 const CallExpr *E) {
286 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
287 QualType Op0Ty = E->getArg(Arg: 0)->getType();
288 llvm::Type *ResType = CGF.IntTy;
289 uint64_t NumElements = 0;
290 if (Op0->getType()->isVectorTy()) {
291 NumElements =
292 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
293 ResType =
294 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
295 }
296 if (!Op0Ty->hasFloatingRepresentation())
297 llvm_unreachable("f32tof16 operand must have a float representation");
298
299 if (CGF.CGM.getTriple().isDXIL())
300 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
301 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
302 Name: "hlsl.f32tof16");
303
304 if (CGF.CGM.getTriple().isSPIRV()) {
305 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
306 // Int16 and Float16 capabilities
307 auto *PackType =
308 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
309
310 if (NumElements == 0) {
311 // a scalar input - simply insert the scalar in the first element
312 // of the 2 element float vector
313 Value *Float2 = Constant::getNullValue(Ty: PackType);
314 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
315 Value *Result = CGF.Builder.CreateIntrinsic(
316 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
317 return Result;
318 }
319
320 // a vector input - build a congruent output vector by iterating through
321 // the input vector calling packhalf2x16 for each element
322 Value *Result = PoisonValue::get(T: ResType);
323 for (uint64_t I = 0; I < NumElements; I++) {
324 Value *Float2 = Constant::getNullValue(Ty: PackType);
325 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
326 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
327 Value *Res = CGF.Builder.CreateIntrinsic(
328 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
329 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
330 }
331 return Result;
332 }
333
334 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
335}
336
337// Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
338// Set the memory scope: Workgroup for groupshared, otherwise Device.
339static llvm::SyncScope::ID getHLSLAtomicScope(CodeGenFunction &CGF,
340 const LValue &DestLV) {
341 StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared
342 ? "workgroup"
343 : "device";
344 return CGF.getLLVMContext().getOrInsertSyncScopeID(SSN: ScopeName);
345}
346
347// The destination can name one element of a vector, as in `buf[0].z` or
348// `gs[i]`. `LValue::getAddress` gives the address of the whole vector for such
349// an lvalue, so index into the vector to get the address of the element. Sema
350// rejects a multi-element swizzle, so the access is always a single element.
351static Address getHLSLAtomicDestAddr(CodeGenFunction &CGF,
352 const LValue &DestLV) {
353 if (!DestLV.isVectorElt() && !DestLV.isExtVectorElt())
354 return DestLV.getAddress();
355
356 Address VecAddr = DestLV.isVectorElt() ? DestLV.getVectorAddress()
357 : DestLV.getExtVectorAddress();
358 Value *Idx = DestLV.isVectorElt()
359 ? DestLV.getVectorIdx()
360 : llvm::ConstantInt::get(Ty: CGF.SizeTy,
361 V: CodeGenFunction::getAccessedFieldNo(
362 Idx: 0, Elts: DestLV.getExtVectorElts()));
363
364 // A vector-element lvalue reports the type of the whole vector, so take the
365 // element type from the address. HLSL also treats a scalar as a one-element
366 // vector, in which case the address already has the element type.
367 llvm::Type *VecTy = VecAddr.getElementType();
368 llvm::Type *ElemTy = VecTy->isVectorTy()
369 ? cast<llvm::VectorType>(Val: VecTy)->getElementType()
370 : VecTy;
371 return CGF.Builder.CreateGEP(CGF, Addr: VecAddr.withElementType(ElemTy), Index: Idx);
372}
373
374static LValue getHLSLAtomicOriginalValueLValue(CodeGenFunction &CGF,
375 const Expr *E,
376 CallArgList &Args) {
377 if (const auto *OutArg = dyn_cast<HLSLOutArgExpr>(Val: E))
378 return CGF.EmitHLSLOutArgExpr(E: OutArg, Args, Ty: OutArg->getType());
379 return CGF.EmitLValue(E);
380}
381
382static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
383 llvm::AtomicRMWInst::BinOp Op) {
384 // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
385 // either DXIL or SPIR-V.
386 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
387 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
388 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
389 [[maybe_unused]] QualType ValTy = E->getArg(Arg: 1)->getType();
390 if (Op == llvm::AtomicRMWInst::Xchg)
391 assert((ValTy->isIntegerType() || ValTy->isFloatingType()) &&
392 "InterlockedExchange value operand must be an integer or a float");
393 else
394 assert(ValTy->isIntegerType() &&
395 "Intrinsic InterlockedOp value operand must be an integer");
396
397 llvm::SyncScope::ID SSID = getHLSLAtomicScope(CGF, DestLV);
398
399 llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
400 Op, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
401
402 // The 3-arg overload writes the old value (the RMW's return value) into
403 // the `original_value` reference parameter.
404 if (E->getNumArgs() == 3) {
405 CallArgList Args;
406 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 2), Args);
407 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
408 CGF.EmitWritebacks(Args);
409 }
410 return Call;
411}
412
413// Emit `cmpxchg` for InterlockedCompareStore and InterlockedCompareExchange.
414// Compare-exchange also reports the previous value.
415static Value *handleInterlockedCompareOp(CodeGenFunction &CGF,
416 const CallExpr *E) {
417 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
418 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
419 Value *Compare = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
420 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
421
422 // `cmpxchg` takes an integer or a pointer, so the float-bitwise operations
423 // work on the bit pattern of the float. This is what those operations mean,
424 // and DXIL and SPIR-V both need the integer form.
425 llvm::Type *FloatTy = nullptr;
426 if (Compare->getType()->isFloatingPointTy()) {
427 FloatTy = Compare->getType();
428 llvm::Type *IntTy =
429 CGF.Builder.getIntNTy(N: FloatTy->getPrimitiveSizeInBits());
430 Compare = CGF.Builder.CreateBitCast(V: Compare, DestTy: IntTy);
431 Val = CGF.Builder.CreateBitCast(V: Val, DestTy: IntTy);
432 DestAddr = DestAddr.withElementType(ElemTy: IntTy);
433 }
434
435 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
436 Addr: DestAddr, Cmp: Compare, New: Val, SuccessOrdering: llvm::AtomicOrdering::Monotonic,
437 FailureOrdering: llvm::AtomicOrdering::Monotonic, SSID: getHLSLAtomicScope(CGF, DestLV));
438
439 // Compare-store reports nothing, so it leaves the `cmpxchg` result unused.
440 if (E->getNumArgs() < 4)
441 return Pair;
442
443 // `cmpxchg` yields a { previous value, success } pair. HLSL reports only the
444 // previous value, through the `original_value` reference parameter.
445 Value *Original = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
446 if (FloatTy)
447 Original = CGF.Builder.CreateBitCast(V: Original, DestTy: FloatTy);
448 CallArgList Args;
449 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 3), Args);
450 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Original), Dst: OrigLV);
451 CGF.EmitWritebacks(Args);
452 return Original;
453}
454
455static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
456 LValue &Stride) {
457 // Figure out the stride of the buffer elements from the handle type.
458 auto *HandleTy =
459 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
460 QualType ElementTy = HandleTy->getContainedType();
461 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
462 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
463}
464
465// Return dot product intrinsic that corresponds to the QT scalar type
466static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
467 if (QT->isFloatingType())
468 return RT.getFDotIntrinsic();
469 if (QT->isSignedIntegerType())
470 return RT.getSDotIntrinsic();
471 assert(QT->isUnsignedIntegerType());
472 return RT.getUDotIntrinsic();
473}
474
475static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
476 if (QT->hasSignedIntegerRepresentation()) {
477 return RT.getFirstBitSHighIntrinsic();
478 }
479
480 assert(QT->hasUnsignedIntegerRepresentation());
481 return RT.getFirstBitUHighIntrinsic();
482}
483
484// Return wave active sum that corresponds to the QT scalar type
485static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
486 QualType QT) {
487 switch (Arch) {
488 case llvm::Triple::spirv:
489 return Intrinsic::spv_wave_reduce_sum;
490 case llvm::Triple::dxil: {
491 if (QT->isUnsignedIntegerType())
492 return Intrinsic::dx_wave_reduce_usum;
493 return Intrinsic::dx_wave_reduce_sum;
494 }
495 default:
496 llvm_unreachable("Intrinsic WaveActiveSum"
497 " not supported by target architecture");
498 }
499}
500
501// Return wave active product that corresponds to the QT scalar type
502static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
503 QualType QT) {
504 switch (Arch) {
505 case llvm::Triple::spirv:
506 return Intrinsic::spv_wave_product;
507 case llvm::Triple::dxil: {
508 if (QT->isUnsignedIntegerType())
509 return Intrinsic::dx_wave_uproduct;
510 return Intrinsic::dx_wave_product;
511 }
512 default:
513 llvm_unreachable("Intrinsic WaveActiveProduct"
514 " not supported by target architecture");
515 }
516}
517
518static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
519 switch (Arch) {
520 case llvm::Triple::spirv:
521 return Intrinsic::spv_subgroup_prefix_bit_count;
522 case llvm::Triple::dxil: {
523 return Intrinsic::dx_wave_prefix_bit_count;
524 }
525 default:
526 llvm_unreachable(
527 "WavePrefixOp instruction not supported by target architecture");
528 }
529}
530
531// Return wave prefix sum that corresponds to the QT scalar type
532static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
533 QualType QT) {
534 switch (Arch) {
535 case llvm::Triple::spirv:
536 return Intrinsic::spv_wave_prefix_sum;
537 case llvm::Triple::dxil: {
538 if (QT->isUnsignedIntegerType())
539 return Intrinsic::dx_wave_prefix_usum;
540 return Intrinsic::dx_wave_prefix_sum;
541 }
542 default:
543 llvm_unreachable("Intrinsic WavePrefixSum"
544 " not supported by target architecture");
545 }
546}
547
548// Return wave prefix product that corresponds to the QT scalar type
549static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
550 QualType QT) {
551 switch (Arch) {
552 case llvm::Triple::spirv:
553 return Intrinsic::spv_wave_prefix_product;
554 case llvm::Triple::dxil: {
555 if (QT->isUnsignedIntegerType())
556 return Intrinsic::dx_wave_prefix_uproduct;
557 return Intrinsic::dx_wave_prefix_product;
558 }
559 default:
560 llvm_unreachable("Intrinsic WavePrefixProduct"
561 " not supported by target architecture");
562 }
563}
564
565// Returns the mangled name for a builtin function that the SPIR-V backend
566// will expand into a spec Constant.
567static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
568 ASTContext &Context) {
569 // The parameter types for our conceptual intrinsic function.
570 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
571
572 // Create a temporary FunctionDecl for the builtin fuction. It won't be
573 // added to the AST.
574 FunctionProtoType::ExtProtoInfo EPI;
575 QualType FnType =
576 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
577 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
578 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
579 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
580 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
581
582 // Attach the created parameter declarations to the function declaration.
583 SmallVector<ParmVarDecl *, 2> ParamDecls;
584 for (QualType ParamType : ClangParamTypes) {
585 ParmVarDecl *PD = ParmVarDecl::Create(
586 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
587 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
588 /*DefaultArg*/ DefArg: nullptr);
589 ParamDecls.push_back(Elt: PD);
590 }
591 FnDeclForMangling->setParams(ParamDecls);
592
593 // Get the mangled name.
594 std::string Name;
595 llvm::raw_string_ostream MangledNameStream(Name);
596 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
597 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
598 MangledNameStream.flush();
599
600 return Name;
601}
602
603static const HLSLAttributedResourceType *
604getHandleAttributedType(QualType HandleQT) {
605 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
606 return RT;
607 // If the expr is a texture/sampler record (or similar), peel to __handle.
608 if (const HLSLAttributedResourceType *RT =
609 HLSLAttributedResourceType::findHandleTypeOnResource(
610 RT: HandleQT.getTypePtr()))
611 return RT;
612 llvm_unreachable("attributed handle type not found");
613}
614
615static const HLSLAttributedResourceType *
616getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
617 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
618}
619
620static const FieldDecl *getResourceHandleField(QualType ResourceTy) {
621 const CXXRecordDecl *ResourceDecl = ResourceTy->getAsCXXRecordDecl();
622 assert(ResourceDecl && "resource must be a record type");
623
624 IdentifierInfo &II = ResourceDecl->getASTContext().Idents.get(Name: "__handle");
625 for (const Decl *D : ResourceDecl->lookup(Name: &II))
626 if (const auto *Field = dyn_cast<FieldDecl>(Val: D))
627 return Field;
628
629 llvm_unreachable("resource handle field not found");
630}
631
632static llvm::Type *getOffsetType(CodeGenModule &CGM,
633 const HLSLAttributedResourceType *RT) {
634 const auto &Attrs = RT->getAttrs();
635 unsigned OffsetSize =
636 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
637 llvm::Type *Int32Ty = CGM.Int32Ty;
638 if (OffsetSize == 1)
639 return Int32Ty;
640 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
641}
642
643static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
644 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
645 if (E->getNumArgs() > OffsetArgIndex)
646 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
647
648 return llvm::Constant::getNullValue(Ty: OffsetTy);
649}
650
651static Value *emitHlslSampleOffset(CodeGenFunction &CGF, const CallExpr *E,
652 const HLSLAttributedResourceType *RT,
653 unsigned OffsetArgIndex) {
654 llvm::Type *OffsetTy = getOffsetType(CGM&: CGF.CGM, RT);
655 if (!clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension))
656 return llvm::Constant::getNullValue(Ty: OffsetTy);
657 return emitHlslOffset(CGF, E, OffsetArgIndex, OffsetTy);
658}
659
660static unsigned getHlslClampArgIndex(const HLSLAttributedResourceType *RT,
661 unsigned OffsetArgIndex) {
662 return clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension)
663 ? OffsetArgIndex + 1
664 : OffsetArgIndex;
665}
666
667static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
668 unsigned IntrinsicID, unsigned NumRetComps,
669 bool HasLod) {
670 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
671
672 SmallVector<Value *> Args{Handle};
673 if (HasLod)
674 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
675
676 Value *DimValue =
677 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
678
679 Value *LastStore = nullptr;
680 unsigned ArgIndex = HasLod ? 2 : 1;
681 for (unsigned i = 0; i < NumRetComps; ++i) {
682 const Expr *Arg = E->getArg(Arg: ArgIndex++);
683 LValue DimOut = CGF.EmitLValue(E: Arg);
684 Value *Elem = DimValue;
685 if (NumRetComps > 1)
686 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
687
688 // Handle float casting if needed
689 if (Arg->getType()->isFloatingType())
690 Elem = CGF.Builder.CreateUIToFP(
691 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
692
693 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
694 }
695 return LastStore;
696}
697
698static llvm::Type *getAggregateType(llvm::Type *ScalarTy, QualType ArgTy) {
699 if (auto *MatTy = ArgTy->getAs<ConstantMatrixType>())
700 return llvm::VectorType::get(
701 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: MatTy->getNumElementsFlattened()));
702 if (auto *VecTy = ArgTy->getAs<clang::VectorType>())
703 return llvm::VectorType::get(
704 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()));
705 return ScalarTy;
706}
707
708Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
709 const CallExpr *E,
710 ReturnValueSlot ReturnValue) {
711 if (!getLangOpts().HLSL)
712 return nullptr;
713
714 switch (BuiltinID) {
715 case Builtin::BI__builtin_hlsl_adduint64: {
716 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
717 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
718 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
719 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
720 assert(Arg0Ty == E->getArg(1)->getType() &&
721 "AddUint64 operand types must match");
722 assert(Arg0Ty->hasIntegerRepresentation() &&
723 "AddUint64 operands must have an integer representation");
724 assert((NumElements == 2 || NumElements == 4) &&
725 "AddUint64 operands must have 2 or 4 elements");
726
727 llvm::Value *LowA;
728 llvm::Value *HighA;
729 llvm::Value *LowB;
730 llvm::Value *HighB;
731
732 // Obtain low and high words of inputs A and B
733 if (NumElements == 2) {
734 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
735 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
736 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
737 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
738 } else {
739 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
740 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
741 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
742 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
743 }
744
745 // Use an uadd_with_overflow to compute the sum of low words and obtain a
746 // carry value
747 llvm::Value *Carry;
748 llvm::Value *LowSum = EmitOverflowIntrinsic(
749 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
750 llvm::Value *ZExtCarry =
751 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
752
753 // Sum the high words and the carry
754 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
755 llvm::Value *HighSumPlusCarry =
756 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
757
758 if (NumElements == 4) {
759 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
760 Name: "hlsl.AddUint64");
761 }
762
763 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
764 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
765 Name: "hlsl.AddUint64.upto0");
766 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
767 Name: "hlsl.AddUint64");
768 return Result;
769 }
770 case Builtin::BI__builtin_hlsl_resource_getpointer:
771 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
772 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
773 bool IsIndexed =
774 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
775 E->getNumArgs() > 1;
776
777 llvm::Type *RetTy = ConvertType(T: E->getType());
778 llvm::Function *IntrFn = nullptr;
779 llvm::CallInst *CI = nullptr;
780 if (IsIndexed) {
781 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
782 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
783 M: &CGM.getModule(),
784 id: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
785 OverloadTys: {RetTy, HandleOp->getType(), IndexOp->getType()});
786 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp, IndexOp});
787 } else {
788 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
789 M: &CGM.getModule(),
790 id: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
791 OverloadTys: {RetTy, HandleOp->getType()});
792 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp});
793 }
794 CI->setCallingConv(IntrFn->getCallingConv());
795 return CI;
796 }
797 case Builtin::BI__builtin_hlsl_transpose_if_memory_is_row_major: {
798 const Expr *ValueExpr = E->getArg(Arg: 0);
799 if (hasAggregateEvaluationKind(T: ValueExpr->getType())) {
800 EmitAnyExprToMem(E: ValueExpr, Location: ReturnValue.getAddress(),
801 Quals: ValueExpr->getType().getQualifiers(), /*IsInit=*/IsInitializer: true);
802 return ReturnValue.getAddress().getBasePointer();
803 }
804
805 Value *ValueOp = EmitScalarExpr(E: ValueExpr);
806 const auto *MatTy = ValueExpr->getType()->getAs<ConstantMatrixType>();
807 if (!MatTy || !getLangOpts().HLSLSpvUseLegacyBufferMatrixOrder)
808 return ValueOp;
809
810 bool IsLoad =
811 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getBoolValue();
812 unsigned Rows = MatTy->getNumRows();
813 unsigned Columns = MatTy->getNumColumns();
814 llvm::MatrixBuilder MB(Builder);
815 return IsLoad ? MB.CreateMatrixTranspose(Matrix: ValueOp, Rows: Columns, Columns: Rows)
816 : MB.CreateMatrixTranspose(Matrix: ValueOp, Rows, Columns);
817 }
818 case Builtin::BI__builtin_hlsl_resource_sample: {
819 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
820 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
821 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
822 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
823
824 SmallVector<Value *, 4> Args;
825 Args.push_back(Elt: HandleOp);
826 Args.push_back(Elt: SamplerOp);
827 Args.push_back(Elt: CoordOp);
828 constexpr unsigned OffsetIdx = 3;
829 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
830
831 llvm::Type *RetTy = ConvertType(T: E->getType());
832 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
833 if (E->getNumArgs() <= ClampIdx)
834 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args,
835 RetTy);
836
837 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
838 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(),
839 Args, RetTy);
840 }
841 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
842 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
843 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
844 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
845 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
846 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
847
848 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
849 Args.push_back(Elt: HandleOp);
850 Args.push_back(Elt: SamplerOp);
851 Args.push_back(Elt: CoordOp);
852 Args.push_back(Elt: BiasOp);
853 constexpr unsigned OffsetIdx = 4;
854 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
855
856 llvm::Type *RetTy = ConvertType(T: E->getType());
857 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
858 if (E->getNumArgs() <= ClampIdx)
859 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(),
860 Args, RetTy);
861
862 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
863 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(),
864 Args, RetTy);
865 }
866 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
867 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
868 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
869 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
870 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
871 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
872 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
873
874 SmallVector<Value *, 7> Args;
875 Args.push_back(Elt: HandleOp);
876 Args.push_back(Elt: SamplerOp);
877 Args.push_back(Elt: CoordOp);
878 Args.push_back(Elt: DDXOp);
879 Args.push_back(Elt: DDYOp);
880 constexpr unsigned OffsetIdx = 5;
881 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
882
883 llvm::Type *RetTy = ConvertType(T: E->getType());
884
885 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
886 if (E->getNumArgs() <= ClampIdx)
887 return Builder.CreateIntrinsic(
888 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
889
890 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
891 return Builder.CreateIntrinsic(
892 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
893 }
894 case Builtin::BI__builtin_hlsl_resource_sample_level: {
895 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
896 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
897 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
898 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
899 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
900
901 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
902 Args.push_back(Elt: HandleOp);
903 Args.push_back(Elt: SamplerOp);
904 Args.push_back(Elt: CoordOp);
905 Args.push_back(Elt: LODOp);
906 constexpr unsigned OffsetIdx = 4;
907 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
908
909 llvm::Type *RetTy = ConvertType(T: E->getType());
910 return Builder.CreateIntrinsic(
911 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
912 }
913 case Builtin::BI__builtin_hlsl_resource_load_level: {
914 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
915 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
916 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
917
918 const auto &Attrs = RT->getAttrs();
919
920 Value *CoordOp = nullptr;
921 Value *LODOp = nullptr;
922 if (Attrs.ResourceClass == llvm::dxil::ResourceClass::UAV) {
923 // A UAV descriptor binds a single mip slice, so a RWTexture location is
924 // all coordinate and there is no mip level to select.
925 CoordOp = CoordLODOp;
926 LODOp = llvm::PoisonValue::get(T: Int32Ty);
927 } else {
928 // Split CoordLOD into Coord and LOD. 1D resources use a scalar
929 // coordinate rather than a 1-element vector.
930 unsigned CoordSize =
931 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension) +
932 (Attrs.IsArray ? 1 : 0);
933 assert(cast<llvm::FixedVectorType>(CoordLODOp->getType())
934 ->getNumElements() == CoordSize + 1 &&
935 "CoordLOD must have one element per coordinate, plus the level");
936
937 if (CoordSize == 1) {
938 CoordOp = Builder.CreateExtractElement(Vec: CoordLODOp, Idx: uint64_t(0),
939 Name: "hlsl.load.coord");
940 } else {
941 SmallVector<int, 4> Mask;
942 for (unsigned I = 0; I < CoordSize; ++I)
943 Mask.push_back(Elt: I);
944 CoordOp =
945 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
946 }
947 LODOp =
948 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: CoordSize, Name: "hlsl.load.lod");
949 }
950
951 SmallVector<Value *, 4> Args;
952 Args.push_back(Elt: HandleOp);
953 Args.push_back(Elt: CoordOp);
954 Args.push_back(Elt: LODOp);
955 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
956
957 llvm::Type *RetTy = ConvertType(T: E->getType());
958 return Builder.CreateIntrinsic(
959 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
960 }
961 case Builtin::BI__builtin_hlsl_resource_load_ms: {
962 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
963 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 1));
964 Value *SampleOp = EmitScalarExpr(E: E->getArg(Arg: 2));
965 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
966
967 SmallVector<Value *, 4> Args;
968 Args.push_back(Elt: HandleOp);
969 Args.push_back(Elt: CoordOp);
970 Args.push_back(Elt: SampleOp);
971 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
972
973 llvm::Type *RetTy = ConvertType(T: E->getType());
974 return Builder.CreateIntrinsic(
975 RetTy, ID: CGM.getHLSLRuntime().getLoadMSIntrinsic(), Args);
976 }
977 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
978 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
979 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
980 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
981 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
982 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
983
984 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
985 Args.push_back(Elt: HandleOp);
986 Args.push_back(Elt: SamplerOp);
987 Args.push_back(Elt: CoordOp);
988 Args.push_back(Elt: CmpOp);
989 constexpr unsigned OffsetIdx = 4;
990 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
991
992 llvm::Type *RetTy = ConvertType(T: E->getType());
993 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
994 if (E->getNumArgs() <= ClampIdx)
995 return Builder.CreateIntrinsic(
996 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
997
998 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
999 return Builder.CreateIntrinsic(
1000 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
1001 }
1002 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
1003 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1004 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1005 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1006 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1007 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1008
1009 SmallVector<Value *, 5> Args;
1010 Args.push_back(Elt: HandleOp);
1011 Args.push_back(Elt: SamplerOp);
1012 Args.push_back(Elt: CoordOp);
1013 Args.push_back(Elt: CmpOp);
1014 constexpr unsigned OffsetIdx = 4;
1015 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
1016
1017 llvm::Type *RetTy = ConvertType(T: E->getType());
1018 return Builder.CreateIntrinsic(
1019 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
1020 }
1021 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
1022 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1023 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1024 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1025
1026 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
1027 Args: {HandleOp, SamplerOp, CoordOp},
1028 RetTy: ConvertType(T: E->getType()));
1029 }
1030 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
1031 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1032 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1033 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1034
1035 return EmitIntrinsicCall(
1036 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
1037 Args: {HandleOp, SamplerOp, CoordOp}, RetTy: ConvertType(T: E->getType()));
1038 }
1039 case Builtin::BI__builtin_hlsl_resource_gather: {
1040 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1041 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1042 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1043 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1044 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1045
1046 SmallVector<Value *, 5> Args;
1047 Args.push_back(Elt: HandleOp);
1048 Args.push_back(Elt: SamplerOp);
1049 Args.push_back(Elt: CoordOp);
1050 Args.push_back(Elt: ComponentOp);
1051 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
1052
1053 llvm::Type *RetTy = ConvertType(T: E->getType());
1054 return Builder.CreateIntrinsic(
1055 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
1056 }
1057 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
1058 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1059 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1060 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1061 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1062
1063 SmallVector<Value *, 6> Args;
1064 Args.push_back(Elt: HandleOp);
1065 Args.push_back(Elt: SamplerOp);
1066 Args.push_back(Elt: CoordOp);
1067 Args.push_back(Elt: CompareOp);
1068
1069 if (CGM.getTarget().getTriple().isDXIL()) {
1070 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1071 Args.push_back(Elt: ComponentOp);
1072 }
1073
1074 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1075 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
1076
1077 llvm::Type *RetTy = ConvertType(T: E->getType());
1078 return Builder.CreateIntrinsic(
1079 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
1080 }
1081 case Builtin::BI__builtin_hlsl_resource_load_with_status:
1082 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
1083 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1084 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1085
1086 // Get the *address* of the status argument to write to it by reference
1087 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
1088 Address StatusAddr = StatusLVal.getAddress();
1089
1090 QualType HandleTy = E->getArg(Arg: 0)->getType();
1091 const HLSLAttributedResourceType *RT =
1092 HandleTy->getAs<HLSLAttributedResourceType>();
1093 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
1094 "Only DXIL currently implements load with status");
1095
1096 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
1097 ? llvm::Intrinsic::dx_resource_load_rawbuffer
1098 : llvm::Intrinsic::dx_resource_load_typedbuffer;
1099
1100 llvm::Type *DataTy = ConvertType(T: E->getType());
1101 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
1102 Elements: {DataTy, Builder.getInt1Ty()});
1103
1104 SmallVector<Value *, 3> Args;
1105 Args.push_back(Elt: HandleOp);
1106 Args.push_back(Elt: IndexOp);
1107
1108 if (RT->isRaw()) {
1109 Value *Offset = Builder.getInt32(C: 0);
1110 // The offset parameter needs to be poison for ByteAddressBuffer
1111 if (!RT->isStructured())
1112 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
1113 Args.push_back(Elt: Offset);
1114 }
1115
1116 // The load intrinsics give us a (T value, i1 status) pair -
1117 // shepherd these into the return value and out reference respectively.
1118 Value *ResRet =
1119 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
1120 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
1121 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
1122 Value *ExtendedStatus =
1123 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
1124 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
1125
1126 return LoadedValue;
1127 }
1128 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
1129 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1130 return llvm::PoisonValue::get(T: HandleTy);
1131 }
1132 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
1133 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1134 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1135 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1136 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1137 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1138 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1139 llvm::Intrinsic::ID IntrinsicID =
1140 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1141 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
1142 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1143 }
1144 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
1145 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1146 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1147 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1148 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1149 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1150 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1151 llvm::Intrinsic::ID IntrinsicID =
1152 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1153 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
1154 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1155 }
1156 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
1157 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1158 if (!CGM.getTriple().isSPIRV())
1159 return MainHandle;
1160
1161 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1162 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1163 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1164 llvm::Intrinsic::ID IntrinsicID =
1165 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
1166 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
1167 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1168 Args);
1169 }
1170 case Builtin::BI__builtin_hlsl_resource_handlefromheap: {
1171 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1172 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1173 llvm::Intrinsic::ID IntrinsicID =
1174 CGM.getHLSLRuntime().getCreateHandleFromHeapIntrinsic();
1175 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args: {IndexOp});
1176 }
1177 case Builtin::BI__builtin_hlsl_resource_counterhandlefromheap: {
1178 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1179 if (!CGM.getTriple().isSPIRV())
1180 return MainHandle;
1181
1182 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1183 llvm::Intrinsic::ID IntrinsicID =
1184 llvm::Intrinsic::spv_resource_counterhandlefromheap;
1185 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1186 Args: {MainHandle});
1187 }
1188
1189 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
1190 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1191 llvm::Type *RetTy = ConvertType(T: E->getType());
1192 return Builder.CreateIntrinsic(
1193 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
1194 Args: ArrayRef<Value *>{IndexOp});
1195 }
1196 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
1197 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
1198 return emitGetDimensions(CGF&: *this, E,
1199 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
1200 NumRetComps: 1, /*HasLod=*/false);
1201 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
1202 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
1203 return emitGetDimensions(CGF&: *this, E,
1204 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1205 NumRetComps: 2, /*HasLod=*/false);
1206 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1207 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1208 return emitGetDimensions(
1209 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1210 /*HasLod=*/true);
1211 case Builtin::BI__builtin_hlsl_resource_getstride: {
1212 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1213 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1214 }
1215 case Builtin::BI__builtin_hlsl_all: {
1216 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1217 return Builder.CreateIntrinsic(
1218 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1219 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1220 Name: "hlsl.all");
1221 }
1222 case Builtin::BI__builtin_hlsl_and: {
1223 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1224 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1225 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1226 }
1227 case Builtin::BI__builtin_hlsl_or: {
1228 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1229 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1230 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1231 }
1232 case Builtin::BI__builtin_hlsl_any: {
1233 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1234 return Builder.CreateIntrinsic(
1235 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1236 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1237 Name: "hlsl.any");
1238 }
1239 case Builtin::BI__builtin_hlsl_asdouble:
1240 return handleAsDoubleBuiltin(CGF&: *this, E);
1241 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1242 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1243 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1244 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1245
1246 QualType Ty = E->getArg(Arg: 0)->getType();
1247 if (auto *VecTy = Ty->getAs<VectorType>())
1248 Ty = VecTy->getElementType();
1249
1250 Intrinsic::ID Intr;
1251 if (Ty->isFloatingType()) {
1252 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1253 } else if (Ty->isUnsignedIntegerType()) {
1254 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1255 } else {
1256 assert(Ty->isSignedIntegerType());
1257 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1258 }
1259 return Builder.CreateIntrinsic(
1260 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1261 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1262 }
1263 case Builtin::BI__builtin_hlsl_dot: {
1264 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1265 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1266 llvm::Type *T0 = Op0->getType();
1267 llvm::Type *T1 = Op1->getType();
1268
1269 // If the arguments are scalars, just emit a multiply
1270 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1271 if (T0->isFloatingPointTy())
1272 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1273
1274 if (T0->isIntegerTy())
1275 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1276
1277 llvm_unreachable(
1278 "Scalar dot product is only supported on ints and floats.");
1279 }
1280 // For vectors, validate types and emit the appropriate intrinsic
1281 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1282 E->getArg(1)->getType()) &&
1283 "Dot product operands must have the same type.");
1284
1285 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1286 assert(VecTy0 && "Dot product argument must be a vector.");
1287
1288 return Builder.CreateIntrinsic(
1289 /*ReturnType=*/RetTy: T0->getScalarType(),
1290 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1291 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1292 }
1293 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1294 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1295 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1296 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1297
1298 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1299 // Note that the argument order disagrees between the builtin and the
1300 // intrinsic here.
1301 return Builder.CreateIntrinsic(
1302 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1303 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1304 }
1305 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1306 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1307 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1308 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1309
1310 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1311 // Note that the argument order disagrees between the builtin and the
1312 // intrinsic here.
1313 return Builder.CreateIntrinsic(
1314 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1315 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1316 }
1317 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1318 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1319
1320 return Builder.CreateIntrinsic(
1321 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1322 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1323 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1324 }
1325 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1326 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1327
1328 return Builder.CreateIntrinsic(
1329 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1330 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1331 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1332 }
1333 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1334 return handleElementwiseF16ToF32(CGF&: *this, E);
1335 }
1336 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1337 return handleElementwiseF32ToF16(CGF&: *this, E);
1338 }
1339 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1340 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1341 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1342 llvm_unreachable("frac operand must have a float representation");
1343 return Builder.CreateIntrinsic(
1344 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1345 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1346 }
1347 case Builtin::BI__builtin_hlsl_elementwise_isfinite: {
1348 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1349 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1350 llvm_unreachable("isfinite operand must have a float representation");
1351 llvm::Type *retType = getAggregateType(
1352 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1353 return Builder.CreateIntrinsic(
1354 RetTy: retType, ID: CGM.getHLSLRuntime().getIsFiniteIntrinsic(),
1355 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isfinite");
1356 }
1357 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1358 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1359 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1360 llvm_unreachable("isinf operand must have a float representation");
1361 llvm::Type *retType = getAggregateType(
1362 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1363 return Builder.CreateIntrinsic(
1364 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1365 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1366 }
1367 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1368 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1369 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1370 llvm_unreachable("isnan operand must have a float representation");
1371 llvm::Type *retType = getAggregateType(
1372 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1373 return Builder.CreateIntrinsic(
1374 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1375 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1376 }
1377 case Builtin::BI__builtin_hlsl_mad: {
1378 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1379 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1380 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1381 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1382 return Builder.CreateIntrinsic(
1383 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1384 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1385
1386 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1387 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1388 return Builder.CreateIntrinsic(
1389 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1390 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1391
1392 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1393 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1394 }
1395 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1396 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1397 return Builder.CreateIntrinsic(
1398 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1399 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1400
1401 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1402 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1403 }
1404 case Builtin::BI__builtin_hlsl_mul: {
1405 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1406 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1407 QualType QTy0 = E->getArg(Arg: 0)->getType();
1408 QualType QTy1 = E->getArg(Arg: 1)->getType();
1409
1410 bool IsVec0 = QTy0->isVectorType();
1411 bool IsVec1 = QTy1->isVectorType();
1412 bool IsMat0 = QTy0->isConstantMatrixType();
1413 bool IsMat1 = QTy1->isConstantMatrixType();
1414
1415 llvm::MatrixBuilder MB(Builder);
1416 if (IsVec0 && IsMat1) {
1417 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1418 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1419 unsigned Cols = MatTy->getNumColumns();
1420 assert(N == MatTy->getNumRows() &&
1421 "vector length must match matrix row count");
1422 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1423 }
1424 if (IsMat0 && IsVec1) {
1425 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1426 unsigned Rows = MatTy->getNumRows();
1427 unsigned Cols = MatTy->getNumColumns();
1428 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1429 "vector length must match matrix column count");
1430 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1431 }
1432 assert(IsMat0 && IsMat1);
1433 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1434 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1435 unsigned Rows0 = MatTy0->getNumRows();
1436 unsigned Cols0 = MatTy0->getNumColumns();
1437 unsigned Cols1 = MatTy1->getNumColumns();
1438 assert(Cols0 == MatTy1->getNumRows() &&
1439 "inner matrix dimensions must match for multiplication");
1440 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1441 }
1442 case Builtin::BI__builtin_hlsl_transpose: {
1443 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1444 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1445 unsigned Rows = MatTy->getNumRows();
1446 unsigned Cols = MatTy->getNumColumns();
1447 if (auto *Transpose = dyn_cast<CallInst>(Val: Op0);
1448 Transpose &&
1449 Transpose->getIntrinsicID() == Intrinsic::matrix_transpose &&
1450 Transpose->use_empty() &&
1451 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 1))->getZExtValue() ==
1452 Cols &&
1453 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 2))->getZExtValue() ==
1454 Rows) {
1455 Value *Result = Transpose->getArgOperand(i: 0);
1456 Transpose->eraseFromParent();
1457 return Result;
1458 }
1459 llvm::MatrixBuilder MB(Builder);
1460 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1461 }
1462 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1463 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1464 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1465 llvm_unreachable("rcp operand must have a float representation");
1466 llvm::Type *Ty = Op0->getType();
1467 llvm::Type *EltTy = Ty->getScalarType();
1468 Constant *One = Ty->isVectorTy()
1469 ? ConstantVector::getSplat(
1470 EC: ElementCount::getFixed(
1471 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1472 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1473 : ConstantFP::get(Ty: EltTy, V: 1.0);
1474 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1475 }
1476 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1477 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1478 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1479 llvm_unreachable("rsqrt operand must have a float representation");
1480 return Builder.CreateIntrinsic(
1481 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1482 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1483 }
1484 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1485 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1486 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1487 "saturate operand must have a float representation");
1488 return Builder.CreateIntrinsic(
1489 /*ReturnType=*/RetTy: Op0->getType(),
1490 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1491 FMFSource: nullptr, Name: "hlsl.saturate");
1492 }
1493 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1494 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1495 assert(Op->getType()->isIntegerTy(1) &&
1496 "WavePrefixBitCount operand must be a boolean type");
1497
1498 Intrinsic::ID IID =
1499 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1500
1501 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1502 }
1503 case Builtin::BI__builtin_hlsl_select: {
1504 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1505 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1506 Value *OpTrue =
1507 RValTrue.isScalar()
1508 ? RValTrue.getScalarVal()
1509 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1510 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1511 Value *OpFalse =
1512 RValFalse.isScalar()
1513 ? RValFalse.getScalarVal()
1514 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1515
1516 unsigned NumElements = 0;
1517 if (auto *VTy = E->getType()->getAs<VectorType>())
1518 NumElements = VTy->getNumElements();
1519 else if (auto *MTy = E->getType()->getAs<ConstantMatrixType>())
1520 NumElements = MTy->getNumElementsFlattened();
1521 if (NumElements) {
1522 if (!OpTrue->getType()->isVectorTy())
1523 OpTrue = Builder.CreateVectorSplat(NumElts: NumElements, V: OpTrue, Name: "splat");
1524 if (!OpFalse->getType()->isVectorTy())
1525 OpFalse = Builder.CreateVectorSplat(NumElts: NumElements, V: OpFalse, Name: "splat");
1526 }
1527
1528 Value *SelectVal =
1529 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1530 if (!RValTrue.isScalar())
1531 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1532 IsVolatile: ReturnValue.isVolatile());
1533
1534 return SelectVal;
1535 }
1536 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1537 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1538
1539 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1540 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1541 }
1542 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1543 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1544 assert(Op->getType()->isIntegerTy(1) &&
1545 "Intrinsic WaveActiveAllTrue operand must be a bool");
1546
1547 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1548 return EmitIntrinsicCall(ID, Args: {Op});
1549 }
1550 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1551 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1552 assert(Op->getType()->isIntegerTy(1) &&
1553 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1554
1555 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1556 return EmitIntrinsicCall(ID, Args: {Op});
1557 }
1558 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1559 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1560 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1561 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1562 "representation");
1563
1564 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1565 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1566 Name: "hlsl.wave.active.bit.or");
1567 }
1568 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1569 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1570 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1571 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1572 "representation");
1573
1574 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1575 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1576 Name: "hlsl.wave.active.bit.xor");
1577 }
1578 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1579 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1580 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1581 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1582 "representation");
1583
1584 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1585 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1586 Name: "hlsl.wave.active.bit.and");
1587 }
1588 case Builtin::BI__builtin_hlsl_interlocked_add: {
1589 // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
1590 // up the raw instruction (DXIL routes it via `dx.resource.atomic.binop`
1591 // in DXILResourceAccess for resource pointers, SPIR-V lowers via
1592 // selectAtomicRMW). No intermediate intrinsic.
1593 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Add);
1594 }
1595 case Builtin::BI__builtin_hlsl_interlocked_and: {
1596 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::And);
1597 }
1598 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange:
1599 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange_float_bitwise:
1600 case Builtin::BI__builtin_hlsl_interlocked_compare_store:
1601 case Builtin::BI__builtin_hlsl_interlocked_compare_store_float_bitwise: {
1602 return handleInterlockedCompareOp(CGF&: *this, E);
1603 }
1604 case Builtin::BI__builtin_hlsl_interlocked_exchange: {
1605 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xchg);
1606 }
1607 case Builtin::BI__builtin_hlsl_interlocked_max: {
1608 llvm::AtomicRMWInst::BinOp Op =
1609 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1610 ? llvm::AtomicRMWInst::Max
1611 : llvm::AtomicRMWInst::UMax;
1612 return handleInterlockedOp(CGF&: *this, E, Op);
1613 }
1614 case Builtin::BI__builtin_hlsl_interlocked_min: {
1615 llvm::AtomicRMWInst::BinOp Op =
1616 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1617 ? llvm::AtomicRMWInst::Min
1618 : llvm::AtomicRMWInst::UMin;
1619 return handleInterlockedOp(CGF&: *this, E, Op);
1620 }
1621 case Builtin::BI__builtin_hlsl_interlocked_or: {
1622 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Or);
1623 }
1624 case Builtin::BI__builtin_hlsl_interlocked_xor: {
1625 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xor);
1626 }
1627 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1628 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1629 assert(Op->getType()->isIntegerTy(1) &&
1630 "Intrinsic WaveActiveBallot operand must be a bool");
1631
1632 return handleHlslWaveActiveBallot(CGF&: *this, E);
1633 }
1634 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1635 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1636 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1637 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1638 }
1639 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1640 // Due to the use of variadic arguments, explicitly retrieve argument
1641 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1642 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1643 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1644
1645 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1646 Name: "hlsl.wave.active.sum");
1647 }
1648 case Builtin::BI__builtin_hlsl_wave_active_product: {
1649 // Due to the use of variadic arguments, explicitly retrieve argument
1650 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1651 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1652 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1653
1654 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1655 Name: "hlsl.wave.active.product");
1656 }
1657 case Builtin::BI__builtin_hlsl_wave_active_max: {
1658 // Due to the use of variadic arguments, explicitly retrieve argument
1659 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1660 QualType QT = E->getArg(Arg: 0)->getType();
1661 Intrinsic::ID IID;
1662 if (QT->isUnsignedIntegerType())
1663 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1664 else
1665 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1666
1667 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1668 Name: "hlsl.wave.active.max");
1669 }
1670 case Builtin::BI__builtin_hlsl_wave_active_min: {
1671 // Due to the use of variadic arguments, explicitly retrieve argument
1672 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1673 QualType QT = E->getArg(Arg: 0)->getType();
1674 Intrinsic::ID IID;
1675 if (QT->isUnsignedIntegerType())
1676 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1677 else
1678 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1679
1680 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1681 Name: "hlsl.wave.active.min");
1682 }
1683 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1684 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1685 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1686 // for the DirectX intrinsic and the demangled builtin name
1687 switch (CGM.getTarget().getTriple().getArch()) {
1688 case llvm::Triple::dxil:
1689 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1690 case llvm::Triple::spirv:
1691 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1692 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1693 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1694 default:
1695 llvm_unreachable(
1696 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1697 }
1698 }
1699 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1700 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1701 return EmitIntrinsicCall(ID);
1702 }
1703 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1704 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1705 return EmitIntrinsicCall(ID);
1706 }
1707 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1708 // Due to the use of variadic arguments we must explicitly retrieve them and
1709 // create our function type.
1710 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1711 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1712 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1713 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1714 Name: "hlsl.wave.readlane");
1715 }
1716 case Builtin::BI__builtin_hlsl_wave_read_lane_first: {
1717 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1718 return EmitIntrinsicCall(
1719 ID: CGM.getHLSLRuntime().getWaveReadLaneFirstIntrinsic(),
1720 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr}, Name: "hlsl.wave.readlane.first");
1721 }
1722 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1723 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1724 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1725 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1726 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1727 Name: "hlsl.wave.prefix.sum");
1728 }
1729 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1730 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1731 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1732 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1733 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1734 Name: "hlsl.wave.prefix.product");
1735 }
1736 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1737 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1738 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1739 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1740 Name: "hlsl.quad.read.across.x");
1741 }
1742 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1743 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1744 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1745 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1746 Name: "hlsl.quad.read.across.y");
1747 }
1748 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1749 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1750 Intrinsic::ID ID =
1751 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1752 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1753 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1754 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.diagonal");
1755 }
1756 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1757 auto *Arg0 = E->getArg(Arg: 0);
1758 Value *Op0 = EmitScalarExpr(E: Arg0);
1759 llvm::Type *Xty = Op0->getType();
1760 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1761 if (Xty->isVectorTy()) {
1762 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1763 retType = llvm::VectorType::get(
1764 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1765 }
1766 assert((Arg0->getType()->hasFloatingRepresentation() ||
1767 Arg0->getType()->hasIntegerRepresentation()) &&
1768 "sign operand must have a float or int representation");
1769
1770 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1771 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1772 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1773 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1774 }
1775
1776 return Builder.CreateIntrinsic(
1777 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1778 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1779 }
1780 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1781 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1782 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1783 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1784 return Builder.CreateIntrinsic(
1785 /*ReturnType=*/RetTy: Offset->getType(),
1786 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1787 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1788 }
1789 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1790
1791 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1792 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1793 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1794 "asuint operands types mismatch");
1795 return handleHlslSplitdouble(E, CGF: this);
1796 }
1797 case Builtin::BI__builtin_hlsl_elementwise_clip:
1798 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1799 "clip operands types mismatch");
1800 return handleHlslClip(E, CGF: this);
1801 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1802 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1803 return EmitIntrinsicCall(ID);
1804 }
1805 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1806 Intrinsic::ID ID =
1807 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1808 return EmitIntrinsicCall(ID);
1809 }
1810 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1811 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1812 return EmitIntrinsicCall(ID);
1813 }
1814 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1815 Intrinsic::ID ID =
1816 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1817 return EmitIntrinsicCall(ID);
1818 }
1819 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1820 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1821 return EmitIntrinsicCall(ID);
1822 }
1823 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1824 Intrinsic::ID ID =
1825 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1826 return EmitIntrinsicCall(ID);
1827 }
1828 case Builtin::BI__builtin_hlsl_barrier: {
1829 std::optional<llvm::APSInt> SemanticFlagsConstant =
1830 E->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
1831 assert(SemanticFlagsConstant && "expected constant semantic flags");
1832 constexpr uint64_t GroupScope =
1833 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::GroupScope);
1834 constexpr uint64_t DeviceScope =
1835 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::DeviceScope);
1836 constexpr uint64_t ScopeMask = GroupScope | DeviceScope;
1837 uint64_t SemanticFlagsValue = SemanticFlagsConstant->getZExtValue();
1838 // DEVICE_SCOPE includes GROUP_SCOPE. Remove GROUP_SCOPE when both are
1839 // set.
1840 if ((SemanticFlagsValue & ScopeMask) == ScopeMask)
1841 SemanticFlagsValue &= ~GroupScope;
1842 Value *SemanticFlags = llvm::ConstantInt::get(
1843 Ty: ConvertType(T: E->getArg(Arg: 1)->getType()), V: SemanticFlagsValue);
1844
1845 if (E->getArg(Arg: 0)->getType()->isUnsignedIntegerType()) {
1846 std::optional<llvm::APSInt> MemoryFlagsConstant =
1847 E->getArg(Arg: 0)->getIntegerConstantExpr(Ctx: getContext());
1848 assert(MemoryFlagsConstant && "expected constant memory flags");
1849 uint64_t MemoryFlagsValue = MemoryFlagsConstant->getZExtValue();
1850 llvm::Triple::EnvironmentType Stage =
1851 getTarget().getTriple().getEnvironment();
1852 if (Stage == llvm::Triple::Library)
1853 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
1854 if (const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>())
1855 Stage = ShaderAttr->getType();
1856 MemoryFlagsValue =
1857 getEffectiveBarrierMemoryFlags(MemoryFlags: MemoryFlagsValue, Stage);
1858 Value *MemoryFlags = llvm::ConstantInt::get(
1859 Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), V: MemoryFlagsValue);
1860 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryTypeIntrinsic();
1861 return EmitIntrinsicCall(ID, Types: {},
1862 Args: ArrayRef<Value *>{MemoryFlags, SemanticFlags});
1863 }
1864
1865 const FieldDecl *HandleField =
1866 getResourceHandleField(ResourceTy: E->getArg(Arg: 0)->getType());
1867 LValue Resource = EmitLValue(E: E->getArg(Arg: 0));
1868 LValue Handle = EmitLValueForField(Base: Resource, Field: HandleField);
1869 Value *HandleValue =
1870 EmitLoadOfLValue(V: Handle, Loc: E->getArg(Arg: 0)->getExprLoc()).getScalarVal();
1871 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryHandleIntrinsic();
1872 return EmitIntrinsicCall(ID, Types: ArrayRef<llvm::Type *>{HandleValue->getType()},
1873 Args: ArrayRef<Value *>{HandleValue, SemanticFlags});
1874 }
1875 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1876 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1877 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1878 llvm_unreachable("ddx_coarse operand must have a float representation");
1879 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1880 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1881 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1882 Name: "hlsl.ddx.coarse");
1883 }
1884 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1885 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1886 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1887 llvm_unreachable("ddy_coarse operand must have a float representation");
1888 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1889 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1890 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1891 Name: "hlsl.ddy.coarse");
1892 }
1893 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1894 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1895 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1896 llvm_unreachable("ddx_fine operand must have a float representation");
1897 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1898 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1899 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1900 Name: "hlsl.ddx.fine");
1901 }
1902 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1903 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1904 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1905 llvm_unreachable("ddy_fine operand must have a float representation");
1906 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1907 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1908 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1909 Name: "hlsl.ddy.fine");
1910 }
1911 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1912 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1913 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1914 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1915 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1916 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1917 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1918 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1919 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1920 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1921 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1922 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1923 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1924 llvm::Value *Args[] = {SpecId, DefaultVal};
1925 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1926 }
1927 }
1928 return nullptr;
1929}
1930
1931llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1932 const clang::QualType &SpecConstantType) {
1933
1934 // Find or create the declaration for the function.
1935 llvm::Module *M = &CGM.getModule();
1936 std::string MangledName =
1937 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1938 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1939
1940 if (!SpecConstantFn) {
1941 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1942 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1943 llvm::Type *ArgTypes[] = {IntType, RetTy};
1944 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1945 SpecConstantFn = llvm::Function::Create(
1946 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1947 }
1948 return SpecConstantFn;
1949}
1950