1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/IR/MatrixBuilder.h"
20#include "llvm/Support/DXILABI.h"
21
22using namespace clang;
23using namespace CodeGen;
24using namespace llvm;
25
26static uint64_t
27getEffectiveBarrierMemoryFlags(uint64_t MemoryFlags,
28 llvm::Triple::EnvironmentType Stage) {
29 constexpr uint64_t AllMemory =
30 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::ValidMask);
31 if (MemoryFlags != AllMemory || Stage == llvm::Triple::Library ||
32 Stage == llvm::Triple::UnknownEnvironment)
33 return MemoryFlags;
34
35 constexpr uint64_t GroupSharedMemory =
36 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::GroupSharedMemory);
37 constexpr uint64_t NodeMemory =
38 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::NodeMemory);
39 const bool HasVisibleGroup = Stage == llvm::Triple::Compute ||
40 Stage == llvm::Triple::Mesh ||
41 Stage == llvm::Triple::Amplification;
42 if (!HasVisibleGroup)
43 MemoryFlags &= ~GroupSharedMemory;
44 // Clang has no node shader stage yet, so no known stage exposes node memory.
45 return MemoryFlags & ~NodeMemory;
46}
47
48static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
49 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
50 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
51 "asdouble operands types mismatch");
52 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
53 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
54
55 llvm::Type *ResultType = CGF.DoubleTy;
56 int N = 1;
57 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
58 N = VTy->getNumElements();
59 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
60 }
61
62 if (CGF.CGM.getTarget().getTriple().isDXIL())
63 return CGF.Builder.CreateIntrinsic(
64 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
65 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
66
67 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
68 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
69 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
70 }
71
72 llvm::SmallVector<int> Mask;
73 for (int i = 0; i < N; i++) {
74 Mask.push_back(Elt: i);
75 Mask.push_back(Elt: i + N);
76 }
77
78 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
79
80 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
81}
82
83static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
84 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
85
86 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
87 Value *CMP;
88 Value *LastInstr;
89
90 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
91 FZeroConst = ConstantVector::getSplat(
92 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
93 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
94 CMP = CGF->Builder.CreateIntrinsic(
95 RetTy: CGF->Builder.getInt1Ty(), ID: Intrinsic::vector_reduce_or, Args: {FCompInst});
96 } else {
97 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
98 }
99
100 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
101 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
102 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
103 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
104 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
105
106 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
107
108 CGF->Builder.SetInsertPoint(LT0);
109
110 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
111
112 LastInstr = CGF->Builder.CreateBr(Dest: End);
113 CGF->Builder.SetInsertPoint(End);
114 } else {
115 llvm_unreachable("Backend Codegen not supported.");
116 }
117
118 return LastInstr;
119}
120
121static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
122 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
123 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
124 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
125
126 CallArgList Args;
127 LValue Op1TmpLValue =
128 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
129 LValue Op2TmpLValue =
130 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
131
132 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
133 Args.reverseWritebacks();
134
135 Value *LowBits = nullptr;
136 Value *HighBits = nullptr;
137
138 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
139 llvm::Type *RetElementTy = CGF->Int32Ty;
140 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
141 RetElementTy = llvm::VectorType::get(
142 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
143 else if (auto *Op0MatTy =
144 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
145 RetElementTy = llvm::VectorType::get(
146 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
147 Op0MatTy->getNumColumns()));
148
149 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
150
151 Value *CI = CGF->Builder.CreateIntrinsic(
152 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
153
154 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
155 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
156 } else {
157 // For Non DXIL targets we generate the instructions.
158
159 if (!Op0->getType()->isVectorTy()) {
160 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
161 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
162
163 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
164 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
165 } else {
166 int NumElements = 1;
167 if (const auto *VecTy =
168 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
169 NumElements = VecTy->getNumElements();
170 else if (const auto *MatTy =
171 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
172 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
173
174 FixedVectorType *Uint32VecTy =
175 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
176 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
177 if (NumElements == 1) {
178 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
179 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
180 } else {
181 SmallVector<int> EvenMask, OddMask;
182 for (int I = 0, E = NumElements; I != E; ++I) {
183 EvenMask.push_back(Elt: I * 2);
184 OddMask.push_back(Elt: I * 2 + 1);
185 }
186 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
187 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
188 }
189 }
190 }
191 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
192 auto *LastInst =
193 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
194 CGF->EmitWritebacks(Args);
195 return LastInst;
196}
197
198static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
199 const CallExpr *E) {
200 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
201 llvm::Type *I32 = CGF.Int32Ty;
202
203 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
204 [[maybe_unused]] llvm::StructType *Struct4I32 =
205 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
206
207 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
208 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
209 Value *StructVal =
210 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
211 assert(StructVal->getType() == Struct4I32 &&
212 "dx.wave.ballot must return {i32,i32,i32,i32}");
213
214 // Reassemble struct to <4 x i32>
215 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
216 for (unsigned I = 0; I < 4; ++I) {
217 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
218 VecVal =
219 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
220 }
221
222 return VecVal;
223 }
224
225 if (CGF.CGM.getTarget().getTriple().isSPIRV())
226 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
227
228 llvm_unreachable(
229 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
230}
231
232static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
233 const CallExpr *E) {
234 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
235 QualType Op0Ty = E->getArg(Arg: 0)->getType();
236 llvm::Type *ResType = CGF.FloatTy;
237 uint64_t NumElements = 0;
238 if (Op0->getType()->isVectorTy()) {
239 NumElements =
240 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
241 ResType =
242 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
243 }
244 if (!Op0Ty->hasUnsignedIntegerRepresentation())
245 llvm_unreachable(
246 "f16tof32 operand must have an unsigned int representation");
247
248 if (CGF.CGM.getTriple().isDXIL())
249 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
250 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
251 Name: "hlsl.f16tof32");
252
253 if (CGF.CGM.getTriple().isSPIRV()) {
254 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
255 // Int16 and Float16 capabilities
256 auto *UnpackType =
257 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
258
259 if (NumElements == 0) {
260 // a scalar input - simply extract the first element of the unpacked
261 // vector
262 Value *Unpack = CGF.Builder.CreateIntrinsic(
263 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
264 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
265 }
266
267 // a vector input - build a congruent output vector by iterating through
268 // the input vector calling unpackhalf2x16 for each element
269 Value *Result = PoisonValue::get(T: ResType);
270 for (uint64_t I = 0; I < NumElements; I++) {
271 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
272 Value *Unpack = CGF.Builder.CreateIntrinsic(
273 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
274 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
275 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
276 }
277 return Result;
278 }
279
280 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
281}
282
283static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
284 const CallExpr *E) {
285 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
286 QualType Op0Ty = E->getArg(Arg: 0)->getType();
287 llvm::Type *ResType = CGF.IntTy;
288 uint64_t NumElements = 0;
289 if (Op0->getType()->isVectorTy()) {
290 NumElements =
291 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
292 ResType =
293 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
294 }
295 if (!Op0Ty->hasFloatingRepresentation())
296 llvm_unreachable("f32tof16 operand must have a float representation");
297
298 if (CGF.CGM.getTriple().isDXIL())
299 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
300 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
301 Name: "hlsl.f32tof16");
302
303 if (CGF.CGM.getTriple().isSPIRV()) {
304 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
305 // Int16 and Float16 capabilities
306 auto *PackType =
307 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
308
309 if (NumElements == 0) {
310 // a scalar input - simply insert the scalar in the first element
311 // of the 2 element float vector
312 Value *Float2 = Constant::getNullValue(Ty: PackType);
313 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
314 Value *Result = CGF.Builder.CreateIntrinsic(
315 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
316 return Result;
317 }
318
319 // a vector input - build a congruent output vector by iterating through
320 // the input vector calling packhalf2x16 for each element
321 Value *Result = PoisonValue::get(T: ResType);
322 for (uint64_t I = 0; I < NumElements; I++) {
323 Value *Float2 = Constant::getNullValue(Ty: PackType);
324 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
325 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
326 Value *Res = CGF.Builder.CreateIntrinsic(
327 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
328 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
329 }
330 return Result;
331 }
332
333 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
334}
335
336// Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
337// Set the memory scope: Workgroup for groupshared, otherwise Device.
338static llvm::SyncScope::ID getHLSLAtomicScope(CodeGenFunction &CGF,
339 const LValue &DestLV) {
340 StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared
341 ? "workgroup"
342 : "device";
343 return CGF.getLLVMContext().getOrInsertSyncScopeID(SSN: ScopeName);
344}
345
346// The destination can name one element of a vector, as in `buf[0].z` or
347// `gs[i]`. `LValue::getAddress` gives the address of the whole vector for such
348// an lvalue, so index into the vector to get the address of the element. Sema
349// rejects a multi-element swizzle, so the access is always a single element.
350static Address getHLSLAtomicDestAddr(CodeGenFunction &CGF,
351 const LValue &DestLV) {
352 if (!DestLV.isVectorElt() && !DestLV.isExtVectorElt())
353 return DestLV.getAddress();
354
355 Address VecAddr = DestLV.isVectorElt() ? DestLV.getVectorAddress()
356 : DestLV.getExtVectorAddress();
357 Value *Idx = DestLV.isVectorElt()
358 ? DestLV.getVectorIdx()
359 : llvm::ConstantInt::get(Ty: CGF.SizeTy,
360 V: CodeGenFunction::getAccessedFieldNo(
361 Idx: 0, Elts: DestLV.getExtVectorElts()));
362
363 // A vector-element lvalue reports the type of the whole vector, so take the
364 // element type from the address. HLSL also treats a scalar as a one-element
365 // vector, in which case the address already has the element type.
366 llvm::Type *VecTy = VecAddr.getElementType();
367 llvm::Type *ElemTy = VecTy->isVectorTy()
368 ? cast<llvm::VectorType>(Val: VecTy)->getElementType()
369 : VecTy;
370 return CGF.Builder.CreateGEP(CGF, Addr: VecAddr.withElementType(ElemTy), Index: Idx);
371}
372
373static LValue getHLSLAtomicOriginalValueLValue(CodeGenFunction &CGF,
374 const Expr *E,
375 CallArgList &Args) {
376 if (const auto *OutArg = dyn_cast<HLSLOutArgExpr>(Val: E))
377 return CGF.EmitHLSLOutArgExpr(E: OutArg, Args, Ty: OutArg->getType());
378 return CGF.EmitLValue(E);
379}
380
381static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
382 llvm::AtomicRMWInst::BinOp Op) {
383 // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
384 // either DXIL or SPIR-V.
385 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
386 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
387 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
388 [[maybe_unused]] QualType ValTy = E->getArg(Arg: 1)->getType();
389 if (Op == llvm::AtomicRMWInst::Xchg)
390 assert((ValTy->isIntegerType() || ValTy->isFloatingType()) &&
391 "InterlockedExchange value operand must be an integer or a float");
392 else
393 assert(ValTy->isIntegerType() &&
394 "Intrinsic InterlockedOp value operand must be an integer");
395
396 llvm::SyncScope::ID SSID = getHLSLAtomicScope(CGF, DestLV);
397
398 llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
399 Op, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
400
401 // The 3-arg overload writes the old value (the RMW's return value) into
402 // the `original_value` reference parameter.
403 if (E->getNumArgs() == 3) {
404 CallArgList Args;
405 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 2), Args);
406 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
407 CGF.EmitWritebacks(Args);
408 }
409 return Call;
410}
411
412// Emit `cmpxchg` for InterlockedCompareStore and InterlockedCompareExchange.
413// Compare-exchange also reports the previous value.
414static Value *handleInterlockedCompareOp(CodeGenFunction &CGF,
415 const CallExpr *E) {
416 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
417 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
418 Value *Compare = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
419 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
420
421 // `cmpxchg` takes an integer or a pointer, so the float-bitwise operations
422 // work on the bit pattern of the float. This is what those operations mean,
423 // and DXIL and SPIR-V both need the integer form.
424 llvm::Type *FloatTy = nullptr;
425 if (Compare->getType()->isFloatingPointTy()) {
426 FloatTy = Compare->getType();
427 llvm::Type *IntTy =
428 CGF.Builder.getIntNTy(N: FloatTy->getPrimitiveSizeInBits());
429 Compare = CGF.Builder.CreateBitCast(V: Compare, DestTy: IntTy);
430 Val = CGF.Builder.CreateBitCast(V: Val, DestTy: IntTy);
431 DestAddr = DestAddr.withElementType(ElemTy: IntTy);
432 }
433
434 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
435 Addr: DestAddr, Cmp: Compare, New: Val, SuccessOrdering: llvm::AtomicOrdering::Monotonic,
436 FailureOrdering: llvm::AtomicOrdering::Monotonic, SSID: getHLSLAtomicScope(CGF, DestLV));
437
438 // Compare-store reports nothing, so it leaves the `cmpxchg` result unused.
439 if (E->getNumArgs() < 4)
440 return Pair;
441
442 // `cmpxchg` yields a { previous value, success } pair. HLSL reports only the
443 // previous value, through the `original_value` reference parameter.
444 Value *Original = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
445 if (FloatTy)
446 Original = CGF.Builder.CreateBitCast(V: Original, DestTy: FloatTy);
447 CallArgList Args;
448 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 3), Args);
449 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Original), Dst: OrigLV);
450 CGF.EmitWritebacks(Args);
451 return Original;
452}
453
454static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
455 LValue &Stride) {
456 // Figure out the stride of the buffer elements from the handle type.
457 auto *HandleTy =
458 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
459 QualType ElementTy = HandleTy->getContainedType();
460 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
461 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
462}
463
464// Return dot product intrinsic that corresponds to the QT scalar type
465static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
466 if (QT->isFloatingType())
467 return RT.getFDotIntrinsic();
468 if (QT->isSignedIntegerType())
469 return RT.getSDotIntrinsic();
470 assert(QT->isUnsignedIntegerType());
471 return RT.getUDotIntrinsic();
472}
473
474static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
475 if (QT->hasSignedIntegerRepresentation()) {
476 return RT.getFirstBitSHighIntrinsic();
477 }
478
479 assert(QT->hasUnsignedIntegerRepresentation());
480 return RT.getFirstBitUHighIntrinsic();
481}
482
483// Return wave active sum that corresponds to the QT scalar type
484static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
485 QualType QT) {
486 switch (Arch) {
487 case llvm::Triple::spirv:
488 return Intrinsic::spv_wave_reduce_sum;
489 case llvm::Triple::dxil: {
490 if (QT->isUnsignedIntegerType())
491 return Intrinsic::dx_wave_reduce_usum;
492 return Intrinsic::dx_wave_reduce_sum;
493 }
494 default:
495 llvm_unreachable("Intrinsic WaveActiveSum"
496 " not supported by target architecture");
497 }
498}
499
500// Return wave active product that corresponds to the QT scalar type
501static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
502 QualType QT) {
503 switch (Arch) {
504 case llvm::Triple::spirv:
505 return Intrinsic::spv_wave_product;
506 case llvm::Triple::dxil: {
507 if (QT->isUnsignedIntegerType())
508 return Intrinsic::dx_wave_uproduct;
509 return Intrinsic::dx_wave_product;
510 }
511 default:
512 llvm_unreachable("Intrinsic WaveActiveProduct"
513 " not supported by target architecture");
514 }
515}
516
517static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
518 switch (Arch) {
519 case llvm::Triple::spirv:
520 return Intrinsic::spv_subgroup_prefix_bit_count;
521 case llvm::Triple::dxil: {
522 return Intrinsic::dx_wave_prefix_bit_count;
523 }
524 default:
525 llvm_unreachable(
526 "WavePrefixOp instruction not supported by target architecture");
527 }
528}
529
530// Return wave prefix sum that corresponds to the QT scalar type
531static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
532 QualType QT) {
533 switch (Arch) {
534 case llvm::Triple::spirv:
535 return Intrinsic::spv_wave_prefix_sum;
536 case llvm::Triple::dxil: {
537 if (QT->isUnsignedIntegerType())
538 return Intrinsic::dx_wave_prefix_usum;
539 return Intrinsic::dx_wave_prefix_sum;
540 }
541 default:
542 llvm_unreachable("Intrinsic WavePrefixSum"
543 " not supported by target architecture");
544 }
545}
546
547// Return wave prefix product that corresponds to the QT scalar type
548static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
549 QualType QT) {
550 switch (Arch) {
551 case llvm::Triple::spirv:
552 return Intrinsic::spv_wave_prefix_product;
553 case llvm::Triple::dxil: {
554 if (QT->isUnsignedIntegerType())
555 return Intrinsic::dx_wave_prefix_uproduct;
556 return Intrinsic::dx_wave_prefix_product;
557 }
558 default:
559 llvm_unreachable("Intrinsic WavePrefixProduct"
560 " not supported by target architecture");
561 }
562}
563
564// Returns the mangled name for a builtin function that the SPIR-V backend
565// will expand into a spec Constant.
566static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
567 ASTContext &Context) {
568 // The parameter types for our conceptual intrinsic function.
569 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
570
571 // Create a temporary FunctionDecl for the builtin fuction. It won't be
572 // added to the AST.
573 FunctionProtoType::ExtProtoInfo EPI;
574 QualType FnType =
575 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
576 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
577 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
578 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
579 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
580
581 // Attach the created parameter declarations to the function declaration.
582 SmallVector<ParmVarDecl *, 2> ParamDecls;
583 for (QualType ParamType : ClangParamTypes) {
584 ParmVarDecl *PD = ParmVarDecl::Create(
585 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
586 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
587 /*DefaultArg*/ DefArg: nullptr);
588 ParamDecls.push_back(Elt: PD);
589 }
590 FnDeclForMangling->setParams(ParamDecls);
591
592 // Get the mangled name.
593 std::string Name;
594 llvm::raw_string_ostream MangledNameStream(Name);
595 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
596 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
597 MangledNameStream.flush();
598
599 return Name;
600}
601
602static const HLSLAttributedResourceType *
603getHandleAttributedType(QualType HandleQT) {
604 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
605 return RT;
606 // If the expr is a texture/sampler record (or similar), peel to __handle.
607 if (const HLSLAttributedResourceType *RT =
608 HLSLAttributedResourceType::findHandleTypeOnResource(
609 RT: HandleQT.getTypePtr()))
610 return RT;
611 llvm_unreachable("attributed handle type not found");
612}
613
614static const HLSLAttributedResourceType *
615getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
616 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
617}
618
619static const FieldDecl *getResourceHandleField(QualType ResourceTy) {
620 const CXXRecordDecl *ResourceDecl = ResourceTy->getAsCXXRecordDecl();
621 assert(ResourceDecl && "resource must be a record type");
622
623 IdentifierInfo &II = ResourceDecl->getASTContext().Idents.get(Name: "__handle");
624 for (const Decl *D : ResourceDecl->lookup(Name: &II))
625 if (const auto *Field = dyn_cast<FieldDecl>(Val: D))
626 return Field;
627
628 llvm_unreachable("resource handle field not found");
629}
630
631static llvm::Type *getOffsetType(CodeGenModule &CGM,
632 const HLSLAttributedResourceType *RT) {
633 const auto &Attrs = RT->getAttrs();
634 unsigned OffsetSize =
635 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
636 llvm::Type *Int32Ty = CGM.Int32Ty;
637 if (OffsetSize == 1)
638 return Int32Ty;
639 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
640}
641
642static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
643 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
644 if (E->getNumArgs() > OffsetArgIndex)
645 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
646
647 return llvm::Constant::getNullValue(Ty: OffsetTy);
648}
649
650static Value *emitHlslSampleOffset(CodeGenFunction &CGF, const CallExpr *E,
651 const HLSLAttributedResourceType *RT,
652 unsigned OffsetArgIndex) {
653 llvm::Type *OffsetTy = getOffsetType(CGM&: CGF.CGM, RT);
654 if (!clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension))
655 return llvm::Constant::getNullValue(Ty: OffsetTy);
656 return emitHlslOffset(CGF, E, OffsetArgIndex, OffsetTy);
657}
658
659static unsigned getHlslClampArgIndex(const HLSLAttributedResourceType *RT,
660 unsigned OffsetArgIndex) {
661 return clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension)
662 ? OffsetArgIndex + 1
663 : OffsetArgIndex;
664}
665
666static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
667 unsigned IntrinsicID, unsigned NumRetComps,
668 bool HasLod) {
669 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
670
671 SmallVector<Value *> Args{Handle};
672 if (HasLod)
673 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
674
675 Value *DimValue =
676 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
677
678 Value *LastStore = nullptr;
679 unsigned ArgIndex = HasLod ? 2 : 1;
680 for (unsigned i = 0; i < NumRetComps; ++i) {
681 const Expr *Arg = E->getArg(Arg: ArgIndex++);
682 LValue DimOut = CGF.EmitLValue(E: Arg);
683 Value *Elem = DimValue;
684 if (NumRetComps > 1)
685 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
686
687 // Handle float casting if needed
688 if (Arg->getType()->isFloatingType())
689 Elem = CGF.Builder.CreateUIToFP(
690 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
691
692 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
693 }
694 return LastStore;
695}
696
697static llvm::Type *getAggregateType(llvm::Type *ScalarTy, QualType ArgTy) {
698 if (auto *MatTy = ArgTy->getAs<ConstantMatrixType>())
699 return llvm::VectorType::get(
700 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: MatTy->getNumElementsFlattened()));
701 if (auto *VecTy = ArgTy->getAs<clang::VectorType>())
702 return llvm::VectorType::get(
703 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()));
704 return ScalarTy;
705}
706
707Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
708 const CallExpr *E,
709 ReturnValueSlot ReturnValue) {
710 if (!getLangOpts().HLSL)
711 return nullptr;
712
713 switch (BuiltinID) {
714 case Builtin::BI__builtin_hlsl_adduint64: {
715 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
716 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
717 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
718 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
719 assert(Arg0Ty == E->getArg(1)->getType() &&
720 "AddUint64 operand types must match");
721 assert(Arg0Ty->hasIntegerRepresentation() &&
722 "AddUint64 operands must have an integer representation");
723 assert((NumElements == 2 || NumElements == 4) &&
724 "AddUint64 operands must have 2 or 4 elements");
725
726 llvm::Value *LowA;
727 llvm::Value *HighA;
728 llvm::Value *LowB;
729 llvm::Value *HighB;
730
731 // Obtain low and high words of inputs A and B
732 if (NumElements == 2) {
733 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
734 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
735 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
736 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
737 } else {
738 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
739 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
740 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
741 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
742 }
743
744 // Use an uadd_with_overflow to compute the sum of low words and obtain a
745 // carry value
746 llvm::Value *Carry;
747 llvm::Value *LowSum = EmitOverflowIntrinsic(
748 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
749 llvm::Value *ZExtCarry =
750 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
751
752 // Sum the high words and the carry
753 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
754 llvm::Value *HighSumPlusCarry =
755 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
756
757 if (NumElements == 4) {
758 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
759 Name: "hlsl.AddUint64");
760 }
761
762 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
763 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
764 Name: "hlsl.AddUint64.upto0");
765 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
766 Name: "hlsl.AddUint64");
767 return Result;
768 }
769 case Builtin::BI__builtin_hlsl_resource_getpointer:
770 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
771 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
772 bool IsIndexed =
773 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
774 E->getNumArgs() > 1;
775
776 llvm::Type *RetTy = ConvertType(T: E->getType());
777 if (IsIndexed) {
778 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
779 return EmitIntrinsicCall(
780 ID: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
781 Types: {RetTy, HandleOp->getType(), IndexOp->getType()},
782 Args: {HandleOp, IndexOp});
783 }
784
785 return EmitIntrinsicCall(
786 ID: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
787 Types: {RetTy, HandleOp->getType()}, Args: {HandleOp});
788 }
789 case Builtin::BI__builtin_hlsl_transpose_if_memory_is_row_major: {
790 const Expr *ValueExpr = E->getArg(Arg: 0);
791 if (hasAggregateEvaluationKind(T: ValueExpr->getType())) {
792 EmitAnyExprToMem(E: ValueExpr, Location: ReturnValue.getAddress(),
793 Quals: ValueExpr->getType().getQualifiers(), /*IsInit=*/IsInitializer: true);
794 return ReturnValue.getAddress().getBasePointer();
795 }
796
797 Value *ValueOp = EmitScalarExpr(E: ValueExpr);
798 const auto *MatTy = ValueExpr->getType()->getAs<ConstantMatrixType>();
799 if (!MatTy || !getLangOpts().HLSLSpvUseLegacyBufferMatrixOrder)
800 return ValueOp;
801
802 bool IsLoad =
803 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getBoolValue();
804 unsigned Rows = MatTy->getNumRows();
805 unsigned Columns = MatTy->getNumColumns();
806 llvm::MatrixBuilder MB(Builder);
807 return IsLoad ? MB.CreateMatrixTranspose(Matrix: ValueOp, Rows: Columns, Columns: Rows)
808 : MB.CreateMatrixTranspose(Matrix: ValueOp, Rows, Columns);
809 }
810 case Builtin::BI__builtin_hlsl_resource_sample: {
811 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
812 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
813 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
814 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
815
816 SmallVector<Value *, 4> Args;
817 Args.push_back(Elt: HandleOp);
818 Args.push_back(Elt: SamplerOp);
819 Args.push_back(Elt: CoordOp);
820 constexpr unsigned OffsetIdx = 3;
821 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
822
823 llvm::Type *RetTy = ConvertType(T: E->getType());
824 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
825 if (E->getNumArgs() <= ClampIdx)
826 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args,
827 RetTy);
828
829 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
830 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(),
831 Args, RetTy);
832 }
833 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
834 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
835 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
836 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
837 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
838 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
839
840 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
841 Args.push_back(Elt: HandleOp);
842 Args.push_back(Elt: SamplerOp);
843 Args.push_back(Elt: CoordOp);
844 Args.push_back(Elt: BiasOp);
845 constexpr unsigned OffsetIdx = 4;
846 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
847
848 llvm::Type *RetTy = ConvertType(T: E->getType());
849 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
850 if (E->getNumArgs() <= ClampIdx)
851 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(),
852 Args, RetTy);
853
854 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
855 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(),
856 Args, RetTy);
857 }
858 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
859 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
860 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
861 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
862 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
863 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
864 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
865
866 SmallVector<Value *, 7> Args;
867 Args.push_back(Elt: HandleOp);
868 Args.push_back(Elt: SamplerOp);
869 Args.push_back(Elt: CoordOp);
870 Args.push_back(Elt: DDXOp);
871 Args.push_back(Elt: DDYOp);
872 constexpr unsigned OffsetIdx = 5;
873 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
874
875 llvm::Type *RetTy = ConvertType(T: E->getType());
876
877 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
878 if (E->getNumArgs() <= ClampIdx)
879 return Builder.CreateIntrinsic(
880 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
881
882 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
883 return Builder.CreateIntrinsic(
884 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
885 }
886 case Builtin::BI__builtin_hlsl_resource_sample_level: {
887 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
888 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
889 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
890 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
891 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
892
893 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
894 Args.push_back(Elt: HandleOp);
895 Args.push_back(Elt: SamplerOp);
896 Args.push_back(Elt: CoordOp);
897 Args.push_back(Elt: LODOp);
898 constexpr unsigned OffsetIdx = 4;
899 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
900
901 llvm::Type *RetTy = ConvertType(T: E->getType());
902 return Builder.CreateIntrinsic(
903 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
904 }
905 case Builtin::BI__builtin_hlsl_resource_load_level: {
906 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
907 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
908 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
909
910 const auto &Attrs = RT->getAttrs();
911
912 Value *CoordOp = nullptr;
913 Value *LODOp = nullptr;
914 if (Attrs.ResourceClass == llvm::dxil::ResourceClass::UAV) {
915 // A UAV descriptor binds a single mip slice, so a RWTexture location is
916 // all coordinate and there is no mip level to select.
917 CoordOp = CoordLODOp;
918 LODOp = llvm::PoisonValue::get(T: Int32Ty);
919 } else {
920 // Split CoordLOD into Coord and LOD. 1D resources use a scalar
921 // coordinate rather than a 1-element vector.
922 unsigned CoordSize =
923 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension) +
924 (Attrs.IsArray ? 1 : 0);
925 assert(cast<llvm::FixedVectorType>(CoordLODOp->getType())
926 ->getNumElements() == CoordSize + 1 &&
927 "CoordLOD must have one element per coordinate, plus the level");
928
929 if (CoordSize == 1) {
930 CoordOp = Builder.CreateExtractElement(Vec: CoordLODOp, Idx: uint64_t(0),
931 Name: "hlsl.load.coord");
932 } else {
933 SmallVector<int, 4> Mask;
934 for (unsigned I = 0; I < CoordSize; ++I)
935 Mask.push_back(Elt: I);
936 CoordOp =
937 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
938 }
939 LODOp =
940 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: CoordSize, Name: "hlsl.load.lod");
941 }
942
943 SmallVector<Value *, 4> Args;
944 Args.push_back(Elt: HandleOp);
945 Args.push_back(Elt: CoordOp);
946 Args.push_back(Elt: LODOp);
947 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
948
949 llvm::Type *RetTy = ConvertType(T: E->getType());
950 return Builder.CreateIntrinsic(
951 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
952 }
953 case Builtin::BI__builtin_hlsl_resource_load_ms: {
954 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
955 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 1));
956 Value *SampleOp = EmitScalarExpr(E: E->getArg(Arg: 2));
957 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
958
959 SmallVector<Value *, 4> Args;
960 Args.push_back(Elt: HandleOp);
961 Args.push_back(Elt: CoordOp);
962 Args.push_back(Elt: SampleOp);
963 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
964
965 llvm::Type *RetTy = ConvertType(T: E->getType());
966 return Builder.CreateIntrinsic(
967 RetTy, ID: CGM.getHLSLRuntime().getLoadMSIntrinsic(), Args);
968 }
969 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
970 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
971 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
972 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
973 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
974 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
975
976 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
977 Args.push_back(Elt: HandleOp);
978 Args.push_back(Elt: SamplerOp);
979 Args.push_back(Elt: CoordOp);
980 Args.push_back(Elt: CmpOp);
981 constexpr unsigned OffsetIdx = 4;
982 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
983
984 llvm::Type *RetTy = ConvertType(T: E->getType());
985 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
986 if (E->getNumArgs() <= ClampIdx)
987 return Builder.CreateIntrinsic(
988 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
989
990 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
991 return Builder.CreateIntrinsic(
992 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
993 }
994 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
995 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
996 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
997 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
998 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
999 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1000
1001 SmallVector<Value *, 5> Args;
1002 Args.push_back(Elt: HandleOp);
1003 Args.push_back(Elt: SamplerOp);
1004 Args.push_back(Elt: CoordOp);
1005 Args.push_back(Elt: CmpOp);
1006 constexpr unsigned OffsetIdx = 4;
1007 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
1008
1009 llvm::Type *RetTy = ConvertType(T: E->getType());
1010 return Builder.CreateIntrinsic(
1011 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
1012 }
1013 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
1014 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1015 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1016 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1017
1018 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
1019 Args: {HandleOp, SamplerOp, CoordOp},
1020 RetTy: ConvertType(T: E->getType()));
1021 }
1022 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
1023 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1024 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1025 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1026
1027 return EmitIntrinsicCall(
1028 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
1029 Args: {HandleOp, SamplerOp, CoordOp}, RetTy: ConvertType(T: E->getType()));
1030 }
1031 case Builtin::BI__builtin_hlsl_resource_gather: {
1032 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1033 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1034 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1035 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1036 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1037
1038 SmallVector<Value *, 5> Args;
1039 Args.push_back(Elt: HandleOp);
1040 Args.push_back(Elt: SamplerOp);
1041 Args.push_back(Elt: CoordOp);
1042 Args.push_back(Elt: ComponentOp);
1043 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
1044
1045 llvm::Type *RetTy = ConvertType(T: E->getType());
1046 return Builder.CreateIntrinsic(
1047 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
1048 }
1049 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
1050 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1051 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1052 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1053 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1054
1055 SmallVector<Value *, 6> Args;
1056 Args.push_back(Elt: HandleOp);
1057 Args.push_back(Elt: SamplerOp);
1058 Args.push_back(Elt: CoordOp);
1059 Args.push_back(Elt: CompareOp);
1060
1061 if (CGM.getTarget().getTriple().isDXIL()) {
1062 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1063 Args.push_back(Elt: ComponentOp);
1064 }
1065
1066 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1067 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
1068
1069 llvm::Type *RetTy = ConvertType(T: E->getType());
1070 return Builder.CreateIntrinsic(
1071 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
1072 }
1073 case Builtin::BI__builtin_hlsl_resource_load_with_status:
1074 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
1075 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1076 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1077
1078 // Get the *address* of the status argument to write to it by reference
1079 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
1080 Address StatusAddr = StatusLVal.getAddress();
1081
1082 QualType HandleTy = E->getArg(Arg: 0)->getType();
1083 const HLSLAttributedResourceType *RT =
1084 HandleTy->getAs<HLSLAttributedResourceType>();
1085 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
1086 "Only DXIL currently implements load with status");
1087
1088 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
1089 ? llvm::Intrinsic::dx_resource_load_rawbuffer
1090 : llvm::Intrinsic::dx_resource_load_typedbuffer;
1091
1092 llvm::Type *DataTy = ConvertType(T: E->getType());
1093 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
1094 Elements: {DataTy, Builder.getInt1Ty()});
1095
1096 SmallVector<Value *, 3> Args;
1097 Args.push_back(Elt: HandleOp);
1098 Args.push_back(Elt: IndexOp);
1099
1100 if (RT->isRaw()) {
1101 Value *Offset = Builder.getInt32(C: 0);
1102 // The offset parameter needs to be poison for ByteAddressBuffer
1103 if (!RT->isStructured())
1104 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
1105 Args.push_back(Elt: Offset);
1106 }
1107
1108 // The load intrinsics give us a (T value, i1 status) pair -
1109 // shepherd these into the return value and out reference respectively.
1110 Value *ResRet =
1111 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
1112 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
1113 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
1114 Value *ExtendedStatus =
1115 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
1116 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
1117
1118 return LoadedValue;
1119 }
1120 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
1121 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1122 return llvm::PoisonValue::get(T: HandleTy);
1123 }
1124 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
1125 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1126 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1127 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1128 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1129 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1130 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1131 llvm::Intrinsic::ID IntrinsicID =
1132 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1133 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
1134 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1135 }
1136 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
1137 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1138 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1139 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1140 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1141 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1142 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1143 llvm::Intrinsic::ID IntrinsicID =
1144 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1145 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
1146 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1147 }
1148 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
1149 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1150 if (!CGM.getTriple().isSPIRV())
1151 return MainHandle;
1152
1153 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1154 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1155 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1156 llvm::Intrinsic::ID IntrinsicID =
1157 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
1158 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
1159 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1160 Args);
1161 }
1162 case Builtin::BI__builtin_hlsl_resource_handlefromheap: {
1163 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1164 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1165 llvm::Intrinsic::ID IntrinsicID =
1166 CGM.getHLSLRuntime().getCreateHandleFromHeapIntrinsic();
1167 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args: {IndexOp});
1168 }
1169 case Builtin::BI__builtin_hlsl_resource_counterhandlefromheap: {
1170 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1171 if (!CGM.getTriple().isSPIRV())
1172 return MainHandle;
1173
1174 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1175 llvm::Intrinsic::ID IntrinsicID =
1176 llvm::Intrinsic::spv_resource_counterhandlefromheap;
1177 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1178 Args: {MainHandle});
1179 }
1180
1181 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
1182 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1183 llvm::Type *RetTy = ConvertType(T: E->getType());
1184 return Builder.CreateIntrinsic(
1185 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
1186 Args: ArrayRef<Value *>{IndexOp});
1187 }
1188 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
1189 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
1190 return emitGetDimensions(CGF&: *this, E,
1191 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
1192 NumRetComps: 1, /*HasLod=*/false);
1193 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
1194 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
1195 return emitGetDimensions(CGF&: *this, E,
1196 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1197 NumRetComps: 2, /*HasLod=*/false);
1198 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1199 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1200 return emitGetDimensions(
1201 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1202 /*HasLod=*/true);
1203 case Builtin::BI__builtin_hlsl_resource_getstride: {
1204 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1205 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1206 }
1207 case Builtin::BI__builtin_hlsl_and: {
1208 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1209 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1210 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1211 }
1212 case Builtin::BI__builtin_hlsl_or: {
1213 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1214 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1215 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1216 }
1217 case Builtin::BI__builtin_hlsl_asdouble:
1218 return handleAsDoubleBuiltin(CGF&: *this, E);
1219 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1220 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1221 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1222 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1223
1224 QualType Ty = E->getArg(Arg: 0)->getType();
1225 if (auto *VecTy = Ty->getAs<VectorType>())
1226 Ty = VecTy->getElementType();
1227 else if (auto *MatTy = Ty->getAs<ConstantMatrixType>())
1228 Ty = MatTy->getElementType();
1229
1230 Intrinsic::ID Intr;
1231 if (Ty->isFloatingType()) {
1232 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1233 } else if (Ty->isUnsignedIntegerType()) {
1234 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1235 } else {
1236 assert(Ty->isSignedIntegerType());
1237 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1238 }
1239 return Builder.CreateIntrinsic(
1240 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1241 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1242 }
1243 case Builtin::BI__builtin_hlsl_dot: {
1244 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1245 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1246 llvm::Type *T0 = Op0->getType();
1247 llvm::Type *T1 = Op1->getType();
1248
1249 // If the arguments are scalars, just emit a multiply
1250 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1251 if (T0->isFloatingPointTy())
1252 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1253
1254 if (T0->isIntegerTy())
1255 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1256
1257 llvm_unreachable(
1258 "Scalar dot product is only supported on ints and floats.");
1259 }
1260 // For vectors, validate types and emit the appropriate intrinsic
1261 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1262 E->getArg(1)->getType()) &&
1263 "Dot product operands must have the same type.");
1264
1265 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1266 assert(VecTy0 && "Dot product argument must be a vector.");
1267
1268 return Builder.CreateIntrinsic(
1269 /*ReturnType=*/RetTy: T0->getScalarType(),
1270 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1271 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1272 }
1273 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1274 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1275 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1276 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1277
1278 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1279 // Note that the argument order disagrees between the builtin and the
1280 // intrinsic here.
1281 return Builder.CreateIntrinsic(
1282 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1283 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1284 }
1285 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1286 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1287 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1288 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1289
1290 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1291 // Note that the argument order disagrees between the builtin and the
1292 // intrinsic here.
1293 return Builder.CreateIntrinsic(
1294 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1295 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1296 }
1297 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1298 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1299
1300 return Builder.CreateIntrinsic(
1301 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1302 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1303 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1304 }
1305 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1306 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1307
1308 return Builder.CreateIntrinsic(
1309 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1310 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1311 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1312 }
1313 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1314 return handleElementwiseF16ToF32(CGF&: *this, E);
1315 }
1316 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1317 return handleElementwiseF32ToF16(CGF&: *this, E);
1318 }
1319 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1320 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1321 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1322 llvm_unreachable("frac operand must have a float representation");
1323 return Builder.CreateIntrinsic(
1324 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1325 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1326 }
1327 case Builtin::BI__builtin_hlsl_elementwise_isfinite: {
1328 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1329 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1330 llvm_unreachable("isfinite operand must have a float representation");
1331 llvm::Type *retType = getAggregateType(
1332 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1333 return Builder.CreateIntrinsic(
1334 RetTy: retType, ID: CGM.getHLSLRuntime().getIsFiniteIntrinsic(),
1335 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isfinite");
1336 }
1337 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1338 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1339 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1340 llvm_unreachable("isinf operand must have a float representation");
1341 llvm::Type *retType = getAggregateType(
1342 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1343 return Builder.CreateIntrinsic(
1344 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1345 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1346 }
1347 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1348 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1349 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1350 llvm_unreachable("isnan operand must have a float representation");
1351 llvm::Type *retType = getAggregateType(
1352 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1353 return Builder.CreateIntrinsic(
1354 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1355 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1356 }
1357 case Builtin::BI__builtin_hlsl_mad: {
1358 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1359 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1360 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1361 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1362 return Builder.CreateIntrinsic(
1363 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1364 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1365
1366 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1367 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1368 return Builder.CreateIntrinsic(
1369 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1370 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1371
1372 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1373 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1374 }
1375 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1376 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1377 return Builder.CreateIntrinsic(
1378 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1379 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1380
1381 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1382 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1383 }
1384 case Builtin::BI__builtin_hlsl_mul: {
1385 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1386 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1387 QualType QTy0 = E->getArg(Arg: 0)->getType();
1388 QualType QTy1 = E->getArg(Arg: 1)->getType();
1389
1390 bool IsVec0 = QTy0->isVectorType();
1391 bool IsVec1 = QTy1->isVectorType();
1392 bool IsMat0 = QTy0->isConstantMatrixType();
1393 bool IsMat1 = QTy1->isConstantMatrixType();
1394
1395 llvm::MatrixBuilder MB(Builder);
1396 if (IsVec0 && IsMat1) {
1397 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1398 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1399 unsigned Cols = MatTy->getNumColumns();
1400 assert(N == MatTy->getNumRows() &&
1401 "vector length must match matrix row count");
1402 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1403 }
1404 if (IsMat0 && IsVec1) {
1405 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1406 unsigned Rows = MatTy->getNumRows();
1407 unsigned Cols = MatTy->getNumColumns();
1408 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1409 "vector length must match matrix column count");
1410 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1411 }
1412 assert(IsMat0 && IsMat1);
1413 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1414 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1415 unsigned Rows0 = MatTy0->getNumRows();
1416 unsigned Cols0 = MatTy0->getNumColumns();
1417 unsigned Cols1 = MatTy1->getNumColumns();
1418 assert(Cols0 == MatTy1->getNumRows() &&
1419 "inner matrix dimensions must match for multiplication");
1420 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1421 }
1422 case Builtin::BI__builtin_hlsl_transpose: {
1423 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1424 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1425 unsigned Rows = MatTy->getNumRows();
1426 unsigned Cols = MatTy->getNumColumns();
1427 if (auto *Transpose = dyn_cast<CallInst>(Val: Op0);
1428 Transpose &&
1429 Transpose->getIntrinsicID() == Intrinsic::matrix_transpose &&
1430 Transpose->use_empty() &&
1431 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 1))->getZExtValue() ==
1432 Cols &&
1433 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 2))->getZExtValue() ==
1434 Rows) {
1435 Value *Result = Transpose->getArgOperand(i: 0);
1436 Transpose->eraseFromParent();
1437 return Result;
1438 }
1439 llvm::MatrixBuilder MB(Builder);
1440 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1441 }
1442 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1443 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1444 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1445 llvm_unreachable("rcp operand must have a float representation");
1446 llvm::Type *Ty = Op0->getType();
1447 llvm::Type *EltTy = Ty->getScalarType();
1448 Constant *One = Ty->isVectorTy()
1449 ? ConstantVector::getSplat(
1450 EC: ElementCount::getFixed(
1451 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1452 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1453 : ConstantFP::get(Ty: EltTy, V: 1.0);
1454 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1455 }
1456 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1457 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1458 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1459 llvm_unreachable("rsqrt operand must have a float representation");
1460 return Builder.CreateIntrinsic(
1461 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1462 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1463 }
1464 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1465 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1466 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1467 "saturate operand must have a float representation");
1468 return Builder.CreateIntrinsic(
1469 /*ReturnType=*/RetTy: Op0->getType(),
1470 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1471 FMFSource: nullptr, Name: "hlsl.saturate");
1472 }
1473 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1474 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1475 assert(Op->getType()->isIntegerTy(1) &&
1476 "WavePrefixBitCount operand must be a boolean type");
1477
1478 Intrinsic::ID IID =
1479 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1480
1481 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1482 }
1483 case Builtin::BI__builtin_hlsl_select: {
1484 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1485 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1486 Value *OpTrue =
1487 RValTrue.isScalar()
1488 ? RValTrue.getScalarVal()
1489 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1490 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1491 Value *OpFalse =
1492 RValFalse.isScalar()
1493 ? RValFalse.getScalarVal()
1494 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1495
1496 unsigned NumElements = 0;
1497 if (auto *VTy = E->getType()->getAs<VectorType>())
1498 NumElements = VTy->getNumElements();
1499 else if (auto *MTy = E->getType()->getAs<ConstantMatrixType>())
1500 NumElements = MTy->getNumElementsFlattened();
1501 if (NumElements) {
1502 if (!OpTrue->getType()->isVectorTy())
1503 OpTrue = Builder.CreateVectorSplat(NumElts: NumElements, V: OpTrue, Name: "splat");
1504 if (!OpFalse->getType()->isVectorTy())
1505 OpFalse = Builder.CreateVectorSplat(NumElts: NumElements, V: OpFalse, Name: "splat");
1506 }
1507
1508 Value *SelectVal =
1509 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1510 if (!RValTrue.isScalar())
1511 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1512 IsVolatile: ReturnValue.isVolatile());
1513
1514 return SelectVal;
1515 }
1516 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1517 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1518
1519 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1520 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1521 }
1522 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1523 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1524 assert(Op->getType()->isIntegerTy(1) &&
1525 "Intrinsic WaveActiveAllTrue operand must be a bool");
1526
1527 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1528 return EmitIntrinsicCall(ID, Args: {Op});
1529 }
1530 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1531 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1532 assert(Op->getType()->isIntegerTy(1) &&
1533 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1534
1535 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1536 return EmitIntrinsicCall(ID, Args: {Op});
1537 }
1538 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1539 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1540 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1541 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1542 "representation");
1543
1544 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1545 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1546 Name: "hlsl.wave.active.bit.or");
1547 }
1548 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1549 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1550 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1551 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1552 "representation");
1553
1554 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1555 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1556 Name: "hlsl.wave.active.bit.xor");
1557 }
1558 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1559 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1560 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1561 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1562 "representation");
1563
1564 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1565 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1566 Name: "hlsl.wave.active.bit.and");
1567 }
1568 case Builtin::BI__builtin_hlsl_interlocked_add: {
1569 // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
1570 // up the raw instruction (DXIL routes it via `dx.resource.atomic.binop`
1571 // in DXILResourceAccess for resource pointers, SPIR-V lowers via
1572 // selectAtomicRMW). No intermediate intrinsic.
1573 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Add);
1574 }
1575 case Builtin::BI__builtin_hlsl_interlocked_and: {
1576 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::And);
1577 }
1578 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange:
1579 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange_float_bitwise:
1580 case Builtin::BI__builtin_hlsl_interlocked_compare_store:
1581 case Builtin::BI__builtin_hlsl_interlocked_compare_store_float_bitwise: {
1582 return handleInterlockedCompareOp(CGF&: *this, E);
1583 }
1584 case Builtin::BI__builtin_hlsl_interlocked_exchange: {
1585 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xchg);
1586 }
1587 case Builtin::BI__builtin_hlsl_interlocked_max: {
1588 llvm::AtomicRMWInst::BinOp Op =
1589 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1590 ? llvm::AtomicRMWInst::Max
1591 : llvm::AtomicRMWInst::UMax;
1592 return handleInterlockedOp(CGF&: *this, E, Op);
1593 }
1594 case Builtin::BI__builtin_hlsl_interlocked_min: {
1595 llvm::AtomicRMWInst::BinOp Op =
1596 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1597 ? llvm::AtomicRMWInst::Min
1598 : llvm::AtomicRMWInst::UMin;
1599 return handleInterlockedOp(CGF&: *this, E, Op);
1600 }
1601 case Builtin::BI__builtin_hlsl_interlocked_or: {
1602 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Or);
1603 }
1604 case Builtin::BI__builtin_hlsl_interlocked_xor: {
1605 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xor);
1606 }
1607 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1608 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1609 assert(Op->getType()->isIntegerTy(1) &&
1610 "Intrinsic WaveActiveBallot operand must be a bool");
1611
1612 return handleHlslWaveActiveBallot(CGF&: *this, E);
1613 }
1614 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1615 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1616 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1617 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1618 }
1619 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1620 // Due to the use of variadic arguments, explicitly retrieve argument
1621 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1622 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1623 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1624
1625 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1626 Name: "hlsl.wave.active.sum");
1627 }
1628 case Builtin::BI__builtin_hlsl_wave_active_product: {
1629 // Due to the use of variadic arguments, explicitly retrieve argument
1630 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1631 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1632 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1633
1634 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1635 Name: "hlsl.wave.active.product");
1636 }
1637 case Builtin::BI__builtin_hlsl_wave_active_max: {
1638 // Due to the use of variadic arguments, explicitly retrieve argument
1639 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1640 QualType QT = E->getArg(Arg: 0)->getType();
1641 Intrinsic::ID IID;
1642 if (QT->isUnsignedIntegerType())
1643 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1644 else
1645 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1646
1647 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1648 Name: "hlsl.wave.active.max");
1649 }
1650 case Builtin::BI__builtin_hlsl_wave_active_min: {
1651 // Due to the use of variadic arguments, explicitly retrieve argument
1652 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1653 QualType QT = E->getArg(Arg: 0)->getType();
1654 Intrinsic::ID IID;
1655 if (QT->isUnsignedIntegerType())
1656 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1657 else
1658 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1659
1660 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1661 Name: "hlsl.wave.active.min");
1662 }
1663 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1664 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1665 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1666 // for the DirectX intrinsic and the demangled builtin name
1667 switch (CGM.getTarget().getTriple().getArch()) {
1668 case llvm::Triple::dxil:
1669 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1670 case llvm::Triple::spirv:
1671 return EmitRuntimeCall(
1672 callee: CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1673 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false,
1674 /*AssumeConvergent=*/true),
1675 args: {}, name: "", /*IsConvergent=*/true);
1676 default:
1677 llvm_unreachable(
1678 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1679 }
1680 }
1681 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1682 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1683 return EmitIntrinsicCall(ID);
1684 }
1685 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1686 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1687 return EmitIntrinsicCall(ID);
1688 }
1689 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1690 // Due to the use of variadic arguments we must explicitly retrieve them and
1691 // create our function type.
1692 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1693 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1694 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1695 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1696 Name: "hlsl.wave.readlane");
1697 }
1698 case Builtin::BI__builtin_hlsl_wave_read_lane_first: {
1699 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1700 return EmitIntrinsicCall(
1701 ID: CGM.getHLSLRuntime().getWaveReadLaneFirstIntrinsic(),
1702 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr}, Name: "hlsl.wave.readlane.first");
1703 }
1704 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1705 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1706 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1707 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1708 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1709 Name: "hlsl.wave.prefix.sum");
1710 }
1711 case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits:
1712 case Builtin::BI__builtin_hlsl_wave_multi_prefix_product:
1713 case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum:
1714 case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
1715 case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_or:
1716 case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
1717 Value *ValueOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1718 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 1));
1719 SmallVector<Value *, 5> Args{ValueOp};
1720 for (unsigned I = 0; I != 4; ++I)
1721 Args.push_back(Elt: Builder.CreateExtractElement(Vec: Mask, Idx: I));
1722 if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits)
1723 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_multi_prefix_bit_count, Args,
1724 Name: "hlsl.wave.multi.prefix.bit.count");
1725 if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product) {
1726 Intrinsic::ID IID =
1727 E->getArg(Arg: 0)->getType()->hasUnsignedIntegerRepresentation()
1728 ? CGM.getHLSLRuntime().getWaveMultiPrefixUProductIntrinsic()
1729 : CGM.getHLSLRuntime().getWaveMultiPrefixProductIntrinsic();
1730 return EmitIntrinsicCall(ID: IID, Types: {ValueOp->getType()}, Args,
1731 Name: "hlsl.wave.multi.prefix.product");
1732 }
1733 if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum) {
1734 Intrinsic::ID IID =
1735 E->getArg(Arg: 0)->getType()->hasUnsignedIntegerRepresentation()
1736 ? CGM.getHLSLRuntime().getWaveMultiPrefixUSumIntrinsic()
1737 : CGM.getHLSLRuntime().getWaveMultiPrefixSumIntrinsic();
1738 return EmitIntrinsicCall(ID: IID, Types: {ValueOp->getType()}, Args,
1739 Name: "hlsl.wave.multi.prefix.sum");
1740 }
1741 if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
1742 return EmitIntrinsicCall(
1743 ID: CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
1744 Types: {ValueOp->getType()}, Args, Name: "hlsl.wave.multi.prefix.bit.and");
1745 if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_or)
1746 return EmitIntrinsicCall(
1747 ID: CGM.getHLSLRuntime().getWaveMultiPrefixBitOrIntrinsic(),
1748 Types: {ValueOp->getType()}, Args, Name: "hlsl.wave.multi.prefix.bit.or");
1749 return EmitIntrinsicCall(
1750 ID: CGM.getHLSLRuntime().getWaveMultiPrefixBitXorIntrinsic(),
1751 Types: {ValueOp->getType()}, Args, Name: "hlsl.wave.multi.prefix.bit.xor");
1752 }
1753 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1754 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1755 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1756 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1757 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1758 Name: "hlsl.wave.prefix.product");
1759 }
1760 case Builtin::BI__builtin_hlsl_quad_all: {
1761 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1762 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadAllIntrinsic();
1763 return EmitIntrinsicCall(ID, Args: {Op});
1764 }
1765 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1766 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1767 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1768 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1769 Name: "hlsl.quad.read.across.x");
1770 }
1771 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1772 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1773 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1774 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1775 Name: "hlsl.quad.read.across.y");
1776 }
1777 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1778 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1779 Intrinsic::ID ID =
1780 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1781 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1782 Name: "hlsl.quad.read.across.diagonal");
1783 }
1784 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1785 auto *Arg0 = E->getArg(Arg: 0);
1786 Value *Op0 = EmitScalarExpr(E: Arg0);
1787 llvm::Type *Xty = Op0->getType();
1788 llvm::Type *retType = getAggregateType(
1789 ScalarTy: llvm::Type::getInt32Ty(C&: this->getLLVMContext()), ArgTy: Arg0->getType());
1790 assert((Arg0->getType()->hasFloatingRepresentation() ||
1791 Arg0->getType()->hasIntegerRepresentation()) &&
1792 "sign operand must have a float or int representation");
1793
1794 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1795 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1796 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1797 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1798 }
1799
1800 return Builder.CreateIntrinsic(
1801 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1802 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1803 }
1804 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1805 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1806 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1807 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1808 return Builder.CreateIntrinsic(
1809 /*ReturnType=*/RetTy: Offset->getType(),
1810 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1811 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1812 }
1813 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1814
1815 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1816 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1817 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1818 "asuint operands types mismatch");
1819 return handleHlslSplitdouble(E, CGF: this);
1820 }
1821 case Builtin::BI__builtin_hlsl_elementwise_clip:
1822 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1823 "clip operands types mismatch");
1824 return handleHlslClip(E, CGF: this);
1825 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1826 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1827 return EmitIntrinsicCall(ID);
1828 }
1829 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1830 Intrinsic::ID ID =
1831 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1832 return EmitIntrinsicCall(ID);
1833 }
1834 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1835 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1836 return EmitIntrinsicCall(ID);
1837 }
1838 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1839 Intrinsic::ID ID =
1840 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1841 return EmitIntrinsicCall(ID);
1842 }
1843 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1844 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1845 return EmitIntrinsicCall(ID);
1846 }
1847 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1848 Intrinsic::ID ID =
1849 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1850 return EmitIntrinsicCall(ID);
1851 }
1852 case Builtin::BI__builtin_hlsl_barrier: {
1853 std::optional<llvm::APSInt> SemanticFlagsConstant =
1854 E->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
1855 assert(SemanticFlagsConstant && "expected constant semantic flags");
1856 constexpr uint64_t GroupScope =
1857 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::GroupScope);
1858 constexpr uint64_t DeviceScope =
1859 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::DeviceScope);
1860 constexpr uint64_t ScopeMask = GroupScope | DeviceScope;
1861 uint64_t SemanticFlagsValue = SemanticFlagsConstant->getZExtValue();
1862 // DEVICE_SCOPE includes GROUP_SCOPE. Remove GROUP_SCOPE when both are
1863 // set.
1864 if ((SemanticFlagsValue & ScopeMask) == ScopeMask)
1865 SemanticFlagsValue &= ~GroupScope;
1866 Value *SemanticFlags = llvm::ConstantInt::get(
1867 Ty: ConvertType(T: E->getArg(Arg: 1)->getType()), V: SemanticFlagsValue);
1868
1869 if (E->getArg(Arg: 0)->getType()->isUnsignedIntegerType()) {
1870 std::optional<llvm::APSInt> MemoryFlagsConstant =
1871 E->getArg(Arg: 0)->getIntegerConstantExpr(Ctx: getContext());
1872 assert(MemoryFlagsConstant && "expected constant memory flags");
1873 uint64_t MemoryFlagsValue = MemoryFlagsConstant->getZExtValue();
1874 llvm::Triple::EnvironmentType Stage =
1875 getTarget().getTriple().getEnvironment();
1876 if (Stage == llvm::Triple::Library)
1877 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
1878 if (const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>())
1879 Stage = ShaderAttr->getType();
1880 MemoryFlagsValue =
1881 getEffectiveBarrierMemoryFlags(MemoryFlags: MemoryFlagsValue, Stage);
1882 Value *MemoryFlags = llvm::ConstantInt::get(
1883 Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), V: MemoryFlagsValue);
1884 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryTypeIntrinsic();
1885 return EmitIntrinsicCall(ID, Types: {},
1886 Args: ArrayRef<Value *>{MemoryFlags, SemanticFlags});
1887 }
1888
1889 const FieldDecl *HandleField =
1890 getResourceHandleField(ResourceTy: E->getArg(Arg: 0)->getType());
1891 LValue Resource = EmitLValue(E: E->getArg(Arg: 0));
1892 LValue Handle = EmitLValueForField(Base: Resource, Field: HandleField);
1893 Value *HandleValue =
1894 EmitLoadOfLValue(V: Handle, Loc: E->getArg(Arg: 0)->getExprLoc()).getScalarVal();
1895 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryHandleIntrinsic();
1896 return EmitIntrinsicCall(ID, Types: ArrayRef<llvm::Type *>{HandleValue->getType()},
1897 Args: ArrayRef<Value *>{HandleValue, SemanticFlags});
1898 }
1899 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1900 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1901 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1902 llvm_unreachable("ddx_coarse operand must have a float representation");
1903 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1904 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1905 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1906 Name: "hlsl.ddx.coarse");
1907 }
1908 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1909 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1910 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1911 llvm_unreachable("ddy_coarse operand must have a float representation");
1912 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1913 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1914 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1915 Name: "hlsl.ddy.coarse");
1916 }
1917 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1918 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1919 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1920 llvm_unreachable("ddx_fine operand must have a float representation");
1921 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1922 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1923 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1924 Name: "hlsl.ddx.fine");
1925 }
1926 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1927 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1928 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1929 llvm_unreachable("ddy_fine operand must have a float representation");
1930 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1931 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1932 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1933 Name: "hlsl.ddy.fine");
1934 }
1935 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1936 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1937 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1938 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1939 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1940 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1941 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1942 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1943 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1944 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1945 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1946 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1947 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1948 llvm::Value *Args[] = {SpecId, DefaultVal};
1949 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1950 }
1951 }
1952 return nullptr;
1953}
1954
1955llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1956 const clang::QualType &SpecConstantType) {
1957
1958 // Find or create the declaration for the function.
1959 llvm::Module *M = &CGM.getModule();
1960 std::string MangledName =
1961 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1962 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1963
1964 if (!SpecConstantFn) {
1965 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1966 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1967 llvm::Type *ArgTypes[] = {IntType, RetTy};
1968 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1969 SpecConstantFn = llvm::Function::Create(
1970 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1971 }
1972 return SpecConstantFn;
1973}
1974