1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/IR/MatrixBuilder.h"
20#include "llvm/Support/DXILABI.h"
21
22using namespace clang;
23using namespace CodeGen;
24using namespace llvm;
25
26static uint64_t
27getEffectiveBarrierMemoryFlags(uint64_t MemoryFlags,
28 llvm::Triple::EnvironmentType Stage) {
29 constexpr uint64_t AllMemory =
30 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::ValidMask);
31 if (MemoryFlags != AllMemory || Stage == llvm::Triple::Library ||
32 Stage == llvm::Triple::UnknownEnvironment)
33 return MemoryFlags;
34
35 constexpr uint64_t GroupSharedMemory =
36 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::GroupSharedMemory);
37 constexpr uint64_t NodeMemory =
38 llvm::to_underlying(E: llvm::dxil::BarrierMemoryTypeFlag::NodeMemory);
39 const bool HasVisibleGroup = Stage == llvm::Triple::Compute ||
40 Stage == llvm::Triple::Mesh ||
41 Stage == llvm::Triple::Amplification;
42 if (!HasVisibleGroup)
43 MemoryFlags &= ~GroupSharedMemory;
44 // Clang has no node shader stage yet, so no known stage exposes node memory.
45 return MemoryFlags & ~NodeMemory;
46}
47
48static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
49 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
50 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
51 "asdouble operands types mismatch");
52 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
53 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
54
55 llvm::Type *ResultType = CGF.DoubleTy;
56 int N = 1;
57 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
58 N = VTy->getNumElements();
59 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
60 }
61
62 if (CGF.CGM.getTarget().getTriple().isDXIL())
63 return CGF.Builder.CreateIntrinsic(
64 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
65 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
66
67 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
68 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
69 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
70 }
71
72 llvm::SmallVector<int> Mask;
73 for (int i = 0; i < N; i++) {
74 Mask.push_back(Elt: i);
75 Mask.push_back(Elt: i + N);
76 }
77
78 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
79
80 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
81}
82
83static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
84 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
85
86 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
87 Value *CMP;
88 Value *LastInstr;
89
90 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
91 FZeroConst = ConstantVector::getSplat(
92 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
93 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
94 CMP = CGF->Builder.CreateIntrinsic(
95 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
96 Args: {FCompInst});
97 } else {
98 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
99 }
100
101 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
102 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
103 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
104 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
105 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
106
107 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
108
109 CGF->Builder.SetInsertPoint(LT0);
110
111 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
112
113 LastInstr = CGF->Builder.CreateBr(Dest: End);
114 CGF->Builder.SetInsertPoint(End);
115 } else {
116 llvm_unreachable("Backend Codegen not supported.");
117 }
118
119 return LastInstr;
120}
121
122static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
123 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
124 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
125 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
126
127 CallArgList Args;
128 LValue Op1TmpLValue =
129 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
130 LValue Op2TmpLValue =
131 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
132
133 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
134 Args.reverseWritebacks();
135
136 Value *LowBits = nullptr;
137 Value *HighBits = nullptr;
138
139 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
140 llvm::Type *RetElementTy = CGF->Int32Ty;
141 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
142 RetElementTy = llvm::VectorType::get(
143 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
144 else if (auto *Op0MatTy =
145 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
146 RetElementTy = llvm::VectorType::get(
147 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
148 Op0MatTy->getNumColumns()));
149
150 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
151
152 Value *CI = CGF->Builder.CreateIntrinsic(
153 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
154
155 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
156 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
157 } else {
158 // For Non DXIL targets we generate the instructions.
159
160 if (!Op0->getType()->isVectorTy()) {
161 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
162 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
163
164 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
165 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
166 } else {
167 int NumElements = 1;
168 if (const auto *VecTy =
169 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
170 NumElements = VecTy->getNumElements();
171 else if (const auto *MatTy =
172 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
173 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
174
175 FixedVectorType *Uint32VecTy =
176 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
177 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
178 if (NumElements == 1) {
179 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
180 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
181 } else {
182 SmallVector<int> EvenMask, OddMask;
183 for (int I = 0, E = NumElements; I != E; ++I) {
184 EvenMask.push_back(Elt: I * 2);
185 OddMask.push_back(Elt: I * 2 + 1);
186 }
187 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
188 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
189 }
190 }
191 }
192 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
193 auto *LastInst =
194 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
195 CGF->EmitWritebacks(Args);
196 return LastInst;
197}
198
199static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
200 const CallExpr *E) {
201 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
202 llvm::Type *I32 = CGF.Int32Ty;
203
204 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
205 [[maybe_unused]] llvm::StructType *Struct4I32 =
206 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
207
208 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
209 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
210 Value *StructVal =
211 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
212 assert(StructVal->getType() == Struct4I32 &&
213 "dx.wave.ballot must return {i32,i32,i32,i32}");
214
215 // Reassemble struct to <4 x i32>
216 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
217 for (unsigned I = 0; I < 4; ++I) {
218 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
219 VecVal =
220 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
221 }
222
223 return VecVal;
224 }
225
226 if (CGF.CGM.getTarget().getTriple().isSPIRV())
227 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
228
229 llvm_unreachable(
230 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
231}
232
233static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
234 const CallExpr *E) {
235 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
236 QualType Op0Ty = E->getArg(Arg: 0)->getType();
237 llvm::Type *ResType = CGF.FloatTy;
238 uint64_t NumElements = 0;
239 if (Op0->getType()->isVectorTy()) {
240 NumElements =
241 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
242 ResType =
243 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
244 }
245 if (!Op0Ty->hasUnsignedIntegerRepresentation())
246 llvm_unreachable(
247 "f16tof32 operand must have an unsigned int representation");
248
249 if (CGF.CGM.getTriple().isDXIL())
250 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
251 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
252 Name: "hlsl.f16tof32");
253
254 if (CGF.CGM.getTriple().isSPIRV()) {
255 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
256 // Int16 and Float16 capabilities
257 auto *UnpackType =
258 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
259
260 if (NumElements == 0) {
261 // a scalar input - simply extract the first element of the unpacked
262 // vector
263 Value *Unpack = CGF.Builder.CreateIntrinsic(
264 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
265 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
266 }
267
268 // a vector input - build a congruent output vector by iterating through
269 // the input vector calling unpackhalf2x16 for each element
270 Value *Result = PoisonValue::get(T: ResType);
271 for (uint64_t I = 0; I < NumElements; I++) {
272 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
273 Value *Unpack = CGF.Builder.CreateIntrinsic(
274 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
275 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
276 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
277 }
278 return Result;
279 }
280
281 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
282}
283
284static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
285 const CallExpr *E) {
286 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
287 QualType Op0Ty = E->getArg(Arg: 0)->getType();
288 llvm::Type *ResType = CGF.IntTy;
289 uint64_t NumElements = 0;
290 if (Op0->getType()->isVectorTy()) {
291 NumElements =
292 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
293 ResType =
294 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
295 }
296 if (!Op0Ty->hasFloatingRepresentation())
297 llvm_unreachable("f32tof16 operand must have a float representation");
298
299 if (CGF.CGM.getTriple().isDXIL())
300 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
301 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
302 Name: "hlsl.f32tof16");
303
304 if (CGF.CGM.getTriple().isSPIRV()) {
305 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
306 // Int16 and Float16 capabilities
307 auto *PackType =
308 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
309
310 if (NumElements == 0) {
311 // a scalar input - simply insert the scalar in the first element
312 // of the 2 element float vector
313 Value *Float2 = Constant::getNullValue(Ty: PackType);
314 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
315 Value *Result = CGF.Builder.CreateIntrinsic(
316 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
317 return Result;
318 }
319
320 // a vector input - build a congruent output vector by iterating through
321 // the input vector calling packhalf2x16 for each element
322 Value *Result = PoisonValue::get(T: ResType);
323 for (uint64_t I = 0; I < NumElements; I++) {
324 Value *Float2 = Constant::getNullValue(Ty: PackType);
325 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
326 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
327 Value *Res = CGF.Builder.CreateIntrinsic(
328 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
329 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
330 }
331 return Result;
332 }
333
334 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
335}
336
337// Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
338// Set the memory scope: Workgroup for groupshared, otherwise Device.
339static llvm::SyncScope::ID getHLSLAtomicScope(CodeGenFunction &CGF,
340 const LValue &DestLV) {
341 StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared
342 ? "workgroup"
343 : "device";
344 return CGF.getLLVMContext().getOrInsertSyncScopeID(SSN: ScopeName);
345}
346
347// The destination can name one element of a vector, as in `buf[0].z` or
348// `gs[i]`. `LValue::getAddress` gives the address of the whole vector for such
349// an lvalue, so index into the vector to get the address of the element. Sema
350// rejects a multi-element swizzle, so the access is always a single element.
351static Address getHLSLAtomicDestAddr(CodeGenFunction &CGF,
352 const LValue &DestLV) {
353 if (!DestLV.isVectorElt() && !DestLV.isExtVectorElt())
354 return DestLV.getAddress();
355
356 Address VecAddr = DestLV.isVectorElt() ? DestLV.getVectorAddress()
357 : DestLV.getExtVectorAddress();
358 Value *Idx = DestLV.isVectorElt()
359 ? DestLV.getVectorIdx()
360 : llvm::ConstantInt::get(Ty: CGF.SizeTy,
361 V: CodeGenFunction::getAccessedFieldNo(
362 Idx: 0, Elts: DestLV.getExtVectorElts()));
363
364 // A vector-element lvalue reports the type of the whole vector, so take the
365 // element type from the address. HLSL also treats a scalar as a one-element
366 // vector, in which case the address already has the element type.
367 llvm::Type *VecTy = VecAddr.getElementType();
368 llvm::Type *ElemTy = VecTy->isVectorTy()
369 ? cast<llvm::VectorType>(Val: VecTy)->getElementType()
370 : VecTy;
371 return CGF.Builder.CreateGEP(CGF, Addr: VecAddr.withElementType(ElemTy), Index: Idx);
372}
373
374static LValue getHLSLAtomicOriginalValueLValue(CodeGenFunction &CGF,
375 const Expr *E,
376 CallArgList &Args) {
377 if (const auto *OutArg = dyn_cast<HLSLOutArgExpr>(Val: E))
378 return CGF.EmitHLSLOutArgExpr(E: OutArg, Args, Ty: OutArg->getType());
379 return CGF.EmitLValue(E);
380}
381
382static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
383 llvm::AtomicRMWInst::BinOp Op) {
384 // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
385 // either DXIL or SPIR-V.
386 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
387 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
388 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
389 [[maybe_unused]] QualType ValTy = E->getArg(Arg: 1)->getType();
390 if (Op == llvm::AtomicRMWInst::Xchg)
391 assert((ValTy->isIntegerType() || ValTy->isFloatingType()) &&
392 "InterlockedExchange value operand must be an integer or a float");
393 else
394 assert(ValTy->isIntegerType() &&
395 "Intrinsic InterlockedOp value operand must be an integer");
396
397 llvm::SyncScope::ID SSID = getHLSLAtomicScope(CGF, DestLV);
398
399 llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
400 Op, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
401
402 // The 3-arg overload writes the old value (the RMW's return value) into
403 // the `original_value` reference parameter.
404 if (E->getNumArgs() == 3) {
405 CallArgList Args;
406 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 2), Args);
407 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
408 CGF.EmitWritebacks(Args);
409 }
410 return Call;
411}
412
413// Emit `cmpxchg` for InterlockedCompareStore and InterlockedCompareExchange.
414// Compare-exchange also reports the previous value.
415static Value *handleInterlockedCompareOp(CodeGenFunction &CGF,
416 const CallExpr *E) {
417 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
418 Address DestAddr = getHLSLAtomicDestAddr(CGF, DestLV);
419 Value *Compare = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
420 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
421
422 // `cmpxchg` takes an integer or a pointer, so the float-bitwise operations
423 // work on the bit pattern of the float. This is what those operations mean,
424 // and DXIL and SPIR-V both need the integer form.
425 llvm::Type *FloatTy = nullptr;
426 if (Compare->getType()->isFloatingPointTy()) {
427 FloatTy = Compare->getType();
428 llvm::Type *IntTy =
429 CGF.Builder.getIntNTy(N: FloatTy->getPrimitiveSizeInBits());
430 Compare = CGF.Builder.CreateBitCast(V: Compare, DestTy: IntTy);
431 Val = CGF.Builder.CreateBitCast(V: Val, DestTy: IntTy);
432 DestAddr = DestAddr.withElementType(ElemTy: IntTy);
433 }
434
435 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
436 Addr: DestAddr, Cmp: Compare, New: Val, SuccessOrdering: llvm::AtomicOrdering::Monotonic,
437 FailureOrdering: llvm::AtomicOrdering::Monotonic, SSID: getHLSLAtomicScope(CGF, DestLV));
438
439 // Compare-store reports nothing, so it leaves the `cmpxchg` result unused.
440 if (E->getNumArgs() < 4)
441 return Pair;
442
443 // `cmpxchg` yields a { previous value, success } pair. HLSL reports only the
444 // previous value, through the `original_value` reference parameter.
445 Value *Original = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
446 if (FloatTy)
447 Original = CGF.Builder.CreateBitCast(V: Original, DestTy: FloatTy);
448 CallArgList Args;
449 LValue OrigLV = getHLSLAtomicOriginalValueLValue(CGF, E: E->getArg(Arg: 3), Args);
450 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Original), Dst: OrigLV);
451 CGF.EmitWritebacks(Args);
452 return Original;
453}
454
455static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
456 LValue &Stride) {
457 // Figure out the stride of the buffer elements from the handle type.
458 auto *HandleTy =
459 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
460 QualType ElementTy = HandleTy->getContainedType();
461 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
462 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
463}
464
465// Return dot product intrinsic that corresponds to the QT scalar type
466static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
467 if (QT->isFloatingType())
468 return RT.getFDotIntrinsic();
469 if (QT->isSignedIntegerType())
470 return RT.getSDotIntrinsic();
471 assert(QT->isUnsignedIntegerType());
472 return RT.getUDotIntrinsic();
473}
474
475static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
476 if (QT->hasSignedIntegerRepresentation()) {
477 return RT.getFirstBitSHighIntrinsic();
478 }
479
480 assert(QT->hasUnsignedIntegerRepresentation());
481 return RT.getFirstBitUHighIntrinsic();
482}
483
484// Return wave active sum that corresponds to the QT scalar type
485static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
486 QualType QT) {
487 switch (Arch) {
488 case llvm::Triple::spirv:
489 return Intrinsic::spv_wave_reduce_sum;
490 case llvm::Triple::dxil: {
491 if (QT->isUnsignedIntegerType())
492 return Intrinsic::dx_wave_reduce_usum;
493 return Intrinsic::dx_wave_reduce_sum;
494 }
495 default:
496 llvm_unreachable("Intrinsic WaveActiveSum"
497 " not supported by target architecture");
498 }
499}
500
501// Return wave active product that corresponds to the QT scalar type
502static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
503 QualType QT) {
504 switch (Arch) {
505 case llvm::Triple::spirv:
506 return Intrinsic::spv_wave_product;
507 case llvm::Triple::dxil: {
508 if (QT->isUnsignedIntegerType())
509 return Intrinsic::dx_wave_uproduct;
510 return Intrinsic::dx_wave_product;
511 }
512 default:
513 llvm_unreachable("Intrinsic WaveActiveProduct"
514 " not supported by target architecture");
515 }
516}
517
518static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
519 switch (Arch) {
520 case llvm::Triple::spirv:
521 return Intrinsic::spv_subgroup_prefix_bit_count;
522 case llvm::Triple::dxil: {
523 return Intrinsic::dx_wave_prefix_bit_count;
524 }
525 default:
526 llvm_unreachable(
527 "WavePrefixOp instruction not supported by target architecture");
528 }
529}
530
531// Return wave prefix sum that corresponds to the QT scalar type
532static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
533 QualType QT) {
534 switch (Arch) {
535 case llvm::Triple::spirv:
536 return Intrinsic::spv_wave_prefix_sum;
537 case llvm::Triple::dxil: {
538 if (QT->isUnsignedIntegerType())
539 return Intrinsic::dx_wave_prefix_usum;
540 return Intrinsic::dx_wave_prefix_sum;
541 }
542 default:
543 llvm_unreachable("Intrinsic WavePrefixSum"
544 " not supported by target architecture");
545 }
546}
547
548// Return wave prefix product that corresponds to the QT scalar type
549static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
550 QualType QT) {
551 switch (Arch) {
552 case llvm::Triple::spirv:
553 return Intrinsic::spv_wave_prefix_product;
554 case llvm::Triple::dxil: {
555 if (QT->isUnsignedIntegerType())
556 return Intrinsic::dx_wave_prefix_uproduct;
557 return Intrinsic::dx_wave_prefix_product;
558 }
559 default:
560 llvm_unreachable("Intrinsic WavePrefixProduct"
561 " not supported by target architecture");
562 }
563}
564
565// Returns the mangled name for a builtin function that the SPIR-V backend
566// will expand into a spec Constant.
567static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
568 ASTContext &Context) {
569 // The parameter types for our conceptual intrinsic function.
570 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
571
572 // Create a temporary FunctionDecl for the builtin fuction. It won't be
573 // added to the AST.
574 FunctionProtoType::ExtProtoInfo EPI;
575 QualType FnType =
576 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
577 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
578 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
579 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
580 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
581
582 // Attach the created parameter declarations to the function declaration.
583 SmallVector<ParmVarDecl *, 2> ParamDecls;
584 for (QualType ParamType : ClangParamTypes) {
585 ParmVarDecl *PD = ParmVarDecl::Create(
586 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
587 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
588 /*DefaultArg*/ DefArg: nullptr);
589 ParamDecls.push_back(Elt: PD);
590 }
591 FnDeclForMangling->setParams(ParamDecls);
592
593 // Get the mangled name.
594 std::string Name;
595 llvm::raw_string_ostream MangledNameStream(Name);
596 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
597 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
598 MangledNameStream.flush();
599
600 return Name;
601}
602
603static const HLSLAttributedResourceType *
604getHandleAttributedType(QualType HandleQT) {
605 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
606 return RT;
607 // If the expr is a texture/sampler record (or similar), peel to __handle.
608 if (const HLSLAttributedResourceType *RT =
609 HLSLAttributedResourceType::findHandleTypeOnResource(
610 RT: HandleQT.getTypePtr()))
611 return RT;
612 llvm_unreachable("attributed handle type not found");
613}
614
615static const HLSLAttributedResourceType *
616getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
617 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
618}
619
620static const FieldDecl *getResourceHandleField(QualType ResourceTy) {
621 const CXXRecordDecl *ResourceDecl = ResourceTy->getAsCXXRecordDecl();
622 assert(ResourceDecl && "resource must be a record type");
623
624 IdentifierInfo &II = ResourceDecl->getASTContext().Idents.get(Name: "__handle");
625 for (const Decl *D : ResourceDecl->lookup(Name: &II))
626 if (const auto *Field = dyn_cast<FieldDecl>(Val: D))
627 return Field;
628
629 llvm_unreachable("resource handle field not found");
630}
631
632static llvm::Type *getOffsetType(CodeGenModule &CGM,
633 const HLSLAttributedResourceType *RT) {
634 const auto &Attrs = RT->getAttrs();
635 unsigned OffsetSize =
636 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
637 llvm::Type *Int32Ty = CGM.Int32Ty;
638 if (OffsetSize == 1)
639 return Int32Ty;
640 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
641}
642
643static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
644 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
645 if (E->getNumArgs() > OffsetArgIndex)
646 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
647
648 return llvm::Constant::getNullValue(Ty: OffsetTy);
649}
650
651static Value *emitHlslSampleOffset(CodeGenFunction &CGF, const CallExpr *E,
652 const HLSLAttributedResourceType *RT,
653 unsigned OffsetArgIndex) {
654 llvm::Type *OffsetTy = getOffsetType(CGM&: CGF.CGM, RT);
655 if (!clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension))
656 return llvm::Constant::getNullValue(Ty: OffsetTy);
657 return emitHlslOffset(CGF, E, OffsetArgIndex, OffsetTy);
658}
659
660static unsigned getHlslClampArgIndex(const HLSLAttributedResourceType *RT,
661 unsigned OffsetArgIndex) {
662 return clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension)
663 ? OffsetArgIndex + 1
664 : OffsetArgIndex;
665}
666
667static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
668 unsigned IntrinsicID, unsigned NumRetComps,
669 bool HasLod) {
670 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
671
672 SmallVector<Value *> Args{Handle};
673 if (HasLod)
674 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
675
676 Value *DimValue =
677 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
678
679 Value *LastStore = nullptr;
680 unsigned ArgIndex = HasLod ? 2 : 1;
681 for (unsigned i = 0; i < NumRetComps; ++i) {
682 const Expr *Arg = E->getArg(Arg: ArgIndex++);
683 LValue DimOut = CGF.EmitLValue(E: Arg);
684 Value *Elem = DimValue;
685 if (NumRetComps > 1)
686 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
687
688 // Handle float casting if needed
689 if (Arg->getType()->isFloatingType())
690 Elem = CGF.Builder.CreateUIToFP(
691 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
692
693 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
694 }
695 return LastStore;
696}
697
698static llvm::Type *getAggregateType(llvm::Type *ScalarTy, QualType ArgTy) {
699 if (auto *MatTy = ArgTy->getAs<ConstantMatrixType>())
700 return llvm::VectorType::get(
701 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: MatTy->getNumElementsFlattened()));
702 if (auto *VecTy = ArgTy->getAs<clang::VectorType>())
703 return llvm::VectorType::get(
704 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()));
705 return ScalarTy;
706}
707
708Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
709 const CallExpr *E,
710 ReturnValueSlot ReturnValue) {
711 if (!getLangOpts().HLSL)
712 return nullptr;
713
714 switch (BuiltinID) {
715 case Builtin::BI__builtin_hlsl_adduint64: {
716 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
717 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
718 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
719 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
720 assert(Arg0Ty == E->getArg(1)->getType() &&
721 "AddUint64 operand types must match");
722 assert(Arg0Ty->hasIntegerRepresentation() &&
723 "AddUint64 operands must have an integer representation");
724 assert((NumElements == 2 || NumElements == 4) &&
725 "AddUint64 operands must have 2 or 4 elements");
726
727 llvm::Value *LowA;
728 llvm::Value *HighA;
729 llvm::Value *LowB;
730 llvm::Value *HighB;
731
732 // Obtain low and high words of inputs A and B
733 if (NumElements == 2) {
734 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
735 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
736 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
737 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
738 } else {
739 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
740 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
741 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
742 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
743 }
744
745 // Use an uadd_with_overflow to compute the sum of low words and obtain a
746 // carry value
747 llvm::Value *Carry;
748 llvm::Value *LowSum = EmitOverflowIntrinsic(
749 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
750 llvm::Value *ZExtCarry =
751 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
752
753 // Sum the high words and the carry
754 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
755 llvm::Value *HighSumPlusCarry =
756 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
757
758 if (NumElements == 4) {
759 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
760 Name: "hlsl.AddUint64");
761 }
762
763 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
764 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
765 Name: "hlsl.AddUint64.upto0");
766 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
767 Name: "hlsl.AddUint64");
768 return Result;
769 }
770 case Builtin::BI__builtin_hlsl_resource_getpointer:
771 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
772 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
773 bool IsIndexed =
774 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
775 E->getNumArgs() > 1;
776
777 llvm::Type *RetTy = ConvertType(T: E->getType());
778 if (IsIndexed) {
779 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
780 return EmitIntrinsicCall(
781 ID: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
782 Types: {RetTy, HandleOp->getType(), IndexOp->getType()},
783 Args: {HandleOp, IndexOp});
784 }
785
786 return EmitIntrinsicCall(
787 ID: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
788 Types: {RetTy, HandleOp->getType()}, Args: {HandleOp});
789 }
790 case Builtin::BI__builtin_hlsl_transpose_if_memory_is_row_major: {
791 const Expr *ValueExpr = E->getArg(Arg: 0);
792 if (hasAggregateEvaluationKind(T: ValueExpr->getType())) {
793 EmitAnyExprToMem(E: ValueExpr, Location: ReturnValue.getAddress(),
794 Quals: ValueExpr->getType().getQualifiers(), /*IsInit=*/IsInitializer: true);
795 return ReturnValue.getAddress().getBasePointer();
796 }
797
798 Value *ValueOp = EmitScalarExpr(E: ValueExpr);
799 const auto *MatTy = ValueExpr->getType()->getAs<ConstantMatrixType>();
800 if (!MatTy || !getLangOpts().HLSLSpvUseLegacyBufferMatrixOrder)
801 return ValueOp;
802
803 bool IsLoad =
804 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getBoolValue();
805 unsigned Rows = MatTy->getNumRows();
806 unsigned Columns = MatTy->getNumColumns();
807 llvm::MatrixBuilder MB(Builder);
808 return IsLoad ? MB.CreateMatrixTranspose(Matrix: ValueOp, Rows: Columns, Columns: Rows)
809 : MB.CreateMatrixTranspose(Matrix: ValueOp, Rows, Columns);
810 }
811 case Builtin::BI__builtin_hlsl_resource_sample: {
812 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
813 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
814 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
815 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
816
817 SmallVector<Value *, 4> Args;
818 Args.push_back(Elt: HandleOp);
819 Args.push_back(Elt: SamplerOp);
820 Args.push_back(Elt: CoordOp);
821 constexpr unsigned OffsetIdx = 3;
822 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
823
824 llvm::Type *RetTy = ConvertType(T: E->getType());
825 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
826 if (E->getNumArgs() <= ClampIdx)
827 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args,
828 RetTy);
829
830 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
831 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(),
832 Args, RetTy);
833 }
834 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
835 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
836 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
837 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
838 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
839 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
840
841 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
842 Args.push_back(Elt: HandleOp);
843 Args.push_back(Elt: SamplerOp);
844 Args.push_back(Elt: CoordOp);
845 Args.push_back(Elt: BiasOp);
846 constexpr unsigned OffsetIdx = 4;
847 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
848
849 llvm::Type *RetTy = ConvertType(T: E->getType());
850 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
851 if (E->getNumArgs() <= ClampIdx)
852 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(),
853 Args, RetTy);
854
855 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
856 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(),
857 Args, RetTy);
858 }
859 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
860 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
861 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
862 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
863 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
864 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
865 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
866
867 SmallVector<Value *, 7> Args;
868 Args.push_back(Elt: HandleOp);
869 Args.push_back(Elt: SamplerOp);
870 Args.push_back(Elt: CoordOp);
871 Args.push_back(Elt: DDXOp);
872 Args.push_back(Elt: DDYOp);
873 constexpr unsigned OffsetIdx = 5;
874 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
875
876 llvm::Type *RetTy = ConvertType(T: E->getType());
877
878 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
879 if (E->getNumArgs() <= ClampIdx)
880 return Builder.CreateIntrinsic(
881 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
882
883 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
884 return Builder.CreateIntrinsic(
885 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
886 }
887 case Builtin::BI__builtin_hlsl_resource_sample_level: {
888 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
889 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
890 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
891 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
892 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
893
894 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
895 Args.push_back(Elt: HandleOp);
896 Args.push_back(Elt: SamplerOp);
897 Args.push_back(Elt: CoordOp);
898 Args.push_back(Elt: LODOp);
899 constexpr unsigned OffsetIdx = 4;
900 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
901
902 llvm::Type *RetTy = ConvertType(T: E->getType());
903 return Builder.CreateIntrinsic(
904 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
905 }
906 case Builtin::BI__builtin_hlsl_resource_load_level: {
907 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
908 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
909 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
910
911 const auto &Attrs = RT->getAttrs();
912
913 Value *CoordOp = nullptr;
914 Value *LODOp = nullptr;
915 if (Attrs.ResourceClass == llvm::dxil::ResourceClass::UAV) {
916 // A UAV descriptor binds a single mip slice, so a RWTexture location is
917 // all coordinate and there is no mip level to select.
918 CoordOp = CoordLODOp;
919 LODOp = llvm::PoisonValue::get(T: Int32Ty);
920 } else {
921 // Split CoordLOD into Coord and LOD. 1D resources use a scalar
922 // coordinate rather than a 1-element vector.
923 unsigned CoordSize =
924 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension) +
925 (Attrs.IsArray ? 1 : 0);
926 assert(cast<llvm::FixedVectorType>(CoordLODOp->getType())
927 ->getNumElements() == CoordSize + 1 &&
928 "CoordLOD must have one element per coordinate, plus the level");
929
930 if (CoordSize == 1) {
931 CoordOp = Builder.CreateExtractElement(Vec: CoordLODOp, Idx: uint64_t(0),
932 Name: "hlsl.load.coord");
933 } else {
934 SmallVector<int, 4> Mask;
935 for (unsigned I = 0; I < CoordSize; ++I)
936 Mask.push_back(Elt: I);
937 CoordOp =
938 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
939 }
940 LODOp =
941 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: CoordSize, Name: "hlsl.load.lod");
942 }
943
944 SmallVector<Value *, 4> Args;
945 Args.push_back(Elt: HandleOp);
946 Args.push_back(Elt: CoordOp);
947 Args.push_back(Elt: LODOp);
948 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
949
950 llvm::Type *RetTy = ConvertType(T: E->getType());
951 return Builder.CreateIntrinsic(
952 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
953 }
954 case Builtin::BI__builtin_hlsl_resource_load_ms: {
955 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
956 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 1));
957 Value *SampleOp = EmitScalarExpr(E: E->getArg(Arg: 2));
958 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
959
960 SmallVector<Value *, 4> Args;
961 Args.push_back(Elt: HandleOp);
962 Args.push_back(Elt: CoordOp);
963 Args.push_back(Elt: SampleOp);
964 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
965
966 llvm::Type *RetTy = ConvertType(T: E->getType());
967 return Builder.CreateIntrinsic(
968 RetTy, ID: CGM.getHLSLRuntime().getLoadMSIntrinsic(), Args);
969 }
970 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
971 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
972 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
973 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
974 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
975 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
976
977 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
978 Args.push_back(Elt: HandleOp);
979 Args.push_back(Elt: SamplerOp);
980 Args.push_back(Elt: CoordOp);
981 Args.push_back(Elt: CmpOp);
982 constexpr unsigned OffsetIdx = 4;
983 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
984
985 llvm::Type *RetTy = ConvertType(T: E->getType());
986 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
987 if (E->getNumArgs() <= ClampIdx)
988 return Builder.CreateIntrinsic(
989 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
990
991 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
992 return Builder.CreateIntrinsic(
993 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
994 }
995 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
996 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
997 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
998 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
999 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1000 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1001
1002 SmallVector<Value *, 5> Args;
1003 Args.push_back(Elt: HandleOp);
1004 Args.push_back(Elt: SamplerOp);
1005 Args.push_back(Elt: CoordOp);
1006 Args.push_back(Elt: CmpOp);
1007 constexpr unsigned OffsetIdx = 4;
1008 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
1009
1010 llvm::Type *RetTy = ConvertType(T: E->getType());
1011 return Builder.CreateIntrinsic(
1012 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
1013 }
1014 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
1015 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1016 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1017 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1018
1019 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
1020 Args: {HandleOp, SamplerOp, CoordOp},
1021 RetTy: ConvertType(T: E->getType()));
1022 }
1023 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
1024 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1025 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1026 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1027
1028 return EmitIntrinsicCall(
1029 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
1030 Args: {HandleOp, SamplerOp, CoordOp}, RetTy: ConvertType(T: E->getType()));
1031 }
1032 case Builtin::BI__builtin_hlsl_resource_gather: {
1033 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1034 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1035 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1036 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1037 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1038
1039 SmallVector<Value *, 5> Args;
1040 Args.push_back(Elt: HandleOp);
1041 Args.push_back(Elt: SamplerOp);
1042 Args.push_back(Elt: CoordOp);
1043 Args.push_back(Elt: ComponentOp);
1044 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
1045
1046 llvm::Type *RetTy = ConvertType(T: E->getType());
1047 return Builder.CreateIntrinsic(
1048 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
1049 }
1050 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
1051 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1052 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1053 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1054 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1055
1056 SmallVector<Value *, 6> Args;
1057 Args.push_back(Elt: HandleOp);
1058 Args.push_back(Elt: SamplerOp);
1059 Args.push_back(Elt: CoordOp);
1060 Args.push_back(Elt: CompareOp);
1061
1062 if (CGM.getTarget().getTriple().isDXIL()) {
1063 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1064 Args.push_back(Elt: ComponentOp);
1065 }
1066
1067 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
1068 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
1069
1070 llvm::Type *RetTy = ConvertType(T: E->getType());
1071 return Builder.CreateIntrinsic(
1072 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
1073 }
1074 case Builtin::BI__builtin_hlsl_resource_load_with_status:
1075 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
1076 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1077 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1078
1079 // Get the *address* of the status argument to write to it by reference
1080 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
1081 Address StatusAddr = StatusLVal.getAddress();
1082
1083 QualType HandleTy = E->getArg(Arg: 0)->getType();
1084 const HLSLAttributedResourceType *RT =
1085 HandleTy->getAs<HLSLAttributedResourceType>();
1086 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
1087 "Only DXIL currently implements load with status");
1088
1089 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
1090 ? llvm::Intrinsic::dx_resource_load_rawbuffer
1091 : llvm::Intrinsic::dx_resource_load_typedbuffer;
1092
1093 llvm::Type *DataTy = ConvertType(T: E->getType());
1094 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
1095 Elements: {DataTy, Builder.getInt1Ty()});
1096
1097 SmallVector<Value *, 3> Args;
1098 Args.push_back(Elt: HandleOp);
1099 Args.push_back(Elt: IndexOp);
1100
1101 if (RT->isRaw()) {
1102 Value *Offset = Builder.getInt32(C: 0);
1103 // The offset parameter needs to be poison for ByteAddressBuffer
1104 if (!RT->isStructured())
1105 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
1106 Args.push_back(Elt: Offset);
1107 }
1108
1109 // The load intrinsics give us a (T value, i1 status) pair -
1110 // shepherd these into the return value and out reference respectively.
1111 Value *ResRet =
1112 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
1113 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
1114 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
1115 Value *ExtendedStatus =
1116 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
1117 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
1118
1119 return LoadedValue;
1120 }
1121 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
1122 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1123 return llvm::PoisonValue::get(T: HandleTy);
1124 }
1125 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
1126 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1127 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1128 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1129 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1130 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1131 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1132 llvm::Intrinsic::ID IntrinsicID =
1133 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1134 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
1135 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1136 }
1137 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
1138 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1139 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1140 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1141 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1142 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1143 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1144 llvm::Intrinsic::ID IntrinsicID =
1145 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1146 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
1147 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1148 }
1149 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
1150 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1151 if (!CGM.getTriple().isSPIRV())
1152 return MainHandle;
1153
1154 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1155 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1156 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1157 llvm::Intrinsic::ID IntrinsicID =
1158 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
1159 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
1160 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1161 Args);
1162 }
1163 case Builtin::BI__builtin_hlsl_resource_handlefromheap: {
1164 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1165 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1166 llvm::Intrinsic::ID IntrinsicID =
1167 CGM.getHLSLRuntime().getCreateHandleFromHeapIntrinsic();
1168 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args: {IndexOp});
1169 }
1170 case Builtin::BI__builtin_hlsl_resource_counterhandlefromheap: {
1171 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1172 if (!CGM.getTriple().isSPIRV())
1173 return MainHandle;
1174
1175 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1176 llvm::Intrinsic::ID IntrinsicID =
1177 llvm::Intrinsic::spv_resource_counterhandlefromheap;
1178 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1179 Args: {MainHandle});
1180 }
1181
1182 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
1183 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1184 llvm::Type *RetTy = ConvertType(T: E->getType());
1185 return Builder.CreateIntrinsic(
1186 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
1187 Args: ArrayRef<Value *>{IndexOp});
1188 }
1189 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
1190 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
1191 return emitGetDimensions(CGF&: *this, E,
1192 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
1193 NumRetComps: 1, /*HasLod=*/false);
1194 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
1195 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
1196 return emitGetDimensions(CGF&: *this, E,
1197 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1198 NumRetComps: 2, /*HasLod=*/false);
1199 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1200 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1201 return emitGetDimensions(
1202 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1203 /*HasLod=*/true);
1204 case Builtin::BI__builtin_hlsl_resource_getstride: {
1205 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1206 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1207 }
1208 case Builtin::BI__builtin_hlsl_all: {
1209 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1210 return Builder.CreateIntrinsic(
1211 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1212 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1213 Name: "hlsl.all");
1214 }
1215 case Builtin::BI__builtin_hlsl_and: {
1216 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1217 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1218 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1219 }
1220 case Builtin::BI__builtin_hlsl_or: {
1221 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1222 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1223 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1224 }
1225 case Builtin::BI__builtin_hlsl_any: {
1226 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1227 return Builder.CreateIntrinsic(
1228 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1229 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1230 Name: "hlsl.any");
1231 }
1232 case Builtin::BI__builtin_hlsl_asdouble:
1233 return handleAsDoubleBuiltin(CGF&: *this, E);
1234 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1235 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1236 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1237 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1238
1239 QualType Ty = E->getArg(Arg: 0)->getType();
1240 if (auto *VecTy = Ty->getAs<VectorType>())
1241 Ty = VecTy->getElementType();
1242
1243 Intrinsic::ID Intr;
1244 if (Ty->isFloatingType()) {
1245 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1246 } else if (Ty->isUnsignedIntegerType()) {
1247 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1248 } else {
1249 assert(Ty->isSignedIntegerType());
1250 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1251 }
1252 return Builder.CreateIntrinsic(
1253 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1254 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1255 }
1256 case Builtin::BI__builtin_hlsl_dot: {
1257 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1258 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1259 llvm::Type *T0 = Op0->getType();
1260 llvm::Type *T1 = Op1->getType();
1261
1262 // If the arguments are scalars, just emit a multiply
1263 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1264 if (T0->isFloatingPointTy())
1265 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1266
1267 if (T0->isIntegerTy())
1268 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1269
1270 llvm_unreachable(
1271 "Scalar dot product is only supported on ints and floats.");
1272 }
1273 // For vectors, validate types and emit the appropriate intrinsic
1274 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1275 E->getArg(1)->getType()) &&
1276 "Dot product operands must have the same type.");
1277
1278 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1279 assert(VecTy0 && "Dot product argument must be a vector.");
1280
1281 return Builder.CreateIntrinsic(
1282 /*ReturnType=*/RetTy: T0->getScalarType(),
1283 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1284 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1285 }
1286 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1287 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1288 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1289 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1290
1291 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1292 // Note that the argument order disagrees between the builtin and the
1293 // intrinsic here.
1294 return Builder.CreateIntrinsic(
1295 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1296 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1297 }
1298 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1299 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1300 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1301 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1302
1303 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1304 // Note that the argument order disagrees between the builtin and the
1305 // intrinsic here.
1306 return Builder.CreateIntrinsic(
1307 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1308 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1309 }
1310 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1311 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1312
1313 return Builder.CreateIntrinsic(
1314 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1315 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1316 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1317 }
1318 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1319 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1320
1321 return Builder.CreateIntrinsic(
1322 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1323 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1324 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1325 }
1326 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1327 return handleElementwiseF16ToF32(CGF&: *this, E);
1328 }
1329 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1330 return handleElementwiseF32ToF16(CGF&: *this, E);
1331 }
1332 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1333 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1334 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1335 llvm_unreachable("frac operand must have a float representation");
1336 return Builder.CreateIntrinsic(
1337 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1338 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1339 }
1340 case Builtin::BI__builtin_hlsl_elementwise_isfinite: {
1341 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1342 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1343 llvm_unreachable("isfinite operand must have a float representation");
1344 llvm::Type *retType = getAggregateType(
1345 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1346 return Builder.CreateIntrinsic(
1347 RetTy: retType, ID: CGM.getHLSLRuntime().getIsFiniteIntrinsic(),
1348 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isfinite");
1349 }
1350 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1351 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1352 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1353 llvm_unreachable("isinf operand must have a float representation");
1354 llvm::Type *retType = getAggregateType(
1355 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1356 return Builder.CreateIntrinsic(
1357 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1358 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1359 }
1360 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1361 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1362 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1363 llvm_unreachable("isnan operand must have a float representation");
1364 llvm::Type *retType = getAggregateType(
1365 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1366 return Builder.CreateIntrinsic(
1367 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1368 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1369 }
1370 case Builtin::BI__builtin_hlsl_mad: {
1371 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1372 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1373 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1374 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1375 return Builder.CreateIntrinsic(
1376 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1377 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1378
1379 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1380 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1381 return Builder.CreateIntrinsic(
1382 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1383 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1384
1385 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1386 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1387 }
1388 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1389 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1390 return Builder.CreateIntrinsic(
1391 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1392 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1393
1394 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1395 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1396 }
1397 case Builtin::BI__builtin_hlsl_mul: {
1398 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1399 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1400 QualType QTy0 = E->getArg(Arg: 0)->getType();
1401 QualType QTy1 = E->getArg(Arg: 1)->getType();
1402
1403 bool IsVec0 = QTy0->isVectorType();
1404 bool IsVec1 = QTy1->isVectorType();
1405 bool IsMat0 = QTy0->isConstantMatrixType();
1406 bool IsMat1 = QTy1->isConstantMatrixType();
1407
1408 llvm::MatrixBuilder MB(Builder);
1409 if (IsVec0 && IsMat1) {
1410 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1411 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1412 unsigned Cols = MatTy->getNumColumns();
1413 assert(N == MatTy->getNumRows() &&
1414 "vector length must match matrix row count");
1415 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1416 }
1417 if (IsMat0 && IsVec1) {
1418 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1419 unsigned Rows = MatTy->getNumRows();
1420 unsigned Cols = MatTy->getNumColumns();
1421 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1422 "vector length must match matrix column count");
1423 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1424 }
1425 assert(IsMat0 && IsMat1);
1426 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1427 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1428 unsigned Rows0 = MatTy0->getNumRows();
1429 unsigned Cols0 = MatTy0->getNumColumns();
1430 unsigned Cols1 = MatTy1->getNumColumns();
1431 assert(Cols0 == MatTy1->getNumRows() &&
1432 "inner matrix dimensions must match for multiplication");
1433 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1434 }
1435 case Builtin::BI__builtin_hlsl_transpose: {
1436 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1437 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1438 unsigned Rows = MatTy->getNumRows();
1439 unsigned Cols = MatTy->getNumColumns();
1440 if (auto *Transpose = dyn_cast<CallInst>(Val: Op0);
1441 Transpose &&
1442 Transpose->getIntrinsicID() == Intrinsic::matrix_transpose &&
1443 Transpose->use_empty() &&
1444 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 1))->getZExtValue() ==
1445 Cols &&
1446 cast<ConstantInt>(Val: Transpose->getArgOperand(i: 2))->getZExtValue() ==
1447 Rows) {
1448 Value *Result = Transpose->getArgOperand(i: 0);
1449 Transpose->eraseFromParent();
1450 return Result;
1451 }
1452 llvm::MatrixBuilder MB(Builder);
1453 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1454 }
1455 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1456 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1457 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1458 llvm_unreachable("rcp operand must have a float representation");
1459 llvm::Type *Ty = Op0->getType();
1460 llvm::Type *EltTy = Ty->getScalarType();
1461 Constant *One = Ty->isVectorTy()
1462 ? ConstantVector::getSplat(
1463 EC: ElementCount::getFixed(
1464 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1465 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1466 : ConstantFP::get(Ty: EltTy, V: 1.0);
1467 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1468 }
1469 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1470 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1471 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1472 llvm_unreachable("rsqrt operand must have a float representation");
1473 return Builder.CreateIntrinsic(
1474 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1475 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1476 }
1477 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1478 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1479 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1480 "saturate operand must have a float representation");
1481 return Builder.CreateIntrinsic(
1482 /*ReturnType=*/RetTy: Op0->getType(),
1483 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1484 FMFSource: nullptr, Name: "hlsl.saturate");
1485 }
1486 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1487 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1488 assert(Op->getType()->isIntegerTy(1) &&
1489 "WavePrefixBitCount operand must be a boolean type");
1490
1491 Intrinsic::ID IID =
1492 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1493
1494 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1495 }
1496 case Builtin::BI__builtin_hlsl_select: {
1497 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1498 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1499 Value *OpTrue =
1500 RValTrue.isScalar()
1501 ? RValTrue.getScalarVal()
1502 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1503 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1504 Value *OpFalse =
1505 RValFalse.isScalar()
1506 ? RValFalse.getScalarVal()
1507 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1508
1509 unsigned NumElements = 0;
1510 if (auto *VTy = E->getType()->getAs<VectorType>())
1511 NumElements = VTy->getNumElements();
1512 else if (auto *MTy = E->getType()->getAs<ConstantMatrixType>())
1513 NumElements = MTy->getNumElementsFlattened();
1514 if (NumElements) {
1515 if (!OpTrue->getType()->isVectorTy())
1516 OpTrue = Builder.CreateVectorSplat(NumElts: NumElements, V: OpTrue, Name: "splat");
1517 if (!OpFalse->getType()->isVectorTy())
1518 OpFalse = Builder.CreateVectorSplat(NumElts: NumElements, V: OpFalse, Name: "splat");
1519 }
1520
1521 Value *SelectVal =
1522 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1523 if (!RValTrue.isScalar())
1524 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1525 IsVolatile: ReturnValue.isVolatile());
1526
1527 return SelectVal;
1528 }
1529 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1530 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1531
1532 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1533 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1534 }
1535 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1536 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1537 assert(Op->getType()->isIntegerTy(1) &&
1538 "Intrinsic WaveActiveAllTrue operand must be a bool");
1539
1540 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1541 return EmitIntrinsicCall(ID, Args: {Op});
1542 }
1543 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1544 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1545 assert(Op->getType()->isIntegerTy(1) &&
1546 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1547
1548 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1549 return EmitIntrinsicCall(ID, Args: {Op});
1550 }
1551 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1552 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1553 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1554 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1555 "representation");
1556
1557 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1558 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1559 Name: "hlsl.wave.active.bit.or");
1560 }
1561 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1562 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1563 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1564 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1565 "representation");
1566
1567 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1568 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1569 Name: "hlsl.wave.active.bit.xor");
1570 }
1571 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1572 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1573 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1574 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1575 "representation");
1576
1577 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1578 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1579 Name: "hlsl.wave.active.bit.and");
1580 }
1581 case Builtin::BI__builtin_hlsl_interlocked_add: {
1582 // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
1583 // up the raw instruction (DXIL routes it via `dx.resource.atomic.binop`
1584 // in DXILResourceAccess for resource pointers, SPIR-V lowers via
1585 // selectAtomicRMW). No intermediate intrinsic.
1586 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Add);
1587 }
1588 case Builtin::BI__builtin_hlsl_interlocked_and: {
1589 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::And);
1590 }
1591 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange:
1592 case Builtin::BI__builtin_hlsl_interlocked_compare_exchange_float_bitwise:
1593 case Builtin::BI__builtin_hlsl_interlocked_compare_store:
1594 case Builtin::BI__builtin_hlsl_interlocked_compare_store_float_bitwise: {
1595 return handleInterlockedCompareOp(CGF&: *this, E);
1596 }
1597 case Builtin::BI__builtin_hlsl_interlocked_exchange: {
1598 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xchg);
1599 }
1600 case Builtin::BI__builtin_hlsl_interlocked_max: {
1601 llvm::AtomicRMWInst::BinOp Op =
1602 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1603 ? llvm::AtomicRMWInst::Max
1604 : llvm::AtomicRMWInst::UMax;
1605 return handleInterlockedOp(CGF&: *this, E, Op);
1606 }
1607 case Builtin::BI__builtin_hlsl_interlocked_min: {
1608 llvm::AtomicRMWInst::BinOp Op =
1609 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1610 ? llvm::AtomicRMWInst::Min
1611 : llvm::AtomicRMWInst::UMin;
1612 return handleInterlockedOp(CGF&: *this, E, Op);
1613 }
1614 case Builtin::BI__builtin_hlsl_interlocked_or: {
1615 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Or);
1616 }
1617 case Builtin::BI__builtin_hlsl_interlocked_xor: {
1618 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xor);
1619 }
1620 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1621 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1622 assert(Op->getType()->isIntegerTy(1) &&
1623 "Intrinsic WaveActiveBallot operand must be a bool");
1624
1625 return handleHlslWaveActiveBallot(CGF&: *this, E);
1626 }
1627 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1628 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1629 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1630 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1631 }
1632 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1633 // Due to the use of variadic arguments, explicitly retrieve argument
1634 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1635 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1636 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1637
1638 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1639 Name: "hlsl.wave.active.sum");
1640 }
1641 case Builtin::BI__builtin_hlsl_wave_active_product: {
1642 // Due to the use of variadic arguments, explicitly retrieve argument
1643 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1644 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1645 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1646
1647 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1648 Name: "hlsl.wave.active.product");
1649 }
1650 case Builtin::BI__builtin_hlsl_wave_active_max: {
1651 // Due to the use of variadic arguments, explicitly retrieve argument
1652 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1653 QualType QT = E->getArg(Arg: 0)->getType();
1654 Intrinsic::ID IID;
1655 if (QT->isUnsignedIntegerType())
1656 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1657 else
1658 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1659
1660 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1661 Name: "hlsl.wave.active.max");
1662 }
1663 case Builtin::BI__builtin_hlsl_wave_active_min: {
1664 // Due to the use of variadic arguments, explicitly retrieve argument
1665 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1666 QualType QT = E->getArg(Arg: 0)->getType();
1667 Intrinsic::ID IID;
1668 if (QT->isUnsignedIntegerType())
1669 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1670 else
1671 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1672
1673 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1674 Name: "hlsl.wave.active.min");
1675 }
1676 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1677 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1678 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1679 // for the DirectX intrinsic and the demangled builtin name
1680 switch (CGM.getTarget().getTriple().getArch()) {
1681 case llvm::Triple::dxil:
1682 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1683 case llvm::Triple::spirv:
1684 return EmitRuntimeCall(
1685 callee: CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1686 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false,
1687 /*AssumeConvergent=*/true),
1688 args: {}, name: "", /*IsConvergent=*/true);
1689 default:
1690 llvm_unreachable(
1691 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1692 }
1693 }
1694 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1695 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1696 return EmitIntrinsicCall(ID);
1697 }
1698 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1699 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1700 return EmitIntrinsicCall(ID);
1701 }
1702 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1703 // Due to the use of variadic arguments we must explicitly retrieve them and
1704 // create our function type.
1705 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1706 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1707 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1708 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1709 Name: "hlsl.wave.readlane");
1710 }
1711 case Builtin::BI__builtin_hlsl_wave_read_lane_first: {
1712 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1713 return EmitIntrinsicCall(
1714 ID: CGM.getHLSLRuntime().getWaveReadLaneFirstIntrinsic(),
1715 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr}, Name: "hlsl.wave.readlane.first");
1716 }
1717 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1718 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1719 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1720 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1721 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1722 Name: "hlsl.wave.prefix.sum");
1723 }
1724 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1725 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1726 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1727 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1728 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1729 Name: "hlsl.wave.prefix.product");
1730 }
1731 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1732 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1733 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1734 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1735 Name: "hlsl.quad.read.across.x");
1736 }
1737 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1738 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1739 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1740 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1741 Name: "hlsl.quad.read.across.y");
1742 }
1743 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1744 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1745 Intrinsic::ID ID =
1746 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1747 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1748 Name: "hlsl.quad.read.across.diagonal");
1749 }
1750 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1751 auto *Arg0 = E->getArg(Arg: 0);
1752 Value *Op0 = EmitScalarExpr(E: Arg0);
1753 llvm::Type *Xty = Op0->getType();
1754 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1755 if (Xty->isVectorTy()) {
1756 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1757 retType = llvm::VectorType::get(
1758 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1759 }
1760 assert((Arg0->getType()->hasFloatingRepresentation() ||
1761 Arg0->getType()->hasIntegerRepresentation()) &&
1762 "sign operand must have a float or int representation");
1763
1764 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1765 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1766 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1767 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1768 }
1769
1770 return Builder.CreateIntrinsic(
1771 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1772 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1773 }
1774 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1775 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1776 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1777 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1778 return Builder.CreateIntrinsic(
1779 /*ReturnType=*/RetTy: Offset->getType(),
1780 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1781 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1782 }
1783 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1784
1785 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1786 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1787 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1788 "asuint operands types mismatch");
1789 return handleHlslSplitdouble(E, CGF: this);
1790 }
1791 case Builtin::BI__builtin_hlsl_elementwise_clip:
1792 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1793 "clip operands types mismatch");
1794 return handleHlslClip(E, CGF: this);
1795 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1796 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1797 return EmitIntrinsicCall(ID);
1798 }
1799 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1800 Intrinsic::ID ID =
1801 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1802 return EmitIntrinsicCall(ID);
1803 }
1804 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1805 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1806 return EmitIntrinsicCall(ID);
1807 }
1808 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1809 Intrinsic::ID ID =
1810 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1811 return EmitIntrinsicCall(ID);
1812 }
1813 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1814 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1815 return EmitIntrinsicCall(ID);
1816 }
1817 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1818 Intrinsic::ID ID =
1819 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1820 return EmitIntrinsicCall(ID);
1821 }
1822 case Builtin::BI__builtin_hlsl_barrier: {
1823 std::optional<llvm::APSInt> SemanticFlagsConstant =
1824 E->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
1825 assert(SemanticFlagsConstant && "expected constant semantic flags");
1826 constexpr uint64_t GroupScope =
1827 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::GroupScope);
1828 constexpr uint64_t DeviceScope =
1829 llvm::to_underlying(E: llvm::dxil::BarrierSemanticFlag::DeviceScope);
1830 constexpr uint64_t ScopeMask = GroupScope | DeviceScope;
1831 uint64_t SemanticFlagsValue = SemanticFlagsConstant->getZExtValue();
1832 // DEVICE_SCOPE includes GROUP_SCOPE. Remove GROUP_SCOPE when both are
1833 // set.
1834 if ((SemanticFlagsValue & ScopeMask) == ScopeMask)
1835 SemanticFlagsValue &= ~GroupScope;
1836 Value *SemanticFlags = llvm::ConstantInt::get(
1837 Ty: ConvertType(T: E->getArg(Arg: 1)->getType()), V: SemanticFlagsValue);
1838
1839 if (E->getArg(Arg: 0)->getType()->isUnsignedIntegerType()) {
1840 std::optional<llvm::APSInt> MemoryFlagsConstant =
1841 E->getArg(Arg: 0)->getIntegerConstantExpr(Ctx: getContext());
1842 assert(MemoryFlagsConstant && "expected constant memory flags");
1843 uint64_t MemoryFlagsValue = MemoryFlagsConstant->getZExtValue();
1844 llvm::Triple::EnvironmentType Stage =
1845 getTarget().getTriple().getEnvironment();
1846 if (Stage == llvm::Triple::Library)
1847 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
1848 if (const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>())
1849 Stage = ShaderAttr->getType();
1850 MemoryFlagsValue =
1851 getEffectiveBarrierMemoryFlags(MemoryFlags: MemoryFlagsValue, Stage);
1852 Value *MemoryFlags = llvm::ConstantInt::get(
1853 Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), V: MemoryFlagsValue);
1854 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryTypeIntrinsic();
1855 return EmitIntrinsicCall(ID, Types: {},
1856 Args: ArrayRef<Value *>{MemoryFlags, SemanticFlags});
1857 }
1858
1859 const FieldDecl *HandleField =
1860 getResourceHandleField(ResourceTy: E->getArg(Arg: 0)->getType());
1861 LValue Resource = EmitLValue(E: E->getArg(Arg: 0));
1862 LValue Handle = EmitLValueForField(Base: Resource, Field: HandleField);
1863 Value *HandleValue =
1864 EmitLoadOfLValue(V: Handle, Loc: E->getArg(Arg: 0)->getExprLoc()).getScalarVal();
1865 Intrinsic::ID ID = CGM.getHLSLRuntime().getBarrierByMemoryHandleIntrinsic();
1866 return EmitIntrinsicCall(ID, Types: ArrayRef<llvm::Type *>{HandleValue->getType()},
1867 Args: ArrayRef<Value *>{HandleValue, SemanticFlags});
1868 }
1869 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1870 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1871 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1872 llvm_unreachable("ddx_coarse operand must have a float representation");
1873 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1874 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1875 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1876 Name: "hlsl.ddx.coarse");
1877 }
1878 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1879 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1880 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1881 llvm_unreachable("ddy_coarse operand must have a float representation");
1882 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1883 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1884 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1885 Name: "hlsl.ddy.coarse");
1886 }
1887 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1888 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1889 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1890 llvm_unreachable("ddx_fine operand must have a float representation");
1891 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1892 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1893 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1894 Name: "hlsl.ddx.fine");
1895 }
1896 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1897 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1898 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1899 llvm_unreachable("ddy_fine operand must have a float representation");
1900 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1901 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1902 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1903 Name: "hlsl.ddy.fine");
1904 }
1905 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1906 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1907 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1908 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1909 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1910 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1911 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1912 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1913 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1914 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1915 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1916 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1917 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1918 llvm::Value *Args[] = {SpecId, DefaultVal};
1919 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1920 }
1921 }
1922 return nullptr;
1923}
1924
1925llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1926 const clang::QualType &SpecConstantType) {
1927
1928 // Find or create the declaration for the function.
1929 llvm::Module *M = &CGM.getModule();
1930 std::string MangledName =
1931 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1932 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1933
1934 if (!SpecConstantFn) {
1935 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1936 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1937 llvm::Type *ArgTypes[] = {IntType, RetTy};
1938 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1939 SpecConstantFn = llvm::Function::Create(
1940 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1941 }
1942 return SpecConstantFn;
1943}
1944