1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16
17using namespace clang;
18using namespace CodeGen;
19using namespace llvm;
20
21static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
22 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
23 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
24 "asdouble operands types mismatch");
25 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
26 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
27
28 llvm::Type *ResultType = CGF.DoubleTy;
29 int N = 1;
30 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
31 N = VTy->getNumElements();
32 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
33 }
34
35 if (CGF.CGM.getTarget().getTriple().isDXIL())
36 return CGF.Builder.CreateIntrinsic(
37 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
38 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
39
40 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
41 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
42 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
43 }
44
45 llvm::SmallVector<int> Mask;
46 for (int i = 0; i < N; i++) {
47 Mask.push_back(Elt: i);
48 Mask.push_back(Elt: i + N);
49 }
50
51 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
52
53 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
54}
55
56static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
57 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
58
59 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
60 Value *CMP;
61 Value *LastInstr;
62
63 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
64 FZeroConst = ConstantVector::getSplat(
65 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
66 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
67 CMP = CGF->Builder.CreateIntrinsic(
68 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
69 Args: {FCompInst});
70 } else {
71 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
72 }
73
74 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
75 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
76 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
77 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
78 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
79
80 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
81
82 CGF->Builder.SetInsertPoint(LT0);
83
84 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
85
86 LastInstr = CGF->Builder.CreateBr(Dest: End);
87 CGF->Builder.SetInsertPoint(End);
88 } else {
89 llvm_unreachable("Backend Codegen not supported.");
90 }
91
92 return LastInstr;
93}
94
95static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
96 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
97 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
98 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
99
100 CallArgList Args;
101 LValue Op1TmpLValue =
102 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
103 LValue Op2TmpLValue =
104 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
105
106 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
107 Args.reverseWritebacks();
108
109 Value *LowBits = nullptr;
110 Value *HighBits = nullptr;
111
112 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
113 llvm::Type *RetElementTy = CGF->Int32Ty;
114 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
115 RetElementTy = llvm::VectorType::get(
116 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
117 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
118
119 CallInst *CI = CGF->Builder.CreateIntrinsic(
120 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
121
122 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
123 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
124 } else {
125 // For Non DXIL targets we generate the instructions.
126
127 if (!Op0->getType()->isVectorTy()) {
128 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
129 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
130
131 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
132 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
133 } else {
134 int NumElements = 1;
135 if (const auto *VecTy =
136 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
137 NumElements = VecTy->getNumElements();
138
139 FixedVectorType *Uint32VecTy =
140 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
141 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
142 if (NumElements == 1) {
143 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
144 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
145 } else {
146 SmallVector<int> EvenMask, OddMask;
147 for (int I = 0, E = NumElements; I != E; ++I) {
148 EvenMask.push_back(Elt: I * 2);
149 OddMask.push_back(Elt: I * 2 + 1);
150 }
151 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
152 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
153 }
154 }
155 }
156 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
157 auto *LastInst =
158 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
159 CGF->EmitWritebacks(Args);
160 return LastInst;
161}
162
163static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
164 const CallExpr *E) {
165 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
166 llvm::Type *I32 = CGF.Int32Ty;
167
168 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
169 [[maybe_unused]] llvm::StructType *Struct4I32 =
170 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
171
172 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
173 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
174 llvm::Function *Fn = CGF.CGM.getIntrinsic(IID: Intrinsic::dx_wave_ballot, Tys: {I32});
175
176 Value *StructVal = CGF.EmitRuntimeCall(callee: Fn, args: Cond);
177 assert(StructVal->getType() == Struct4I32 &&
178 "dx.wave.ballot must return {i32,i32,i32,i32}");
179
180 // Reassemble struct to <4 x i32>
181 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
182 for (unsigned I = 0; I < 4; ++I) {
183 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
184 VecVal =
185 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
186 }
187
188 return VecVal;
189 }
190
191 if (CGF.CGM.getTarget().getTriple().isSPIRV())
192 return CGF.EmitRuntimeCall(
193 callee: CGF.CGM.getIntrinsic(IID: Intrinsic::spv_subgroup_ballot), args: Cond);
194
195 llvm_unreachable(
196 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
197}
198
199static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
200 const CallExpr *E) {
201 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
202 QualType Op0Ty = E->getArg(Arg: 0)->getType();
203 llvm::Type *ResType = CGF.FloatTy;
204 uint64_t NumElements = 0;
205 if (Op0->getType()->isVectorTy()) {
206 NumElements =
207 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
208 ResType =
209 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
210 }
211 if (!Op0Ty->hasUnsignedIntegerRepresentation())
212 llvm_unreachable(
213 "f16tof32 operand must have an unsigned int representation");
214
215 if (CGF.CGM.getTriple().isDXIL())
216 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
217 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
218 Name: "hlsl.f16tof32");
219
220 if (CGF.CGM.getTriple().isSPIRV()) {
221 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
222 // Int16 and Float16 capabilities
223 auto *UnpackType =
224 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
225
226 if (NumElements == 0) {
227 // a scalar input - simply extract the first element of the unpacked
228 // vector
229 Value *Unpack = CGF.Builder.CreateIntrinsic(
230 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
231 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
232 }
233
234 // a vector input - build a congruent output vector by iterating through
235 // the input vector calling unpackhalf2x16 for each element
236 Value *Result = PoisonValue::get(T: ResType);
237 for (uint64_t I = 0; I < NumElements; I++) {
238 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
239 Value *Unpack = CGF.Builder.CreateIntrinsic(
240 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
241 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
242 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
243 }
244 return Result;
245 }
246
247 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
248}
249
250static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
251 const CallExpr *E) {
252 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
253 QualType Op0Ty = E->getArg(Arg: 0)->getType();
254 llvm::Type *ResType = CGF.IntTy;
255 uint64_t NumElements = 0;
256 if (Op0->getType()->isVectorTy()) {
257 NumElements =
258 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
259 ResType =
260 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
261 }
262 if (!Op0Ty->hasFloatingRepresentation())
263 llvm_unreachable("f32tof16 operand must have a float representation");
264
265 if (CGF.CGM.getTriple().isDXIL())
266 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
267 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
268 Name: "hlsl.f32tof16");
269
270 if (CGF.CGM.getTriple().isSPIRV()) {
271 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
272 // Int16 and Float16 capabilities
273 auto *PackType =
274 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
275
276 if (NumElements == 0) {
277 // a scalar input - simply insert the scalar in the first element
278 // of the 2 element float vector
279 Value *Float2 = Constant::getNullValue(Ty: PackType);
280 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
281 Value *Result = CGF.Builder.CreateIntrinsic(
282 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
283 return Result;
284 }
285
286 // a vector input - build a congruent output vector by iterating through
287 // the input vector calling packhalf2x16 for each element
288 Value *Result = PoisonValue::get(T: ResType);
289 for (uint64_t I = 0; I < NumElements; I++) {
290 Value *Float2 = Constant::getNullValue(Ty: PackType);
291 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
292 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
293 Value *Res = CGF.Builder.CreateIntrinsic(
294 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
295 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
296 }
297 return Result;
298 }
299
300 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
301}
302
303static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
304 LValue &Stride) {
305 // Figure out the stride of the buffer elements from the handle type.
306 auto *HandleTy =
307 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
308 QualType ElementTy = HandleTy->getContainedType();
309 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
310 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
311}
312
313// Return dot product intrinsic that corresponds to the QT scalar type
314static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
315 if (QT->isFloatingType())
316 return RT.getFDotIntrinsic();
317 if (QT->isSignedIntegerType())
318 return RT.getSDotIntrinsic();
319 assert(QT->isUnsignedIntegerType());
320 return RT.getUDotIntrinsic();
321}
322
323static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
324 if (QT->hasSignedIntegerRepresentation()) {
325 return RT.getFirstBitSHighIntrinsic();
326 }
327
328 assert(QT->hasUnsignedIntegerRepresentation());
329 return RT.getFirstBitUHighIntrinsic();
330}
331
332// Return wave active sum that corresponds to the QT scalar type
333static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
334 CGHLSLRuntime &RT, QualType QT) {
335 switch (Arch) {
336 case llvm::Triple::spirv:
337 return Intrinsic::spv_wave_reduce_sum;
338 case llvm::Triple::dxil: {
339 if (QT->isUnsignedIntegerType())
340 return Intrinsic::dx_wave_reduce_usum;
341 return Intrinsic::dx_wave_reduce_sum;
342 }
343 default:
344 llvm_unreachable("Intrinsic WaveActiveSum"
345 " not supported by target architecture");
346 }
347}
348
349static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
350 switch (Arch) {
351 case llvm::Triple::spirv:
352 return Intrinsic::spv_subgroup_prefix_bit_count;
353 case llvm::Triple::dxil: {
354 return Intrinsic::dx_wave_prefix_bit_count;
355 }
356 default:
357 llvm_unreachable(
358 "WavePrefixOp instruction not supported by target architecture");
359 }
360}
361
362// Return wave prefix sum that corresponds to the QT scalar type
363static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
364 QualType QT) {
365 switch (Arch) {
366 case llvm::Triple::spirv:
367 return Intrinsic::spv_wave_prefix_sum;
368 case llvm::Triple::dxil: {
369 if (QT->isUnsignedIntegerType())
370 return Intrinsic::dx_wave_prefix_usum;
371 return Intrinsic::dx_wave_prefix_sum;
372 }
373 default:
374 llvm_unreachable("Intrinsic WavePrefixSum"
375 " not supported by target architecture");
376 }
377}
378
379// Return wave prefix product that corresponds to the QT scalar type
380static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
381 QualType QT) {
382 switch (Arch) {
383 case llvm::Triple::spirv:
384 return Intrinsic::spv_wave_prefix_product;
385 case llvm::Triple::dxil: {
386 if (QT->isUnsignedIntegerType())
387 return Intrinsic::dx_wave_prefix_uproduct;
388 return Intrinsic::dx_wave_prefix_product;
389 }
390 default:
391 llvm_unreachable("Intrinsic WavePrefixProduct"
392 " not supported by target architecture");
393 }
394}
395
396// Returns the mangled name for a builtin function that the SPIR-V backend
397// will expand into a spec Constant.
398static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
399 ASTContext &Context) {
400 // The parameter types for our conceptual intrinsic function.
401 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
402
403 // Create a temporary FunctionDecl for the builtin fuction. It won't be
404 // added to the AST.
405 FunctionProtoType::ExtProtoInfo EPI;
406 QualType FnType =
407 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
408 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
409 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
410 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
411 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
412
413 // Attach the created parameter declarations to the function declaration.
414 SmallVector<ParmVarDecl *, 2> ParamDecls;
415 for (QualType ParamType : ClangParamTypes) {
416 ParmVarDecl *PD = ParmVarDecl::Create(
417 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
418 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
419 /*DefaultArg*/ DefArg: nullptr);
420 ParamDecls.push_back(Elt: PD);
421 }
422 FnDeclForMangling->setParams(ParamDecls);
423
424 // Get the mangled name.
425 std::string Name;
426 llvm::raw_string_ostream MangledNameStream(Name);
427 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
428 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
429 MangledNameStream.flush();
430
431 return Name;
432}
433
434static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
435 unsigned OffsetArgIndex) {
436 if (E->getNumArgs() > OffsetArgIndex)
437 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
438
439 llvm::Type *CoordTy = CGF.ConvertType(T: E->getArg(Arg: 2)->getType());
440 llvm::Type *Int32Ty = CGF.Int32Ty;
441 llvm::Type *OffsetTy = Int32Ty;
442 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: CoordTy))
443 OffsetTy = llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: VT->getNumElements());
444 return llvm::Constant::getNullValue(Ty: OffsetTy);
445}
446
447static Value *emitHlslClamp(CodeGenFunction &CGF, const CallExpr *E,
448 unsigned ClampArgIndex) {
449 Value *Clamp = CGF.EmitScalarExpr(E: E->getArg(Arg: ClampArgIndex));
450 // The builtin is defined with variadic arguments, so the clamp parameter
451 // might have been promoted to double. The intrinsic requires a 32-bit
452 // float.
453 if (Clamp->getType() != CGF.Builder.getFloatTy())
454 Clamp = CGF.Builder.CreateFPCast(V: Clamp, DestTy: CGF.Builder.getFloatTy());
455 return Clamp;
456}
457
458Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
459 const CallExpr *E,
460 ReturnValueSlot ReturnValue) {
461 if (!getLangOpts().HLSL)
462 return nullptr;
463
464 switch (BuiltinID) {
465 case Builtin::BI__builtin_hlsl_adduint64: {
466 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
467 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
468 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
469 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
470 assert(Arg0Ty == E->getArg(1)->getType() &&
471 "AddUint64 operand types must match");
472 assert(Arg0Ty->hasIntegerRepresentation() &&
473 "AddUint64 operands must have an integer representation");
474 assert((NumElements == 2 || NumElements == 4) &&
475 "AddUint64 operands must have 2 or 4 elements");
476
477 llvm::Value *LowA;
478 llvm::Value *HighA;
479 llvm::Value *LowB;
480 llvm::Value *HighB;
481
482 // Obtain low and high words of inputs A and B
483 if (NumElements == 2) {
484 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
485 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
486 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
487 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
488 } else {
489 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
490 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
491 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
492 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
493 }
494
495 // Use an uadd_with_overflow to compute the sum of low words and obtain a
496 // carry value
497 llvm::Value *Carry;
498 llvm::Value *LowSum = EmitOverflowIntrinsic(
499 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
500 llvm::Value *ZExtCarry =
501 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
502
503 // Sum the high words and the carry
504 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
505 llvm::Value *HighSumPlusCarry =
506 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
507
508 if (NumElements == 4) {
509 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
510 Name: "hlsl.AddUint64");
511 }
512
513 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
514 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
515 Name: "hlsl.AddUint64.upto0");
516 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
517 Name: "hlsl.AddUint64");
518 return Result;
519 }
520 case Builtin::BI__builtin_hlsl_resource_getpointer:
521 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
522 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
523 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
524
525 llvm::Type *RetTy = ConvertType(T: E->getType());
526 return Builder.CreateIntrinsic(
527 RetTy, ID: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
528 Args: ArrayRef<Value *>{HandleOp, IndexOp});
529 }
530 case Builtin::BI__builtin_hlsl_resource_sample: {
531 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
532 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
533 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
534
535 SmallVector<Value *, 4> Args;
536 Args.push_back(Elt: HandleOp);
537 Args.push_back(Elt: SamplerOp);
538 Args.push_back(Elt: CoordOp);
539 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3));
540
541 llvm::Type *RetTy = ConvertType(T: E->getType());
542 if (E->getNumArgs() <= 4) {
543 return Builder.CreateIntrinsic(
544 RetTy, ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args);
545 }
546
547 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 4));
548 return Builder.CreateIntrinsic(
549 RetTy, ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(), Args);
550 }
551 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
552 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
553 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
554 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
555 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
556 if (BiasOp->getType() != Builder.getFloatTy())
557 BiasOp = Builder.CreateFPCast(V: BiasOp, DestTy: Builder.getFloatTy());
558
559 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
560 Args.push_back(Elt: HandleOp);
561 Args.push_back(Elt: SamplerOp);
562 Args.push_back(Elt: CoordOp);
563 Args.push_back(Elt: BiasOp);
564 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4));
565
566 llvm::Type *RetTy = ConvertType(T: E->getType());
567 if (E->getNumArgs() <= 5)
568 return Builder.CreateIntrinsic(
569 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(), Args);
570
571 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
572 return Builder.CreateIntrinsic(
573 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(), Args);
574 }
575 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
576 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
577 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
578 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
579 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
580 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
581
582 SmallVector<Value *, 7> Args;
583 Args.push_back(Elt: HandleOp);
584 Args.push_back(Elt: SamplerOp);
585 Args.push_back(Elt: CoordOp);
586 Args.push_back(Elt: DDXOp);
587 Args.push_back(Elt: DDYOp);
588 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5));
589
590 llvm::Type *RetTy = ConvertType(T: E->getType());
591
592 if (E->getNumArgs() <= 6) {
593 return Builder.CreateIntrinsic(
594 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
595 }
596
597 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 6));
598 return Builder.CreateIntrinsic(
599 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
600 }
601 case Builtin::BI__builtin_hlsl_resource_sample_level: {
602 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
603 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
604 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
605 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
606 if (LODOp->getType() != Builder.getFloatTy())
607 LODOp = Builder.CreateFPCast(V: LODOp, DestTy: Builder.getFloatTy());
608
609 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
610 Args.push_back(Elt: HandleOp);
611 Args.push_back(Elt: SamplerOp);
612 Args.push_back(Elt: CoordOp);
613 Args.push_back(Elt: LODOp);
614 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4));
615
616 llvm::Type *RetTy = ConvertType(T: E->getType());
617 return Builder.CreateIntrinsic(
618 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
619 }
620 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
621 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
622 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
623 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
624 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
625 if (CmpOp->getType() != Builder.getFloatTy())
626 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
627
628 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
629 Args.push_back(Elt: HandleOp);
630 Args.push_back(Elt: SamplerOp);
631 Args.push_back(Elt: CoordOp);
632 Args.push_back(Elt: CmpOp);
633 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4));
634
635 llvm::Type *RetTy = ConvertType(T: E->getType());
636 if (E->getNumArgs() <= 5) {
637 return Builder.CreateIntrinsic(
638 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
639 }
640
641 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
642 return Builder.CreateIntrinsic(
643 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
644 }
645 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
646 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
647 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
648 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
649 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
650 if (CmpOp->getType() != Builder.getFloatTy())
651 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
652
653 SmallVector<Value *, 5> Args;
654 Args.push_back(Elt: HandleOp);
655 Args.push_back(Elt: SamplerOp);
656 Args.push_back(Elt: CoordOp);
657 Args.push_back(Elt: CmpOp);
658
659 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4));
660
661 llvm::Type *RetTy = ConvertType(T: E->getType());
662 return Builder.CreateIntrinsic(
663 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
664 }
665 case Builtin::BI__builtin_hlsl_resource_load_with_status:
666 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
667 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
668 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
669
670 // Get the *address* of the status argument to write to it by reference
671 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
672 Address StatusAddr = StatusLVal.getAddress();
673
674 QualType HandleTy = E->getArg(Arg: 0)->getType();
675 const HLSLAttributedResourceType *RT =
676 HandleTy->getAs<HLSLAttributedResourceType>();
677 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
678 "Only DXIL currently implements load with status");
679
680 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
681 ? llvm::Intrinsic::dx_resource_load_rawbuffer
682 : llvm::Intrinsic::dx_resource_load_typedbuffer;
683
684 llvm::Type *DataTy = ConvertType(T: E->getType());
685 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
686 Elements: {DataTy, Builder.getInt1Ty()});
687
688 SmallVector<Value *, 3> Args;
689 Args.push_back(Elt: HandleOp);
690 Args.push_back(Elt: IndexOp);
691
692 if (RT->isRaw()) {
693 Value *Offset = Builder.getInt32(C: 0);
694 // The offset parameter needs to be poison for ByteAddressBuffer
695 if (!RT->isStructured())
696 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
697 Args.push_back(Elt: Offset);
698 }
699
700 // The load intrinsics give us a (T value, i1 status) pair -
701 // shepherd these into the return value and out reference respectively.
702 Value *ResRet =
703 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
704 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
705 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
706 Value *ExtendedStatus =
707 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
708 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
709
710 return LoadedValue;
711 }
712 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
713 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
714 return llvm::PoisonValue::get(T: HandleTy);
715 }
716 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
717 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
718 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
719 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
720 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
721 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
722 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
723 llvm::Intrinsic::ID IntrinsicID =
724 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
725 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
726 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
727 }
728 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
729 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
730 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
731 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
732 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
733 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
734 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
735 llvm::Intrinsic::ID IntrinsicID =
736 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
737 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
738 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
739 }
740 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
741 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
742 if (!CGM.getTriple().isSPIRV())
743 return MainHandle;
744
745 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
746 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
747 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
748 llvm::Intrinsic::ID IntrinsicID =
749 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
750 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
751 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
752 }
753 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
754 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
755 llvm::Type *RetTy = ConvertType(T: E->getType());
756 return Builder.CreateIntrinsic(
757 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
758 Args: ArrayRef<Value *>{IndexOp});
759 }
760 case Builtin::BI__builtin_hlsl_resource_getdimensions_x: {
761 Value *Handle = EmitScalarExpr(E: E->getArg(Arg: 0));
762 LValue Dim = EmitLValue(E: E->getArg(Arg: 1));
763 llvm::Type *RetTy = llvm::Type::getInt32Ty(C&: getLLVMContext());
764 Value *DimValue = Builder.CreateIntrinsic(
765 RetTy, ID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
766 Args: ArrayRef<Value *>{Handle});
767 return Builder.CreateStore(Val: DimValue, Addr: Dim.getAddress());
768 }
769 case Builtin::BI__builtin_hlsl_resource_getstride: {
770 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
771 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
772 }
773 case Builtin::BI__builtin_hlsl_all: {
774 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
775 return Builder.CreateIntrinsic(
776 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
777 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
778 Name: "hlsl.all");
779 }
780 case Builtin::BI__builtin_hlsl_and: {
781 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
782 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
783 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
784 }
785 case Builtin::BI__builtin_hlsl_or: {
786 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
787 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
788 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
789 }
790 case Builtin::BI__builtin_hlsl_any: {
791 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
792 return Builder.CreateIntrinsic(
793 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
794 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
795 Name: "hlsl.any");
796 }
797 case Builtin::BI__builtin_hlsl_asdouble:
798 return handleAsDoubleBuiltin(CGF&: *this, E);
799 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
800 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
801 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
802 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
803
804 QualType Ty = E->getArg(Arg: 0)->getType();
805 if (auto *VecTy = Ty->getAs<VectorType>())
806 Ty = VecTy->getElementType();
807
808 Intrinsic::ID Intr;
809 if (Ty->isFloatingType()) {
810 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
811 } else if (Ty->isUnsignedIntegerType()) {
812 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
813 } else {
814 assert(Ty->isSignedIntegerType());
815 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
816 }
817 return Builder.CreateIntrinsic(
818 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
819 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
820 }
821 case Builtin::BI__builtin_hlsl_crossf16:
822 case Builtin::BI__builtin_hlsl_crossf32: {
823 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
824 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
825 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
826 E->getArg(1)->getType()->hasFloatingRepresentation() &&
827 "cross operands must have a float representation");
828 // make sure each vector has exactly 3 elements
829 assert(
830 E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
831 E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
832 "input vectors must have 3 elements each");
833 return Builder.CreateIntrinsic(
834 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getCrossIntrinsic(),
835 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.cross");
836 }
837 case Builtin::BI__builtin_hlsl_dot: {
838 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
839 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
840 llvm::Type *T0 = Op0->getType();
841 llvm::Type *T1 = Op1->getType();
842
843 // If the arguments are scalars, just emit a multiply
844 if (!T0->isVectorTy() && !T1->isVectorTy()) {
845 if (T0->isFloatingPointTy())
846 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
847
848 if (T0->isIntegerTy())
849 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
850
851 llvm_unreachable(
852 "Scalar dot product is only supported on ints and floats.");
853 }
854 // For vectors, validate types and emit the appropriate intrinsic
855 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
856 E->getArg(1)->getType()) &&
857 "Dot product operands must have the same type.");
858
859 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
860 assert(VecTy0 && "Dot product argument must be a vector.");
861
862 return Builder.CreateIntrinsic(
863 /*ReturnType=*/RetTy: T0->getScalarType(),
864 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
865 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
866 }
867 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
868 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
869 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
870 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
871
872 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
873 // Note that the argument order disagrees between the builtin and the
874 // intrinsic here.
875 return Builder.CreateIntrinsic(
876 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
877 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
878 }
879 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
880 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
881 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
882 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
883
884 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
885 // Note that the argument order disagrees between the builtin and the
886 // intrinsic here.
887 return Builder.CreateIntrinsic(
888 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
889 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
890 }
891 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
892 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
893
894 return Builder.CreateIntrinsic(
895 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
896 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
897 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
898 }
899 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
900 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
901
902 return Builder.CreateIntrinsic(
903 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
904 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
905 FMFSource: nullptr, Name: "hlsl.firstbitlow");
906 }
907 case Builtin::BI__builtin_hlsl_lerp: {
908 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
909 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
910 Value *S = EmitScalarExpr(E: E->getArg(Arg: 2));
911 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
912 llvm_unreachable("lerp operand must have a float representation");
913 return Builder.CreateIntrinsic(
914 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getLerpIntrinsic(),
915 Args: ArrayRef<Value *>{X, Y, S}, FMFSource: nullptr, Name: "hlsl.lerp");
916 }
917 case Builtin::BI__builtin_hlsl_normalize: {
918 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
919
920 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
921 "normalize operand must have a float representation");
922
923 return Builder.CreateIntrinsic(
924 /*ReturnType=*/RetTy: X->getType(),
925 ID: CGM.getHLSLRuntime().getNormalizeIntrinsic(), Args: ArrayRef<Value *>{X},
926 FMFSource: nullptr, Name: "hlsl.normalize");
927 }
928 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
929 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
930
931 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
932 "degree operand must have a float representation");
933
934 return Builder.CreateIntrinsic(
935 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getDegreesIntrinsic(),
936 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.degrees");
937 }
938 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
939 return handleElementwiseF16ToF32(CGF&: *this, E);
940 }
941 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
942 return handleElementwiseF32ToF16(CGF&: *this, E);
943 }
944 case Builtin::BI__builtin_hlsl_elementwise_frac: {
945 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
946 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
947 llvm_unreachable("frac operand must have a float representation");
948 return Builder.CreateIntrinsic(
949 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
950 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
951 }
952 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
953 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
954 llvm::Type *Xty = Op0->getType();
955 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
956 if (Xty->isVectorTy()) {
957 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
958 retType = llvm::VectorType::get(
959 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
960 }
961 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
962 llvm_unreachable("isinf operand must have a float representation");
963 return Builder.CreateIntrinsic(
964 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
965 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
966 }
967 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
968 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
969 llvm::Type *Xty = Op0->getType();
970 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
971 if (Xty->isVectorTy()) {
972 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
973 retType = llvm::VectorType::get(
974 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
975 }
976 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
977 llvm_unreachable("isnan operand must have a float representation");
978 return Builder.CreateIntrinsic(
979 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
980 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
981 }
982 case Builtin::BI__builtin_hlsl_mad: {
983 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
984 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
985 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
986 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
987 return Builder.CreateIntrinsic(
988 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
989 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
990
991 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
992 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
993 return Builder.CreateIntrinsic(
994 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
995 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
996
997 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
998 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
999 }
1000 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1001 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1002 return Builder.CreateIntrinsic(
1003 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1004 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1005
1006 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1007 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1008 }
1009 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1010 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1011 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1012 llvm_unreachable("rcp operand must have a float representation");
1013 llvm::Type *Ty = Op0->getType();
1014 llvm::Type *EltTy = Ty->getScalarType();
1015 Constant *One = Ty->isVectorTy()
1016 ? ConstantVector::getSplat(
1017 EC: ElementCount::getFixed(
1018 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1019 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1020 : ConstantFP::get(Ty: EltTy, V: 1.0);
1021 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1022 }
1023 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1024 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1025 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1026 llvm_unreachable("rsqrt operand must have a float representation");
1027 return Builder.CreateIntrinsic(
1028 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1029 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1030 }
1031 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1032 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1033 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1034 "saturate operand must have a float representation");
1035 return Builder.CreateIntrinsic(
1036 /*ReturnType=*/RetTy: Op0->getType(),
1037 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1038 FMFSource: nullptr, Name: "hlsl.saturate");
1039 }
1040 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1041 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1042 assert(Op->getType()->isIntegerTy(1) &&
1043 "WavePrefixBitCount operand must be a boolean type");
1044
1045 Intrinsic::ID IID =
1046 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1047
1048 return EmitRuntimeCall(
1049 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: IID), args: ArrayRef{Op},
1050 name: "hlsl.wave.prefix.bit.count");
1051 }
1052 case Builtin::BI__builtin_hlsl_select: {
1053 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1054 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1055 Value *OpTrue =
1056 RValTrue.isScalar()
1057 ? RValTrue.getScalarVal()
1058 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1059 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1060 Value *OpFalse =
1061 RValFalse.isScalar()
1062 ? RValFalse.getScalarVal()
1063 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1064 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1065 if (!OpTrue->getType()->isVectorTy())
1066 OpTrue =
1067 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1068 if (!OpFalse->getType()->isVectorTy())
1069 OpFalse =
1070 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1071 }
1072
1073 Value *SelectVal =
1074 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1075 if (!RValTrue.isScalar())
1076 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1077 IsVolatile: ReturnValue.isVolatile());
1078
1079 return SelectVal;
1080 }
1081 case Builtin::BI__builtin_hlsl_step: {
1082 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1083 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1084 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1085 E->getArg(1)->getType()->hasFloatingRepresentation() &&
1086 "step operands must have a float representation");
1087 return Builder.CreateIntrinsic(
1088 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getStepIntrinsic(),
1089 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.step");
1090 }
1091 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1092 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1093 assert(Op->getType()->isIntegerTy(1) &&
1094 "Intrinsic WaveActiveAllTrue operand must be a bool");
1095
1096 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1097 return EmitRuntimeCall(
1098 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1099 }
1100 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1101 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1102 assert(Op->getType()->isIntegerTy(1) &&
1103 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1104
1105 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1106 return EmitRuntimeCall(
1107 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1108 }
1109 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1110 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1111 assert(Op->getType()->isIntegerTy(1) &&
1112 "Intrinsic WaveActiveBallot operand must be a bool");
1113
1114 return handleHlslWaveActiveBallot(CGF&: *this, E);
1115 }
1116 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1117 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1118 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1119 return EmitRuntimeCall(
1120 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID),
1121 args: ArrayRef{OpExpr});
1122 }
1123 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1124 // Due to the use of variadic arguments, explicitly retreive argument
1125 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1126 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1127 Arch: getTarget().getTriple().getArch(), RT&: CGM.getHLSLRuntime(),
1128 QT: E->getArg(Arg: 0)->getType());
1129
1130 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1131 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1132 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.sum");
1133 }
1134 case Builtin::BI__builtin_hlsl_wave_active_max: {
1135 // Due to the use of variadic arguments, explicitly retreive argument
1136 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1137 QualType QT = E->getArg(Arg: 0)->getType();
1138 Intrinsic::ID IID;
1139 if (QT->isUnsignedIntegerType())
1140 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1141 else
1142 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1143
1144 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1145 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1146 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.max");
1147 }
1148 case Builtin::BI__builtin_hlsl_wave_active_min: {
1149 // Due to the use of variadic arguments, explicitly retreive argument
1150 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1151 QualType QT = E->getArg(Arg: 0)->getType();
1152 Intrinsic::ID IID;
1153 if (QT->isUnsignedIntegerType())
1154 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1155 else
1156 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1157
1158 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1159 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1160 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.min");
1161 }
1162 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1163 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1164 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1165 // for the DirectX intrinsic and the demangled builtin name
1166 switch (CGM.getTarget().getTriple().getArch()) {
1167 case llvm::Triple::dxil:
1168 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1169 M: &CGM.getModule(), id: Intrinsic::dx_wave_getlaneindex));
1170 case llvm::Triple::spirv:
1171 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1172 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1173 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1174 default:
1175 llvm_unreachable(
1176 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1177 }
1178 }
1179 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1180 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1181 return EmitRuntimeCall(
1182 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1183 }
1184 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1185 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1186 return EmitRuntimeCall(
1187 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1188 }
1189 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1190 // Due to the use of variadic arguments we must explicitly retreive them and
1191 // create our function type.
1192 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1193 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1194 return EmitRuntimeCall(
1195 callee: Intrinsic::getOrInsertDeclaration(
1196 M: &CGM.getModule(), id: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1197 Tys: {OpExpr->getType()}),
1198 args: ArrayRef{OpExpr, OpIndex}, name: "hlsl.wave.readlane");
1199 }
1200 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1201 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1202 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1203 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1204 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1205 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1206 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.sum");
1207 }
1208 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1209 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1210 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1211 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1212 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1213 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1214 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.product");
1215 }
1216 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1217 auto *Arg0 = E->getArg(Arg: 0);
1218 Value *Op0 = EmitScalarExpr(E: Arg0);
1219 llvm::Type *Xty = Op0->getType();
1220 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1221 if (Xty->isVectorTy()) {
1222 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1223 retType = llvm::VectorType::get(
1224 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1225 }
1226 assert((Arg0->getType()->hasFloatingRepresentation() ||
1227 Arg0->getType()->hasIntegerRepresentation()) &&
1228 "sign operand must have a float or int representation");
1229
1230 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1231 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1232 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1233 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1234 }
1235
1236 return Builder.CreateIntrinsic(
1237 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1238 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1239 }
1240 case Builtin::BI__builtin_hlsl_elementwise_radians: {
1241 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1242 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1243 "radians operand must have a float representation");
1244 return Builder.CreateIntrinsic(
1245 /*ReturnType=*/RetTy: Op0->getType(),
1246 ID: CGM.getHLSLRuntime().getRadiansIntrinsic(), Args: ArrayRef<Value *>{Op0},
1247 FMFSource: nullptr, Name: "hlsl.radians");
1248 }
1249 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1250 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1251 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1252 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1253 return Builder.CreateIntrinsic(
1254 /*ReturnType=*/RetTy: Offset->getType(),
1255 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1256 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1257 }
1258 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1259
1260 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1261 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1262 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1263 "asuint operands types mismatch");
1264 return handleHlslSplitdouble(E, CGF: this);
1265 }
1266 case Builtin::BI__builtin_hlsl_elementwise_clip:
1267 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1268 "clip operands types mismatch");
1269 return handleHlslClip(E, CGF: this);
1270 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1271 Intrinsic::ID ID =
1272 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1273 return EmitRuntimeCall(
1274 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1275 }
1276 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1277 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1278 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1279 llvm_unreachable("ddx_coarse operand must have a float representation");
1280 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1281 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1282 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1283 Name: "hlsl.ddx.coarse");
1284 }
1285 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1286 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1287 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1288 llvm_unreachable("ddy_coarse operand must have a float representation");
1289 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1290 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1291 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1292 Name: "hlsl.ddy.coarse");
1293 }
1294 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1295 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1296 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1297 llvm_unreachable("ddx_fine operand must have a float representation");
1298 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1299 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1300 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1301 Name: "hlsl.ddx.fine");
1302 }
1303 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1304 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1305 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1306 llvm_unreachable("ddy_fine operand must have a float representation");
1307 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1308 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1309 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1310 Name: "hlsl.ddy.fine");
1311 }
1312 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1313 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1314 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1315 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1316 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1317 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1318 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1319 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1320 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1321 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1322 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1323 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1324 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1325 llvm::Value *Args[] = {SpecId, DefaultVal};
1326 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1327 }
1328 }
1329 return nullptr;
1330}
1331
1332llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1333 const clang::QualType &SpecConstantType) {
1334
1335 // Find or create the declaration for the function.
1336 llvm::Module *M = &CGM.getModule();
1337 std::string MangledName =
1338 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1339 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1340
1341 if (!SpecConstantFn) {
1342 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1343 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1344 llvm::Type *ArgTypes[] = {IntType, RetTy};
1345 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1346 SpecConstantFn = llvm::Function::Create(
1347 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1348 }
1349 return SpecConstantFn;
1350}
1351