1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "llvm/IR/MatrixBuilder.h"
17
18using namespace clang;
19using namespace CodeGen;
20using namespace llvm;
21
22static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
23 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
24 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
25 "asdouble operands types mismatch");
26 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
27 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
28
29 llvm::Type *ResultType = CGF.DoubleTy;
30 int N = 1;
31 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
32 N = VTy->getNumElements();
33 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
34 }
35
36 if (CGF.CGM.getTarget().getTriple().isDXIL())
37 return CGF.Builder.CreateIntrinsic(
38 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
39 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
40
41 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
42 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
43 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
44 }
45
46 llvm::SmallVector<int> Mask;
47 for (int i = 0; i < N; i++) {
48 Mask.push_back(Elt: i);
49 Mask.push_back(Elt: i + N);
50 }
51
52 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
53
54 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
55}
56
57static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
58 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
59
60 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
61 Value *CMP;
62 Value *LastInstr;
63
64 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
65 FZeroConst = ConstantVector::getSplat(
66 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
67 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
68 CMP = CGF->Builder.CreateIntrinsic(
69 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
70 Args: {FCompInst});
71 } else {
72 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
73 }
74
75 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
76 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
77 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
78 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
79 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
80
81 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
82
83 CGF->Builder.SetInsertPoint(LT0);
84
85 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
86
87 LastInstr = CGF->Builder.CreateBr(Dest: End);
88 CGF->Builder.SetInsertPoint(End);
89 } else {
90 llvm_unreachable("Backend Codegen not supported.");
91 }
92
93 return LastInstr;
94}
95
96static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
97 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
98 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
99 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
100
101 CallArgList Args;
102 LValue Op1TmpLValue =
103 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
104 LValue Op2TmpLValue =
105 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
106
107 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
108 Args.reverseWritebacks();
109
110 Value *LowBits = nullptr;
111 Value *HighBits = nullptr;
112
113 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
114 llvm::Type *RetElementTy = CGF->Int32Ty;
115 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
116 RetElementTy = llvm::VectorType::get(
117 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
118 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
119
120 CallInst *CI = CGF->Builder.CreateIntrinsic(
121 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
122
123 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
124 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
125 } else {
126 // For Non DXIL targets we generate the instructions.
127
128 if (!Op0->getType()->isVectorTy()) {
129 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
130 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
131
132 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
133 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
134 } else {
135 int NumElements = 1;
136 if (const auto *VecTy =
137 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
138 NumElements = VecTy->getNumElements();
139
140 FixedVectorType *Uint32VecTy =
141 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
142 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
143 if (NumElements == 1) {
144 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
145 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
146 } else {
147 SmallVector<int> EvenMask, OddMask;
148 for (int I = 0, E = NumElements; I != E; ++I) {
149 EvenMask.push_back(Elt: I * 2);
150 OddMask.push_back(Elt: I * 2 + 1);
151 }
152 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
153 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
154 }
155 }
156 }
157 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
158 auto *LastInst =
159 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
160 CGF->EmitWritebacks(Args);
161 return LastInst;
162}
163
164static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
165 const CallExpr *E) {
166 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
167 llvm::Type *I32 = CGF.Int32Ty;
168
169 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
170 [[maybe_unused]] llvm::StructType *Struct4I32 =
171 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
172
173 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
174 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
175 llvm::Function *Fn = CGF.CGM.getIntrinsic(IID: Intrinsic::dx_wave_ballot, Tys: {I32});
176
177 Value *StructVal = CGF.EmitRuntimeCall(callee: Fn, args: Cond);
178 assert(StructVal->getType() == Struct4I32 &&
179 "dx.wave.ballot must return {i32,i32,i32,i32}");
180
181 // Reassemble struct to <4 x i32>
182 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
183 for (unsigned I = 0; I < 4; ++I) {
184 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
185 VecVal =
186 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
187 }
188
189 return VecVal;
190 }
191
192 if (CGF.CGM.getTarget().getTriple().isSPIRV())
193 return CGF.EmitRuntimeCall(
194 callee: CGF.CGM.getIntrinsic(IID: Intrinsic::spv_subgroup_ballot), args: Cond);
195
196 llvm_unreachable(
197 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
198}
199
200static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
201 const CallExpr *E) {
202 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
203 QualType Op0Ty = E->getArg(Arg: 0)->getType();
204 llvm::Type *ResType = CGF.FloatTy;
205 uint64_t NumElements = 0;
206 if (Op0->getType()->isVectorTy()) {
207 NumElements =
208 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
209 ResType =
210 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
211 }
212 if (!Op0Ty->hasUnsignedIntegerRepresentation())
213 llvm_unreachable(
214 "f16tof32 operand must have an unsigned int representation");
215
216 if (CGF.CGM.getTriple().isDXIL())
217 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
218 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
219 Name: "hlsl.f16tof32");
220
221 if (CGF.CGM.getTriple().isSPIRV()) {
222 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
223 // Int16 and Float16 capabilities
224 auto *UnpackType =
225 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
226
227 if (NumElements == 0) {
228 // a scalar input - simply extract the first element of the unpacked
229 // vector
230 Value *Unpack = CGF.Builder.CreateIntrinsic(
231 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
232 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
233 }
234
235 // a vector input - build a congruent output vector by iterating through
236 // the input vector calling unpackhalf2x16 for each element
237 Value *Result = PoisonValue::get(T: ResType);
238 for (uint64_t I = 0; I < NumElements; I++) {
239 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
240 Value *Unpack = CGF.Builder.CreateIntrinsic(
241 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
242 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
243 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
244 }
245 return Result;
246 }
247
248 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
249}
250
251static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
252 const CallExpr *E) {
253 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
254 QualType Op0Ty = E->getArg(Arg: 0)->getType();
255 llvm::Type *ResType = CGF.IntTy;
256 uint64_t NumElements = 0;
257 if (Op0->getType()->isVectorTy()) {
258 NumElements =
259 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
260 ResType =
261 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
262 }
263 if (!Op0Ty->hasFloatingRepresentation())
264 llvm_unreachable("f32tof16 operand must have a float representation");
265
266 if (CGF.CGM.getTriple().isDXIL())
267 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
268 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
269 Name: "hlsl.f32tof16");
270
271 if (CGF.CGM.getTriple().isSPIRV()) {
272 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
273 // Int16 and Float16 capabilities
274 auto *PackType =
275 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
276
277 if (NumElements == 0) {
278 // a scalar input - simply insert the scalar in the first element
279 // of the 2 element float vector
280 Value *Float2 = Constant::getNullValue(Ty: PackType);
281 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
282 Value *Result = CGF.Builder.CreateIntrinsic(
283 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
284 return Result;
285 }
286
287 // a vector input - build a congruent output vector by iterating through
288 // the input vector calling packhalf2x16 for each element
289 Value *Result = PoisonValue::get(T: ResType);
290 for (uint64_t I = 0; I < NumElements; I++) {
291 Value *Float2 = Constant::getNullValue(Ty: PackType);
292 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
293 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
294 Value *Res = CGF.Builder.CreateIntrinsic(
295 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
296 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
297 }
298 return Result;
299 }
300
301 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
302}
303
304static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
305 LValue &Stride) {
306 // Figure out the stride of the buffer elements from the handle type.
307 auto *HandleTy =
308 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
309 QualType ElementTy = HandleTy->getContainedType();
310 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
311 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
312}
313
314// Return dot product intrinsic that corresponds to the QT scalar type
315static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
316 if (QT->isFloatingType())
317 return RT.getFDotIntrinsic();
318 if (QT->isSignedIntegerType())
319 return RT.getSDotIntrinsic();
320 assert(QT->isUnsignedIntegerType());
321 return RT.getUDotIntrinsic();
322}
323
324static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
325 if (QT->hasSignedIntegerRepresentation()) {
326 return RT.getFirstBitSHighIntrinsic();
327 }
328
329 assert(QT->hasUnsignedIntegerRepresentation());
330 return RT.getFirstBitUHighIntrinsic();
331}
332
333// Return wave active sum that corresponds to the QT scalar type
334static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
335 QualType QT) {
336 switch (Arch) {
337 case llvm::Triple::spirv:
338 return Intrinsic::spv_wave_reduce_sum;
339 case llvm::Triple::dxil: {
340 if (QT->isUnsignedIntegerType())
341 return Intrinsic::dx_wave_reduce_usum;
342 return Intrinsic::dx_wave_reduce_sum;
343 }
344 default:
345 llvm_unreachable("Intrinsic WaveActiveSum"
346 " not supported by target architecture");
347 }
348}
349
350// Return wave active product that corresponds to the QT scalar type
351static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
352 QualType QT) {
353 switch (Arch) {
354 case llvm::Triple::spirv:
355 return Intrinsic::spv_wave_product;
356 case llvm::Triple::dxil: {
357 if (QT->isUnsignedIntegerType())
358 return Intrinsic::dx_wave_uproduct;
359 return Intrinsic::dx_wave_product;
360 }
361 default:
362 llvm_unreachable("Intrinsic WaveActiveProduct"
363 " not supported by target architecture");
364 }
365}
366
367static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
368 switch (Arch) {
369 case llvm::Triple::spirv:
370 return Intrinsic::spv_subgroup_prefix_bit_count;
371 case llvm::Triple::dxil: {
372 return Intrinsic::dx_wave_prefix_bit_count;
373 }
374 default:
375 llvm_unreachable(
376 "WavePrefixOp instruction not supported by target architecture");
377 }
378}
379
380// Return wave prefix sum that corresponds to the QT scalar type
381static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
382 QualType QT) {
383 switch (Arch) {
384 case llvm::Triple::spirv:
385 return Intrinsic::spv_wave_prefix_sum;
386 case llvm::Triple::dxil: {
387 if (QT->isUnsignedIntegerType())
388 return Intrinsic::dx_wave_prefix_usum;
389 return Intrinsic::dx_wave_prefix_sum;
390 }
391 default:
392 llvm_unreachable("Intrinsic WavePrefixSum"
393 " not supported by target architecture");
394 }
395}
396
397// Return wave prefix product that corresponds to the QT scalar type
398static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
399 QualType QT) {
400 switch (Arch) {
401 case llvm::Triple::spirv:
402 return Intrinsic::spv_wave_prefix_product;
403 case llvm::Triple::dxil: {
404 if (QT->isUnsignedIntegerType())
405 return Intrinsic::dx_wave_prefix_uproduct;
406 return Intrinsic::dx_wave_prefix_product;
407 }
408 default:
409 llvm_unreachable("Intrinsic WavePrefixProduct"
410 " not supported by target architecture");
411 }
412}
413
414// Returns the mangled name for a builtin function that the SPIR-V backend
415// will expand into a spec Constant.
416static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
417 ASTContext &Context) {
418 // The parameter types for our conceptual intrinsic function.
419 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
420
421 // Create a temporary FunctionDecl for the builtin fuction. It won't be
422 // added to the AST.
423 FunctionProtoType::ExtProtoInfo EPI;
424 QualType FnType =
425 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
426 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
427 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
428 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
429 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
430
431 // Attach the created parameter declarations to the function declaration.
432 SmallVector<ParmVarDecl *, 2> ParamDecls;
433 for (QualType ParamType : ClangParamTypes) {
434 ParmVarDecl *PD = ParmVarDecl::Create(
435 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
436 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
437 /*DefaultArg*/ DefArg: nullptr);
438 ParamDecls.push_back(Elt: PD);
439 }
440 FnDeclForMangling->setParams(ParamDecls);
441
442 // Get the mangled name.
443 std::string Name;
444 llvm::raw_string_ostream MangledNameStream(Name);
445 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
446 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
447 MangledNameStream.flush();
448
449 return Name;
450}
451
452static llvm::Type *getOffsetType(CodeGenModule &CGM, llvm::Type *CoordTy) {
453 llvm::Type *Int32Ty = CGM.Int32Ty;
454 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: CoordTy))
455 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: VT->getNumElements());
456 return Int32Ty;
457}
458
459static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
460 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
461 if (E->getNumArgs() > OffsetArgIndex)
462 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
463
464 return llvm::Constant::getNullValue(Ty: OffsetTy);
465}
466
467static Value *emitHlslClamp(CodeGenFunction &CGF, const CallExpr *E,
468 unsigned ClampArgIndex) {
469 Value *Clamp = CGF.EmitScalarExpr(E: E->getArg(Arg: ClampArgIndex));
470 // The builtin is defined with variadic arguments, so the clamp parameter
471 // might have been promoted to double. The intrinsic requires a 32-bit
472 // float.
473 if (Clamp->getType() != CGF.Builder.getFloatTy())
474 Clamp = CGF.Builder.CreateFPCast(V: Clamp, DestTy: CGF.Builder.getFloatTy());
475 return Clamp;
476}
477
478static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
479 unsigned IntrinsicID, unsigned NumRetComps,
480 bool HasLod) {
481 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
482
483 SmallVector<Value *> Args{Handle};
484 if (HasLod)
485 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
486
487 Value *DimValue =
488 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, Types: {Handle->getType()}, Args);
489
490 Value *LastStore = nullptr;
491 unsigned ArgIndex = HasLod ? 2 : 1;
492 for (unsigned i = 0; i < NumRetComps; ++i) {
493 const Expr *Arg = E->getArg(Arg: ArgIndex++);
494 LValue DimOut = CGF.EmitLValue(E: Arg);
495 Value *Elem = DimValue;
496 if (NumRetComps > 1)
497 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
498
499 // Handle float casting if needed
500 if (Arg->getType()->isFloatingType())
501 Elem = CGF.Builder.CreateUIToFP(
502 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
503
504 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
505 }
506 return LastStore;
507}
508
509Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
510 const CallExpr *E,
511 ReturnValueSlot ReturnValue) {
512 if (!getLangOpts().HLSL)
513 return nullptr;
514
515 switch (BuiltinID) {
516 case Builtin::BI__builtin_hlsl_adduint64: {
517 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
518 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
519 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
520 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
521 assert(Arg0Ty == E->getArg(1)->getType() &&
522 "AddUint64 operand types must match");
523 assert(Arg0Ty->hasIntegerRepresentation() &&
524 "AddUint64 operands must have an integer representation");
525 assert((NumElements == 2 || NumElements == 4) &&
526 "AddUint64 operands must have 2 or 4 elements");
527
528 llvm::Value *LowA;
529 llvm::Value *HighA;
530 llvm::Value *LowB;
531 llvm::Value *HighB;
532
533 // Obtain low and high words of inputs A and B
534 if (NumElements == 2) {
535 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
536 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
537 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
538 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
539 } else {
540 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
541 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
542 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
543 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
544 }
545
546 // Use an uadd_with_overflow to compute the sum of low words and obtain a
547 // carry value
548 llvm::Value *Carry;
549 llvm::Value *LowSum = EmitOverflowIntrinsic(
550 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
551 llvm::Value *ZExtCarry =
552 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
553
554 // Sum the high words and the carry
555 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
556 llvm::Value *HighSumPlusCarry =
557 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
558
559 if (NumElements == 4) {
560 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
561 Name: "hlsl.AddUint64");
562 }
563
564 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
565 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
566 Name: "hlsl.AddUint64.upto0");
567 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
568 Name: "hlsl.AddUint64");
569 return Result;
570 }
571 case Builtin::BI__builtin_hlsl_resource_getpointer:
572 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
573 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
574 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
575
576 llvm::Type *RetTy = ConvertType(T: E->getType());
577 return Builder.CreateIntrinsic(
578 RetTy, ID: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
579 Args: ArrayRef<Value *>{HandleOp, IndexOp});
580 }
581 case Builtin::BI__builtin_hlsl_resource_sample: {
582 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
583 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
584 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
585
586 SmallVector<Value *, 4> Args;
587 Args.push_back(Elt: HandleOp);
588 Args.push_back(Elt: SamplerOp);
589 Args.push_back(Elt: CoordOp);
590 Args.push_back(
591 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
592
593 llvm::Type *RetTy = ConvertType(T: E->getType());
594 if (E->getNumArgs() <= 4) {
595 return Builder.CreateIntrinsic(
596 RetTy, ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args);
597 }
598
599 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 4));
600 return Builder.CreateIntrinsic(
601 RetTy, ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(), Args);
602 }
603 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
604 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
605 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
606 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
607 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
608 if (BiasOp->getType() != Builder.getFloatTy())
609 BiasOp = Builder.CreateFPCast(V: BiasOp, DestTy: Builder.getFloatTy());
610
611 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
612 Args.push_back(Elt: HandleOp);
613 Args.push_back(Elt: SamplerOp);
614 Args.push_back(Elt: CoordOp);
615 Args.push_back(Elt: BiasOp);
616 Args.push_back(
617 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
618
619 llvm::Type *RetTy = ConvertType(T: E->getType());
620 if (E->getNumArgs() <= 5)
621 return Builder.CreateIntrinsic(
622 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(), Args);
623
624 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
625 return Builder.CreateIntrinsic(
626 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(), Args);
627 }
628 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
629 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
630 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
631 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
632 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
633 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
634
635 SmallVector<Value *, 7> Args;
636 Args.push_back(Elt: HandleOp);
637 Args.push_back(Elt: SamplerOp);
638 Args.push_back(Elt: CoordOp);
639 Args.push_back(Elt: DDXOp);
640 Args.push_back(Elt: DDYOp);
641 Args.push_back(
642 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
643
644 llvm::Type *RetTy = ConvertType(T: E->getType());
645
646 if (E->getNumArgs() <= 6) {
647 return Builder.CreateIntrinsic(
648 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
649 }
650
651 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 6));
652 return Builder.CreateIntrinsic(
653 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
654 }
655 case Builtin::BI__builtin_hlsl_resource_sample_level: {
656 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
657 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
658 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
659 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
660 if (LODOp->getType() != Builder.getFloatTy())
661 LODOp = Builder.CreateFPCast(V: LODOp, DestTy: Builder.getFloatTy());
662
663 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
664 Args.push_back(Elt: HandleOp);
665 Args.push_back(Elt: SamplerOp);
666 Args.push_back(Elt: CoordOp);
667 Args.push_back(Elt: LODOp);
668 Args.push_back(
669 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
670
671 llvm::Type *RetTy = ConvertType(T: E->getType());
672 return Builder.CreateIntrinsic(
673 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
674 }
675 case Builtin::BI__builtin_hlsl_resource_load_level: {
676 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
677 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
678
679 auto *CoordLODVecTy = cast<llvm::FixedVectorType>(Val: CoordLODOp->getType());
680 unsigned NumElts = CoordLODVecTy->getNumElements();
681 assert(NumElts >= 2 && "CoordLOD must have at least 2 elements");
682
683 // Split CoordLOD into Coord and LOD
684 SmallVector<int, 4> Mask;
685 for (unsigned I = 0; I < NumElts - 1; ++I)
686 Mask.push_back(Elt: I);
687
688 Value *CoordOp =
689 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
690 Value *LODOp =
691 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: NumElts - 1, Name: "hlsl.load.lod");
692
693 SmallVector<Value *, 4> Args;
694 Args.push_back(Elt: HandleOp);
695 Args.push_back(Elt: CoordOp);
696 Args.push_back(Elt: LODOp);
697 Args.push_back(
698 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
699
700 llvm::Type *RetTy = ConvertType(T: E->getType());
701 return Builder.CreateIntrinsic(
702 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
703 }
704 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
705 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
706 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
707 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
708 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
709 if (CmpOp->getType() != Builder.getFloatTy())
710 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
711
712 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
713 Args.push_back(Elt: HandleOp);
714 Args.push_back(Elt: SamplerOp);
715 Args.push_back(Elt: CoordOp);
716 Args.push_back(Elt: CmpOp);
717 Args.push_back(
718 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
719
720 llvm::Type *RetTy = ConvertType(T: E->getType());
721 if (E->getNumArgs() <= 5) {
722 return Builder.CreateIntrinsic(
723 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
724 }
725
726 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
727 return Builder.CreateIntrinsic(
728 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
729 }
730 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
731 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
732 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
733 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
734 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
735 if (CmpOp->getType() != Builder.getFloatTy())
736 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
737
738 SmallVector<Value *, 5> Args;
739 Args.push_back(Elt: HandleOp);
740 Args.push_back(Elt: SamplerOp);
741 Args.push_back(Elt: CoordOp);
742 Args.push_back(Elt: CmpOp);
743
744 Args.push_back(
745 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
746
747 llvm::Type *RetTy = ConvertType(T: E->getType());
748 return Builder.CreateIntrinsic(
749 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
750 }
751 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
752 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
753 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
754 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
755
756 return Builder.CreateIntrinsic(
757 RetTy: ConvertType(T: E->getType()),
758 ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
759 Args: {HandleOp, SamplerOp, CoordOp});
760 }
761 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
762 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
763 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
764 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
765
766 return Builder.CreateIntrinsic(
767 RetTy: ConvertType(T: E->getType()),
768 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
769 Args: {HandleOp, SamplerOp, CoordOp});
770 }
771 case Builtin::BI__builtin_hlsl_resource_gather: {
772 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
773 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
774 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
775 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
776 if (ComponentOp->getType() != Builder.getInt32Ty())
777 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
778 /*isSigned=*/false);
779
780 SmallVector<Value *, 5> Args;
781 Args.push_back(Elt: HandleOp);
782 Args.push_back(Elt: SamplerOp);
783 Args.push_back(Elt: CoordOp);
784 Args.push_back(Elt: ComponentOp);
785 Args.push_back(
786 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
787
788 llvm::Type *RetTy = ConvertType(T: E->getType());
789 return Builder.CreateIntrinsic(
790 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
791 }
792 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
793 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
794 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
795 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
796 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
797 if (CompareOp->getType() != Builder.getFloatTy())
798 CompareOp = Builder.CreateFPCast(V: CompareOp, DestTy: Builder.getFloatTy());
799
800 SmallVector<Value *, 6> Args;
801 Args.push_back(Elt: HandleOp);
802 Args.push_back(Elt: SamplerOp);
803 Args.push_back(Elt: CoordOp);
804 Args.push_back(Elt: CompareOp);
805
806 if (CGM.getTarget().getTriple().isDXIL()) {
807 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
808 if (ComponentOp->getType() != Builder.getInt32Ty())
809 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
810 /*isSigned=*/false);
811 Args.push_back(Elt: ComponentOp);
812 }
813
814 Args.push_back(
815 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
816
817 llvm::Type *RetTy = ConvertType(T: E->getType());
818 return Builder.CreateIntrinsic(
819 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
820 }
821 case Builtin::BI__builtin_hlsl_resource_load_with_status:
822 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
823 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
824 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
825
826 // Get the *address* of the status argument to write to it by reference
827 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
828 Address StatusAddr = StatusLVal.getAddress();
829
830 QualType HandleTy = E->getArg(Arg: 0)->getType();
831 const HLSLAttributedResourceType *RT =
832 HandleTy->getAs<HLSLAttributedResourceType>();
833 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
834 "Only DXIL currently implements load with status");
835
836 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
837 ? llvm::Intrinsic::dx_resource_load_rawbuffer
838 : llvm::Intrinsic::dx_resource_load_typedbuffer;
839
840 llvm::Type *DataTy = ConvertType(T: E->getType());
841 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
842 Elements: {DataTy, Builder.getInt1Ty()});
843
844 SmallVector<Value *, 3> Args;
845 Args.push_back(Elt: HandleOp);
846 Args.push_back(Elt: IndexOp);
847
848 if (RT->isRaw()) {
849 Value *Offset = Builder.getInt32(C: 0);
850 // The offset parameter needs to be poison for ByteAddressBuffer
851 if (!RT->isStructured())
852 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
853 Args.push_back(Elt: Offset);
854 }
855
856 // The load intrinsics give us a (T value, i1 status) pair -
857 // shepherd these into the return value and out reference respectively.
858 Value *ResRet =
859 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
860 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
861 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
862 Value *ExtendedStatus =
863 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
864 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
865
866 return LoadedValue;
867 }
868 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
869 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
870 return llvm::PoisonValue::get(T: HandleTy);
871 }
872 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
873 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
874 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
875 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
876 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
877 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
878 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
879 llvm::Intrinsic::ID IntrinsicID =
880 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
881 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
882 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
883 }
884 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
885 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
886 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
887 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
888 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
889 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
890 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
891 llvm::Intrinsic::ID IntrinsicID =
892 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
893 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
894 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
895 }
896 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
897 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
898 if (!CGM.getTriple().isSPIRV())
899 return MainHandle;
900
901 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
902 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
903 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
904 llvm::Intrinsic::ID IntrinsicID =
905 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
906 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
907 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
908 }
909 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
910 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
911 llvm::Type *RetTy = ConvertType(T: E->getType());
912 return Builder.CreateIntrinsic(
913 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
914 Args: ArrayRef<Value *>{IndexOp});
915 }
916 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
917 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
918 return emitGetDimensions(CGF&: *this, E,
919 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
920 NumRetComps: 1, /*HasLod=*/false);
921 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
922 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
923 return emitGetDimensions(CGF&: *this, E,
924 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
925 NumRetComps: 2, /*HasLod=*/false);
926 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
927 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
928 return emitGetDimensions(
929 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
930 /*HasLod=*/true);
931 case Builtin::BI__builtin_hlsl_resource_getstride: {
932 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
933 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
934 }
935 case Builtin::BI__builtin_hlsl_all: {
936 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
937 return Builder.CreateIntrinsic(
938 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
939 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
940 Name: "hlsl.all");
941 }
942 case Builtin::BI__builtin_hlsl_and: {
943 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
944 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
945 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
946 }
947 case Builtin::BI__builtin_hlsl_or: {
948 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
949 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
950 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
951 }
952 case Builtin::BI__builtin_hlsl_any: {
953 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
954 return Builder.CreateIntrinsic(
955 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
956 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
957 Name: "hlsl.any");
958 }
959 case Builtin::BI__builtin_hlsl_asdouble:
960 return handleAsDoubleBuiltin(CGF&: *this, E);
961 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
962 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
963 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
964 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
965
966 QualType Ty = E->getArg(Arg: 0)->getType();
967 if (auto *VecTy = Ty->getAs<VectorType>())
968 Ty = VecTy->getElementType();
969
970 Intrinsic::ID Intr;
971 if (Ty->isFloatingType()) {
972 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
973 } else if (Ty->isUnsignedIntegerType()) {
974 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
975 } else {
976 assert(Ty->isSignedIntegerType());
977 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
978 }
979 return Builder.CreateIntrinsic(
980 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
981 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
982 }
983 case Builtin::BI__builtin_hlsl_crossf16:
984 case Builtin::BI__builtin_hlsl_crossf32: {
985 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
986 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
987 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
988 E->getArg(1)->getType()->hasFloatingRepresentation() &&
989 "cross operands must have a float representation");
990 // make sure each vector has exactly 3 elements
991 assert(
992 E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
993 E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
994 "input vectors must have 3 elements each");
995 return Builder.CreateIntrinsic(
996 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getCrossIntrinsic(),
997 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.cross");
998 }
999 case Builtin::BI__builtin_hlsl_dot: {
1000 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1001 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1002 llvm::Type *T0 = Op0->getType();
1003 llvm::Type *T1 = Op1->getType();
1004
1005 // If the arguments are scalars, just emit a multiply
1006 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1007 if (T0->isFloatingPointTy())
1008 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1009
1010 if (T0->isIntegerTy())
1011 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1012
1013 llvm_unreachable(
1014 "Scalar dot product is only supported on ints and floats.");
1015 }
1016 // For vectors, validate types and emit the appropriate intrinsic
1017 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1018 E->getArg(1)->getType()) &&
1019 "Dot product operands must have the same type.");
1020
1021 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1022 assert(VecTy0 && "Dot product argument must be a vector.");
1023
1024 return Builder.CreateIntrinsic(
1025 /*ReturnType=*/RetTy: T0->getScalarType(),
1026 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1027 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1028 }
1029 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1030 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1031 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1032 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1033
1034 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1035 // Note that the argument order disagrees between the builtin and the
1036 // intrinsic here.
1037 return Builder.CreateIntrinsic(
1038 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1039 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1040 }
1041 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1042 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1043 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1044 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1045
1046 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1047 // Note that the argument order disagrees between the builtin and the
1048 // intrinsic here.
1049 return Builder.CreateIntrinsic(
1050 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1051 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1052 }
1053 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1054 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1055
1056 return Builder.CreateIntrinsic(
1057 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1058 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1059 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1060 }
1061 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1062 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1063
1064 return Builder.CreateIntrinsic(
1065 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1066 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1067 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1068 }
1069 case Builtin::BI__builtin_hlsl_lerp: {
1070 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1071 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1072 Value *S = EmitScalarExpr(E: E->getArg(Arg: 2));
1073 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1074 llvm_unreachable("lerp operand must have a float representation");
1075 return Builder.CreateIntrinsic(
1076 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getLerpIntrinsic(),
1077 Args: ArrayRef<Value *>{X, Y, S}, FMFSource: nullptr, Name: "hlsl.lerp");
1078 }
1079 case Builtin::BI__builtin_hlsl_normalize: {
1080 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1081
1082 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1083 "normalize operand must have a float representation");
1084
1085 return Builder.CreateIntrinsic(
1086 /*ReturnType=*/RetTy: X->getType(),
1087 ID: CGM.getHLSLRuntime().getNormalizeIntrinsic(), Args: ArrayRef<Value *>{X},
1088 FMFSource: nullptr, Name: "hlsl.normalize");
1089 }
1090 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
1091 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1092
1093 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1094 "degree operand must have a float representation");
1095
1096 return Builder.CreateIntrinsic(
1097 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getDegreesIntrinsic(),
1098 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.degrees");
1099 }
1100 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1101 return handleElementwiseF16ToF32(CGF&: *this, E);
1102 }
1103 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1104 return handleElementwiseF32ToF16(CGF&: *this, E);
1105 }
1106 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1107 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1108 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1109 llvm_unreachable("frac operand must have a float representation");
1110 return Builder.CreateIntrinsic(
1111 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1112 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1113 }
1114 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1115 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1116 llvm::Type *Xty = Op0->getType();
1117 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1118 if (Xty->isVectorTy()) {
1119 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1120 retType = llvm::VectorType::get(
1121 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1122 }
1123 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1124 llvm_unreachable("isinf operand must have a float representation");
1125 return Builder.CreateIntrinsic(
1126 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1127 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1128 }
1129 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1130 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1131 llvm::Type *Xty = Op0->getType();
1132 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1133 if (Xty->isVectorTy()) {
1134 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1135 retType = llvm::VectorType::get(
1136 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1137 }
1138 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1139 llvm_unreachable("isnan operand must have a float representation");
1140 return Builder.CreateIntrinsic(
1141 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1142 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1143 }
1144 case Builtin::BI__builtin_hlsl_mad: {
1145 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1146 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1147 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1148 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1149 return Builder.CreateIntrinsic(
1150 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1151 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1152
1153 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1154 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1155 return Builder.CreateIntrinsic(
1156 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1157 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1158
1159 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1160 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1161 }
1162 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1163 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1164 return Builder.CreateIntrinsic(
1165 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1166 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1167
1168 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1169 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1170 }
1171 case Builtin::BI__builtin_hlsl_mul: {
1172 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1173 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1174 QualType QTy0 = E->getArg(Arg: 0)->getType();
1175 QualType QTy1 = E->getArg(Arg: 1)->getType();
1176
1177 bool IsVec0 = QTy0->isVectorType();
1178 bool IsVec1 = QTy1->isVectorType();
1179 bool IsMat0 = QTy0->isConstantMatrixType();
1180 bool IsMat1 = QTy1->isConstantMatrixType();
1181
1182 // The matrix multiply intrinsic only operates on column-major order
1183 // matrices. Therefore matrix memory layout transforms must be inserted
1184 // before and after matrix multiply intrinsics.
1185 bool IsRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
1186 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
1187
1188 llvm::MatrixBuilder MB(Builder);
1189 if (IsVec0 && IsMat1) {
1190 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1191 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1192 unsigned Rows = MatTy->getNumRows();
1193 unsigned Cols = MatTy->getNumColumns();
1194 assert(N == Rows && "vector length must match matrix row count");
1195 if (IsRowMajor)
1196 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows, Columns: Cols);
1197 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1198 }
1199 if (IsMat0 && IsVec1) {
1200 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1201 unsigned Rows = MatTy->getNumRows();
1202 unsigned Cols = MatTy->getNumColumns();
1203 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1204 "vector length must match matrix column count");
1205 if (IsRowMajor)
1206 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows, Columns: Cols);
1207 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1208 }
1209 assert(IsMat0 && IsMat1);
1210 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1211 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1212 unsigned Rows0 = MatTy0->getNumRows();
1213 unsigned Rows1 = MatTy1->getNumRows();
1214 unsigned Cols0 = MatTy0->getNumColumns();
1215 unsigned Cols1 = MatTy1->getNumColumns();
1216 assert(Cols0 == Rows1 &&
1217 "inner matrix dimensions must match for multiplication");
1218 if (IsRowMajor) {
1219 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows: Rows0, Columns: Cols0);
1220 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows: Rows1, Columns: Cols1);
1221 }
1222 Value *Result =
1223 MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1224 if (IsRowMajor)
1225 Result = MB.CreateColumnMajorToRowMajorTransform(Matrix: Result, Rows: Rows0, Columns: Cols1);
1226 return Result;
1227 }
1228 case Builtin::BI__builtin_hlsl_transpose: {
1229 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1230 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1231 unsigned Rows = MatTy->getNumRows();
1232 unsigned Cols = MatTy->getNumColumns();
1233 llvm::MatrixBuilder MB(Builder);
1234 // The matrix transpose intrinsic operates on column-major matrices.
1235 // For row-major, a row-major RxC matrix is equivalent to a column-major
1236 // CxR matrix, so transposing with swapped dimensions produces the correct
1237 // row-major CxR result directly.
1238 bool IsRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
1239 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
1240 if (IsRowMajor)
1241 return MB.CreateMatrixTranspose(Matrix: Op0, Rows: Cols, Columns: Rows);
1242 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1243 }
1244 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1245 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1246 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1247 llvm_unreachable("rcp operand must have a float representation");
1248 llvm::Type *Ty = Op0->getType();
1249 llvm::Type *EltTy = Ty->getScalarType();
1250 Constant *One = Ty->isVectorTy()
1251 ? ConstantVector::getSplat(
1252 EC: ElementCount::getFixed(
1253 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1254 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1255 : ConstantFP::get(Ty: EltTy, V: 1.0);
1256 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1257 }
1258 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1259 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1260 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1261 llvm_unreachable("rsqrt operand must have a float representation");
1262 return Builder.CreateIntrinsic(
1263 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1264 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1265 }
1266 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1267 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1268 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1269 "saturate operand must have a float representation");
1270 return Builder.CreateIntrinsic(
1271 /*ReturnType=*/RetTy: Op0->getType(),
1272 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1273 FMFSource: nullptr, Name: "hlsl.saturate");
1274 }
1275 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1276 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1277 assert(Op->getType()->isIntegerTy(1) &&
1278 "WavePrefixBitCount operand must be a boolean type");
1279
1280 Intrinsic::ID IID =
1281 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1282
1283 return EmitRuntimeCall(
1284 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: IID), args: ArrayRef{Op},
1285 name: "hlsl.wave.prefix.bit.count");
1286 }
1287 case Builtin::BI__builtin_hlsl_select: {
1288 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1289 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1290 Value *OpTrue =
1291 RValTrue.isScalar()
1292 ? RValTrue.getScalarVal()
1293 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1294 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1295 Value *OpFalse =
1296 RValFalse.isScalar()
1297 ? RValFalse.getScalarVal()
1298 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1299 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1300 if (!OpTrue->getType()->isVectorTy())
1301 OpTrue =
1302 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1303 if (!OpFalse->getType()->isVectorTy())
1304 OpFalse =
1305 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1306 }
1307
1308 Value *SelectVal =
1309 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1310 if (!RValTrue.isScalar())
1311 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1312 IsVolatile: ReturnValue.isVolatile());
1313
1314 return SelectVal;
1315 }
1316 case Builtin::BI__builtin_hlsl_step: {
1317 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1318 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1319 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1320 E->getArg(1)->getType()->hasFloatingRepresentation() &&
1321 "step operands must have a float representation");
1322 return Builder.CreateIntrinsic(
1323 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getStepIntrinsic(),
1324 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.step");
1325 }
1326 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1327 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1328
1329 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1330 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1331 M: &CGM.getModule(), id: ID, OverloadTys: {Op->getType()}),
1332 args: {Op});
1333 }
1334 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1335 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1336 assert(Op->getType()->isIntegerTy(1) &&
1337 "Intrinsic WaveActiveAllTrue operand must be a bool");
1338
1339 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1340 return EmitRuntimeCall(
1341 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1342 }
1343 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1344 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1345 assert(Op->getType()->isIntegerTy(1) &&
1346 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1347
1348 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1349 return EmitRuntimeCall(
1350 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1351 }
1352 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1353 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1354 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1355 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1356 "representation");
1357
1358 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1359 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1360 M: &CGM.getModule(), id: ID, OverloadTys: {Op->getType()}),
1361 args: ArrayRef{Op}, name: "hlsl.wave.active.bit.or");
1362 }
1363 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1364 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1365 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1366 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1367 "representation");
1368
1369 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1370 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1371 M: &CGM.getModule(), id: ID, OverloadTys: {Op->getType()}),
1372 args: ArrayRef{Op}, name: "hlsl.wave.active.bit.xor");
1373 }
1374 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1375 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1376 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1377 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1378 "representation");
1379
1380 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1381 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1382 M: &CGM.getModule(), id: ID, OverloadTys: {Op->getType()}),
1383 args: ArrayRef{Op}, name: "hlsl.wave.active.bit.and");
1384 }
1385 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1386 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1387 assert(Op->getType()->isIntegerTy(1) &&
1388 "Intrinsic WaveActiveBallot operand must be a bool");
1389
1390 return handleHlslWaveActiveBallot(CGF&: *this, E);
1391 }
1392 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1393 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1394 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1395 return EmitRuntimeCall(
1396 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID),
1397 args: ArrayRef{OpExpr});
1398 }
1399 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1400 // Due to the use of variadic arguments, explicitly retrieve argument
1401 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1402 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1403 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1404
1405 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1406 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1407 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.sum");
1408 }
1409 case Builtin::BI__builtin_hlsl_wave_active_product: {
1410 // Due to the use of variadic arguments, explicitly retrieve argument
1411 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1412 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1413 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1414
1415 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1416 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1417 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.product");
1418 }
1419 case Builtin::BI__builtin_hlsl_wave_active_max: {
1420 // Due to the use of variadic arguments, explicitly retrieve argument
1421 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1422 QualType QT = E->getArg(Arg: 0)->getType();
1423 Intrinsic::ID IID;
1424 if (QT->isUnsignedIntegerType())
1425 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1426 else
1427 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1428
1429 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1430 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1431 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.max");
1432 }
1433 case Builtin::BI__builtin_hlsl_wave_active_min: {
1434 // Due to the use of variadic arguments, explicitly retrieve argument
1435 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1436 QualType QT = E->getArg(Arg: 0)->getType();
1437 Intrinsic::ID IID;
1438 if (QT->isUnsignedIntegerType())
1439 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1440 else
1441 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1442
1443 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1444 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1445 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.min");
1446 }
1447 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1448 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1449 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1450 // for the DirectX intrinsic and the demangled builtin name
1451 switch (CGM.getTarget().getTriple().getArch()) {
1452 case llvm::Triple::dxil:
1453 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1454 M: &CGM.getModule(), id: Intrinsic::dx_wave_getlaneindex));
1455 case llvm::Triple::spirv:
1456 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1457 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1458 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1459 default:
1460 llvm_unreachable(
1461 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1462 }
1463 }
1464 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1465 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1466 return EmitRuntimeCall(
1467 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1468 }
1469 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1470 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1471 return EmitRuntimeCall(
1472 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1473 }
1474 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1475 // Due to the use of variadic arguments we must explicitly retrieve them and
1476 // create our function type.
1477 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1478 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1479 return EmitRuntimeCall(
1480 callee: Intrinsic::getOrInsertDeclaration(
1481 M: &CGM.getModule(), id: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1482 OverloadTys: {OpExpr->getType()}),
1483 args: ArrayRef{OpExpr, OpIndex}, name: "hlsl.wave.readlane");
1484 }
1485 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1486 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1487 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1488 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1489 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1490 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1491 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.sum");
1492 }
1493 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1494 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1495 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1496 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1497 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1498 M: &CGM.getModule(), id: IID, OverloadTys: {OpExpr->getType()}),
1499 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.product");
1500 }
1501 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1502 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1503 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1504 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1505 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1506 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.x");
1507 }
1508 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1509 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1510 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1511 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1512 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1513 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.y");
1514 }
1515 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1516 auto *Arg0 = E->getArg(Arg: 0);
1517 Value *Op0 = EmitScalarExpr(E: Arg0);
1518 llvm::Type *Xty = Op0->getType();
1519 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1520 if (Xty->isVectorTy()) {
1521 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1522 retType = llvm::VectorType::get(
1523 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1524 }
1525 assert((Arg0->getType()->hasFloatingRepresentation() ||
1526 Arg0->getType()->hasIntegerRepresentation()) &&
1527 "sign operand must have a float or int representation");
1528
1529 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1530 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1531 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1532 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1533 }
1534
1535 return Builder.CreateIntrinsic(
1536 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1537 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1538 }
1539 case Builtin::BI__builtin_hlsl_elementwise_radians: {
1540 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1541 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1542 "radians operand must have a float representation");
1543 return Builder.CreateIntrinsic(
1544 /*ReturnType=*/RetTy: Op0->getType(),
1545 ID: CGM.getHLSLRuntime().getRadiansIntrinsic(), Args: ArrayRef<Value *>{Op0},
1546 FMFSource: nullptr, Name: "hlsl.radians");
1547 }
1548 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1549 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1550 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1551 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1552 return Builder.CreateIntrinsic(
1553 /*ReturnType=*/RetTy: Offset->getType(),
1554 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1555 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1556 }
1557 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1558
1559 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1560 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1561 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1562 "asuint operands types mismatch");
1563 return handleHlslSplitdouble(E, CGF: this);
1564 }
1565 case Builtin::BI__builtin_hlsl_elementwise_clip:
1566 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1567 "clip operands types mismatch");
1568 return handleHlslClip(E, CGF: this);
1569 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1570 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1571 return EmitRuntimeCall(
1572 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1573 }
1574 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1575 Intrinsic::ID ID =
1576 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1577 return EmitRuntimeCall(
1578 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1579 }
1580 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1581 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1582 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1583 llvm_unreachable("ddx_coarse operand must have a float representation");
1584 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1585 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1586 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1587 Name: "hlsl.ddx.coarse");
1588 }
1589 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1590 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1591 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1592 llvm_unreachable("ddy_coarse operand must have a float representation");
1593 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1594 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1595 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1596 Name: "hlsl.ddy.coarse");
1597 }
1598 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1599 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1600 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1601 llvm_unreachable("ddx_fine operand must have a float representation");
1602 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1603 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1604 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1605 Name: "hlsl.ddx.fine");
1606 }
1607 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1608 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1609 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1610 llvm_unreachable("ddy_fine operand must have a float representation");
1611 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1612 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1613 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1614 Name: "hlsl.ddy.fine");
1615 }
1616 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1617 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1618 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1619 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1620 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1621 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1622 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1623 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1624 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1625 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1626 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1627 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1628 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1629 llvm::Value *Args[] = {SpecId, DefaultVal};
1630 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1631 }
1632 }
1633 return nullptr;
1634}
1635
1636llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1637 const clang::QualType &SpecConstantType) {
1638
1639 // Find or create the declaration for the function.
1640 llvm::Module *M = &CGM.getModule();
1641 std::string MangledName =
1642 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1643 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1644
1645 if (!SpecConstantFn) {
1646 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1647 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1648 llvm::Type *ArgTypes[] = {IntType, RetTy};
1649 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1650 SpecConstantFn = llvm::Function::Create(
1651 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1652 }
1653 return SpecConstantFn;
1654}
1655