1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "llvm/IR/MatrixBuilder.h"
17
18using namespace clang;
19using namespace CodeGen;
20using namespace llvm;
21
22static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
23 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
24 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
25 "asdouble operands types mismatch");
26 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
27 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
28
29 llvm::Type *ResultType = CGF.DoubleTy;
30 int N = 1;
31 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
32 N = VTy->getNumElements();
33 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
34 }
35
36 if (CGF.CGM.getTarget().getTriple().isDXIL())
37 return CGF.Builder.CreateIntrinsic(
38 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
39 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
40
41 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
42 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
43 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
44 }
45
46 llvm::SmallVector<int> Mask;
47 for (int i = 0; i < N; i++) {
48 Mask.push_back(Elt: i);
49 Mask.push_back(Elt: i + N);
50 }
51
52 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
53
54 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
55}
56
57static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
58 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
59
60 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
61 Value *CMP;
62 Value *LastInstr;
63
64 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
65 FZeroConst = ConstantVector::getSplat(
66 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
67 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
68 CMP = CGF->Builder.CreateIntrinsic(
69 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
70 Args: {FCompInst});
71 } else {
72 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
73 }
74
75 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
76 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
77 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
78 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
79 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
80
81 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
82
83 CGF->Builder.SetInsertPoint(LT0);
84
85 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
86
87 LastInstr = CGF->Builder.CreateBr(Dest: End);
88 CGF->Builder.SetInsertPoint(End);
89 } else {
90 llvm_unreachable("Backend Codegen not supported.");
91 }
92
93 return LastInstr;
94}
95
96static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
97 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
98 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
99 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
100
101 CallArgList Args;
102 LValue Op1TmpLValue =
103 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
104 LValue Op2TmpLValue =
105 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
106
107 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
108 Args.reverseWritebacks();
109
110 Value *LowBits = nullptr;
111 Value *HighBits = nullptr;
112
113 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
114 llvm::Type *RetElementTy = CGF->Int32Ty;
115 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
116 RetElementTy = llvm::VectorType::get(
117 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
118 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
119
120 CallInst *CI = CGF->Builder.CreateIntrinsic(
121 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
122
123 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
124 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
125 } else {
126 // For Non DXIL targets we generate the instructions.
127
128 if (!Op0->getType()->isVectorTy()) {
129 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
130 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
131
132 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
133 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
134 } else {
135 int NumElements = 1;
136 if (const auto *VecTy =
137 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
138 NumElements = VecTy->getNumElements();
139
140 FixedVectorType *Uint32VecTy =
141 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
142 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
143 if (NumElements == 1) {
144 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
145 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
146 } else {
147 SmallVector<int> EvenMask, OddMask;
148 for (int I = 0, E = NumElements; I != E; ++I) {
149 EvenMask.push_back(Elt: I * 2);
150 OddMask.push_back(Elt: I * 2 + 1);
151 }
152 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
153 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
154 }
155 }
156 }
157 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
158 auto *LastInst =
159 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
160 CGF->EmitWritebacks(Args);
161 return LastInst;
162}
163
164static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
165 const CallExpr *E) {
166 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
167 llvm::Type *I32 = CGF.Int32Ty;
168
169 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
170 [[maybe_unused]] llvm::StructType *Struct4I32 =
171 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
172
173 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
174 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
175 llvm::Function *Fn = CGF.CGM.getIntrinsic(IID: Intrinsic::dx_wave_ballot, Tys: {I32});
176
177 Value *StructVal = CGF.EmitRuntimeCall(callee: Fn, args: Cond);
178 assert(StructVal->getType() == Struct4I32 &&
179 "dx.wave.ballot must return {i32,i32,i32,i32}");
180
181 // Reassemble struct to <4 x i32>
182 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
183 for (unsigned I = 0; I < 4; ++I) {
184 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
185 VecVal =
186 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
187 }
188
189 return VecVal;
190 }
191
192 if (CGF.CGM.getTarget().getTriple().isSPIRV())
193 return CGF.EmitRuntimeCall(
194 callee: CGF.CGM.getIntrinsic(IID: Intrinsic::spv_subgroup_ballot), args: Cond);
195
196 llvm_unreachable(
197 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
198}
199
200static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
201 const CallExpr *E) {
202 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
203 QualType Op0Ty = E->getArg(Arg: 0)->getType();
204 llvm::Type *ResType = CGF.FloatTy;
205 uint64_t NumElements = 0;
206 if (Op0->getType()->isVectorTy()) {
207 NumElements =
208 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
209 ResType =
210 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
211 }
212 if (!Op0Ty->hasUnsignedIntegerRepresentation())
213 llvm_unreachable(
214 "f16tof32 operand must have an unsigned int representation");
215
216 if (CGF.CGM.getTriple().isDXIL())
217 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
218 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
219 Name: "hlsl.f16tof32");
220
221 if (CGF.CGM.getTriple().isSPIRV()) {
222 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
223 // Int16 and Float16 capabilities
224 auto *UnpackType =
225 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
226
227 if (NumElements == 0) {
228 // a scalar input - simply extract the first element of the unpacked
229 // vector
230 Value *Unpack = CGF.Builder.CreateIntrinsic(
231 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
232 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
233 }
234
235 // a vector input - build a congruent output vector by iterating through
236 // the input vector calling unpackhalf2x16 for each element
237 Value *Result = PoisonValue::get(T: ResType);
238 for (uint64_t I = 0; I < NumElements; I++) {
239 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
240 Value *Unpack = CGF.Builder.CreateIntrinsic(
241 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
242 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
243 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
244 }
245 return Result;
246 }
247
248 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
249}
250
251static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
252 const CallExpr *E) {
253 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
254 QualType Op0Ty = E->getArg(Arg: 0)->getType();
255 llvm::Type *ResType = CGF.IntTy;
256 uint64_t NumElements = 0;
257 if (Op0->getType()->isVectorTy()) {
258 NumElements =
259 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
260 ResType =
261 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
262 }
263 if (!Op0Ty->hasFloatingRepresentation())
264 llvm_unreachable("f32tof16 operand must have a float representation");
265
266 if (CGF.CGM.getTriple().isDXIL())
267 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
268 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
269 Name: "hlsl.f32tof16");
270
271 if (CGF.CGM.getTriple().isSPIRV()) {
272 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
273 // Int16 and Float16 capabilities
274 auto *PackType =
275 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
276
277 if (NumElements == 0) {
278 // a scalar input - simply insert the scalar in the first element
279 // of the 2 element float vector
280 Value *Float2 = Constant::getNullValue(Ty: PackType);
281 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
282 Value *Result = CGF.Builder.CreateIntrinsic(
283 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
284 return Result;
285 }
286
287 // a vector input - build a congruent output vector by iterating through
288 // the input vector calling packhalf2x16 for each element
289 Value *Result = PoisonValue::get(T: ResType);
290 for (uint64_t I = 0; I < NumElements; I++) {
291 Value *Float2 = Constant::getNullValue(Ty: PackType);
292 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
293 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
294 Value *Res = CGF.Builder.CreateIntrinsic(
295 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
296 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
297 }
298 return Result;
299 }
300
301 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
302}
303
304static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
305 LValue &Stride) {
306 // Figure out the stride of the buffer elements from the handle type.
307 auto *HandleTy =
308 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
309 QualType ElementTy = HandleTy->getContainedType();
310 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
311 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
312}
313
314// Return dot product intrinsic that corresponds to the QT scalar type
315static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
316 if (QT->isFloatingType())
317 return RT.getFDotIntrinsic();
318 if (QT->isSignedIntegerType())
319 return RT.getSDotIntrinsic();
320 assert(QT->isUnsignedIntegerType());
321 return RT.getUDotIntrinsic();
322}
323
324static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
325 if (QT->hasSignedIntegerRepresentation()) {
326 return RT.getFirstBitSHighIntrinsic();
327 }
328
329 assert(QT->hasUnsignedIntegerRepresentation());
330 return RT.getFirstBitUHighIntrinsic();
331}
332
333// Return wave active sum that corresponds to the QT scalar type
334static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
335 QualType QT) {
336 switch (Arch) {
337 case llvm::Triple::spirv:
338 return Intrinsic::spv_wave_reduce_sum;
339 case llvm::Triple::dxil: {
340 if (QT->isUnsignedIntegerType())
341 return Intrinsic::dx_wave_reduce_usum;
342 return Intrinsic::dx_wave_reduce_sum;
343 }
344 default:
345 llvm_unreachable("Intrinsic WaveActiveSum"
346 " not supported by target architecture");
347 }
348}
349
350// Return wave active product that corresponds to the QT scalar type
351static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
352 QualType QT) {
353 switch (Arch) {
354 case llvm::Triple::spirv:
355 return Intrinsic::spv_wave_product;
356 case llvm::Triple::dxil: {
357 if (QT->isUnsignedIntegerType())
358 return Intrinsic::dx_wave_uproduct;
359 return Intrinsic::dx_wave_product;
360 }
361 default:
362 llvm_unreachable("Intrinsic WaveActiveProduct"
363 " not supported by target architecture");
364 }
365}
366
367static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
368 switch (Arch) {
369 case llvm::Triple::spirv:
370 return Intrinsic::spv_subgroup_prefix_bit_count;
371 case llvm::Triple::dxil: {
372 return Intrinsic::dx_wave_prefix_bit_count;
373 }
374 default:
375 llvm_unreachable(
376 "WavePrefixOp instruction not supported by target architecture");
377 }
378}
379
380// Return wave prefix sum that corresponds to the QT scalar type
381static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
382 QualType QT) {
383 switch (Arch) {
384 case llvm::Triple::spirv:
385 return Intrinsic::spv_wave_prefix_sum;
386 case llvm::Triple::dxil: {
387 if (QT->isUnsignedIntegerType())
388 return Intrinsic::dx_wave_prefix_usum;
389 return Intrinsic::dx_wave_prefix_sum;
390 }
391 default:
392 llvm_unreachable("Intrinsic WavePrefixSum"
393 " not supported by target architecture");
394 }
395}
396
397// Return wave prefix product that corresponds to the QT scalar type
398static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
399 QualType QT) {
400 switch (Arch) {
401 case llvm::Triple::spirv:
402 return Intrinsic::spv_wave_prefix_product;
403 case llvm::Triple::dxil: {
404 if (QT->isUnsignedIntegerType())
405 return Intrinsic::dx_wave_prefix_uproduct;
406 return Intrinsic::dx_wave_prefix_product;
407 }
408 default:
409 llvm_unreachable("Intrinsic WavePrefixProduct"
410 " not supported by target architecture");
411 }
412}
413
414// Returns the mangled name for a builtin function that the SPIR-V backend
415// will expand into a spec Constant.
416static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
417 ASTContext &Context) {
418 // The parameter types for our conceptual intrinsic function.
419 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
420
421 // Create a temporary FunctionDecl for the builtin fuction. It won't be
422 // added to the AST.
423 FunctionProtoType::ExtProtoInfo EPI;
424 QualType FnType =
425 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
426 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
427 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
428 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
429 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
430
431 // Attach the created parameter declarations to the function declaration.
432 SmallVector<ParmVarDecl *, 2> ParamDecls;
433 for (QualType ParamType : ClangParamTypes) {
434 ParmVarDecl *PD = ParmVarDecl::Create(
435 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
436 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
437 /*DefaultArg*/ DefArg: nullptr);
438 ParamDecls.push_back(Elt: PD);
439 }
440 FnDeclForMangling->setParams(ParamDecls);
441
442 // Get the mangled name.
443 std::string Name;
444 llvm::raw_string_ostream MangledNameStream(Name);
445 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
446 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
447 MangledNameStream.flush();
448
449 return Name;
450}
451
452static llvm::Type *getOffsetType(CodeGenModule &CGM, llvm::Type *CoordTy) {
453 llvm::Type *Int32Ty = CGM.Int32Ty;
454 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: CoordTy))
455 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: VT->getNumElements());
456 return Int32Ty;
457}
458
459static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
460 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
461 if (E->getNumArgs() > OffsetArgIndex)
462 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
463
464 return llvm::Constant::getNullValue(Ty: OffsetTy);
465}
466
467static Value *emitHlslClamp(CodeGenFunction &CGF, const CallExpr *E,
468 unsigned ClampArgIndex) {
469 Value *Clamp = CGF.EmitScalarExpr(E: E->getArg(Arg: ClampArgIndex));
470 // The builtin is defined with variadic arguments, so the clamp parameter
471 // might have been promoted to double. The intrinsic requires a 32-bit
472 // float.
473 if (Clamp->getType() != CGF.Builder.getFloatTy())
474 Clamp = CGF.Builder.CreateFPCast(V: Clamp, DestTy: CGF.Builder.getFloatTy());
475 return Clamp;
476}
477
478Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
479 const CallExpr *E,
480 ReturnValueSlot ReturnValue) {
481 if (!getLangOpts().HLSL)
482 return nullptr;
483
484 switch (BuiltinID) {
485 case Builtin::BI__builtin_hlsl_adduint64: {
486 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
487 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
488 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
489 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
490 assert(Arg0Ty == E->getArg(1)->getType() &&
491 "AddUint64 operand types must match");
492 assert(Arg0Ty->hasIntegerRepresentation() &&
493 "AddUint64 operands must have an integer representation");
494 assert((NumElements == 2 || NumElements == 4) &&
495 "AddUint64 operands must have 2 or 4 elements");
496
497 llvm::Value *LowA;
498 llvm::Value *HighA;
499 llvm::Value *LowB;
500 llvm::Value *HighB;
501
502 // Obtain low and high words of inputs A and B
503 if (NumElements == 2) {
504 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
505 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
506 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
507 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
508 } else {
509 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
510 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
511 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
512 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
513 }
514
515 // Use an uadd_with_overflow to compute the sum of low words and obtain a
516 // carry value
517 llvm::Value *Carry;
518 llvm::Value *LowSum = EmitOverflowIntrinsic(
519 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
520 llvm::Value *ZExtCarry =
521 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
522
523 // Sum the high words and the carry
524 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
525 llvm::Value *HighSumPlusCarry =
526 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
527
528 if (NumElements == 4) {
529 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
530 Name: "hlsl.AddUint64");
531 }
532
533 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
534 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
535 Name: "hlsl.AddUint64.upto0");
536 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
537 Name: "hlsl.AddUint64");
538 return Result;
539 }
540 case Builtin::BI__builtin_hlsl_resource_getpointer:
541 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
542 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
543 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
544
545 llvm::Type *RetTy = ConvertType(T: E->getType());
546 return Builder.CreateIntrinsic(
547 RetTy, ID: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
548 Args: ArrayRef<Value *>{HandleOp, IndexOp});
549 }
550 case Builtin::BI__builtin_hlsl_resource_sample: {
551 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
552 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
553 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
554
555 SmallVector<Value *, 4> Args;
556 Args.push_back(Elt: HandleOp);
557 Args.push_back(Elt: SamplerOp);
558 Args.push_back(Elt: CoordOp);
559 Args.push_back(
560 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
561
562 llvm::Type *RetTy = ConvertType(T: E->getType());
563 if (E->getNumArgs() <= 4) {
564 return Builder.CreateIntrinsic(
565 RetTy, ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args);
566 }
567
568 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 4));
569 return Builder.CreateIntrinsic(
570 RetTy, ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(), Args);
571 }
572 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
573 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
574 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
575 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
576 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
577 if (BiasOp->getType() != Builder.getFloatTy())
578 BiasOp = Builder.CreateFPCast(V: BiasOp, DestTy: Builder.getFloatTy());
579
580 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
581 Args.push_back(Elt: HandleOp);
582 Args.push_back(Elt: SamplerOp);
583 Args.push_back(Elt: CoordOp);
584 Args.push_back(Elt: BiasOp);
585 Args.push_back(
586 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
587
588 llvm::Type *RetTy = ConvertType(T: E->getType());
589 if (E->getNumArgs() <= 5)
590 return Builder.CreateIntrinsic(
591 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(), Args);
592
593 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
594 return Builder.CreateIntrinsic(
595 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(), Args);
596 }
597 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
598 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
599 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
600 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
601 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
602 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
603
604 SmallVector<Value *, 7> Args;
605 Args.push_back(Elt: HandleOp);
606 Args.push_back(Elt: SamplerOp);
607 Args.push_back(Elt: CoordOp);
608 Args.push_back(Elt: DDXOp);
609 Args.push_back(Elt: DDYOp);
610 Args.push_back(
611 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
612
613 llvm::Type *RetTy = ConvertType(T: E->getType());
614
615 if (E->getNumArgs() <= 6) {
616 return Builder.CreateIntrinsic(
617 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
618 }
619
620 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 6));
621 return Builder.CreateIntrinsic(
622 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
623 }
624 case Builtin::BI__builtin_hlsl_resource_sample_level: {
625 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
626 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
627 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
628 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
629 if (LODOp->getType() != Builder.getFloatTy())
630 LODOp = Builder.CreateFPCast(V: LODOp, DestTy: Builder.getFloatTy());
631
632 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
633 Args.push_back(Elt: HandleOp);
634 Args.push_back(Elt: SamplerOp);
635 Args.push_back(Elt: CoordOp);
636 Args.push_back(Elt: LODOp);
637 Args.push_back(
638 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
639
640 llvm::Type *RetTy = ConvertType(T: E->getType());
641 return Builder.CreateIntrinsic(
642 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
643 }
644 case Builtin::BI__builtin_hlsl_resource_load_level: {
645 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
646 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
647
648 auto *CoordLODVecTy = cast<llvm::FixedVectorType>(Val: CoordLODOp->getType());
649 unsigned NumElts = CoordLODVecTy->getNumElements();
650 assert(NumElts >= 2 && "CoordLOD must have at least 2 elements");
651
652 // Split CoordLOD into Coord and LOD
653 SmallVector<int, 4> Mask;
654 for (unsigned I = 0; I < NumElts - 1; ++I)
655 Mask.push_back(Elt: I);
656
657 Value *CoordOp =
658 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
659 Value *LODOp =
660 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: NumElts - 1, Name: "hlsl.load.lod");
661
662 SmallVector<Value *, 4> Args;
663 Args.push_back(Elt: HandleOp);
664 Args.push_back(Elt: CoordOp);
665 Args.push_back(Elt: LODOp);
666 Args.push_back(
667 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
668
669 llvm::Type *RetTy = ConvertType(T: E->getType());
670 return Builder.CreateIntrinsic(
671 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
672 }
673 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
674 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
675 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
676 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
677 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
678 if (CmpOp->getType() != Builder.getFloatTy())
679 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
680
681 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
682 Args.push_back(Elt: HandleOp);
683 Args.push_back(Elt: SamplerOp);
684 Args.push_back(Elt: CoordOp);
685 Args.push_back(Elt: CmpOp);
686 Args.push_back(
687 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
688
689 llvm::Type *RetTy = ConvertType(T: E->getType());
690 if (E->getNumArgs() <= 5) {
691 return Builder.CreateIntrinsic(
692 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
693 }
694
695 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
696 return Builder.CreateIntrinsic(
697 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
698 }
699 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
700 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
701 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
702 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
703 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
704 if (CmpOp->getType() != Builder.getFloatTy())
705 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
706
707 SmallVector<Value *, 5> Args;
708 Args.push_back(Elt: HandleOp);
709 Args.push_back(Elt: SamplerOp);
710 Args.push_back(Elt: CoordOp);
711 Args.push_back(Elt: CmpOp);
712
713 Args.push_back(
714 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
715
716 llvm::Type *RetTy = ConvertType(T: E->getType());
717 return Builder.CreateIntrinsic(
718 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
719 }
720 case Builtin::BI__builtin_hlsl_resource_gather: {
721 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
722 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
723 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
724 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
725 if (ComponentOp->getType() != Builder.getInt32Ty())
726 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
727 /*isSigned=*/false);
728
729 SmallVector<Value *, 5> Args;
730 Args.push_back(Elt: HandleOp);
731 Args.push_back(Elt: SamplerOp);
732 Args.push_back(Elt: CoordOp);
733 Args.push_back(Elt: ComponentOp);
734 Args.push_back(
735 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
736
737 llvm::Type *RetTy = ConvertType(T: E->getType());
738 return Builder.CreateIntrinsic(
739 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
740 }
741 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
742 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
743 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
744 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
745 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
746 if (CompareOp->getType() != Builder.getFloatTy())
747 CompareOp = Builder.CreateFPCast(V: CompareOp, DestTy: Builder.getFloatTy());
748
749 SmallVector<Value *, 6> Args;
750 Args.push_back(Elt: HandleOp);
751 Args.push_back(Elt: SamplerOp);
752 Args.push_back(Elt: CoordOp);
753 Args.push_back(Elt: CompareOp);
754
755 if (CGM.getTarget().getTriple().isDXIL()) {
756 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
757 if (ComponentOp->getType() != Builder.getInt32Ty())
758 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
759 /*isSigned=*/false);
760 Args.push_back(Elt: ComponentOp);
761 }
762
763 Args.push_back(
764 Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, CoordTy: CoordOp->getType())));
765
766 llvm::Type *RetTy = ConvertType(T: E->getType());
767 return Builder.CreateIntrinsic(
768 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
769 }
770 case Builtin::BI__builtin_hlsl_resource_load_with_status:
771 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
772 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
773 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
774
775 // Get the *address* of the status argument to write to it by reference
776 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
777 Address StatusAddr = StatusLVal.getAddress();
778
779 QualType HandleTy = E->getArg(Arg: 0)->getType();
780 const HLSLAttributedResourceType *RT =
781 HandleTy->getAs<HLSLAttributedResourceType>();
782 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
783 "Only DXIL currently implements load with status");
784
785 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
786 ? llvm::Intrinsic::dx_resource_load_rawbuffer
787 : llvm::Intrinsic::dx_resource_load_typedbuffer;
788
789 llvm::Type *DataTy = ConvertType(T: E->getType());
790 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
791 Elements: {DataTy, Builder.getInt1Ty()});
792
793 SmallVector<Value *, 3> Args;
794 Args.push_back(Elt: HandleOp);
795 Args.push_back(Elt: IndexOp);
796
797 if (RT->isRaw()) {
798 Value *Offset = Builder.getInt32(C: 0);
799 // The offset parameter needs to be poison for ByteAddressBuffer
800 if (!RT->isStructured())
801 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
802 Args.push_back(Elt: Offset);
803 }
804
805 // The load intrinsics give us a (T value, i1 status) pair -
806 // shepherd these into the return value and out reference respectively.
807 Value *ResRet =
808 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
809 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
810 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
811 Value *ExtendedStatus =
812 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
813 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
814
815 return LoadedValue;
816 }
817 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
818 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
819 return llvm::PoisonValue::get(T: HandleTy);
820 }
821 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
822 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
823 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
824 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
825 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
826 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
827 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
828 llvm::Intrinsic::ID IntrinsicID =
829 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
830 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
831 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
832 }
833 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
834 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
835 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
836 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
837 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
838 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
839 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
840 llvm::Intrinsic::ID IntrinsicID =
841 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
842 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
843 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
844 }
845 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
846 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
847 if (!CGM.getTriple().isSPIRV())
848 return MainHandle;
849
850 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
851 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
852 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
853 llvm::Intrinsic::ID IntrinsicID =
854 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
855 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
856 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
857 }
858 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
859 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
860 llvm::Type *RetTy = ConvertType(T: E->getType());
861 return Builder.CreateIntrinsic(
862 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
863 Args: ArrayRef<Value *>{IndexOp});
864 }
865 case Builtin::BI__builtin_hlsl_resource_getdimensions_x: {
866 Value *Handle = EmitScalarExpr(E: E->getArg(Arg: 0));
867 LValue Dim = EmitLValue(E: E->getArg(Arg: 1));
868 llvm::Type *RetTy = llvm::Type::getInt32Ty(C&: getLLVMContext());
869 Value *DimValue = Builder.CreateIntrinsic(
870 RetTy, ID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
871 Args: ArrayRef<Value *>{Handle});
872 return Builder.CreateStore(Val: DimValue, Addr: Dim.getAddress());
873 }
874 case Builtin::BI__builtin_hlsl_resource_getstride: {
875 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
876 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
877 }
878 case Builtin::BI__builtin_hlsl_all: {
879 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
880 return Builder.CreateIntrinsic(
881 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
882 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
883 Name: "hlsl.all");
884 }
885 case Builtin::BI__builtin_hlsl_and: {
886 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
887 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
888 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
889 }
890 case Builtin::BI__builtin_hlsl_or: {
891 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
892 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
893 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
894 }
895 case Builtin::BI__builtin_hlsl_any: {
896 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
897 return Builder.CreateIntrinsic(
898 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
899 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
900 Name: "hlsl.any");
901 }
902 case Builtin::BI__builtin_hlsl_asdouble:
903 return handleAsDoubleBuiltin(CGF&: *this, E);
904 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
905 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
906 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
907 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
908
909 QualType Ty = E->getArg(Arg: 0)->getType();
910 if (auto *VecTy = Ty->getAs<VectorType>())
911 Ty = VecTy->getElementType();
912
913 Intrinsic::ID Intr;
914 if (Ty->isFloatingType()) {
915 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
916 } else if (Ty->isUnsignedIntegerType()) {
917 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
918 } else {
919 assert(Ty->isSignedIntegerType());
920 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
921 }
922 return Builder.CreateIntrinsic(
923 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
924 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
925 }
926 case Builtin::BI__builtin_hlsl_crossf16:
927 case Builtin::BI__builtin_hlsl_crossf32: {
928 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
929 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
930 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
931 E->getArg(1)->getType()->hasFloatingRepresentation() &&
932 "cross operands must have a float representation");
933 // make sure each vector has exactly 3 elements
934 assert(
935 E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
936 E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
937 "input vectors must have 3 elements each");
938 return Builder.CreateIntrinsic(
939 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getCrossIntrinsic(),
940 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.cross");
941 }
942 case Builtin::BI__builtin_hlsl_dot: {
943 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
944 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
945 llvm::Type *T0 = Op0->getType();
946 llvm::Type *T1 = Op1->getType();
947
948 // If the arguments are scalars, just emit a multiply
949 if (!T0->isVectorTy() && !T1->isVectorTy()) {
950 if (T0->isFloatingPointTy())
951 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
952
953 if (T0->isIntegerTy())
954 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
955
956 llvm_unreachable(
957 "Scalar dot product is only supported on ints and floats.");
958 }
959 // For vectors, validate types and emit the appropriate intrinsic
960 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
961 E->getArg(1)->getType()) &&
962 "Dot product operands must have the same type.");
963
964 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
965 assert(VecTy0 && "Dot product argument must be a vector.");
966
967 return Builder.CreateIntrinsic(
968 /*ReturnType=*/RetTy: T0->getScalarType(),
969 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
970 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
971 }
972 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
973 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
974 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
975 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
976
977 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
978 // Note that the argument order disagrees between the builtin and the
979 // intrinsic here.
980 return Builder.CreateIntrinsic(
981 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
982 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
983 }
984 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
985 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
986 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
987 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
988
989 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
990 // Note that the argument order disagrees between the builtin and the
991 // intrinsic here.
992 return Builder.CreateIntrinsic(
993 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
994 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
995 }
996 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
997 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
998
999 return Builder.CreateIntrinsic(
1000 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1001 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1002 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1003 }
1004 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1005 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1006
1007 return Builder.CreateIntrinsic(
1008 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1009 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1010 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1011 }
1012 case Builtin::BI__builtin_hlsl_lerp: {
1013 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1014 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1015 Value *S = EmitScalarExpr(E: E->getArg(Arg: 2));
1016 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1017 llvm_unreachable("lerp operand must have a float representation");
1018 return Builder.CreateIntrinsic(
1019 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getLerpIntrinsic(),
1020 Args: ArrayRef<Value *>{X, Y, S}, FMFSource: nullptr, Name: "hlsl.lerp");
1021 }
1022 case Builtin::BI__builtin_hlsl_normalize: {
1023 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1024
1025 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1026 "normalize operand must have a float representation");
1027
1028 return Builder.CreateIntrinsic(
1029 /*ReturnType=*/RetTy: X->getType(),
1030 ID: CGM.getHLSLRuntime().getNormalizeIntrinsic(), Args: ArrayRef<Value *>{X},
1031 FMFSource: nullptr, Name: "hlsl.normalize");
1032 }
1033 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
1034 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1035
1036 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1037 "degree operand must have a float representation");
1038
1039 return Builder.CreateIntrinsic(
1040 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getDegreesIntrinsic(),
1041 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.degrees");
1042 }
1043 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1044 return handleElementwiseF16ToF32(CGF&: *this, E);
1045 }
1046 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1047 return handleElementwiseF32ToF16(CGF&: *this, E);
1048 }
1049 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1050 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1051 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1052 llvm_unreachable("frac operand must have a float representation");
1053 return Builder.CreateIntrinsic(
1054 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1055 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1056 }
1057 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1058 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1059 llvm::Type *Xty = Op0->getType();
1060 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1061 if (Xty->isVectorTy()) {
1062 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1063 retType = llvm::VectorType::get(
1064 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1065 }
1066 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1067 llvm_unreachable("isinf operand must have a float representation");
1068 return Builder.CreateIntrinsic(
1069 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1070 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1071 }
1072 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1073 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1074 llvm::Type *Xty = Op0->getType();
1075 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1076 if (Xty->isVectorTy()) {
1077 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1078 retType = llvm::VectorType::get(
1079 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1080 }
1081 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1082 llvm_unreachable("isnan operand must have a float representation");
1083 return Builder.CreateIntrinsic(
1084 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1085 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1086 }
1087 case Builtin::BI__builtin_hlsl_mad: {
1088 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1089 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1090 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1091 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1092 return Builder.CreateIntrinsic(
1093 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1094 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1095
1096 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1097 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1098 return Builder.CreateIntrinsic(
1099 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1100 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1101
1102 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1103 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1104 }
1105 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1106 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1107 return Builder.CreateIntrinsic(
1108 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1109 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1110
1111 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1112 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1113 }
1114 case Builtin::BI__builtin_hlsl_mul: {
1115 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1116 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1117 QualType QTy0 = E->getArg(Arg: 0)->getType();
1118 QualType QTy1 = E->getArg(Arg: 1)->getType();
1119
1120 bool IsVec0 = QTy0->isVectorType();
1121 bool IsVec1 = QTy1->isVectorType();
1122 bool IsMat0 = QTy0->isConstantMatrixType();
1123 bool IsMat1 = QTy1->isConstantMatrixType();
1124
1125 llvm::MatrixBuilder MB(Builder);
1126 if (IsVec0 && IsMat1) {
1127 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1128 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1129 unsigned M = MatTy->getNumColumns();
1130 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: M, Name: "hlsl.mul");
1131 }
1132 if (IsMat0 && IsVec1) {
1133 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1134 unsigned Rows = MatTy->getNumRows();
1135 unsigned Cols = MatTy->getNumColumns();
1136 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1137 }
1138 assert(IsMat0 && IsMat1);
1139 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1140 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1141 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: MatTy0->getNumRows(),
1142 LHSColumns: MatTy0->getNumColumns(),
1143 RHSColumns: MatTy1->getNumColumns(), Name: "hlsl.mul");
1144 }
1145 case Builtin::BI__builtin_hlsl_transpose: {
1146 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1147 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1148 llvm::MatrixBuilder MB(Builder);
1149 return MB.CreateMatrixTranspose(Matrix: Op0, Rows: MatTy->getNumRows(),
1150 Columns: MatTy->getNumColumns());
1151 }
1152 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1153 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1154 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1155 llvm_unreachable("rcp operand must have a float representation");
1156 llvm::Type *Ty = Op0->getType();
1157 llvm::Type *EltTy = Ty->getScalarType();
1158 Constant *One = Ty->isVectorTy()
1159 ? ConstantVector::getSplat(
1160 EC: ElementCount::getFixed(
1161 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1162 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1163 : ConstantFP::get(Ty: EltTy, V: 1.0);
1164 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1165 }
1166 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1167 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1168 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1169 llvm_unreachable("rsqrt operand must have a float representation");
1170 return Builder.CreateIntrinsic(
1171 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1172 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1173 }
1174 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1175 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1176 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1177 "saturate operand must have a float representation");
1178 return Builder.CreateIntrinsic(
1179 /*ReturnType=*/RetTy: Op0->getType(),
1180 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1181 FMFSource: nullptr, Name: "hlsl.saturate");
1182 }
1183 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1184 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1185 assert(Op->getType()->isIntegerTy(1) &&
1186 "WavePrefixBitCount operand must be a boolean type");
1187
1188 Intrinsic::ID IID =
1189 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1190
1191 return EmitRuntimeCall(
1192 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: IID), args: ArrayRef{Op},
1193 name: "hlsl.wave.prefix.bit.count");
1194 }
1195 case Builtin::BI__builtin_hlsl_select: {
1196 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1197 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1198 Value *OpTrue =
1199 RValTrue.isScalar()
1200 ? RValTrue.getScalarVal()
1201 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1202 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1203 Value *OpFalse =
1204 RValFalse.isScalar()
1205 ? RValFalse.getScalarVal()
1206 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1207 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1208 if (!OpTrue->getType()->isVectorTy())
1209 OpTrue =
1210 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1211 if (!OpFalse->getType()->isVectorTy())
1212 OpFalse =
1213 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1214 }
1215
1216 Value *SelectVal =
1217 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1218 if (!RValTrue.isScalar())
1219 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1220 IsVolatile: ReturnValue.isVolatile());
1221
1222 return SelectVal;
1223 }
1224 case Builtin::BI__builtin_hlsl_step: {
1225 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1226 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1227 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1228 E->getArg(1)->getType()->hasFloatingRepresentation() &&
1229 "step operands must have a float representation");
1230 return Builder.CreateIntrinsic(
1231 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getStepIntrinsic(),
1232 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.step");
1233 }
1234 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1235 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1236
1237 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1238 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1239 M: &CGM.getModule(), id: ID, Tys: {Op->getType()}),
1240 args: {Op});
1241 }
1242 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1243 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1244 assert(Op->getType()->isIntegerTy(1) &&
1245 "Intrinsic WaveActiveAllTrue operand must be a bool");
1246
1247 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1248 return EmitRuntimeCall(
1249 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1250 }
1251 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1252 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1253 assert(Op->getType()->isIntegerTy(1) &&
1254 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1255
1256 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1257 return EmitRuntimeCall(
1258 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID), args: {Op});
1259 }
1260 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1261 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1262 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1263 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1264 "representation");
1265
1266 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1267 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1268 M: &CGM.getModule(), id: ID, Tys: {Op->getType()}),
1269 args: ArrayRef{Op}, name: "hlsl.wave.active.bit.or");
1270 }
1271 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1272 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1273 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1274 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1275 "representation");
1276
1277 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1278 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1279 M: &CGM.getModule(), id: ID, Tys: {Op->getType()}),
1280 args: ArrayRef{Op}, name: "hlsl.wave.active.bit.xor");
1281 }
1282 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1283 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1284 assert(Op->getType()->isIntegerTy(1) &&
1285 "Intrinsic WaveActiveBallot operand must be a bool");
1286
1287 return handleHlslWaveActiveBallot(CGF&: *this, E);
1288 }
1289 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1290 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1291 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1292 return EmitRuntimeCall(
1293 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID),
1294 args: ArrayRef{OpExpr});
1295 }
1296 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1297 // Due to the use of variadic arguments, explicitly retrieve argument
1298 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1299 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1300 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1301
1302 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1303 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1304 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.sum");
1305 }
1306 case Builtin::BI__builtin_hlsl_wave_active_product: {
1307 // Due to the use of variadic arguments, explicitly retrieve argument
1308 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1309 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1310 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1311
1312 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1313 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1314 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.product");
1315 }
1316 case Builtin::BI__builtin_hlsl_wave_active_max: {
1317 // Due to the use of variadic arguments, explicitly retrieve argument
1318 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1319 QualType QT = E->getArg(Arg: 0)->getType();
1320 Intrinsic::ID IID;
1321 if (QT->isUnsignedIntegerType())
1322 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1323 else
1324 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1325
1326 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1327 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1328 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.max");
1329 }
1330 case Builtin::BI__builtin_hlsl_wave_active_min: {
1331 // Due to the use of variadic arguments, explicitly retrieve argument
1332 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1333 QualType QT = E->getArg(Arg: 0)->getType();
1334 Intrinsic::ID IID;
1335 if (QT->isUnsignedIntegerType())
1336 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1337 else
1338 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1339
1340 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1341 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1342 args: ArrayRef{OpExpr}, name: "hlsl.wave.active.min");
1343 }
1344 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1345 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1346 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1347 // for the DirectX intrinsic and the demangled builtin name
1348 switch (CGM.getTarget().getTriple().getArch()) {
1349 case llvm::Triple::dxil:
1350 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1351 M: &CGM.getModule(), id: Intrinsic::dx_wave_getlaneindex));
1352 case llvm::Triple::spirv:
1353 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1354 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1355 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1356 default:
1357 llvm_unreachable(
1358 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1359 }
1360 }
1361 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1362 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1363 return EmitRuntimeCall(
1364 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1365 }
1366 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1367 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1368 return EmitRuntimeCall(
1369 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1370 }
1371 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1372 // Due to the use of variadic arguments we must explicitly retrieve them and
1373 // create our function type.
1374 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1375 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1376 return EmitRuntimeCall(
1377 callee: Intrinsic::getOrInsertDeclaration(
1378 M: &CGM.getModule(), id: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1379 Tys: {OpExpr->getType()}),
1380 args: ArrayRef{OpExpr, OpIndex}, name: "hlsl.wave.readlane");
1381 }
1382 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1383 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1384 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1385 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1386 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1387 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1388 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.sum");
1389 }
1390 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1391 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1392 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1393 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1394 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1395 M: &CGM.getModule(), id: IID, Tys: {OpExpr->getType()}),
1396 args: ArrayRef{OpExpr}, name: "hlsl.wave.prefix.product");
1397 }
1398 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1399 auto *Arg0 = E->getArg(Arg: 0);
1400 Value *Op0 = EmitScalarExpr(E: Arg0);
1401 llvm::Type *Xty = Op0->getType();
1402 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1403 if (Xty->isVectorTy()) {
1404 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1405 retType = llvm::VectorType::get(
1406 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1407 }
1408 assert((Arg0->getType()->hasFloatingRepresentation() ||
1409 Arg0->getType()->hasIntegerRepresentation()) &&
1410 "sign operand must have a float or int representation");
1411
1412 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1413 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1414 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1415 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1416 }
1417
1418 return Builder.CreateIntrinsic(
1419 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1420 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1421 }
1422 case Builtin::BI__builtin_hlsl_elementwise_radians: {
1423 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1424 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1425 "radians operand must have a float representation");
1426 return Builder.CreateIntrinsic(
1427 /*ReturnType=*/RetTy: Op0->getType(),
1428 ID: CGM.getHLSLRuntime().getRadiansIntrinsic(), Args: ArrayRef<Value *>{Op0},
1429 FMFSource: nullptr, Name: "hlsl.radians");
1430 }
1431 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1432 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1433 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1434 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1435 return Builder.CreateIntrinsic(
1436 /*ReturnType=*/RetTy: Offset->getType(),
1437 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1438 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1439 }
1440 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1441
1442 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1443 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1444 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1445 "asuint operands types mismatch");
1446 return handleHlslSplitdouble(E, CGF: this);
1447 }
1448 case Builtin::BI__builtin_hlsl_elementwise_clip:
1449 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1450 "clip operands types mismatch");
1451 return handleHlslClip(E, CGF: this);
1452 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1453 Intrinsic::ID ID =
1454 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1455 return EmitRuntimeCall(
1456 callee: Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID));
1457 }
1458 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1459 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1460 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1461 llvm_unreachable("ddx_coarse operand must have a float representation");
1462 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1463 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1464 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1465 Name: "hlsl.ddx.coarse");
1466 }
1467 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1468 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1469 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1470 llvm_unreachable("ddy_coarse operand must have a float representation");
1471 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1472 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1473 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1474 Name: "hlsl.ddy.coarse");
1475 }
1476 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1477 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1478 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1479 llvm_unreachable("ddx_fine operand must have a float representation");
1480 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1481 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1482 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1483 Name: "hlsl.ddx.fine");
1484 }
1485 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1486 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1487 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1488 llvm_unreachable("ddy_fine operand must have a float representation");
1489 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1490 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1491 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1492 Name: "hlsl.ddy.fine");
1493 }
1494 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1495 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1496 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1497 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1498 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1499 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1500 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1501 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1502 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1503 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1504 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1505 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1506 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1507 llvm::Value *Args[] = {SpecId, DefaultVal};
1508 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1509 }
1510 }
1511 return nullptr;
1512}
1513
1514llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1515 const clang::QualType &SpecConstantType) {
1516
1517 // Find or create the declaration for the function.
1518 llvm::Module *M = &CGM.getModule();
1519 std::string MangledName =
1520 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1521 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1522
1523 if (!SpecConstantFn) {
1524 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1525 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1526 llvm::Type *ArgTypes[] = {IntType, RetTy};
1527 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1528 SpecConstantFn = llvm::Function::Create(
1529 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1530 }
1531 return SpecConstantFn;
1532}
1533