1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/IR/MatrixBuilder.h"
19
20using namespace clang;
21using namespace CodeGen;
22using namespace llvm;
23
24static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
25 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
26 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
27 "asdouble operands types mismatch");
28 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
29 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
30
31 llvm::Type *ResultType = CGF.DoubleTy;
32 int N = 1;
33 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
34 N = VTy->getNumElements();
35 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
36 }
37
38 if (CGF.CGM.getTarget().getTriple().isDXIL())
39 return CGF.Builder.CreateIntrinsic(
40 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
41 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
42
43 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
44 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
45 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
46 }
47
48 llvm::SmallVector<int> Mask;
49 for (int i = 0; i < N; i++) {
50 Mask.push_back(Elt: i);
51 Mask.push_back(Elt: i + N);
52 }
53
54 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
55
56 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
57}
58
59static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
60 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
61
62 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
63 Value *CMP;
64 Value *LastInstr;
65
66 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
67 FZeroConst = ConstantVector::getSplat(
68 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
69 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
70 CMP = CGF->Builder.CreateIntrinsic(
71 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
72 Args: {FCompInst});
73 } else {
74 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
75 }
76
77 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
78 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
79 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
80 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
81 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
82
83 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
84
85 CGF->Builder.SetInsertPoint(LT0);
86
87 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
88
89 LastInstr = CGF->Builder.CreateBr(Dest: End);
90 CGF->Builder.SetInsertPoint(End);
91 } else {
92 llvm_unreachable("Backend Codegen not supported.");
93 }
94
95 return LastInstr;
96}
97
98static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
99 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
100 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
101 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
102
103 CallArgList Args;
104 LValue Op1TmpLValue =
105 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
106 LValue Op2TmpLValue =
107 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
108
109 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
110 Args.reverseWritebacks();
111
112 Value *LowBits = nullptr;
113 Value *HighBits = nullptr;
114
115 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
116 llvm::Type *RetElementTy = CGF->Int32Ty;
117 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
118 RetElementTy = llvm::VectorType::get(
119 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
120 else if (auto *Op0MatTy =
121 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
122 RetElementTy = llvm::VectorType::get(
123 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
124 Op0MatTy->getNumColumns()));
125
126 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
127
128 Value *CI = CGF->Builder.CreateIntrinsic(
129 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
130
131 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
132 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
133 } else {
134 // For Non DXIL targets we generate the instructions.
135
136 if (!Op0->getType()->isVectorTy()) {
137 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
138 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
139
140 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
141 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
142 } else {
143 int NumElements = 1;
144 if (const auto *VecTy =
145 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
146 NumElements = VecTy->getNumElements();
147 else if (const auto *MatTy =
148 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
149 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
150
151 FixedVectorType *Uint32VecTy =
152 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
153 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
154 if (NumElements == 1) {
155 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
156 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
157 } else {
158 SmallVector<int> EvenMask, OddMask;
159 for (int I = 0, E = NumElements; I != E; ++I) {
160 EvenMask.push_back(Elt: I * 2);
161 OddMask.push_back(Elt: I * 2 + 1);
162 }
163 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
164 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
165 }
166 }
167 }
168 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
169 auto *LastInst =
170 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
171 CGF->EmitWritebacks(Args);
172 return LastInst;
173}
174
175static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
176 const CallExpr *E) {
177 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
178 llvm::Type *I32 = CGF.Int32Ty;
179
180 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
181 [[maybe_unused]] llvm::StructType *Struct4I32 =
182 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
183
184 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
185 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
186 Value *StructVal =
187 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
188 assert(StructVal->getType() == Struct4I32 &&
189 "dx.wave.ballot must return {i32,i32,i32,i32}");
190
191 // Reassemble struct to <4 x i32>
192 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
193 for (unsigned I = 0; I < 4; ++I) {
194 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
195 VecVal =
196 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
197 }
198
199 return VecVal;
200 }
201
202 if (CGF.CGM.getTarget().getTriple().isSPIRV())
203 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
204
205 llvm_unreachable(
206 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
207}
208
209static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
210 const CallExpr *E) {
211 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
212 QualType Op0Ty = E->getArg(Arg: 0)->getType();
213 llvm::Type *ResType = CGF.FloatTy;
214 uint64_t NumElements = 0;
215 if (Op0->getType()->isVectorTy()) {
216 NumElements =
217 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
218 ResType =
219 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
220 }
221 if (!Op0Ty->hasUnsignedIntegerRepresentation())
222 llvm_unreachable(
223 "f16tof32 operand must have an unsigned int representation");
224
225 if (CGF.CGM.getTriple().isDXIL())
226 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
227 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
228 Name: "hlsl.f16tof32");
229
230 if (CGF.CGM.getTriple().isSPIRV()) {
231 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
232 // Int16 and Float16 capabilities
233 auto *UnpackType =
234 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
235
236 if (NumElements == 0) {
237 // a scalar input - simply extract the first element of the unpacked
238 // vector
239 Value *Unpack = CGF.Builder.CreateIntrinsic(
240 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
241 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
242 }
243
244 // a vector input - build a congruent output vector by iterating through
245 // the input vector calling unpackhalf2x16 for each element
246 Value *Result = PoisonValue::get(T: ResType);
247 for (uint64_t I = 0; I < NumElements; I++) {
248 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
249 Value *Unpack = CGF.Builder.CreateIntrinsic(
250 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
251 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
252 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
253 }
254 return Result;
255 }
256
257 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
258}
259
260static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
261 const CallExpr *E) {
262 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
263 QualType Op0Ty = E->getArg(Arg: 0)->getType();
264 llvm::Type *ResType = CGF.IntTy;
265 uint64_t NumElements = 0;
266 if (Op0->getType()->isVectorTy()) {
267 NumElements =
268 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
269 ResType =
270 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
271 }
272 if (!Op0Ty->hasFloatingRepresentation())
273 llvm_unreachable("f32tof16 operand must have a float representation");
274
275 if (CGF.CGM.getTriple().isDXIL())
276 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
277 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
278 Name: "hlsl.f32tof16");
279
280 if (CGF.CGM.getTriple().isSPIRV()) {
281 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
282 // Int16 and Float16 capabilities
283 auto *PackType =
284 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
285
286 if (NumElements == 0) {
287 // a scalar input - simply insert the scalar in the first element
288 // of the 2 element float vector
289 Value *Float2 = Constant::getNullValue(Ty: PackType);
290 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
291 Value *Result = CGF.Builder.CreateIntrinsic(
292 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
293 return Result;
294 }
295
296 // a vector input - build a congruent output vector by iterating through
297 // the input vector calling packhalf2x16 for each element
298 Value *Result = PoisonValue::get(T: ResType);
299 for (uint64_t I = 0; I < NumElements; I++) {
300 Value *Float2 = Constant::getNullValue(Ty: PackType);
301 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
302 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
303 Value *Res = CGF.Builder.CreateIntrinsic(
304 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
305 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
306 }
307 return Result;
308 }
309
310 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
311}
312
313static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
314 Intrinsic::ID ID, const Twine &Name) {
315 // HLSL signatures (synthesized as overloads in HLSLExternalSemaSource):
316 // void InterlockedOp(groupshared|device T &dest, T value);
317 // void InterlockedOp(groupshared|device T &dest, T value,
318 // T &original_value);
319 // Both `dest` and `original_value` are plain references, so we can use
320 // the underlying lvalue directly without HLSLOutArgExpr unwrapping.
321 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
322 Value *Ptr = DestLV.getAddress().emitRawPointer(CGF);
323 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
324 assert(E->getArg(1)->getType()->isIntegerType() &&
325 "Intrinsic InterlockedOp value operand must be an integer");
326
327 Value *Call = CGF.EmitRuntimeCall(
328 callee: Intrinsic::getOrInsertDeclaration(M: &CGF.CGM.getModule(), id: ID,
329 OverloadTys: {Val->getType(), Ptr->getType()}),
330 args: ArrayRef<Value *>{Ptr, Val}, name: Name);
331
332 // The 3-arg overload writes the old value (the intrinsic's return value)
333 // into the `original_value` reference parameter.
334 if (E->getNumArgs() == 3) {
335 LValue OrigLV = CGF.EmitLValue(E: E->getArg(Arg: 2));
336 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
337 }
338 return Call;
339}
340
341static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
342 LValue &Stride) {
343 // Figure out the stride of the buffer elements from the handle type.
344 auto *HandleTy =
345 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
346 QualType ElementTy = HandleTy->getContainedType();
347 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
348 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
349}
350
351// Return dot product intrinsic that corresponds to the QT scalar type
352static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
353 if (QT->isFloatingType())
354 return RT.getFDotIntrinsic();
355 if (QT->isSignedIntegerType())
356 return RT.getSDotIntrinsic();
357 assert(QT->isUnsignedIntegerType());
358 return RT.getUDotIntrinsic();
359}
360
361static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
362 if (QT->hasSignedIntegerRepresentation()) {
363 return RT.getFirstBitSHighIntrinsic();
364 }
365
366 assert(QT->hasUnsignedIntegerRepresentation());
367 return RT.getFirstBitUHighIntrinsic();
368}
369
370// Return wave active sum that corresponds to the QT scalar type
371static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
372 QualType QT) {
373 switch (Arch) {
374 case llvm::Triple::spirv:
375 return Intrinsic::spv_wave_reduce_sum;
376 case llvm::Triple::dxil: {
377 if (QT->isUnsignedIntegerType())
378 return Intrinsic::dx_wave_reduce_usum;
379 return Intrinsic::dx_wave_reduce_sum;
380 }
381 default:
382 llvm_unreachable("Intrinsic WaveActiveSum"
383 " not supported by target architecture");
384 }
385}
386
387// Return wave active product that corresponds to the QT scalar type
388static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
389 QualType QT) {
390 switch (Arch) {
391 case llvm::Triple::spirv:
392 return Intrinsic::spv_wave_product;
393 case llvm::Triple::dxil: {
394 if (QT->isUnsignedIntegerType())
395 return Intrinsic::dx_wave_uproduct;
396 return Intrinsic::dx_wave_product;
397 }
398 default:
399 llvm_unreachable("Intrinsic WaveActiveProduct"
400 " not supported by target architecture");
401 }
402}
403
404static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
405 switch (Arch) {
406 case llvm::Triple::spirv:
407 return Intrinsic::spv_subgroup_prefix_bit_count;
408 case llvm::Triple::dxil: {
409 return Intrinsic::dx_wave_prefix_bit_count;
410 }
411 default:
412 llvm_unreachable(
413 "WavePrefixOp instruction not supported by target architecture");
414 }
415}
416
417// Return wave prefix sum that corresponds to the QT scalar type
418static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
419 QualType QT) {
420 switch (Arch) {
421 case llvm::Triple::spirv:
422 return Intrinsic::spv_wave_prefix_sum;
423 case llvm::Triple::dxil: {
424 if (QT->isUnsignedIntegerType())
425 return Intrinsic::dx_wave_prefix_usum;
426 return Intrinsic::dx_wave_prefix_sum;
427 }
428 default:
429 llvm_unreachable("Intrinsic WavePrefixSum"
430 " not supported by target architecture");
431 }
432}
433
434// Return wave prefix product that corresponds to the QT scalar type
435static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
436 QualType QT) {
437 switch (Arch) {
438 case llvm::Triple::spirv:
439 return Intrinsic::spv_wave_prefix_product;
440 case llvm::Triple::dxil: {
441 if (QT->isUnsignedIntegerType())
442 return Intrinsic::dx_wave_prefix_uproduct;
443 return Intrinsic::dx_wave_prefix_product;
444 }
445 default:
446 llvm_unreachable("Intrinsic WavePrefixProduct"
447 " not supported by target architecture");
448 }
449}
450
451// Returns the mangled name for a builtin function that the SPIR-V backend
452// will expand into a spec Constant.
453static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
454 ASTContext &Context) {
455 // The parameter types for our conceptual intrinsic function.
456 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
457
458 // Create a temporary FunctionDecl for the builtin fuction. It won't be
459 // added to the AST.
460 FunctionProtoType::ExtProtoInfo EPI;
461 QualType FnType =
462 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
463 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
464 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
465 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
466 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
467
468 // Attach the created parameter declarations to the function declaration.
469 SmallVector<ParmVarDecl *, 2> ParamDecls;
470 for (QualType ParamType : ClangParamTypes) {
471 ParmVarDecl *PD = ParmVarDecl::Create(
472 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
473 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
474 /*DefaultArg*/ DefArg: nullptr);
475 ParamDecls.push_back(Elt: PD);
476 }
477 FnDeclForMangling->setParams(ParamDecls);
478
479 // Get the mangled name.
480 std::string Name;
481 llvm::raw_string_ostream MangledNameStream(Name);
482 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
483 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
484 MangledNameStream.flush();
485
486 return Name;
487}
488
489static const HLSLAttributedResourceType *
490getHandleAttributedType(QualType HandleQT) {
491 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
492 return RT;
493 // If the expr is a texture/sampler record (or similar), peel to __handle.
494 if (const HLSLAttributedResourceType *RT =
495 HLSLAttributedResourceType::findHandleTypeOnResource(
496 RT: HandleQT.getTypePtr()))
497 return RT;
498 llvm_unreachable("attributed handle type not found");
499}
500
501static const HLSLAttributedResourceType *
502getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
503 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
504}
505
506static llvm::Type *getOffsetType(CodeGenModule &CGM,
507 const HLSLAttributedResourceType *RT) {
508 const auto &Attrs = RT->getAttrs();
509 unsigned OffsetSize =
510 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
511 llvm::Type *Int32Ty = CGM.Int32Ty;
512 if (OffsetSize == 1)
513 return Int32Ty;
514 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
515}
516
517static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
518 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
519 if (E->getNumArgs() > OffsetArgIndex)
520 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
521
522 return llvm::Constant::getNullValue(Ty: OffsetTy);
523}
524
525static Value *emitHlslClamp(CodeGenFunction &CGF, const CallExpr *E,
526 unsigned ClampArgIndex) {
527 Value *Clamp = CGF.EmitScalarExpr(E: E->getArg(Arg: ClampArgIndex));
528 // The builtin is defined with variadic arguments, so the clamp parameter
529 // might have been promoted to double. The intrinsic requires a 32-bit
530 // float.
531 if (Clamp->getType() != CGF.Builder.getFloatTy())
532 Clamp = CGF.Builder.CreateFPCast(V: Clamp, DestTy: CGF.Builder.getFloatTy());
533 return Clamp;
534}
535
536static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
537 unsigned IntrinsicID, unsigned NumRetComps,
538 bool HasLod) {
539 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
540
541 SmallVector<Value *> Args{Handle};
542 if (HasLod)
543 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
544
545 Value *DimValue =
546 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
547
548 Value *LastStore = nullptr;
549 unsigned ArgIndex = HasLod ? 2 : 1;
550 for (unsigned i = 0; i < NumRetComps; ++i) {
551 const Expr *Arg = E->getArg(Arg: ArgIndex++);
552 LValue DimOut = CGF.EmitLValue(E: Arg);
553 Value *Elem = DimValue;
554 if (NumRetComps > 1)
555 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
556
557 // Handle float casting if needed
558 if (Arg->getType()->isFloatingType())
559 Elem = CGF.Builder.CreateUIToFP(
560 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
561
562 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
563 }
564 return LastStore;
565}
566
567Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
568 const CallExpr *E,
569 ReturnValueSlot ReturnValue) {
570 if (!getLangOpts().HLSL)
571 return nullptr;
572
573 switch (BuiltinID) {
574 case Builtin::BI__builtin_hlsl_adduint64: {
575 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
576 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
577 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
578 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
579 assert(Arg0Ty == E->getArg(1)->getType() &&
580 "AddUint64 operand types must match");
581 assert(Arg0Ty->hasIntegerRepresentation() &&
582 "AddUint64 operands must have an integer representation");
583 assert((NumElements == 2 || NumElements == 4) &&
584 "AddUint64 operands must have 2 or 4 elements");
585
586 llvm::Value *LowA;
587 llvm::Value *HighA;
588 llvm::Value *LowB;
589 llvm::Value *HighB;
590
591 // Obtain low and high words of inputs A and B
592 if (NumElements == 2) {
593 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
594 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
595 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
596 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
597 } else {
598 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
599 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
600 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
601 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
602 }
603
604 // Use an uadd_with_overflow to compute the sum of low words and obtain a
605 // carry value
606 llvm::Value *Carry;
607 llvm::Value *LowSum = EmitOverflowIntrinsic(
608 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
609 llvm::Value *ZExtCarry =
610 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
611
612 // Sum the high words and the carry
613 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
614 llvm::Value *HighSumPlusCarry =
615 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
616
617 if (NumElements == 4) {
618 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
619 Name: "hlsl.AddUint64");
620 }
621
622 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
623 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
624 Name: "hlsl.AddUint64.upto0");
625 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
626 Name: "hlsl.AddUint64");
627 return Result;
628 }
629 case Builtin::BI__builtin_hlsl_resource_getpointer:
630 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
631 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
632 bool IsIndexed =
633 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
634 E->getNumArgs() > 1;
635
636 llvm::Type *RetTy = ConvertType(T: E->getType());
637 llvm::Function *IntrFn = nullptr;
638 llvm::CallInst *CI = nullptr;
639 if (IsIndexed) {
640 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
641 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
642 M: &CGM.getModule(),
643 id: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
644 OverloadTys: {RetTy, HandleOp->getType(), IndexOp->getType()});
645 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp, IndexOp});
646 } else {
647 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
648 M: &CGM.getModule(),
649 id: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
650 OverloadTys: {RetTy, HandleOp->getType()});
651 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp});
652 }
653 CI->setCallingConv(IntrFn->getCallingConv());
654 return CI;
655 }
656 case Builtin::BI__builtin_hlsl_resource_sample: {
657 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
658 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
659 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
660 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
661
662 SmallVector<Value *, 4> Args;
663 Args.push_back(Elt: HandleOp);
664 Args.push_back(Elt: SamplerOp);
665 Args.push_back(Elt: CoordOp);
666 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
667
668 llvm::Type *RetTy = ConvertType(T: E->getType());
669 if (E->getNumArgs() <= 4) {
670 return Builder.CreateIntrinsic(
671 RetTy, ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args);
672 }
673
674 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 4));
675 return Builder.CreateIntrinsic(
676 RetTy, ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(), Args);
677 }
678 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
679 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
680 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
681 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
682 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
683 if (BiasOp->getType() != Builder.getFloatTy())
684 BiasOp = Builder.CreateFPCast(V: BiasOp, DestTy: Builder.getFloatTy());
685 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
686
687 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
688 Args.push_back(Elt: HandleOp);
689 Args.push_back(Elt: SamplerOp);
690 Args.push_back(Elt: CoordOp);
691 Args.push_back(Elt: BiasOp);
692 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
693
694 llvm::Type *RetTy = ConvertType(T: E->getType());
695 if (E->getNumArgs() <= 5)
696 return Builder.CreateIntrinsic(
697 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(), Args);
698
699 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
700 return Builder.CreateIntrinsic(
701 RetTy, ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(), Args);
702 }
703 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
704 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
705 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
706 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
707 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
708 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
709 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
710
711 SmallVector<Value *, 7> Args;
712 Args.push_back(Elt: HandleOp);
713 Args.push_back(Elt: SamplerOp);
714 Args.push_back(Elt: CoordOp);
715 Args.push_back(Elt: DDXOp);
716 Args.push_back(Elt: DDYOp);
717 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
718
719 llvm::Type *RetTy = ConvertType(T: E->getType());
720
721 if (E->getNumArgs() <= 6) {
722 return Builder.CreateIntrinsic(
723 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
724 }
725
726 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 6));
727 return Builder.CreateIntrinsic(
728 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
729 }
730 case Builtin::BI__builtin_hlsl_resource_sample_level: {
731 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
732 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
733 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
734 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
735 if (LODOp->getType() != Builder.getFloatTy())
736 LODOp = Builder.CreateFPCast(V: LODOp, DestTy: Builder.getFloatTy());
737 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
738
739 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
740 Args.push_back(Elt: HandleOp);
741 Args.push_back(Elt: SamplerOp);
742 Args.push_back(Elt: CoordOp);
743 Args.push_back(Elt: LODOp);
744 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
745
746 llvm::Type *RetTy = ConvertType(T: E->getType());
747 return Builder.CreateIntrinsic(
748 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
749 }
750 case Builtin::BI__builtin_hlsl_resource_load_level: {
751 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
752 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
753
754 auto *CoordLODVecTy = cast<llvm::FixedVectorType>(Val: CoordLODOp->getType());
755 unsigned NumElts = CoordLODVecTy->getNumElements();
756 assert(NumElts >= 2 && "CoordLOD must have at least 2 elements");
757
758 // Split CoordLOD into Coord and LOD
759 SmallVector<int, 4> Mask;
760 for (unsigned I = 0; I < NumElts - 1; ++I)
761 Mask.push_back(Elt: I);
762
763 Value *CoordOp =
764 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
765 Value *LODOp =
766 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: NumElts - 1, Name: "hlsl.load.lod");
767 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
768
769 SmallVector<Value *, 4> Args;
770 Args.push_back(Elt: HandleOp);
771 Args.push_back(Elt: CoordOp);
772 Args.push_back(Elt: LODOp);
773 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
774
775 llvm::Type *RetTy = ConvertType(T: E->getType());
776 return Builder.CreateIntrinsic(
777 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
778 }
779 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
780 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
781 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
782 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
783 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
784 if (CmpOp->getType() != Builder.getFloatTy())
785 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
786 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
787
788 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
789 Args.push_back(Elt: HandleOp);
790 Args.push_back(Elt: SamplerOp);
791 Args.push_back(Elt: CoordOp);
792 Args.push_back(Elt: CmpOp);
793 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
794
795 llvm::Type *RetTy = ConvertType(T: E->getType());
796 if (E->getNumArgs() <= 5) {
797 return Builder.CreateIntrinsic(
798 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
799 }
800
801 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
802 return Builder.CreateIntrinsic(
803 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
804 }
805 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
806 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
807 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
808 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
809 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
810 if (CmpOp->getType() != Builder.getFloatTy())
811 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
812 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
813
814 SmallVector<Value *, 5> Args;
815 Args.push_back(Elt: HandleOp);
816 Args.push_back(Elt: SamplerOp);
817 Args.push_back(Elt: CoordOp);
818 Args.push_back(Elt: CmpOp);
819 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
820
821 llvm::Type *RetTy = ConvertType(T: E->getType());
822 return Builder.CreateIntrinsic(
823 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
824 }
825 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
826 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
827 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
828 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
829
830 return Builder.CreateIntrinsic(
831 RetTy: ConvertType(T: E->getType()),
832 ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
833 Args: {HandleOp, SamplerOp, CoordOp});
834 }
835 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
836 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
837 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
838 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
839
840 return Builder.CreateIntrinsic(
841 RetTy: ConvertType(T: E->getType()),
842 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
843 Args: {HandleOp, SamplerOp, CoordOp});
844 }
845 case Builtin::BI__builtin_hlsl_resource_gather: {
846 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
847 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
848 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
849 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
850 if (ComponentOp->getType() != Builder.getInt32Ty())
851 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
852 /*isSigned=*/false);
853 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
854
855 SmallVector<Value *, 5> Args;
856 Args.push_back(Elt: HandleOp);
857 Args.push_back(Elt: SamplerOp);
858 Args.push_back(Elt: CoordOp);
859 Args.push_back(Elt: ComponentOp);
860 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
861
862 llvm::Type *RetTy = ConvertType(T: E->getType());
863 return Builder.CreateIntrinsic(
864 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
865 }
866 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
867 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
868 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
869 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
870 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
871 if (CompareOp->getType() != Builder.getFloatTy())
872 CompareOp = Builder.CreateFPCast(V: CompareOp, DestTy: Builder.getFloatTy());
873
874 SmallVector<Value *, 6> Args;
875 Args.push_back(Elt: HandleOp);
876 Args.push_back(Elt: SamplerOp);
877 Args.push_back(Elt: CoordOp);
878 Args.push_back(Elt: CompareOp);
879
880 if (CGM.getTarget().getTriple().isDXIL()) {
881 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
882 if (ComponentOp->getType() != Builder.getInt32Ty())
883 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
884 /*isSigned=*/false);
885 Args.push_back(Elt: ComponentOp);
886 }
887
888 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
889 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
890
891 llvm::Type *RetTy = ConvertType(T: E->getType());
892 return Builder.CreateIntrinsic(
893 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
894 }
895 case Builtin::BI__builtin_hlsl_resource_load_with_status:
896 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
897 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
898 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
899
900 // Get the *address* of the status argument to write to it by reference
901 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
902 Address StatusAddr = StatusLVal.getAddress();
903
904 QualType HandleTy = E->getArg(Arg: 0)->getType();
905 const HLSLAttributedResourceType *RT =
906 HandleTy->getAs<HLSLAttributedResourceType>();
907 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
908 "Only DXIL currently implements load with status");
909
910 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
911 ? llvm::Intrinsic::dx_resource_load_rawbuffer
912 : llvm::Intrinsic::dx_resource_load_typedbuffer;
913
914 llvm::Type *DataTy = ConvertType(T: E->getType());
915 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
916 Elements: {DataTy, Builder.getInt1Ty()});
917
918 SmallVector<Value *, 3> Args;
919 Args.push_back(Elt: HandleOp);
920 Args.push_back(Elt: IndexOp);
921
922 if (RT->isRaw()) {
923 Value *Offset = Builder.getInt32(C: 0);
924 // The offset parameter needs to be poison for ByteAddressBuffer
925 if (!RT->isStructured())
926 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
927 Args.push_back(Elt: Offset);
928 }
929
930 // The load intrinsics give us a (T value, i1 status) pair -
931 // shepherd these into the return value and out reference respectively.
932 Value *ResRet =
933 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
934 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
935 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
936 Value *ExtendedStatus =
937 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
938 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
939
940 return LoadedValue;
941 }
942 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
943 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
944 return llvm::PoisonValue::get(T: HandleTy);
945 }
946 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
947 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
948 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
949 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
950 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
951 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
952 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
953 llvm::Intrinsic::ID IntrinsicID =
954 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
955 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
956 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
957 }
958 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
959 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
960 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
961 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
962 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
963 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
964 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
965 llvm::Intrinsic::ID IntrinsicID =
966 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
967 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
968 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
969 }
970 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
971 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
972 if (!CGM.getTriple().isSPIRV())
973 return MainHandle;
974
975 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
976 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
977 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
978 llvm::Intrinsic::ID IntrinsicID =
979 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
980 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
981 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
982 Args);
983 }
984 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
985 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
986 llvm::Type *RetTy = ConvertType(T: E->getType());
987 return Builder.CreateIntrinsic(
988 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
989 Args: ArrayRef<Value *>{IndexOp});
990 }
991 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
992 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
993 return emitGetDimensions(CGF&: *this, E,
994 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
995 NumRetComps: 1, /*HasLod=*/false);
996 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
997 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
998 return emitGetDimensions(CGF&: *this, E,
999 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1000 NumRetComps: 2, /*HasLod=*/false);
1001 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1002 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1003 return emitGetDimensions(
1004 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1005 /*HasLod=*/true);
1006 case Builtin::BI__builtin_hlsl_resource_getstride: {
1007 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1008 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1009 }
1010 case Builtin::BI__builtin_hlsl_all: {
1011 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1012 return Builder.CreateIntrinsic(
1013 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1014 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1015 Name: "hlsl.all");
1016 }
1017 case Builtin::BI__builtin_hlsl_and: {
1018 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1019 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1020 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1021 }
1022 case Builtin::BI__builtin_hlsl_or: {
1023 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1024 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1025 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1026 }
1027 case Builtin::BI__builtin_hlsl_any: {
1028 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1029 return Builder.CreateIntrinsic(
1030 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1031 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1032 Name: "hlsl.any");
1033 }
1034 case Builtin::BI__builtin_hlsl_asdouble:
1035 return handleAsDoubleBuiltin(CGF&: *this, E);
1036 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1037 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1038 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1039 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1040
1041 QualType Ty = E->getArg(Arg: 0)->getType();
1042 if (auto *VecTy = Ty->getAs<VectorType>())
1043 Ty = VecTy->getElementType();
1044
1045 Intrinsic::ID Intr;
1046 if (Ty->isFloatingType()) {
1047 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1048 } else if (Ty->isUnsignedIntegerType()) {
1049 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1050 } else {
1051 assert(Ty->isSignedIntegerType());
1052 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1053 }
1054 return Builder.CreateIntrinsic(
1055 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1056 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1057 }
1058 case Builtin::BI__builtin_hlsl_crossf16:
1059 case Builtin::BI__builtin_hlsl_crossf32: {
1060 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1061 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1062 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1063 E->getArg(1)->getType()->hasFloatingRepresentation() &&
1064 "cross operands must have a float representation");
1065 // make sure each vector has exactly 3 elements
1066 assert(
1067 E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
1068 E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
1069 "input vectors must have 3 elements each");
1070 return Builder.CreateIntrinsic(
1071 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getCrossIntrinsic(),
1072 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.cross");
1073 }
1074 case Builtin::BI__builtin_hlsl_dot: {
1075 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1076 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1077 llvm::Type *T0 = Op0->getType();
1078 llvm::Type *T1 = Op1->getType();
1079
1080 // If the arguments are scalars, just emit a multiply
1081 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1082 if (T0->isFloatingPointTy())
1083 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1084
1085 if (T0->isIntegerTy())
1086 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1087
1088 llvm_unreachable(
1089 "Scalar dot product is only supported on ints and floats.");
1090 }
1091 // For vectors, validate types and emit the appropriate intrinsic
1092 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1093 E->getArg(1)->getType()) &&
1094 "Dot product operands must have the same type.");
1095
1096 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1097 assert(VecTy0 && "Dot product argument must be a vector.");
1098
1099 return Builder.CreateIntrinsic(
1100 /*ReturnType=*/RetTy: T0->getScalarType(),
1101 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1102 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1103 }
1104 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1105 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1106 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1107 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1108
1109 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1110 // Note that the argument order disagrees between the builtin and the
1111 // intrinsic here.
1112 return Builder.CreateIntrinsic(
1113 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1114 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1115 }
1116 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1117 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1118 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1119 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1120
1121 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1122 // Note that the argument order disagrees between the builtin and the
1123 // intrinsic here.
1124 return Builder.CreateIntrinsic(
1125 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1126 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1127 }
1128 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1129 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1130
1131 return Builder.CreateIntrinsic(
1132 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1133 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1134 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1135 }
1136 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1137 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1138
1139 return Builder.CreateIntrinsic(
1140 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1141 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1142 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1143 }
1144 case Builtin::BI__builtin_hlsl_lerp: {
1145 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1146 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1147 Value *S = EmitScalarExpr(E: E->getArg(Arg: 2));
1148 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1149 llvm_unreachable("lerp operand must have a float representation");
1150 return Builder.CreateIntrinsic(
1151 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getLerpIntrinsic(),
1152 Args: ArrayRef<Value *>{X, Y, S}, FMFSource: nullptr, Name: "hlsl.lerp");
1153 }
1154 case Builtin::BI__builtin_hlsl_normalize: {
1155 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1156
1157 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1158 "normalize operand must have a float representation");
1159
1160 return Builder.CreateIntrinsic(
1161 /*ReturnType=*/RetTy: X->getType(),
1162 ID: CGM.getHLSLRuntime().getNormalizeIntrinsic(), Args: ArrayRef<Value *>{X},
1163 FMFSource: nullptr, Name: "hlsl.normalize");
1164 }
1165 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
1166 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1167
1168 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1169 "degree operand must have a float representation");
1170
1171 return Builder.CreateIntrinsic(
1172 /*ReturnType=*/RetTy: X->getType(), ID: CGM.getHLSLRuntime().getDegreesIntrinsic(),
1173 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.degrees");
1174 }
1175 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1176 return handleElementwiseF16ToF32(CGF&: *this, E);
1177 }
1178 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1179 return handleElementwiseF32ToF16(CGF&: *this, E);
1180 }
1181 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1182 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1183 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1184 llvm_unreachable("frac operand must have a float representation");
1185 return Builder.CreateIntrinsic(
1186 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1187 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1188 }
1189 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1190 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1191 llvm::Type *Xty = Op0->getType();
1192 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1193 if (Xty->isVectorTy()) {
1194 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1195 retType = llvm::VectorType::get(
1196 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1197 }
1198 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1199 llvm_unreachable("isinf operand must have a float representation");
1200 return Builder.CreateIntrinsic(
1201 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1202 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1203 }
1204 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1205 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1206 llvm::Type *Xty = Op0->getType();
1207 llvm::Type *retType = llvm::Type::getInt1Ty(C&: this->getLLVMContext());
1208 if (Xty->isVectorTy()) {
1209 auto *XVecTy = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1210 retType = llvm::VectorType::get(
1211 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1212 }
1213 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1214 llvm_unreachable("isnan operand must have a float representation");
1215 return Builder.CreateIntrinsic(
1216 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1217 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1218 }
1219 case Builtin::BI__builtin_hlsl_mad: {
1220 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1221 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1222 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1223 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1224 return Builder.CreateIntrinsic(
1225 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1226 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1227
1228 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1229 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1230 return Builder.CreateIntrinsic(
1231 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1232 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1233
1234 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1235 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1236 }
1237 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1238 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1239 return Builder.CreateIntrinsic(
1240 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1241 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1242
1243 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1244 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1245 }
1246 case Builtin::BI__builtin_hlsl_mul: {
1247 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1248 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1249 QualType QTy0 = E->getArg(Arg: 0)->getType();
1250 QualType QTy1 = E->getArg(Arg: 1)->getType();
1251
1252 bool IsVec0 = QTy0->isVectorType();
1253 bool IsVec1 = QTy1->isVectorType();
1254 bool IsMat0 = QTy0->isConstantMatrixType();
1255 bool IsMat1 = QTy1->isConstantMatrixType();
1256
1257 // The matrix multiply intrinsic only operates on column-major order
1258 // matrices. Therefore matrix memory layout transforms must be inserted
1259 // before and after matrix multiply intrinsics.
1260 // Use whichever operand is a matrix to discover its declared layout.
1261 bool IsRowMajorMat0 = IsMat0 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy0);
1262 bool IsRowMajorMat1 = IsMat1 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy1);
1263
1264 llvm::MatrixBuilder MB(Builder);
1265 if (IsVec0 && IsMat1) {
1266 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1267 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1268 unsigned Rows = MatTy->getNumRows();
1269 unsigned Cols = MatTy->getNumColumns();
1270 assert(N == Rows && "vector length must match matrix row count");
1271 if (IsRowMajorMat1)
1272 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows, Columns: Cols);
1273 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1274 }
1275 if (IsMat0 && IsVec1) {
1276 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1277 unsigned Rows = MatTy->getNumRows();
1278 unsigned Cols = MatTy->getNumColumns();
1279 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1280 "vector length must match matrix column count");
1281 if (IsRowMajorMat0)
1282 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows, Columns: Cols);
1283 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1284 }
1285 assert(IsMat0 && IsMat1);
1286 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1287 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1288 unsigned Rows0 = MatTy0->getNumRows();
1289 unsigned Rows1 = MatTy1->getNumRows();
1290 unsigned Cols0 = MatTy0->getNumColumns();
1291 unsigned Cols1 = MatTy1->getNumColumns();
1292 assert(Cols0 == Rows1 &&
1293 "inner matrix dimensions must match for multiplication");
1294 if (IsRowMajorMat0)
1295 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows: Rows0, Columns: Cols0);
1296 if (IsRowMajorMat1)
1297 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows: Rows1, Columns: Cols1);
1298
1299 Value *Result =
1300 MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1301
1302 bool IsResultRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1303 if (IsResultRowMajor)
1304 Result = MB.CreateColumnMajorToRowMajorTransform(Matrix: Result, Rows: Rows0, Columns: Cols1);
1305 return Result;
1306 }
1307 case Builtin::BI__builtin_hlsl_transpose: {
1308 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1309 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1310 unsigned Rows = MatTy->getNumRows();
1311 unsigned Cols = MatTy->getNumColumns();
1312 llvm::MatrixBuilder MB(Builder);
1313 // The correct lowering of a transpose depends on both the source layout
1314 // and the result layout.
1315 bool SrcRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getArg(Arg: 0)->getType());
1316 bool DstRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1317 // When the source & result layouts differ, the operand already holds the
1318 // transposed result, ie transpose is a no-op on the underlying vector.
1319 if (SrcRowMajor != DstRowMajor)
1320 return Op0;
1321 // When the source and result share a layout, emit a transpose.
1322 if (SrcRowMajor)
1323 // For row-major operands the dimensions are swapped
1324 return MB.CreateMatrixTranspose(Matrix: Op0, Rows: Cols, Columns: Rows);
1325 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1326 }
1327 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1328 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1329 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1330 llvm_unreachable("rcp operand must have a float representation");
1331 llvm::Type *Ty = Op0->getType();
1332 llvm::Type *EltTy = Ty->getScalarType();
1333 Constant *One = Ty->isVectorTy()
1334 ? ConstantVector::getSplat(
1335 EC: ElementCount::getFixed(
1336 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1337 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1338 : ConstantFP::get(Ty: EltTy, V: 1.0);
1339 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1340 }
1341 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1342 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1343 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1344 llvm_unreachable("rsqrt operand must have a float representation");
1345 return Builder.CreateIntrinsic(
1346 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1347 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1348 }
1349 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1350 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1351 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1352 "saturate operand must have a float representation");
1353 return Builder.CreateIntrinsic(
1354 /*ReturnType=*/RetTy: Op0->getType(),
1355 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1356 FMFSource: nullptr, Name: "hlsl.saturate");
1357 }
1358 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1359 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1360 assert(Op->getType()->isIntegerTy(1) &&
1361 "WavePrefixBitCount operand must be a boolean type");
1362
1363 Intrinsic::ID IID =
1364 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1365
1366 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1367 }
1368 case Builtin::BI__builtin_hlsl_select: {
1369 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1370 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1371 Value *OpTrue =
1372 RValTrue.isScalar()
1373 ? RValTrue.getScalarVal()
1374 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1375 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1376 Value *OpFalse =
1377 RValFalse.isScalar()
1378 ? RValFalse.getScalarVal()
1379 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1380 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1381 if (!OpTrue->getType()->isVectorTy())
1382 OpTrue =
1383 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1384 if (!OpFalse->getType()->isVectorTy())
1385 OpFalse =
1386 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1387 }
1388
1389 Value *SelectVal =
1390 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1391 if (!RValTrue.isScalar())
1392 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1393 IsVolatile: ReturnValue.isVolatile());
1394
1395 return SelectVal;
1396 }
1397 case Builtin::BI__builtin_hlsl_step: {
1398 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1399 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1400 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1401 E->getArg(1)->getType()->hasFloatingRepresentation() &&
1402 "step operands must have a float representation");
1403 return Builder.CreateIntrinsic(
1404 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getStepIntrinsic(),
1405 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.step");
1406 }
1407 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1408 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1409
1410 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1411 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1412 }
1413 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1414 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1415 assert(Op->getType()->isIntegerTy(1) &&
1416 "Intrinsic WaveActiveAllTrue operand must be a bool");
1417
1418 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1419 return EmitIntrinsicCall(ID, Args: {Op});
1420 }
1421 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1422 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1423 assert(Op->getType()->isIntegerTy(1) &&
1424 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1425
1426 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1427 return EmitIntrinsicCall(ID, Args: {Op});
1428 }
1429 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1430 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1431 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1432 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1433 "representation");
1434
1435 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1436 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1437 Name: "hlsl.wave.active.bit.or");
1438 }
1439 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1440 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1441 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1442 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1443 "representation");
1444
1445 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1446 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1447 Name: "hlsl.wave.active.bit.xor");
1448 }
1449 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1450 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1451 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1452 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1453 "representation");
1454
1455 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1456 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1457 Name: "hlsl.wave.active.bit.and");
1458 }
1459 case Builtin::BI__builtin_hlsl_interlocked_add: {
1460 return handleInterlockedOp(
1461 CGF&: *this, E, ID: CGM.getHLSLRuntime().getInterlockedAddIntrinsic(),
1462 Name: "hlsl.interlocked.add");
1463 }
1464 case Builtin::BI__builtin_hlsl_interlocked_or: {
1465 return handleInterlockedOp(CGF&: *this, E,
1466 ID: CGM.getHLSLRuntime().getInterlockedOrIntrinsic(),
1467 Name: "hlsl.interlocked.or");
1468 }
1469 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1470 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1471 assert(Op->getType()->isIntegerTy(1) &&
1472 "Intrinsic WaveActiveBallot operand must be a bool");
1473
1474 return handleHlslWaveActiveBallot(CGF&: *this, E);
1475 }
1476 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1477 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1478 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1479 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1480 }
1481 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1482 // Due to the use of variadic arguments, explicitly retrieve argument
1483 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1484 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1485 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1486
1487 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1488 Name: "hlsl.wave.active.sum");
1489 }
1490 case Builtin::BI__builtin_hlsl_wave_active_product: {
1491 // Due to the use of variadic arguments, explicitly retrieve argument
1492 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1493 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1494 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1495
1496 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1497 Name: "hlsl.wave.active.product");
1498 }
1499 case Builtin::BI__builtin_hlsl_wave_active_max: {
1500 // Due to the use of variadic arguments, explicitly retrieve argument
1501 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1502 QualType QT = E->getArg(Arg: 0)->getType();
1503 Intrinsic::ID IID;
1504 if (QT->isUnsignedIntegerType())
1505 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1506 else
1507 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1508
1509 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1510 Name: "hlsl.wave.active.max");
1511 }
1512 case Builtin::BI__builtin_hlsl_wave_active_min: {
1513 // Due to the use of variadic arguments, explicitly retrieve argument
1514 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1515 QualType QT = E->getArg(Arg: 0)->getType();
1516 Intrinsic::ID IID;
1517 if (QT->isUnsignedIntegerType())
1518 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1519 else
1520 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1521
1522 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1523 Name: "hlsl.wave.active.min");
1524 }
1525 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1526 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1527 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1528 // for the DirectX intrinsic and the demangled builtin name
1529 switch (CGM.getTarget().getTriple().getArch()) {
1530 case llvm::Triple::dxil:
1531 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1532 case llvm::Triple::spirv:
1533 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1534 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1535 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1536 default:
1537 llvm_unreachable(
1538 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1539 }
1540 }
1541 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1542 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1543 return EmitIntrinsicCall(ID);
1544 }
1545 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1546 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1547 return EmitIntrinsicCall(ID);
1548 }
1549 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1550 // Due to the use of variadic arguments we must explicitly retrieve them and
1551 // create our function type.
1552 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1553 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1554 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1555 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1556 Name: "hlsl.wave.readlane");
1557 }
1558 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1559 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1560 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1561 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1562 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1563 Name: "hlsl.wave.prefix.sum");
1564 }
1565 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1566 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1567 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1568 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1569 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1570 Name: "hlsl.wave.prefix.product");
1571 }
1572 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1573 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1574 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1575 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1576 Name: "hlsl.quad.read.across.x");
1577 }
1578 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1579 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1580 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1581 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1582 Name: "hlsl.quad.read.across.y");
1583 }
1584 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1585 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1586 Intrinsic::ID ID =
1587 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1588 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1589 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1590 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.diagonal");
1591 }
1592 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1593 auto *Arg0 = E->getArg(Arg: 0);
1594 Value *Op0 = EmitScalarExpr(E: Arg0);
1595 llvm::Type *Xty = Op0->getType();
1596 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1597 if (Xty->isVectorTy()) {
1598 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1599 retType = llvm::VectorType::get(
1600 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1601 }
1602 assert((Arg0->getType()->hasFloatingRepresentation() ||
1603 Arg0->getType()->hasIntegerRepresentation()) &&
1604 "sign operand must have a float or int representation");
1605
1606 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1607 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1608 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1609 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1610 }
1611
1612 return Builder.CreateIntrinsic(
1613 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1614 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1615 }
1616 case Builtin::BI__builtin_hlsl_elementwise_radians: {
1617 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1618 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1619 "radians operand must have a float representation");
1620 return Builder.CreateIntrinsic(
1621 /*ReturnType=*/RetTy: Op0->getType(),
1622 ID: CGM.getHLSLRuntime().getRadiansIntrinsic(), Args: ArrayRef<Value *>{Op0},
1623 FMFSource: nullptr, Name: "hlsl.radians");
1624 }
1625 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1626 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1627 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1628 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1629 return Builder.CreateIntrinsic(
1630 /*ReturnType=*/RetTy: Offset->getType(),
1631 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1632 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1633 }
1634 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1635
1636 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1637 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1638 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1639 "asuint operands types mismatch");
1640 return handleHlslSplitdouble(E, CGF: this);
1641 }
1642 case Builtin::BI__builtin_hlsl_elementwise_clip:
1643 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1644 "clip operands types mismatch");
1645 return handleHlslClip(E, CGF: this);
1646 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1647 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1648 return EmitIntrinsicCall(ID);
1649 }
1650 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1651 Intrinsic::ID ID =
1652 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1653 return EmitIntrinsicCall(ID);
1654 }
1655 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1656 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1657 return EmitIntrinsicCall(ID);
1658 }
1659 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1660 Intrinsic::ID ID =
1661 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1662 return EmitIntrinsicCall(ID);
1663 }
1664 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1665 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1666 return EmitIntrinsicCall(ID);
1667 }
1668 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1669 Intrinsic::ID ID =
1670 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1671 return EmitIntrinsicCall(ID);
1672 }
1673 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1674 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1675 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1676 llvm_unreachable("ddx_coarse operand must have a float representation");
1677 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1678 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1679 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1680 Name: "hlsl.ddx.coarse");
1681 }
1682 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1683 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1684 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1685 llvm_unreachable("ddy_coarse operand must have a float representation");
1686 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1687 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1688 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1689 Name: "hlsl.ddy.coarse");
1690 }
1691 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1692 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1693 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1694 llvm_unreachable("ddx_fine operand must have a float representation");
1695 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1696 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1697 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1698 Name: "hlsl.ddx.fine");
1699 }
1700 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1701 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1702 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1703 llvm_unreachable("ddy_fine operand must have a float representation");
1704 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1705 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1706 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1707 Name: "hlsl.ddy.fine");
1708 }
1709 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1710 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1711 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1712 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1713 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1714 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1715 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1716 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1717 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1718 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1719 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1720 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1721 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1722 llvm::Value *Args[] = {SpecId, DefaultVal};
1723 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1724 }
1725 }
1726 return nullptr;
1727}
1728
1729llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1730 const clang::QualType &SpecConstantType) {
1731
1732 // Find or create the declaration for the function.
1733 llvm::Module *M = &CGM.getModule();
1734 std::string MangledName =
1735 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1736 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1737
1738 if (!SpecConstantFn) {
1739 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1740 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1741 llvm::Type *ArgTypes[] = {IntType, RetTy};
1742 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1743 SpecConstantFn = llvm::Function::Create(
1744 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1745 }
1746 return SpecConstantFn;
1747}
1748