1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/IR/MatrixBuilder.h"
19
20using namespace clang;
21using namespace CodeGen;
22using namespace llvm;
23
24static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
25 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
26 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
27 "asdouble operands types mismatch");
28 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
29 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
30
31 llvm::Type *ResultType = CGF.DoubleTy;
32 int N = 1;
33 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
34 N = VTy->getNumElements();
35 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
36 }
37
38 if (CGF.CGM.getTarget().getTriple().isDXIL())
39 return CGF.Builder.CreateIntrinsic(
40 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
41 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
42
43 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
44 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
45 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
46 }
47
48 llvm::SmallVector<int> Mask;
49 for (int i = 0; i < N; i++) {
50 Mask.push_back(Elt: i);
51 Mask.push_back(Elt: i + N);
52 }
53
54 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
55
56 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
57}
58
59static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
60 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
61
62 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
63 Value *CMP;
64 Value *LastInstr;
65
66 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
67 FZeroConst = ConstantVector::getSplat(
68 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
69 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
70 CMP = CGF->Builder.CreateIntrinsic(
71 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
72 Args: {FCompInst});
73 } else {
74 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
75 }
76
77 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
78 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
79 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
80 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
81 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
82
83 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
84
85 CGF->Builder.SetInsertPoint(LT0);
86
87 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
88
89 LastInstr = CGF->Builder.CreateBr(Dest: End);
90 CGF->Builder.SetInsertPoint(End);
91 } else {
92 llvm_unreachable("Backend Codegen not supported.");
93 }
94
95 return LastInstr;
96}
97
98static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
99 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
100 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
101 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
102
103 CallArgList Args;
104 LValue Op1TmpLValue =
105 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
106 LValue Op2TmpLValue =
107 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
108
109 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
110 Args.reverseWritebacks();
111
112 Value *LowBits = nullptr;
113 Value *HighBits = nullptr;
114
115 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
116 llvm::Type *RetElementTy = CGF->Int32Ty;
117 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
118 RetElementTy = llvm::VectorType::get(
119 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
120 else if (auto *Op0MatTy =
121 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
122 RetElementTy = llvm::VectorType::get(
123 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
124 Op0MatTy->getNumColumns()));
125
126 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
127
128 Value *CI = CGF->Builder.CreateIntrinsic(
129 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
130
131 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
132 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
133 } else {
134 // For Non DXIL targets we generate the instructions.
135
136 if (!Op0->getType()->isVectorTy()) {
137 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
138 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
139
140 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
141 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
142 } else {
143 int NumElements = 1;
144 if (const auto *VecTy =
145 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
146 NumElements = VecTy->getNumElements();
147 else if (const auto *MatTy =
148 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
149 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
150
151 FixedVectorType *Uint32VecTy =
152 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
153 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
154 if (NumElements == 1) {
155 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
156 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
157 } else {
158 SmallVector<int> EvenMask, OddMask;
159 for (int I = 0, E = NumElements; I != E; ++I) {
160 EvenMask.push_back(Elt: I * 2);
161 OddMask.push_back(Elt: I * 2 + 1);
162 }
163 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
164 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
165 }
166 }
167 }
168 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
169 auto *LastInst =
170 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
171 CGF->EmitWritebacks(Args);
172 return LastInst;
173}
174
175static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
176 const CallExpr *E) {
177 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
178 llvm::Type *I32 = CGF.Int32Ty;
179
180 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
181 [[maybe_unused]] llvm::StructType *Struct4I32 =
182 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
183
184 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
185 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
186 Value *StructVal =
187 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
188 assert(StructVal->getType() == Struct4I32 &&
189 "dx.wave.ballot must return {i32,i32,i32,i32}");
190
191 // Reassemble struct to <4 x i32>
192 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
193 for (unsigned I = 0; I < 4; ++I) {
194 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
195 VecVal =
196 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
197 }
198
199 return VecVal;
200 }
201
202 if (CGF.CGM.getTarget().getTriple().isSPIRV())
203 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
204
205 llvm_unreachable(
206 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
207}
208
209static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
210 const CallExpr *E) {
211 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
212 QualType Op0Ty = E->getArg(Arg: 0)->getType();
213 llvm::Type *ResType = CGF.FloatTy;
214 uint64_t NumElements = 0;
215 if (Op0->getType()->isVectorTy()) {
216 NumElements =
217 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
218 ResType =
219 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
220 }
221 if (!Op0Ty->hasUnsignedIntegerRepresentation())
222 llvm_unreachable(
223 "f16tof32 operand must have an unsigned int representation");
224
225 if (CGF.CGM.getTriple().isDXIL())
226 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
227 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
228 Name: "hlsl.f16tof32");
229
230 if (CGF.CGM.getTriple().isSPIRV()) {
231 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
232 // Int16 and Float16 capabilities
233 auto *UnpackType =
234 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
235
236 if (NumElements == 0) {
237 // a scalar input - simply extract the first element of the unpacked
238 // vector
239 Value *Unpack = CGF.Builder.CreateIntrinsic(
240 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
241 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
242 }
243
244 // a vector input - build a congruent output vector by iterating through
245 // the input vector calling unpackhalf2x16 for each element
246 Value *Result = PoisonValue::get(T: ResType);
247 for (uint64_t I = 0; I < NumElements; I++) {
248 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
249 Value *Unpack = CGF.Builder.CreateIntrinsic(
250 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
251 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
252 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
253 }
254 return Result;
255 }
256
257 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
258}
259
260static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
261 const CallExpr *E) {
262 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
263 QualType Op0Ty = E->getArg(Arg: 0)->getType();
264 llvm::Type *ResType = CGF.IntTy;
265 uint64_t NumElements = 0;
266 if (Op0->getType()->isVectorTy()) {
267 NumElements =
268 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
269 ResType =
270 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
271 }
272 if (!Op0Ty->hasFloatingRepresentation())
273 llvm_unreachable("f32tof16 operand must have a float representation");
274
275 if (CGF.CGM.getTriple().isDXIL())
276 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
277 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
278 Name: "hlsl.f32tof16");
279
280 if (CGF.CGM.getTriple().isSPIRV()) {
281 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
282 // Int16 and Float16 capabilities
283 auto *PackType =
284 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
285
286 if (NumElements == 0) {
287 // a scalar input - simply insert the scalar in the first element
288 // of the 2 element float vector
289 Value *Float2 = Constant::getNullValue(Ty: PackType);
290 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
291 Value *Result = CGF.Builder.CreateIntrinsic(
292 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
293 return Result;
294 }
295
296 // a vector input - build a congruent output vector by iterating through
297 // the input vector calling packhalf2x16 for each element
298 Value *Result = PoisonValue::get(T: ResType);
299 for (uint64_t I = 0; I < NumElements; I++) {
300 Value *Float2 = Constant::getNullValue(Ty: PackType);
301 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
302 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
303 Value *Res = CGF.Builder.CreateIntrinsic(
304 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
305 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
306 }
307 return Result;
308 }
309
310 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
311}
312
313static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
314 llvm::AtomicRMWInst::BinOp Op) {
315 // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
316 // either DXIL or SPIR-V.
317 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
318 Address DestAddr = DestLV.getAddress();
319 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
320 assert(E->getArg(1)->getType()->isIntegerType() &&
321 "Intrinsic InterlockedOp value operand must be an integer");
322
323 // Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
324 // Set the memory scope: Workgroup for groupshared, otherwise Device.
325 StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared
326 ? "workgroup"
327 : "device";
328 llvm::SyncScope::ID SSID =
329 CGF.getLLVMContext().getOrInsertSyncScopeID(SSN: ScopeName);
330
331 llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
332 Op, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
333
334 // The 3-arg overload writes the old value (the RMW's return value) into
335 // the `original_value` reference parameter.
336 if (E->getNumArgs() == 3) {
337 LValue OrigLV = CGF.EmitLValue(E: E->getArg(Arg: 2));
338 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
339 }
340 return Call;
341}
342
343static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
344 LValue &Stride) {
345 // Figure out the stride of the buffer elements from the handle type.
346 auto *HandleTy =
347 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
348 QualType ElementTy = HandleTy->getContainedType();
349 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
350 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
351}
352
353// Return dot product intrinsic that corresponds to the QT scalar type
354static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
355 if (QT->isFloatingType())
356 return RT.getFDotIntrinsic();
357 if (QT->isSignedIntegerType())
358 return RT.getSDotIntrinsic();
359 assert(QT->isUnsignedIntegerType());
360 return RT.getUDotIntrinsic();
361}
362
363static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
364 if (QT->hasSignedIntegerRepresentation()) {
365 return RT.getFirstBitSHighIntrinsic();
366 }
367
368 assert(QT->hasUnsignedIntegerRepresentation());
369 return RT.getFirstBitUHighIntrinsic();
370}
371
372// Return wave active sum that corresponds to the QT scalar type
373static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
374 QualType QT) {
375 switch (Arch) {
376 case llvm::Triple::spirv:
377 return Intrinsic::spv_wave_reduce_sum;
378 case llvm::Triple::dxil: {
379 if (QT->isUnsignedIntegerType())
380 return Intrinsic::dx_wave_reduce_usum;
381 return Intrinsic::dx_wave_reduce_sum;
382 }
383 default:
384 llvm_unreachable("Intrinsic WaveActiveSum"
385 " not supported by target architecture");
386 }
387}
388
389// Return wave active product that corresponds to the QT scalar type
390static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
391 QualType QT) {
392 switch (Arch) {
393 case llvm::Triple::spirv:
394 return Intrinsic::spv_wave_product;
395 case llvm::Triple::dxil: {
396 if (QT->isUnsignedIntegerType())
397 return Intrinsic::dx_wave_uproduct;
398 return Intrinsic::dx_wave_product;
399 }
400 default:
401 llvm_unreachable("Intrinsic WaveActiveProduct"
402 " not supported by target architecture");
403 }
404}
405
406static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
407 switch (Arch) {
408 case llvm::Triple::spirv:
409 return Intrinsic::spv_subgroup_prefix_bit_count;
410 case llvm::Triple::dxil: {
411 return Intrinsic::dx_wave_prefix_bit_count;
412 }
413 default:
414 llvm_unreachable(
415 "WavePrefixOp instruction not supported by target architecture");
416 }
417}
418
419// Return wave prefix sum that corresponds to the QT scalar type
420static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
421 QualType QT) {
422 switch (Arch) {
423 case llvm::Triple::spirv:
424 return Intrinsic::spv_wave_prefix_sum;
425 case llvm::Triple::dxil: {
426 if (QT->isUnsignedIntegerType())
427 return Intrinsic::dx_wave_prefix_usum;
428 return Intrinsic::dx_wave_prefix_sum;
429 }
430 default:
431 llvm_unreachable("Intrinsic WavePrefixSum"
432 " not supported by target architecture");
433 }
434}
435
436// Return wave prefix product that corresponds to the QT scalar type
437static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
438 QualType QT) {
439 switch (Arch) {
440 case llvm::Triple::spirv:
441 return Intrinsic::spv_wave_prefix_product;
442 case llvm::Triple::dxil: {
443 if (QT->isUnsignedIntegerType())
444 return Intrinsic::dx_wave_prefix_uproduct;
445 return Intrinsic::dx_wave_prefix_product;
446 }
447 default:
448 llvm_unreachable("Intrinsic WavePrefixProduct"
449 " not supported by target architecture");
450 }
451}
452
453// Returns the mangled name for a builtin function that the SPIR-V backend
454// will expand into a spec Constant.
455static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
456 ASTContext &Context) {
457 // The parameter types for our conceptual intrinsic function.
458 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
459
460 // Create a temporary FunctionDecl for the builtin fuction. It won't be
461 // added to the AST.
462 FunctionProtoType::ExtProtoInfo EPI;
463 QualType FnType =
464 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
465 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
466 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
467 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
468 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
469
470 // Attach the created parameter declarations to the function declaration.
471 SmallVector<ParmVarDecl *, 2> ParamDecls;
472 for (QualType ParamType : ClangParamTypes) {
473 ParmVarDecl *PD = ParmVarDecl::Create(
474 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
475 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
476 /*DefaultArg*/ DefArg: nullptr);
477 ParamDecls.push_back(Elt: PD);
478 }
479 FnDeclForMangling->setParams(ParamDecls);
480
481 // Get the mangled name.
482 std::string Name;
483 llvm::raw_string_ostream MangledNameStream(Name);
484 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
485 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
486 MangledNameStream.flush();
487
488 return Name;
489}
490
491static const HLSLAttributedResourceType *
492getHandleAttributedType(QualType HandleQT) {
493 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
494 return RT;
495 // If the expr is a texture/sampler record (or similar), peel to __handle.
496 if (const HLSLAttributedResourceType *RT =
497 HLSLAttributedResourceType::findHandleTypeOnResource(
498 RT: HandleQT.getTypePtr()))
499 return RT;
500 llvm_unreachable("attributed handle type not found");
501}
502
503static const HLSLAttributedResourceType *
504getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
505 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
506}
507
508static llvm::Type *getOffsetType(CodeGenModule &CGM,
509 const HLSLAttributedResourceType *RT) {
510 const auto &Attrs = RT->getAttrs();
511 unsigned OffsetSize =
512 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
513 llvm::Type *Int32Ty = CGM.Int32Ty;
514 if (OffsetSize == 1)
515 return Int32Ty;
516 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
517}
518
519static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
520 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
521 if (E->getNumArgs() > OffsetArgIndex)
522 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
523
524 return llvm::Constant::getNullValue(Ty: OffsetTy);
525}
526
527static Value *emitHlslSampleOffset(CodeGenFunction &CGF, const CallExpr *E,
528 const HLSLAttributedResourceType *RT,
529 unsigned OffsetArgIndex) {
530 llvm::Type *OffsetTy = getOffsetType(CGM&: CGF.CGM, RT);
531 if (!clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension))
532 return llvm::Constant::getNullValue(Ty: OffsetTy);
533 return emitHlslOffset(CGF, E, OffsetArgIndex, OffsetTy);
534}
535
536static unsigned getHlslClampArgIndex(const HLSLAttributedResourceType *RT,
537 unsigned OffsetArgIndex) {
538 return clang::hlsl::hasResourceOffset(Dim: RT->getAttrs().ResourceDimension)
539 ? OffsetArgIndex + 1
540 : OffsetArgIndex;
541}
542
543static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
544 unsigned IntrinsicID, unsigned NumRetComps,
545 bool HasLod) {
546 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
547
548 SmallVector<Value *> Args{Handle};
549 if (HasLod)
550 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
551
552 Value *DimValue =
553 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
554
555 Value *LastStore = nullptr;
556 unsigned ArgIndex = HasLod ? 2 : 1;
557 for (unsigned i = 0; i < NumRetComps; ++i) {
558 const Expr *Arg = E->getArg(Arg: ArgIndex++);
559 LValue DimOut = CGF.EmitLValue(E: Arg);
560 Value *Elem = DimValue;
561 if (NumRetComps > 1)
562 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
563
564 // Handle float casting if needed
565 if (Arg->getType()->isFloatingType())
566 Elem = CGF.Builder.CreateUIToFP(
567 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
568
569 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
570 }
571 return LastStore;
572}
573
574static llvm::Type *getAggregateType(llvm::Type *ScalarTy, QualType ArgTy) {
575 if (auto *MatTy = ArgTy->getAs<ConstantMatrixType>())
576 return llvm::VectorType::get(
577 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: MatTy->getNumElementsFlattened()));
578 if (auto *VecTy = ArgTy->getAs<clang::VectorType>())
579 return llvm::VectorType::get(
580 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()));
581 return ScalarTy;
582}
583
584Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
585 const CallExpr *E,
586 ReturnValueSlot ReturnValue) {
587 if (!getLangOpts().HLSL)
588 return nullptr;
589
590 switch (BuiltinID) {
591 case Builtin::BI__builtin_hlsl_adduint64: {
592 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
593 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
594 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
595 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
596 assert(Arg0Ty == E->getArg(1)->getType() &&
597 "AddUint64 operand types must match");
598 assert(Arg0Ty->hasIntegerRepresentation() &&
599 "AddUint64 operands must have an integer representation");
600 assert((NumElements == 2 || NumElements == 4) &&
601 "AddUint64 operands must have 2 or 4 elements");
602
603 llvm::Value *LowA;
604 llvm::Value *HighA;
605 llvm::Value *LowB;
606 llvm::Value *HighB;
607
608 // Obtain low and high words of inputs A and B
609 if (NumElements == 2) {
610 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
611 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
612 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
613 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
614 } else {
615 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
616 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
617 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
618 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
619 }
620
621 // Use an uadd_with_overflow to compute the sum of low words and obtain a
622 // carry value
623 llvm::Value *Carry;
624 llvm::Value *LowSum = EmitOverflowIntrinsic(
625 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
626 llvm::Value *ZExtCarry =
627 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
628
629 // Sum the high words and the carry
630 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
631 llvm::Value *HighSumPlusCarry =
632 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
633
634 if (NumElements == 4) {
635 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
636 Name: "hlsl.AddUint64");
637 }
638
639 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
640 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
641 Name: "hlsl.AddUint64.upto0");
642 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
643 Name: "hlsl.AddUint64");
644 return Result;
645 }
646 case Builtin::BI__builtin_hlsl_resource_getpointer:
647 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
648 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
649 bool IsIndexed =
650 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
651 E->getNumArgs() > 1;
652
653 llvm::Type *RetTy = ConvertType(T: E->getType());
654 llvm::Function *IntrFn = nullptr;
655 llvm::CallInst *CI = nullptr;
656 if (IsIndexed) {
657 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
658 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
659 M: &CGM.getModule(),
660 id: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
661 OverloadTys: {RetTy, HandleOp->getType(), IndexOp->getType()});
662 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp, IndexOp});
663 } else {
664 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
665 M: &CGM.getModule(),
666 id: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
667 OverloadTys: {RetTy, HandleOp->getType()});
668 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp});
669 }
670 CI->setCallingConv(IntrFn->getCallingConv());
671 return CI;
672 }
673 case Builtin::BI__builtin_hlsl_transpose_if_memory_is_row_major: {
674 const Expr *ValueExpr = E->getArg(Arg: 0);
675 if (hasAggregateEvaluationKind(T: ValueExpr->getType())) {
676 EmitAnyExprToMem(E: ValueExpr, Location: ReturnValue.getAddress(),
677 Quals: ValueExpr->getType().getQualifiers(), /*IsInit=*/IsInitializer: true);
678 return ReturnValue.getAddress().getBasePointer();
679 }
680
681 Value *ValueOp = EmitScalarExpr(E: ValueExpr);
682 const auto *MatTy = ValueExpr->getType()->getAs<ConstantMatrixType>();
683 if (!MatTy || !getLangOpts().HLSLSpvUseLegacyBufferMatrixOrder)
684 return ValueOp;
685
686 bool IsLoad =
687 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getBoolValue();
688 unsigned Rows = MatTy->getNumRows();
689 unsigned Columns = MatTy->getNumColumns();
690 llvm::MatrixBuilder MB(Builder);
691 return IsLoad ? MB.CreateMatrixTranspose(Matrix: ValueOp, Rows: Columns, Columns: Rows)
692 : MB.CreateMatrixTranspose(Matrix: ValueOp, Rows, Columns);
693 }
694 case Builtin::BI__builtin_hlsl_resource_sample: {
695 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
696 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
697 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
698 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
699
700 SmallVector<Value *, 4> Args;
701 Args.push_back(Elt: HandleOp);
702 Args.push_back(Elt: SamplerOp);
703 Args.push_back(Elt: CoordOp);
704 constexpr unsigned OffsetIdx = 3;
705 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
706
707 llvm::Type *RetTy = ConvertType(T: E->getType());
708 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
709 if (E->getNumArgs() <= ClampIdx)
710 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args,
711 RetTy);
712
713 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
714 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(),
715 Args, RetTy);
716 }
717 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
718 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
719 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
720 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
721 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
722 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
723
724 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
725 Args.push_back(Elt: HandleOp);
726 Args.push_back(Elt: SamplerOp);
727 Args.push_back(Elt: CoordOp);
728 Args.push_back(Elt: BiasOp);
729 constexpr unsigned OffsetIdx = 4;
730 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
731
732 llvm::Type *RetTy = ConvertType(T: E->getType());
733 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
734 if (E->getNumArgs() <= ClampIdx)
735 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(),
736 Args, RetTy);
737
738 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
739 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(),
740 Args, RetTy);
741 }
742 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
743 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
744 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
745 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
746 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
747 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
748 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
749
750 SmallVector<Value *, 7> Args;
751 Args.push_back(Elt: HandleOp);
752 Args.push_back(Elt: SamplerOp);
753 Args.push_back(Elt: CoordOp);
754 Args.push_back(Elt: DDXOp);
755 Args.push_back(Elt: DDYOp);
756 constexpr unsigned OffsetIdx = 5;
757 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
758
759 llvm::Type *RetTy = ConvertType(T: E->getType());
760
761 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
762 if (E->getNumArgs() <= ClampIdx)
763 return Builder.CreateIntrinsic(
764 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
765
766 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
767 return Builder.CreateIntrinsic(
768 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
769 }
770 case Builtin::BI__builtin_hlsl_resource_sample_level: {
771 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
772 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
773 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
774 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
775 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
776
777 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
778 Args.push_back(Elt: HandleOp);
779 Args.push_back(Elt: SamplerOp);
780 Args.push_back(Elt: CoordOp);
781 Args.push_back(Elt: LODOp);
782 constexpr unsigned OffsetIdx = 4;
783 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
784
785 llvm::Type *RetTy = ConvertType(T: E->getType());
786 return Builder.CreateIntrinsic(
787 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
788 }
789 case Builtin::BI__builtin_hlsl_resource_load_level: {
790 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
791 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
792 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
793
794 const auto &Attrs = RT->getAttrs();
795
796 Value *CoordOp = nullptr;
797 Value *LODOp = nullptr;
798 if (Attrs.ResourceClass == llvm::dxil::ResourceClass::UAV) {
799 // A UAV descriptor binds a single mip slice, so a RWTexture location is
800 // all coordinate and there is no mip level to select.
801 CoordOp = CoordLODOp;
802 LODOp = llvm::PoisonValue::get(T: Int32Ty);
803 } else {
804 // Split CoordLOD into Coord and LOD. 1D resources use a scalar
805 // coordinate rather than a 1-element vector.
806 unsigned CoordSize =
807 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension) +
808 (Attrs.IsArray ? 1 : 0);
809 assert(cast<llvm::FixedVectorType>(CoordLODOp->getType())
810 ->getNumElements() == CoordSize + 1 &&
811 "CoordLOD must have one element per coordinate, plus the level");
812
813 if (CoordSize == 1) {
814 CoordOp = Builder.CreateExtractElement(Vec: CoordLODOp, Idx: uint64_t(0),
815 Name: "hlsl.load.coord");
816 } else {
817 SmallVector<int, 4> Mask;
818 for (unsigned I = 0; I < CoordSize; ++I)
819 Mask.push_back(Elt: I);
820 CoordOp =
821 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
822 }
823 LODOp =
824 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: CoordSize, Name: "hlsl.load.lod");
825 }
826
827 SmallVector<Value *, 4> Args;
828 Args.push_back(Elt: HandleOp);
829 Args.push_back(Elt: CoordOp);
830 Args.push_back(Elt: LODOp);
831 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
832
833 llvm::Type *RetTy = ConvertType(T: E->getType());
834 return Builder.CreateIntrinsic(
835 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
836 }
837 case Builtin::BI__builtin_hlsl_resource_load_ms: {
838 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
839 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 1));
840 Value *SampleOp = EmitScalarExpr(E: E->getArg(Arg: 2));
841 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
842
843 SmallVector<Value *, 4> Args;
844 Args.push_back(Elt: HandleOp);
845 Args.push_back(Elt: CoordOp);
846 Args.push_back(Elt: SampleOp);
847 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
848
849 llvm::Type *RetTy = ConvertType(T: E->getType());
850 return Builder.CreateIntrinsic(
851 RetTy, ID: CGM.getHLSLRuntime().getLoadMSIntrinsic(), Args);
852 }
853 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
854 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
855 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
856 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
857 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
858 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
859
860 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
861 Args.push_back(Elt: HandleOp);
862 Args.push_back(Elt: SamplerOp);
863 Args.push_back(Elt: CoordOp);
864 Args.push_back(Elt: CmpOp);
865 constexpr unsigned OffsetIdx = 4;
866 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
867
868 llvm::Type *RetTy = ConvertType(T: E->getType());
869 const unsigned ClampIdx = getHlslClampArgIndex(RT, OffsetArgIndex: OffsetIdx);
870 if (E->getNumArgs() <= ClampIdx)
871 return Builder.CreateIntrinsic(
872 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
873
874 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: ClampIdx)));
875 return Builder.CreateIntrinsic(
876 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
877 }
878 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
879 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
880 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
881 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
882 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
883 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
884
885 SmallVector<Value *, 5> Args;
886 Args.push_back(Elt: HandleOp);
887 Args.push_back(Elt: SamplerOp);
888 Args.push_back(Elt: CoordOp);
889 Args.push_back(Elt: CmpOp);
890 constexpr unsigned OffsetIdx = 4;
891 Args.push_back(Elt: emitHlslSampleOffset(CGF&: *this, E, RT, OffsetArgIndex: OffsetIdx));
892
893 llvm::Type *RetTy = ConvertType(T: E->getType());
894 return Builder.CreateIntrinsic(
895 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
896 }
897 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
898 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
899 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
900 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
901
902 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
903 Args: {HandleOp, SamplerOp, CoordOp},
904 RetTy: ConvertType(T: E->getType()));
905 }
906 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
907 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
908 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
909 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
910
911 return EmitIntrinsicCall(
912 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
913 Args: {HandleOp, SamplerOp, CoordOp}, RetTy: ConvertType(T: E->getType()));
914 }
915 case Builtin::BI__builtin_hlsl_resource_gather: {
916 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
917 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
918 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
919 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
920 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
921
922 SmallVector<Value *, 5> Args;
923 Args.push_back(Elt: HandleOp);
924 Args.push_back(Elt: SamplerOp);
925 Args.push_back(Elt: CoordOp);
926 Args.push_back(Elt: ComponentOp);
927 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
928
929 llvm::Type *RetTy = ConvertType(T: E->getType());
930 return Builder.CreateIntrinsic(
931 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
932 }
933 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
934 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
935 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
936 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
937 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
938
939 SmallVector<Value *, 6> Args;
940 Args.push_back(Elt: HandleOp);
941 Args.push_back(Elt: SamplerOp);
942 Args.push_back(Elt: CoordOp);
943 Args.push_back(Elt: CompareOp);
944
945 if (CGM.getTarget().getTriple().isDXIL()) {
946 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
947 Args.push_back(Elt: ComponentOp);
948 }
949
950 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
951 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
952
953 llvm::Type *RetTy = ConvertType(T: E->getType());
954 return Builder.CreateIntrinsic(
955 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
956 }
957 case Builtin::BI__builtin_hlsl_resource_load_with_status:
958 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
959 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
960 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
961
962 // Get the *address* of the status argument to write to it by reference
963 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
964 Address StatusAddr = StatusLVal.getAddress();
965
966 QualType HandleTy = E->getArg(Arg: 0)->getType();
967 const HLSLAttributedResourceType *RT =
968 HandleTy->getAs<HLSLAttributedResourceType>();
969 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
970 "Only DXIL currently implements load with status");
971
972 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
973 ? llvm::Intrinsic::dx_resource_load_rawbuffer
974 : llvm::Intrinsic::dx_resource_load_typedbuffer;
975
976 llvm::Type *DataTy = ConvertType(T: E->getType());
977 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
978 Elements: {DataTy, Builder.getInt1Ty()});
979
980 SmallVector<Value *, 3> Args;
981 Args.push_back(Elt: HandleOp);
982 Args.push_back(Elt: IndexOp);
983
984 if (RT->isRaw()) {
985 Value *Offset = Builder.getInt32(C: 0);
986 // The offset parameter needs to be poison for ByteAddressBuffer
987 if (!RT->isStructured())
988 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
989 Args.push_back(Elt: Offset);
990 }
991
992 // The load intrinsics give us a (T value, i1 status) pair -
993 // shepherd these into the return value and out reference respectively.
994 Value *ResRet =
995 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
996 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
997 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
998 Value *ExtendedStatus =
999 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
1000 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
1001
1002 return LoadedValue;
1003 }
1004 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
1005 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1006 return llvm::PoisonValue::get(T: HandleTy);
1007 }
1008 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
1009 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1010 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1011 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1012 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1013 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1014 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1015 llvm::Intrinsic::ID IntrinsicID =
1016 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1017 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
1018 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1019 }
1020 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
1021 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1022 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1023 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1024 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1025 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1026 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1027 llvm::Intrinsic::ID IntrinsicID =
1028 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1029 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
1030 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1031 }
1032 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
1033 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1034 if (!CGM.getTriple().isSPIRV())
1035 return MainHandle;
1036
1037 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1038 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1039 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1040 llvm::Intrinsic::ID IntrinsicID =
1041 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
1042 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
1043 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1044 Args);
1045 }
1046 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
1047 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1048 llvm::Type *RetTy = ConvertType(T: E->getType());
1049 return Builder.CreateIntrinsic(
1050 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
1051 Args: ArrayRef<Value *>{IndexOp});
1052 }
1053 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
1054 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
1055 return emitGetDimensions(CGF&: *this, E,
1056 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
1057 NumRetComps: 1, /*HasLod=*/false);
1058 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
1059 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
1060 return emitGetDimensions(CGF&: *this, E,
1061 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1062 NumRetComps: 2, /*HasLod=*/false);
1063 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1064 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1065 return emitGetDimensions(
1066 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1067 /*HasLod=*/true);
1068 case Builtin::BI__builtin_hlsl_resource_getstride: {
1069 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1070 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1071 }
1072 case Builtin::BI__builtin_hlsl_all: {
1073 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1074 return Builder.CreateIntrinsic(
1075 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1076 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1077 Name: "hlsl.all");
1078 }
1079 case Builtin::BI__builtin_hlsl_and: {
1080 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1081 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1082 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1083 }
1084 case Builtin::BI__builtin_hlsl_or: {
1085 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1086 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1087 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1088 }
1089 case Builtin::BI__builtin_hlsl_any: {
1090 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1091 return Builder.CreateIntrinsic(
1092 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1093 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1094 Name: "hlsl.any");
1095 }
1096 case Builtin::BI__builtin_hlsl_asdouble:
1097 return handleAsDoubleBuiltin(CGF&: *this, E);
1098 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1099 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1100 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1101 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1102
1103 QualType Ty = E->getArg(Arg: 0)->getType();
1104 if (auto *VecTy = Ty->getAs<VectorType>())
1105 Ty = VecTy->getElementType();
1106
1107 Intrinsic::ID Intr;
1108 if (Ty->isFloatingType()) {
1109 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1110 } else if (Ty->isUnsignedIntegerType()) {
1111 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1112 } else {
1113 assert(Ty->isSignedIntegerType());
1114 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1115 }
1116 return Builder.CreateIntrinsic(
1117 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1118 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1119 }
1120 case Builtin::BI__builtin_hlsl_dot: {
1121 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1122 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1123 llvm::Type *T0 = Op0->getType();
1124 llvm::Type *T1 = Op1->getType();
1125
1126 // If the arguments are scalars, just emit a multiply
1127 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1128 if (T0->isFloatingPointTy())
1129 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1130
1131 if (T0->isIntegerTy())
1132 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1133
1134 llvm_unreachable(
1135 "Scalar dot product is only supported on ints and floats.");
1136 }
1137 // For vectors, validate types and emit the appropriate intrinsic
1138 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1139 E->getArg(1)->getType()) &&
1140 "Dot product operands must have the same type.");
1141
1142 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1143 assert(VecTy0 && "Dot product argument must be a vector.");
1144
1145 return Builder.CreateIntrinsic(
1146 /*ReturnType=*/RetTy: T0->getScalarType(),
1147 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1148 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1149 }
1150 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1151 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1152 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1153 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1154
1155 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1156 // Note that the argument order disagrees between the builtin and the
1157 // intrinsic here.
1158 return Builder.CreateIntrinsic(
1159 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1160 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1161 }
1162 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1163 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1164 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1165 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1166
1167 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1168 // Note that the argument order disagrees between the builtin and the
1169 // intrinsic here.
1170 return Builder.CreateIntrinsic(
1171 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1172 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1173 }
1174 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1175 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1176
1177 return Builder.CreateIntrinsic(
1178 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1179 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1180 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1181 }
1182 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1183 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1184
1185 return Builder.CreateIntrinsic(
1186 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1187 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1188 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1189 }
1190 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1191 return handleElementwiseF16ToF32(CGF&: *this, E);
1192 }
1193 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1194 return handleElementwiseF32ToF16(CGF&: *this, E);
1195 }
1196 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1197 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1198 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1199 llvm_unreachable("frac operand must have a float representation");
1200 return Builder.CreateIntrinsic(
1201 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1202 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1203 }
1204 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1205 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1206 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1207 llvm_unreachable("isinf operand must have a float representation");
1208 llvm::Type *retType = getAggregateType(
1209 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1210 return Builder.CreateIntrinsic(
1211 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1212 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1213 }
1214 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1215 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1216 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1217 llvm_unreachable("isnan operand must have a float representation");
1218 llvm::Type *retType = getAggregateType(
1219 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1220 return Builder.CreateIntrinsic(
1221 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1222 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1223 }
1224 case Builtin::BI__builtin_hlsl_mad: {
1225 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1226 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1227 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1228 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1229 return Builder.CreateIntrinsic(
1230 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1231 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1232
1233 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1234 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1235 return Builder.CreateIntrinsic(
1236 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1237 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1238
1239 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1240 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1241 }
1242 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1243 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1244 return Builder.CreateIntrinsic(
1245 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1246 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1247
1248 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1249 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1250 }
1251 case Builtin::BI__builtin_hlsl_mul: {
1252 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1253 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1254 QualType QTy0 = E->getArg(Arg: 0)->getType();
1255 QualType QTy1 = E->getArg(Arg: 1)->getType();
1256
1257 bool IsVec0 = QTy0->isVectorType();
1258 bool IsVec1 = QTy1->isVectorType();
1259 bool IsMat0 = QTy0->isConstantMatrixType();
1260 bool IsMat1 = QTy1->isConstantMatrixType();
1261
1262 // The matrix multiply intrinsic only operates on column-major order
1263 // matrices. Therefore matrix memory layout transforms must be inserted
1264 // before and after matrix multiply intrinsics.
1265 // Use whichever operand is a matrix to discover its declared layout.
1266 bool IsRowMajorMat0 = IsMat0 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy0);
1267 bool IsRowMajorMat1 = IsMat1 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy1);
1268
1269 llvm::MatrixBuilder MB(Builder);
1270 if (IsVec0 && IsMat1) {
1271 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1272 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1273 unsigned Rows = MatTy->getNumRows();
1274 unsigned Cols = MatTy->getNumColumns();
1275 assert(N == Rows && "vector length must match matrix row count");
1276 if (IsRowMajorMat1)
1277 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows, Columns: Cols);
1278 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1279 }
1280 if (IsMat0 && IsVec1) {
1281 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1282 unsigned Rows = MatTy->getNumRows();
1283 unsigned Cols = MatTy->getNumColumns();
1284 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1285 "vector length must match matrix column count");
1286 if (IsRowMajorMat0)
1287 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows, Columns: Cols);
1288 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1289 }
1290 assert(IsMat0 && IsMat1);
1291 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1292 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1293 unsigned Rows0 = MatTy0->getNumRows();
1294 unsigned Rows1 = MatTy1->getNumRows();
1295 unsigned Cols0 = MatTy0->getNumColumns();
1296 unsigned Cols1 = MatTy1->getNumColumns();
1297 assert(Cols0 == Rows1 &&
1298 "inner matrix dimensions must match for multiplication");
1299 if (IsRowMajorMat0)
1300 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows: Rows0, Columns: Cols0);
1301 if (IsRowMajorMat1)
1302 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows: Rows1, Columns: Cols1);
1303
1304 Value *Result =
1305 MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1306
1307 bool IsResultRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1308 if (IsResultRowMajor)
1309 Result = MB.CreateColumnMajorToRowMajorTransform(Matrix: Result, Rows: Rows0, Columns: Cols1);
1310 return Result;
1311 }
1312 case Builtin::BI__builtin_hlsl_transpose: {
1313 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1314 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1315 unsigned Rows = MatTy->getNumRows();
1316 unsigned Cols = MatTy->getNumColumns();
1317 llvm::MatrixBuilder MB(Builder);
1318 // The correct lowering of a transpose depends on both the source layout
1319 // and the result layout.
1320 bool SrcRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getArg(Arg: 0)->getType());
1321 bool DstRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1322 // When the source & result layouts differ, the operand already holds the
1323 // transposed result, ie transpose is a no-op on the underlying vector.
1324 if (SrcRowMajor != DstRowMajor)
1325 return Op0;
1326 // When the source and result share a layout, emit a transpose.
1327 if (SrcRowMajor)
1328 // For row-major operands the dimensions are swapped
1329 return MB.CreateMatrixTranspose(Matrix: Op0, Rows: Cols, Columns: Rows);
1330 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1331 }
1332 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1333 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1334 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1335 llvm_unreachable("rcp operand must have a float representation");
1336 llvm::Type *Ty = Op0->getType();
1337 llvm::Type *EltTy = Ty->getScalarType();
1338 Constant *One = Ty->isVectorTy()
1339 ? ConstantVector::getSplat(
1340 EC: ElementCount::getFixed(
1341 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1342 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1343 : ConstantFP::get(Ty: EltTy, V: 1.0);
1344 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1345 }
1346 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1347 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1348 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1349 llvm_unreachable("rsqrt operand must have a float representation");
1350 return Builder.CreateIntrinsic(
1351 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1352 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1353 }
1354 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1355 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1356 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1357 "saturate operand must have a float representation");
1358 return Builder.CreateIntrinsic(
1359 /*ReturnType=*/RetTy: Op0->getType(),
1360 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1361 FMFSource: nullptr, Name: "hlsl.saturate");
1362 }
1363 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1364 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1365 assert(Op->getType()->isIntegerTy(1) &&
1366 "WavePrefixBitCount operand must be a boolean type");
1367
1368 Intrinsic::ID IID =
1369 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1370
1371 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1372 }
1373 case Builtin::BI__builtin_hlsl_select: {
1374 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1375 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1376 Value *OpTrue =
1377 RValTrue.isScalar()
1378 ? RValTrue.getScalarVal()
1379 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1380 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1381 Value *OpFalse =
1382 RValFalse.isScalar()
1383 ? RValFalse.getScalarVal()
1384 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1385 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1386 if (!OpTrue->getType()->isVectorTy())
1387 OpTrue =
1388 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1389 if (!OpFalse->getType()->isVectorTy())
1390 OpFalse =
1391 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1392 }
1393
1394 Value *SelectVal =
1395 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1396 if (!RValTrue.isScalar())
1397 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1398 IsVolatile: ReturnValue.isVolatile());
1399
1400 return SelectVal;
1401 }
1402 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1403 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1404
1405 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1406 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1407 }
1408 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1409 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1410 assert(Op->getType()->isIntegerTy(1) &&
1411 "Intrinsic WaveActiveAllTrue operand must be a bool");
1412
1413 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1414 return EmitIntrinsicCall(ID, Args: {Op});
1415 }
1416 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1417 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1418 assert(Op->getType()->isIntegerTy(1) &&
1419 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1420
1421 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1422 return EmitIntrinsicCall(ID, Args: {Op});
1423 }
1424 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1425 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1426 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1427 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1428 "representation");
1429
1430 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1431 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1432 Name: "hlsl.wave.active.bit.or");
1433 }
1434 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1435 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1436 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1437 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1438 "representation");
1439
1440 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1441 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1442 Name: "hlsl.wave.active.bit.xor");
1443 }
1444 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1445 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1446 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1447 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1448 "representation");
1449
1450 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1451 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1452 Name: "hlsl.wave.active.bit.and");
1453 }
1454 case Builtin::BI__builtin_hlsl_interlocked_add: {
1455 // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
1456 // up the raw instruction (DXIL routes it via `dx.resource.atomic.binop`
1457 // in DXILResourceAccess for resource pointers, SPIR-V lowers via
1458 // selectAtomicRMW). No intermediate intrinsic.
1459 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Add);
1460 }
1461 case Builtin::BI__builtin_hlsl_interlocked_and: {
1462 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::And);
1463 }
1464 case Builtin::BI__builtin_hlsl_interlocked_min: {
1465 llvm::AtomicRMWInst::BinOp Op =
1466 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1467 ? llvm::AtomicRMWInst::Min
1468 : llvm::AtomicRMWInst::UMin;
1469 return handleInterlockedOp(CGF&: *this, E, Op);
1470 }
1471 case Builtin::BI__builtin_hlsl_interlocked_or: {
1472 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Or);
1473 }
1474 case Builtin::BI__builtin_hlsl_interlocked_xor: {
1475 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xor);
1476 }
1477 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1478 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1479 assert(Op->getType()->isIntegerTy(1) &&
1480 "Intrinsic WaveActiveBallot operand must be a bool");
1481
1482 return handleHlslWaveActiveBallot(CGF&: *this, E);
1483 }
1484 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1485 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1486 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1487 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1488 }
1489 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1490 // Due to the use of variadic arguments, explicitly retrieve argument
1491 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1492 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1493 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1494
1495 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1496 Name: "hlsl.wave.active.sum");
1497 }
1498 case Builtin::BI__builtin_hlsl_wave_active_product: {
1499 // Due to the use of variadic arguments, explicitly retrieve argument
1500 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1501 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1502 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1503
1504 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1505 Name: "hlsl.wave.active.product");
1506 }
1507 case Builtin::BI__builtin_hlsl_wave_active_max: {
1508 // Due to the use of variadic arguments, explicitly retrieve argument
1509 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1510 QualType QT = E->getArg(Arg: 0)->getType();
1511 Intrinsic::ID IID;
1512 if (QT->isUnsignedIntegerType())
1513 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1514 else
1515 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1516
1517 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1518 Name: "hlsl.wave.active.max");
1519 }
1520 case Builtin::BI__builtin_hlsl_wave_active_min: {
1521 // Due to the use of variadic arguments, explicitly retrieve argument
1522 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1523 QualType QT = E->getArg(Arg: 0)->getType();
1524 Intrinsic::ID IID;
1525 if (QT->isUnsignedIntegerType())
1526 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1527 else
1528 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1529
1530 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1531 Name: "hlsl.wave.active.min");
1532 }
1533 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1534 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1535 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1536 // for the DirectX intrinsic and the demangled builtin name
1537 switch (CGM.getTarget().getTriple().getArch()) {
1538 case llvm::Triple::dxil:
1539 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1540 case llvm::Triple::spirv:
1541 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1542 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1543 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1544 default:
1545 llvm_unreachable(
1546 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1547 }
1548 }
1549 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1550 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1551 return EmitIntrinsicCall(ID);
1552 }
1553 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1554 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1555 return EmitIntrinsicCall(ID);
1556 }
1557 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1558 // Due to the use of variadic arguments we must explicitly retrieve them and
1559 // create our function type.
1560 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1561 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1562 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1563 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1564 Name: "hlsl.wave.readlane");
1565 }
1566 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1567 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1568 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1569 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1570 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1571 Name: "hlsl.wave.prefix.sum");
1572 }
1573 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1574 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1575 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1576 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1577 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1578 Name: "hlsl.wave.prefix.product");
1579 }
1580 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1581 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1582 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1583 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1584 Name: "hlsl.quad.read.across.x");
1585 }
1586 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1587 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1588 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1589 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1590 Name: "hlsl.quad.read.across.y");
1591 }
1592 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1593 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1594 Intrinsic::ID ID =
1595 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1596 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1597 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1598 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.diagonal");
1599 }
1600 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1601 auto *Arg0 = E->getArg(Arg: 0);
1602 Value *Op0 = EmitScalarExpr(E: Arg0);
1603 llvm::Type *Xty = Op0->getType();
1604 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1605 if (Xty->isVectorTy()) {
1606 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1607 retType = llvm::VectorType::get(
1608 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1609 }
1610 assert((Arg0->getType()->hasFloatingRepresentation() ||
1611 Arg0->getType()->hasIntegerRepresentation()) &&
1612 "sign operand must have a float or int representation");
1613
1614 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1615 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1616 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1617 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1618 }
1619
1620 return Builder.CreateIntrinsic(
1621 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1622 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1623 }
1624 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1625 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1626 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1627 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1628 return Builder.CreateIntrinsic(
1629 /*ReturnType=*/RetTy: Offset->getType(),
1630 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1631 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1632 }
1633 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1634
1635 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1636 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1637 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1638 "asuint operands types mismatch");
1639 return handleHlslSplitdouble(E, CGF: this);
1640 }
1641 case Builtin::BI__builtin_hlsl_elementwise_clip:
1642 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1643 "clip operands types mismatch");
1644 return handleHlslClip(E, CGF: this);
1645 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1646 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1647 return EmitIntrinsicCall(ID);
1648 }
1649 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1650 Intrinsic::ID ID =
1651 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1652 return EmitIntrinsicCall(ID);
1653 }
1654 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1655 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1656 return EmitIntrinsicCall(ID);
1657 }
1658 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1659 Intrinsic::ID ID =
1660 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1661 return EmitIntrinsicCall(ID);
1662 }
1663 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1664 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1665 return EmitIntrinsicCall(ID);
1666 }
1667 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1668 Intrinsic::ID ID =
1669 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1670 return EmitIntrinsicCall(ID);
1671 }
1672 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1673 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1674 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1675 llvm_unreachable("ddx_coarse operand must have a float representation");
1676 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1677 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1678 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1679 Name: "hlsl.ddx.coarse");
1680 }
1681 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1682 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1683 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1684 llvm_unreachable("ddy_coarse operand must have a float representation");
1685 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1686 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1687 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1688 Name: "hlsl.ddy.coarse");
1689 }
1690 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1691 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1692 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1693 llvm_unreachable("ddx_fine operand must have a float representation");
1694 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1695 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1696 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1697 Name: "hlsl.ddx.fine");
1698 }
1699 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1700 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1701 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1702 llvm_unreachable("ddy_fine operand must have a float representation");
1703 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1704 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1705 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1706 Name: "hlsl.ddy.fine");
1707 }
1708 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1709 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1710 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1711 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1712 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1713 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1714 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1715 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1716 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1717 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1718 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1719 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1720 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1721 llvm::Value *Args[] = {SpecId, DefaultVal};
1722 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1723 }
1724 }
1725 return nullptr;
1726}
1727
1728llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1729 const clang::QualType &SpecConstantType) {
1730
1731 // Find or create the declaration for the function.
1732 llvm::Module *M = &CGM.getModule();
1733 std::string MangledName =
1734 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1735 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1736
1737 if (!SpecConstantFn) {
1738 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1739 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1740 llvm::Type *ArgTypes[] = {IntType, RetTy};
1741 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1742 SpecConstantFn = llvm::Function::Create(
1743 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1744 }
1745 return SpecConstantFn;
1746}
1747