1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit HLSL Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGHLSLRuntime.h"
15#include "CodeGenFunction.h"
16#include "clang/AST/HLSLResource.h"
17#include "clang/AST/MatrixUtils.h"
18#include "llvm/IR/MatrixBuilder.h"
19
20using namespace clang;
21using namespace CodeGen;
22using namespace llvm;
23
24static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
25 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
26 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
27 "asdouble operands types mismatch");
28 Value *OpLowBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
29 Value *OpHighBits = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
30
31 llvm::Type *ResultType = CGF.DoubleTy;
32 int N = 1;
33 if (auto *VTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
34 N = VTy->getNumElements();
35 ResultType = llvm::FixedVectorType::get(ElementType: CGF.DoubleTy, NumElts: N);
36 }
37
38 if (CGF.CGM.getTarget().getTriple().isDXIL())
39 return CGF.Builder.CreateIntrinsic(
40 /*ReturnType=*/RetTy: ResultType, ID: Intrinsic::dx_asdouble,
41 Args: {OpLowBits, OpHighBits}, FMFSource: nullptr, Name: "hlsl.asdouble");
42
43 if (!E->getArg(Arg: 0)->getType()->isVectorType()) {
44 OpLowBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpLowBits);
45 OpHighBits = CGF.Builder.CreateVectorSplat(NumElts: 1, V: OpHighBits);
46 }
47
48 llvm::SmallVector<int> Mask;
49 for (int i = 0; i < N; i++) {
50 Mask.push_back(Elt: i);
51 Mask.push_back(Elt: i + N);
52 }
53
54 Value *BitVec = CGF.Builder.CreateShuffleVector(V1: OpLowBits, V2: OpHighBits, Mask);
55
56 return CGF.Builder.CreateBitCast(V: BitVec, DestTy: ResultType);
57}
58
59static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
60 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
61
62 Constant *FZeroConst = ConstantFP::getZero(Ty: CGF->FloatTy);
63 Value *CMP;
64 Value *LastInstr;
65
66 if (const auto *VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>()) {
67 FZeroConst = ConstantVector::getSplat(
68 EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()), Elt: FZeroConst);
69 auto *FCompInst = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
70 CMP = CGF->Builder.CreateIntrinsic(
71 RetTy: CGF->Builder.getInt1Ty(), ID: CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
72 Args: {FCompInst});
73 } else {
74 CMP = CGF->Builder.CreateFCmpOLT(LHS: Op0, RHS: FZeroConst);
75 }
76
77 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
78 LastInstr = CGF->Builder.CreateIntrinsic(ID: Intrinsic::dx_discard, Args: {CMP});
79 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
80 BasicBlock *LT0 = CGF->createBasicBlock(name: "lt0", parent: CGF->CurFn);
81 BasicBlock *End = CGF->createBasicBlock(name: "end", parent: CGF->CurFn);
82
83 CGF->Builder.CreateCondBr(Cond: CMP, True: LT0, False: End);
84
85 CGF->Builder.SetInsertPoint(LT0);
86
87 CGF->Builder.CreateIntrinsic(ID: Intrinsic::spv_discard, Args: {});
88
89 LastInstr = CGF->Builder.CreateBr(Dest: End);
90 CGF->Builder.SetInsertPoint(End);
91 } else {
92 llvm_unreachable("Backend Codegen not supported.");
93 }
94
95 return LastInstr;
96}
97
98static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
99 Value *Op0 = CGF->EmitScalarExpr(E: E->getArg(Arg: 0));
100 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 1));
101 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(Val: E->getArg(Arg: 2));
102
103 CallArgList Args;
104 LValue Op1TmpLValue =
105 CGF->EmitHLSLOutArgExpr(E: OutArg1, Args, Ty: OutArg1->getType());
106 LValue Op2TmpLValue =
107 CGF->EmitHLSLOutArgExpr(E: OutArg2, Args, Ty: OutArg2->getType());
108
109 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
110 Args.reverseWritebacks();
111
112 Value *LowBits = nullptr;
113 Value *HighBits = nullptr;
114
115 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
116 llvm::Type *RetElementTy = CGF->Int32Ty;
117 if (auto *Op0VecTy = E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
118 RetElementTy = llvm::VectorType::get(
119 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0VecTy->getNumElements()));
120 else if (auto *Op0MatTy =
121 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
122 RetElementTy = llvm::VectorType::get(
123 ElementType: CGF->Int32Ty, EC: ElementCount::getFixed(MinVal: Op0MatTy->getNumRows() *
124 Op0MatTy->getNumColumns()));
125
126 auto *RetTy = llvm::StructType::get(elt1: RetElementTy, elts: RetElementTy);
127
128 Value *CI = CGF->Builder.CreateIntrinsic(
129 RetTy, ID: Intrinsic::dx_splitdouble, Args: {Op0}, FMFSource: nullptr, Name: "hlsl.splitdouble");
130
131 LowBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 0);
132 HighBits = CGF->Builder.CreateExtractValue(Agg: CI, Idxs: 1);
133 } else {
134 // For Non DXIL targets we generate the instructions.
135
136 if (!Op0->getType()->isVectorTy()) {
137 FixedVectorType *DestTy = FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: 2);
138 Value *Bitcast = CGF->Builder.CreateBitCast(V: Op0, DestTy);
139
140 LowBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: (uint64_t)0);
141 HighBits = CGF->Builder.CreateExtractElement(Vec: Bitcast, Idx: 1);
142 } else {
143 int NumElements = 1;
144 if (const auto *VecTy =
145 E->getArg(Arg: 0)->getType()->getAs<clang::VectorType>())
146 NumElements = VecTy->getNumElements();
147 else if (const auto *MatTy =
148 E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>())
149 NumElements = MatTy->getNumRows() * MatTy->getNumColumns();
150
151 FixedVectorType *Uint32VecTy =
152 FixedVectorType::get(ElementType: CGF->Int32Ty, NumElts: NumElements * 2);
153 Value *Uint32Vec = CGF->Builder.CreateBitCast(V: Op0, DestTy: Uint32VecTy);
154 if (NumElements == 1) {
155 LowBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: (uint64_t)0);
156 HighBits = CGF->Builder.CreateExtractElement(Vec: Uint32Vec, Idx: 1);
157 } else {
158 SmallVector<int> EvenMask, OddMask;
159 for (int I = 0, E = NumElements; I != E; ++I) {
160 EvenMask.push_back(Elt: I * 2);
161 OddMask.push_back(Elt: I * 2 + 1);
162 }
163 LowBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: EvenMask);
164 HighBits = CGF->Builder.CreateShuffleVector(V: Uint32Vec, Mask: OddMask);
165 }
166 }
167 }
168 CGF->Builder.CreateStore(Val: LowBits, Addr: Op1TmpLValue.getAddress());
169 auto *LastInst =
170 CGF->Builder.CreateStore(Val: HighBits, Addr: Op2TmpLValue.getAddress());
171 CGF->EmitWritebacks(Args);
172 return LastInst;
173}
174
175static Value *handleHlslWaveActiveBallot(CodeGenFunction &CGF,
176 const CallExpr *E) {
177 Value *Cond = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
178 llvm::Type *I32 = CGF.Int32Ty;
179
180 llvm::Type *Vec4I32 = llvm::FixedVectorType::get(ElementType: I32, NumElts: 4);
181 [[maybe_unused]] llvm::StructType *Struct4I32 =
182 llvm::StructType::get(Context&: CGF.getLLVMContext(), Elements: {I32, I32, I32, I32});
183
184 if (CGF.CGM.getTarget().getTriple().isDXIL()) {
185 // Call DXIL intrinsic: returns { i32, i32, i32, i32 }
186 Value *StructVal =
187 CGF.EmitIntrinsicCall(ID: Intrinsic::dx_wave_ballot, Types: {I32}, Args: {Cond});
188 assert(StructVal->getType() == Struct4I32 &&
189 "dx.wave.ballot must return {i32,i32,i32,i32}");
190
191 // Reassemble struct to <4 x i32>
192 llvm::Value *VecVal = llvm::PoisonValue::get(T: Vec4I32);
193 for (unsigned I = 0; I < 4; ++I) {
194 Value *Elt = CGF.Builder.CreateExtractValue(Agg: StructVal, Idxs: I);
195 VecVal =
196 CGF.Builder.CreateInsertElement(Vec: VecVal, NewElt: Elt, Idx: CGF.Builder.getInt32(C: I));
197 }
198
199 return VecVal;
200 }
201
202 if (CGF.CGM.getTarget().getTriple().isSPIRV())
203 return CGF.EmitIntrinsicCall(ID: Intrinsic::spv_subgroup_ballot, Args: {Cond});
204
205 llvm_unreachable(
206 "WaveActiveBallot is only supported for DXIL and SPIRV targets");
207}
208
209static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,
210 const CallExpr *E) {
211 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
212 QualType Op0Ty = E->getArg(Arg: 0)->getType();
213 llvm::Type *ResType = CGF.FloatTy;
214 uint64_t NumElements = 0;
215 if (Op0->getType()->isVectorTy()) {
216 NumElements =
217 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
218 ResType =
219 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
220 }
221 if (!Op0Ty->hasUnsignedIntegerRepresentation())
222 llvm_unreachable(
223 "f16tof32 operand must have an unsigned int representation");
224
225 if (CGF.CGM.getTriple().isDXIL())
226 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf16tof32,
227 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
228 Name: "hlsl.f16tof32");
229
230 if (CGF.CGM.getTriple().isSPIRV()) {
231 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the
232 // Int16 and Float16 capabilities
233 auto *UnpackType =
234 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
235
236 if (NumElements == 0) {
237 // a scalar input - simply extract the first element of the unpacked
238 // vector
239 Value *Unpack = CGF.Builder.CreateIntrinsic(
240 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{Op0});
241 return CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
242 }
243
244 // a vector input - build a congruent output vector by iterating through
245 // the input vector calling unpackhalf2x16 for each element
246 Value *Result = PoisonValue::get(T: ResType);
247 for (uint64_t I = 0; I < NumElements; I++) {
248 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
249 Value *Unpack = CGF.Builder.CreateIntrinsic(
250 RetTy: UnpackType, ID: Intrinsic::spv_unpackhalf2x16, Args: ArrayRef<Value *>{InVal});
251 Value *Res = CGF.Builder.CreateExtractElement(Vec: Unpack, Idx: (uint64_t)0);
252 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
253 }
254 return Result;
255 }
256
257 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");
258}
259
260static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
261 const CallExpr *E) {
262 Value *Op0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
263 QualType Op0Ty = E->getArg(Arg: 0)->getType();
264 llvm::Type *ResType = CGF.IntTy;
265 uint64_t NumElements = 0;
266 if (Op0->getType()->isVectorTy()) {
267 NumElements =
268 E->getArg(Arg: 0)->getType()->castAs<clang::VectorType>()->getNumElements();
269 ResType =
270 llvm::VectorType::get(ElementType: ResType, EC: ElementCount::getFixed(MinVal: NumElements));
271 }
272 if (!Op0Ty->hasFloatingRepresentation())
273 llvm_unreachable("f32tof16 operand must have a float representation");
274
275 if (CGF.CGM.getTriple().isDXIL())
276 return CGF.Builder.CreateIntrinsic(RetTy: ResType, ID: Intrinsic::dx_legacyf32tof16,
277 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
278 Name: "hlsl.f32tof16");
279
280 if (CGF.CGM.getTriple().isSPIRV()) {
281 // We use the SPIRV PackHalf2x16 operation to avoid the need for the
282 // Int16 and Float16 capabilities
283 auto *PackType =
284 llvm::VectorType::get(ElementType: CGF.FloatTy, EC: ElementCount::getFixed(MinVal: 2));
285
286 if (NumElements == 0) {
287 // a scalar input - simply insert the scalar in the first element
288 // of the 2 element float vector
289 Value *Float2 = Constant::getNullValue(Ty: PackType);
290 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: Op0, Idx: (uint64_t)0);
291 Value *Result = CGF.Builder.CreateIntrinsic(
292 RetTy: ResType, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
293 return Result;
294 }
295
296 // a vector input - build a congruent output vector by iterating through
297 // the input vector calling packhalf2x16 for each element
298 Value *Result = PoisonValue::get(T: ResType);
299 for (uint64_t I = 0; I < NumElements; I++) {
300 Value *Float2 = Constant::getNullValue(Ty: PackType);
301 Value *InVal = CGF.Builder.CreateExtractElement(Vec: Op0, Idx: I);
302 Float2 = CGF.Builder.CreateInsertElement(Vec: Float2, NewElt: InVal, Idx: (uint64_t)0);
303 Value *Res = CGF.Builder.CreateIntrinsic(
304 RetTy: CGF.IntTy, ID: Intrinsic::spv_packhalf2x16, Args: ArrayRef<Value *>{Float2});
305 Result = CGF.Builder.CreateInsertElement(Vec: Result, NewElt: Res, Idx: I);
306 }
307 return Result;
308 }
309
310 llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
311}
312
313static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
314 llvm::AtomicRMWInst::BinOp Op) {
315 // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
316 // either DXIL or SPIR-V.
317 LValue DestLV = CGF.EmitLValue(E: E->getArg(Arg: 0));
318 Address DestAddr = DestLV.getAddress();
319 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
320 assert(E->getArg(1)->getType()->isIntegerType() &&
321 "Intrinsic InterlockedOp value operand must be an integer");
322
323 // Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
324 // Set the memory scope: Workgroup for groupshared, otherwise Device.
325 StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared
326 ? "workgroup"
327 : "device";
328 llvm::SyncScope::ID SSID =
329 CGF.getLLVMContext().getOrInsertSyncScopeID(SSN: ScopeName);
330
331 llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
332 Op, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
333
334 // The 3-arg overload writes the old value (the RMW's return value) into
335 // the `original_value` reference parameter.
336 if (E->getNumArgs() == 3) {
337 LValue OrigLV = CGF.EmitLValue(E: E->getArg(Arg: 2));
338 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Call), Dst: OrigLV);
339 }
340 return Call;
341}
342
343static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
344 LValue &Stride) {
345 // Figure out the stride of the buffer elements from the handle type.
346 auto *HandleTy =
347 cast<HLSLAttributedResourceType>(Val: HandleExpr->getType().getTypePtr());
348 QualType ElementTy = HandleTy->getContainedType();
349 Value *StrideValue = CGF->getTypeSize(Ty: ElementTy);
350 return CGF->Builder.CreateStore(Val: StrideValue, Addr: Stride.getAddress());
351}
352
353// Return dot product intrinsic that corresponds to the QT scalar type
354static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
355 if (QT->isFloatingType())
356 return RT.getFDotIntrinsic();
357 if (QT->isSignedIntegerType())
358 return RT.getSDotIntrinsic();
359 assert(QT->isUnsignedIntegerType());
360 return RT.getUDotIntrinsic();
361}
362
363static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
364 if (QT->hasSignedIntegerRepresentation()) {
365 return RT.getFirstBitSHighIntrinsic();
366 }
367
368 assert(QT->hasUnsignedIntegerRepresentation());
369 return RT.getFirstBitUHighIntrinsic();
370}
371
372// Return wave active sum that corresponds to the QT scalar type
373static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
374 QualType QT) {
375 switch (Arch) {
376 case llvm::Triple::spirv:
377 return Intrinsic::spv_wave_reduce_sum;
378 case llvm::Triple::dxil: {
379 if (QT->isUnsignedIntegerType())
380 return Intrinsic::dx_wave_reduce_usum;
381 return Intrinsic::dx_wave_reduce_sum;
382 }
383 default:
384 llvm_unreachable("Intrinsic WaveActiveSum"
385 " not supported by target architecture");
386 }
387}
388
389// Return wave active product that corresponds to the QT scalar type
390static Intrinsic::ID getWaveActiveProductIntrinsic(llvm::Triple::ArchType Arch,
391 QualType QT) {
392 switch (Arch) {
393 case llvm::Triple::spirv:
394 return Intrinsic::spv_wave_product;
395 case llvm::Triple::dxil: {
396 if (QT->isUnsignedIntegerType())
397 return Intrinsic::dx_wave_uproduct;
398 return Intrinsic::dx_wave_product;
399 }
400 default:
401 llvm_unreachable("Intrinsic WaveActiveProduct"
402 " not supported by target architecture");
403 }
404}
405
406static Intrinsic::ID getPrefixCountBitsIntrinsic(llvm::Triple::ArchType Arch) {
407 switch (Arch) {
408 case llvm::Triple::spirv:
409 return Intrinsic::spv_subgroup_prefix_bit_count;
410 case llvm::Triple::dxil: {
411 return Intrinsic::dx_wave_prefix_bit_count;
412 }
413 default:
414 llvm_unreachable(
415 "WavePrefixOp instruction not supported by target architecture");
416 }
417}
418
419// Return wave prefix sum that corresponds to the QT scalar type
420static Intrinsic::ID getWavePrefixSumIntrinsic(llvm::Triple::ArchType Arch,
421 QualType QT) {
422 switch (Arch) {
423 case llvm::Triple::spirv:
424 return Intrinsic::spv_wave_prefix_sum;
425 case llvm::Triple::dxil: {
426 if (QT->isUnsignedIntegerType())
427 return Intrinsic::dx_wave_prefix_usum;
428 return Intrinsic::dx_wave_prefix_sum;
429 }
430 default:
431 llvm_unreachable("Intrinsic WavePrefixSum"
432 " not supported by target architecture");
433 }
434}
435
436// Return wave prefix product that corresponds to the QT scalar type
437static Intrinsic::ID getWavePrefixProductIntrinsic(llvm::Triple::ArchType Arch,
438 QualType QT) {
439 switch (Arch) {
440 case llvm::Triple::spirv:
441 return Intrinsic::spv_wave_prefix_product;
442 case llvm::Triple::dxil: {
443 if (QT->isUnsignedIntegerType())
444 return Intrinsic::dx_wave_prefix_uproduct;
445 return Intrinsic::dx_wave_prefix_product;
446 }
447 default:
448 llvm_unreachable("Intrinsic WavePrefixProduct"
449 " not supported by target architecture");
450 }
451}
452
453// Returns the mangled name for a builtin function that the SPIR-V backend
454// will expand into a spec Constant.
455static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
456 ASTContext &Context) {
457 // The parameter types for our conceptual intrinsic function.
458 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
459
460 // Create a temporary FunctionDecl for the builtin fuction. It won't be
461 // added to the AST.
462 FunctionProtoType::ExtProtoInfo EPI;
463 QualType FnType =
464 Context.getFunctionType(ResultTy: SpecConstantType, Args: ClangParamTypes, EPI);
465 DeclarationName FuncName = &Context.Idents.get(Name: "__spirv_SpecConstant");
466 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
467 C&: Context, DC: Context.getTranslationUnitDecl(), StartLoc: SourceLocation(),
468 NLoc: SourceLocation(), N: FuncName, T: FnType, /*TSI=*/TInfo: nullptr, SC: SC_Extern);
469
470 // Attach the created parameter declarations to the function declaration.
471 SmallVector<ParmVarDecl *, 2> ParamDecls;
472 for (QualType ParamType : ClangParamTypes) {
473 ParmVarDecl *PD = ParmVarDecl::Create(
474 C&: Context, DC: FnDeclForMangling, StartLoc: SourceLocation(), IdLoc: SourceLocation(),
475 /*IdentifierInfo*/ Id: nullptr, T: ParamType, /*TSI*/ TInfo: nullptr, S: SC_None,
476 /*DefaultArg*/ DefArg: nullptr);
477 ParamDecls.push_back(Elt: PD);
478 }
479 FnDeclForMangling->setParams(ParamDecls);
480
481 // Get the mangled name.
482 std::string Name;
483 llvm::raw_string_ostream MangledNameStream(Name);
484 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
485 Mangler->mangleName(GD: FnDeclForMangling, MangledNameStream);
486 MangledNameStream.flush();
487
488 return Name;
489}
490
491static const HLSLAttributedResourceType *
492getHandleAttributedType(QualType HandleQT) {
493 if (const auto *RT = HandleQT->getAs<HLSLAttributedResourceType>())
494 return RT;
495 // If the expr is a texture/sampler record (or similar), peel to __handle.
496 if (const HLSLAttributedResourceType *RT =
497 HLSLAttributedResourceType::findHandleTypeOnResource(
498 RT: HandleQT.getTypePtr()))
499 return RT;
500 llvm_unreachable("attributed handle type not found");
501}
502
503static const HLSLAttributedResourceType *
504getRequiredHandleType(const CallExpr *E, unsigned ArgNo) {
505 return getHandleAttributedType(HandleQT: E->getArg(Arg: ArgNo)->getType());
506}
507
508static llvm::Type *getOffsetType(CodeGenModule &CGM,
509 const HLSLAttributedResourceType *RT) {
510 const auto &Attrs = RT->getAttrs();
511 unsigned OffsetSize =
512 clang::hlsl::getResourceDimensions(Dim: Attrs.ResourceDimension);
513 llvm::Type *Int32Ty = CGM.Int32Ty;
514 if (OffsetSize == 1)
515 return Int32Ty;
516 return llvm::FixedVectorType::get(ElementType: Int32Ty, NumElts: OffsetSize);
517}
518
519static Value *emitHlslOffset(CodeGenFunction &CGF, const CallExpr *E,
520 unsigned OffsetArgIndex, llvm::Type *OffsetTy) {
521 if (E->getNumArgs() > OffsetArgIndex)
522 return CGF.EmitScalarExpr(E: E->getArg(Arg: OffsetArgIndex));
523
524 return llvm::Constant::getNullValue(Ty: OffsetTy);
525}
526
527static Value *emitHlslClamp(CodeGenFunction &CGF, const CallExpr *E,
528 unsigned ClampArgIndex) {
529 Value *Clamp = CGF.EmitScalarExpr(E: E->getArg(Arg: ClampArgIndex));
530 // The builtin is defined with variadic arguments, so the clamp parameter
531 // might have been promoted to double. The intrinsic requires a 32-bit
532 // float.
533 if (Clamp->getType() != CGF.Builder.getFloatTy())
534 Clamp = CGF.Builder.CreateFPCast(V: Clamp, DestTy: CGF.Builder.getFloatTy());
535 return Clamp;
536}
537
538static Value *emitGetDimensions(CodeGenFunction &CGF, const CallExpr *E,
539 unsigned IntrinsicID, unsigned NumRetComps,
540 bool HasLod) {
541 Value *Handle = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
542
543 SmallVector<Value *> Args{Handle};
544 if (HasLod)
545 Args.push_back(Elt: CGF.EmitScalarExpr(E: E->getArg(Arg: 1)));
546
547 Value *DimValue =
548 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Handle->getType()}, Args);
549
550 Value *LastStore = nullptr;
551 unsigned ArgIndex = HasLod ? 2 : 1;
552 for (unsigned i = 0; i < NumRetComps; ++i) {
553 const Expr *Arg = E->getArg(Arg: ArgIndex++);
554 LValue DimOut = CGF.EmitLValue(E: Arg);
555 Value *Elem = DimValue;
556 if (NumRetComps > 1)
557 Elem = CGF.Builder.CreateExtractElement(Vec: DimValue, Idx: i);
558
559 // Handle float casting if needed
560 if (Arg->getType()->isFloatingType())
561 Elem = CGF.Builder.CreateUIToFP(
562 V: Elem, DestTy: llvm::Type::getFloatTy(C&: CGF.getLLVMContext()));
563
564 LastStore = CGF.Builder.CreateStore(Val: Elem, Addr: DimOut.getAddress());
565 }
566 return LastStore;
567}
568
569static llvm::Type *getAggregateType(llvm::Type *ScalarTy, QualType ArgTy) {
570 if (auto *MatTy = ArgTy->getAs<ConstantMatrixType>())
571 return llvm::VectorType::get(
572 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: MatTy->getNumElementsFlattened()));
573 if (auto *VecTy = ArgTy->getAs<clang::VectorType>())
574 return llvm::VectorType::get(
575 ElementType: ScalarTy, EC: ElementCount::getFixed(MinVal: VecTy->getNumElements()));
576 return ScalarTy;
577}
578
579Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
580 const CallExpr *E,
581 ReturnValueSlot ReturnValue) {
582 if (!getLangOpts().HLSL)
583 return nullptr;
584
585 switch (BuiltinID) {
586 case Builtin::BI__builtin_hlsl_adduint64: {
587 Value *OpA = EmitScalarExpr(E: E->getArg(Arg: 0));
588 Value *OpB = EmitScalarExpr(E: E->getArg(Arg: 1));
589 QualType Arg0Ty = E->getArg(Arg: 0)->getType();
590 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
591 assert(Arg0Ty == E->getArg(1)->getType() &&
592 "AddUint64 operand types must match");
593 assert(Arg0Ty->hasIntegerRepresentation() &&
594 "AddUint64 operands must have an integer representation");
595 assert((NumElements == 2 || NumElements == 4) &&
596 "AddUint64 operands must have 2 or 4 elements");
597
598 llvm::Value *LowA;
599 llvm::Value *HighA;
600 llvm::Value *LowB;
601 llvm::Value *HighB;
602
603 // Obtain low and high words of inputs A and B
604 if (NumElements == 2) {
605 LowA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)0, Name: "LowA");
606 HighA = Builder.CreateExtractElement(Vec: OpA, Idx: (uint64_t)1, Name: "HighA");
607 LowB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)0, Name: "LowB");
608 HighB = Builder.CreateExtractElement(Vec: OpB, Idx: (uint64_t)1, Name: "HighB");
609 } else {
610 LowA = Builder.CreateShuffleVector(V: OpA, Mask: {0, 2}, Name: "LowA");
611 HighA = Builder.CreateShuffleVector(V: OpA, Mask: {1, 3}, Name: "HighA");
612 LowB = Builder.CreateShuffleVector(V: OpB, Mask: {0, 2}, Name: "LowB");
613 HighB = Builder.CreateShuffleVector(V: OpB, Mask: {1, 3}, Name: "HighB");
614 }
615
616 // Use an uadd_with_overflow to compute the sum of low words and obtain a
617 // carry value
618 llvm::Value *Carry;
619 llvm::Value *LowSum = EmitOverflowIntrinsic(
620 CGF&: *this, IntrinsicID: Intrinsic::uadd_with_overflow, X: LowA, Y: LowB, Carry);
621 llvm::Value *ZExtCarry =
622 Builder.CreateZExt(V: Carry, DestTy: HighA->getType(), Name: "CarryZExt");
623
624 // Sum the high words and the carry
625 llvm::Value *HighSum = Builder.CreateAdd(LHS: HighA, RHS: HighB, Name: "HighSum");
626 llvm::Value *HighSumPlusCarry =
627 Builder.CreateAdd(LHS: HighSum, RHS: ZExtCarry, Name: "HighSumPlusCarry");
628
629 if (NumElements == 4) {
630 return Builder.CreateShuffleVector(V1: LowSum, V2: HighSumPlusCarry, Mask: {0, 2, 1, 3},
631 Name: "hlsl.AddUint64");
632 }
633
634 llvm::Value *Result = PoisonValue::get(T: OpA->getType());
635 Result = Builder.CreateInsertElement(Vec: Result, NewElt: LowSum, Idx: (uint64_t)0,
636 Name: "hlsl.AddUint64.upto0");
637 Result = Builder.CreateInsertElement(Vec: Result, NewElt: HighSumPlusCarry, Idx: (uint64_t)1,
638 Name: "hlsl.AddUint64");
639 return Result;
640 }
641 case Builtin::BI__builtin_hlsl_resource_getpointer:
642 case Builtin::BI__builtin_hlsl_resource_getpointer_typed: {
643 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
644 bool IsIndexed =
645 BuiltinID == Builtin::BI__builtin_hlsl_resource_getpointer_typed ||
646 E->getNumArgs() > 1;
647
648 llvm::Type *RetTy = ConvertType(T: E->getType());
649 llvm::Function *IntrFn = nullptr;
650 llvm::CallInst *CI = nullptr;
651 if (IsIndexed) {
652 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
653 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
654 M: &CGM.getModule(),
655 id: CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
656 OverloadTys: {RetTy, HandleOp->getType(), IndexOp->getType()});
657 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp, IndexOp});
658 } else {
659 IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
660 M: &CGM.getModule(),
661 id: CGM.getHLSLRuntime().getCreateResourceGetBasePointerIntrinsic(),
662 OverloadTys: {RetTy, HandleOp->getType()});
663 CI = EmitRuntimeCall(callee: IntrFn, args: {HandleOp});
664 }
665 CI->setCallingConv(IntrFn->getCallingConv());
666 return CI;
667 }
668 case Builtin::BI__builtin_hlsl_transpose_if_memory_is_row_major: {
669 const Expr *ValueExpr = E->getArg(Arg: 0);
670 if (hasAggregateEvaluationKind(T: ValueExpr->getType())) {
671 EmitAnyExprToMem(E: ValueExpr, Location: ReturnValue.getAddress(),
672 Quals: ValueExpr->getType().getQualifiers(), /*IsInit=*/IsInitializer: true);
673 return ReturnValue.getAddress().getBasePointer();
674 }
675
676 Value *ValueOp = EmitScalarExpr(E: ValueExpr);
677 const auto *MatTy = ValueExpr->getType()->getAs<ConstantMatrixType>();
678 if (!MatTy || !getLangOpts().HLSLSpvUseLegacyBufferMatrixOrder)
679 return ValueOp;
680
681 bool IsLoad =
682 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getBoolValue();
683 unsigned Rows = MatTy->getNumRows();
684 unsigned Columns = MatTy->getNumColumns();
685 llvm::MatrixBuilder MB(Builder);
686 return IsLoad ? MB.CreateMatrixTranspose(Matrix: ValueOp, Rows: Columns, Columns: Rows)
687 : MB.CreateMatrixTranspose(Matrix: ValueOp, Rows, Columns);
688 }
689 case Builtin::BI__builtin_hlsl_resource_sample: {
690 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
691 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
692 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
693 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
694
695 SmallVector<Value *, 4> Args;
696 Args.push_back(Elt: HandleOp);
697 Args.push_back(Elt: SamplerOp);
698 Args.push_back(Elt: CoordOp);
699 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
700
701 llvm::Type *RetTy = ConvertType(T: E->getType());
702 if (E->getNumArgs() <= 4) {
703 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleIntrinsic(), Args,
704 RetTy);
705 }
706
707 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 4));
708 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleClampIntrinsic(),
709 Args, RetTy);
710 }
711 case Builtin::BI__builtin_hlsl_resource_sample_bias: {
712 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
713 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
714 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
715 Value *BiasOp = EmitScalarExpr(E: E->getArg(Arg: 3));
716 if (BiasOp->getType() != Builder.getFloatTy())
717 BiasOp = Builder.CreateFPCast(V: BiasOp, DestTy: Builder.getFloatTy());
718 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
719
720 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleBias
721 Args.push_back(Elt: HandleOp);
722 Args.push_back(Elt: SamplerOp);
723 Args.push_back(Elt: CoordOp);
724 Args.push_back(Elt: BiasOp);
725 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
726
727 llvm::Type *RetTy = ConvertType(T: E->getType());
728 if (E->getNumArgs() <= 5) {
729 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasIntrinsic(),
730 Args, RetTy);
731 }
732
733 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
734 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getSampleBiasClampIntrinsic(),
735 Args, RetTy);
736 }
737 case Builtin::BI__builtin_hlsl_resource_sample_grad: {
738 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
739 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
740 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
741 Value *DDXOp = EmitScalarExpr(E: E->getArg(Arg: 3));
742 Value *DDYOp = EmitScalarExpr(E: E->getArg(Arg: 4));
743 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
744
745 SmallVector<Value *, 7> Args;
746 Args.push_back(Elt: HandleOp);
747 Args.push_back(Elt: SamplerOp);
748 Args.push_back(Elt: CoordOp);
749 Args.push_back(Elt: DDXOp);
750 Args.push_back(Elt: DDYOp);
751 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
752
753 llvm::Type *RetTy = ConvertType(T: E->getType());
754
755 if (E->getNumArgs() <= 6) {
756 return Builder.CreateIntrinsic(
757 RetTy, ID: CGM.getHLSLRuntime().getSampleGradIntrinsic(), Args);
758 }
759
760 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 6));
761 return Builder.CreateIntrinsic(
762 RetTy, ID: CGM.getHLSLRuntime().getSampleGradClampIntrinsic(), Args);
763 }
764 case Builtin::BI__builtin_hlsl_resource_sample_level: {
765 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
766 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
767 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
768 Value *LODOp = EmitScalarExpr(E: E->getArg(Arg: 3));
769 if (LODOp->getType() != Builder.getFloatTy())
770 LODOp = Builder.CreateFPCast(V: LODOp, DestTy: Builder.getFloatTy());
771 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
772
773 SmallVector<Value *, 5> Args; // Max 5 arguments for SampleLevel
774 Args.push_back(Elt: HandleOp);
775 Args.push_back(Elt: SamplerOp);
776 Args.push_back(Elt: CoordOp);
777 Args.push_back(Elt: LODOp);
778 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
779
780 llvm::Type *RetTy = ConvertType(T: E->getType());
781 return Builder.CreateIntrinsic(
782 RetTy, ID: CGM.getHLSLRuntime().getSampleLevelIntrinsic(), Args);
783 }
784 case Builtin::BI__builtin_hlsl_resource_load_level: {
785 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
786 Value *CoordLODOp = EmitScalarExpr(E: E->getArg(Arg: 1));
787
788 auto *CoordLODVecTy = cast<llvm::FixedVectorType>(Val: CoordLODOp->getType());
789 unsigned NumElts = CoordLODVecTy->getNumElements();
790 assert(NumElts >= 2 && "CoordLOD must have at least 2 elements");
791
792 // Split CoordLOD into Coord and LOD
793 SmallVector<int, 4> Mask;
794 for (unsigned I = 0; I < NumElts - 1; ++I)
795 Mask.push_back(Elt: I);
796
797 Value *CoordOp =
798 Builder.CreateShuffleVector(V: CoordLODOp, Mask, Name: "hlsl.load.coord");
799 Value *LODOp =
800 Builder.CreateExtractElement(Vec: CoordLODOp, Idx: NumElts - 1, Name: "hlsl.load.lod");
801 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
802
803 SmallVector<Value *, 4> Args;
804 Args.push_back(Elt: HandleOp);
805 Args.push_back(Elt: CoordOp);
806 Args.push_back(Elt: LODOp);
807 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 2, OffsetTy: getOffsetType(CGM, RT)));
808
809 llvm::Type *RetTy = ConvertType(T: E->getType());
810 return Builder.CreateIntrinsic(
811 RetTy, ID: CGM.getHLSLRuntime().getLoadLevelIntrinsic(), Args);
812 }
813 case Builtin::BI__builtin_hlsl_resource_load_ms: {
814 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
815 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 1));
816 Value *SampleOp = EmitScalarExpr(E: E->getArg(Arg: 2));
817 if (SampleOp->getType() != Builder.getInt32Ty())
818 SampleOp = Builder.CreateIntCast(V: SampleOp, DestTy: Builder.getInt32Ty(),
819 /*isSigned=*/true);
820 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
821
822 SmallVector<Value *, 4> Args;
823 Args.push_back(Elt: HandleOp);
824 Args.push_back(Elt: CoordOp);
825 Args.push_back(Elt: SampleOp);
826 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 3, OffsetTy: getOffsetType(CGM, RT)));
827
828 llvm::Type *RetTy = ConvertType(T: E->getType());
829 return Builder.CreateIntrinsic(
830 RetTy, ID: CGM.getHLSLRuntime().getLoadMSIntrinsic(), Args);
831 }
832 case Builtin::BI__builtin_hlsl_resource_sample_cmp: {
833 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
834 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
835 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
836 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
837 if (CmpOp->getType() != Builder.getFloatTy())
838 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
839 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
840
841 SmallVector<Value *, 6> Args; // Max 6 arguments for SampleCmp
842 Args.push_back(Elt: HandleOp);
843 Args.push_back(Elt: SamplerOp);
844 Args.push_back(Elt: CoordOp);
845 Args.push_back(Elt: CmpOp);
846 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
847
848 llvm::Type *RetTy = ConvertType(T: E->getType());
849 if (E->getNumArgs() <= 5) {
850 return Builder.CreateIntrinsic(
851 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpIntrinsic(), Args);
852 }
853
854 Args.push_back(Elt: emitHlslClamp(CGF&: *this, E, ClampArgIndex: 5));
855 return Builder.CreateIntrinsic(
856 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpClampIntrinsic(), Args);
857 }
858 case Builtin::BI__builtin_hlsl_resource_sample_cmp_level_zero: {
859 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
860 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
861 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
862 Value *CmpOp = EmitScalarExpr(E: E->getArg(Arg: 3));
863 if (CmpOp->getType() != Builder.getFloatTy())
864 CmpOp = Builder.CreateFPCast(V: CmpOp, DestTy: Builder.getFloatTy());
865 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
866
867 SmallVector<Value *, 5> Args;
868 Args.push_back(Elt: HandleOp);
869 Args.push_back(Elt: SamplerOp);
870 Args.push_back(Elt: CoordOp);
871 Args.push_back(Elt: CmpOp);
872 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
873
874 llvm::Type *RetTy = ConvertType(T: E->getType());
875 return Builder.CreateIntrinsic(
876 RetTy, ID: CGM.getHLSLRuntime().getSampleCmpLevelZeroIntrinsic(), Args);
877 }
878 case Builtin::BI__builtin_hlsl_resource_calculate_lod: {
879 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
880 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
881 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
882
883 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getCalculateLodIntrinsic(),
884 Args: {HandleOp, SamplerOp, CoordOp},
885 RetTy: ConvertType(T: E->getType()));
886 }
887 case Builtin::BI__builtin_hlsl_resource_calculate_lod_unclamped: {
888 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
889 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
890 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
891
892 return EmitIntrinsicCall(
893 ID: CGM.getHLSLRuntime().getCalculateLodUnclampedIntrinsic(),
894 Args: {HandleOp, SamplerOp, CoordOp}, RetTy: ConvertType(T: E->getType()));
895 }
896 case Builtin::BI__builtin_hlsl_resource_gather: {
897 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
898 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
899 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
900 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 3));
901 if (ComponentOp->getType() != Builder.getInt32Ty())
902 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
903 /*isSigned=*/false);
904 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
905
906 SmallVector<Value *, 5> Args;
907 Args.push_back(Elt: HandleOp);
908 Args.push_back(Elt: SamplerOp);
909 Args.push_back(Elt: CoordOp);
910 Args.push_back(Elt: ComponentOp);
911 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 4, OffsetTy: getOffsetType(CGM, RT)));
912
913 llvm::Type *RetTy = ConvertType(T: E->getType());
914 return Builder.CreateIntrinsic(
915 RetTy, ID: CGM.getHLSLRuntime().getGatherIntrinsic(), Args);
916 }
917 case Builtin::BI__builtin_hlsl_resource_gather_cmp: {
918 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
919 Value *SamplerOp = EmitScalarExpr(E: E->getArg(Arg: 1));
920 Value *CoordOp = EmitScalarExpr(E: E->getArg(Arg: 2));
921 Value *CompareOp = EmitScalarExpr(E: E->getArg(Arg: 3));
922 if (CompareOp->getType() != Builder.getFloatTy())
923 CompareOp = Builder.CreateFPCast(V: CompareOp, DestTy: Builder.getFloatTy());
924
925 SmallVector<Value *, 6> Args;
926 Args.push_back(Elt: HandleOp);
927 Args.push_back(Elt: SamplerOp);
928 Args.push_back(Elt: CoordOp);
929 Args.push_back(Elt: CompareOp);
930
931 if (CGM.getTarget().getTriple().isDXIL()) {
932 Value *ComponentOp = EmitScalarExpr(E: E->getArg(Arg: 4));
933 if (ComponentOp->getType() != Builder.getInt32Ty())
934 ComponentOp = Builder.CreateIntCast(V: ComponentOp, DestTy: Builder.getInt32Ty(),
935 /*isSigned=*/false);
936 Args.push_back(Elt: ComponentOp);
937 }
938
939 const HLSLAttributedResourceType *RT = getRequiredHandleType(E, ArgNo: 0);
940 Args.push_back(Elt: emitHlslOffset(CGF&: *this, E, OffsetArgIndex: 5, OffsetTy: getOffsetType(CGM, RT)));
941
942 llvm::Type *RetTy = ConvertType(T: E->getType());
943 return Builder.CreateIntrinsic(
944 RetTy, ID: CGM.getHLSLRuntime().getGatherCmpIntrinsic(), Args);
945 }
946 case Builtin::BI__builtin_hlsl_resource_load_with_status:
947 case Builtin::BI__builtin_hlsl_resource_load_with_status_typed: {
948 Value *HandleOp = EmitScalarExpr(E: E->getArg(Arg: 0));
949 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 1));
950
951 // Get the *address* of the status argument to write to it by reference
952 LValue StatusLVal = EmitLValue(E: E->getArg(Arg: 2));
953 Address StatusAddr = StatusLVal.getAddress();
954
955 QualType HandleTy = E->getArg(Arg: 0)->getType();
956 const HLSLAttributedResourceType *RT =
957 HandleTy->getAs<HLSLAttributedResourceType>();
958 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&
959 "Only DXIL currently implements load with status");
960
961 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer
962 ? llvm::Intrinsic::dx_resource_load_rawbuffer
963 : llvm::Intrinsic::dx_resource_load_typedbuffer;
964
965 llvm::Type *DataTy = ConvertType(T: E->getType());
966 llvm::Type *RetTy = llvm::StructType::get(Context&: Builder.getContext(),
967 Elements: {DataTy, Builder.getInt1Ty()});
968
969 SmallVector<Value *, 3> Args;
970 Args.push_back(Elt: HandleOp);
971 Args.push_back(Elt: IndexOp);
972
973 if (RT->isRaw()) {
974 Value *Offset = Builder.getInt32(C: 0);
975 // The offset parameter needs to be poison for ByteAddressBuffer
976 if (!RT->isStructured())
977 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
978 Args.push_back(Elt: Offset);
979 }
980
981 // The load intrinsics give us a (T value, i1 status) pair -
982 // shepherd these into the return value and out reference respectively.
983 Value *ResRet =
984 Builder.CreateIntrinsic(RetTy, ID: IntrID, Args, FMFSource: {}, Name: "ld.struct");
985 Value *LoadedValue = Builder.CreateExtractValue(Agg: ResRet, Idxs: {0}, Name: "ld.value");
986 Value *StatusBit = Builder.CreateExtractValue(Agg: ResRet, Idxs: {1}, Name: "ld.status");
987 Value *ExtendedStatus =
988 Builder.CreateZExt(V: StatusBit, DestTy: Builder.getInt32Ty(), Name: "ld.status.ext");
989 Builder.CreateStore(Val: ExtendedStatus, Addr: StatusAddr);
990
991 return LoadedValue;
992 }
993 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
994 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
995 return llvm::PoisonValue::get(T: HandleTy);
996 }
997 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
998 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
999 Value *RegisterOp = EmitScalarExpr(E: E->getArg(Arg: 1));
1000 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1001 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1002 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1003 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1004 llvm::Intrinsic::ID IntrinsicID =
1005 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
1006 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};
1007 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1008 }
1009 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
1010 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1011 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1012 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1013 Value *RangeOp = EmitScalarExpr(E: E->getArg(Arg: 3));
1014 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 4));
1015 Value *Name = EmitScalarExpr(E: E->getArg(Arg: 5));
1016 llvm::Intrinsic::ID IntrinsicID =
1017 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
1018 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};
1019 return Builder.CreateIntrinsic(RetTy: HandleTy, ID: IntrinsicID, Args);
1020 }
1021 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {
1022 Value *MainHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1023 if (!CGM.getTriple().isSPIRV())
1024 return MainHandle;
1025
1026 llvm::Type *HandleTy = CGM.getTypes().ConvertType(T: E->getType());
1027 Value *OrderID = EmitScalarExpr(E: E->getArg(Arg: 1));
1028 Value *SpaceOp = EmitScalarExpr(E: E->getArg(Arg: 2));
1029 llvm::Intrinsic::ID IntrinsicID =
1030 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;
1031 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};
1032 return EmitIntrinsicCall(ID: IntrinsicID, Types: {HandleTy, MainHandle->getType()},
1033 Args);
1034 }
1035 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
1036 Value *IndexOp = EmitScalarExpr(E: E->getArg(Arg: 0));
1037 llvm::Type *RetTy = ConvertType(T: E->getType());
1038 return Builder.CreateIntrinsic(
1039 RetTy, ID: CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
1040 Args: ArrayRef<Value *>{IndexOp});
1041 }
1042 case Builtin::BI__builtin_hlsl_resource_getdimensions_x:
1043 case Builtin::BI__builtin_hlsl_resource_getdimensions_x_float:
1044 return emitGetDimensions(CGF&: *this, E,
1045 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),
1046 NumRetComps: 1, /*HasLod=*/false);
1047 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy:
1048 case Builtin::BI__builtin_hlsl_resource_getdimensions_xy_float:
1049 return emitGetDimensions(CGF&: *this, E,
1050 IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsXYIntrinsic(),
1051 NumRetComps: 2, /*HasLod=*/false);
1052 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy:
1053 case Builtin::BI__builtin_hlsl_resource_getdimensions_levels_xy_float:
1054 return emitGetDimensions(
1055 CGF&: *this, E, IntrinsicID: CGM.getHLSLRuntime().getGetDimensionsLevelsXYIntrinsic(), NumRetComps: 3,
1056 /*HasLod=*/true);
1057 case Builtin::BI__builtin_hlsl_resource_getstride: {
1058 LValue Stride = EmitLValue(E: E->getArg(Arg: 1));
1059 return emitBufferStride(CGF: this, HandleExpr: E->getArg(Arg: 0), Stride);
1060 }
1061 case Builtin::BI__builtin_hlsl_all: {
1062 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1063 return Builder.CreateIntrinsic(
1064 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1065 ID: CGM.getHLSLRuntime().getAllIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1066 Name: "hlsl.all");
1067 }
1068 case Builtin::BI__builtin_hlsl_and: {
1069 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1070 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1071 return Builder.CreateAnd(LHS: Op0, RHS: Op1, Name: "hlsl.and");
1072 }
1073 case Builtin::BI__builtin_hlsl_or: {
1074 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1075 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1076 return Builder.CreateOr(LHS: Op0, RHS: Op1, Name: "hlsl.or");
1077 }
1078 case Builtin::BI__builtin_hlsl_any: {
1079 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1080 return Builder.CreateIntrinsic(
1081 /*ReturnType=*/RetTy: llvm::Type::getInt1Ty(C&: getLLVMContext()),
1082 ID: CGM.getHLSLRuntime().getAnyIntrinsic(), Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1083 Name: "hlsl.any");
1084 }
1085 case Builtin::BI__builtin_hlsl_asdouble:
1086 return handleAsDoubleBuiltin(CGF&: *this, E);
1087 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
1088 Value *OpX = EmitScalarExpr(E: E->getArg(Arg: 0));
1089 Value *OpMin = EmitScalarExpr(E: E->getArg(Arg: 1));
1090 Value *OpMax = EmitScalarExpr(E: E->getArg(Arg: 2));
1091
1092 QualType Ty = E->getArg(Arg: 0)->getType();
1093 if (auto *VecTy = Ty->getAs<VectorType>())
1094 Ty = VecTy->getElementType();
1095
1096 Intrinsic::ID Intr;
1097 if (Ty->isFloatingType()) {
1098 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
1099 } else if (Ty->isUnsignedIntegerType()) {
1100 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
1101 } else {
1102 assert(Ty->isSignedIntegerType());
1103 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
1104 }
1105 return Builder.CreateIntrinsic(
1106 /*ReturnType=*/RetTy: OpX->getType(), ID: Intr,
1107 Args: ArrayRef<Value *>{OpX, OpMin, OpMax}, FMFSource: nullptr, Name: "hlsl.clamp");
1108 }
1109 case Builtin::BI__builtin_hlsl_dot: {
1110 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1111 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1112 llvm::Type *T0 = Op0->getType();
1113 llvm::Type *T1 = Op1->getType();
1114
1115 // If the arguments are scalars, just emit a multiply
1116 if (!T0->isVectorTy() && !T1->isVectorTy()) {
1117 if (T0->isFloatingPointTy())
1118 return Builder.CreateFMul(L: Op0, R: Op1, Name: "hlsl.dot");
1119
1120 if (T0->isIntegerTy())
1121 return Builder.CreateMul(LHS: Op0, RHS: Op1, Name: "hlsl.dot");
1122
1123 llvm_unreachable(
1124 "Scalar dot product is only supported on ints and floats.");
1125 }
1126 // For vectors, validate types and emit the appropriate intrinsic
1127 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),
1128 E->getArg(1)->getType()) &&
1129 "Dot product operands must have the same type.");
1130
1131 auto *VecTy0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
1132 assert(VecTy0 && "Dot product argument must be a vector.");
1133
1134 return Builder.CreateIntrinsic(
1135 /*ReturnType=*/RetTy: T0->getScalarType(),
1136 ID: getDotProductIntrinsic(RT&: CGM.getHLSLRuntime(), QT: VecTy0->getElementType()),
1137 Args: ArrayRef<Value *>{Op0, Op1}, FMFSource: nullptr, Name: "hlsl.dot");
1138 }
1139 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
1140 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1141 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1142 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1143
1144 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
1145 // Note that the argument order disagrees between the builtin and the
1146 // intrinsic here.
1147 return Builder.CreateIntrinsic(
1148 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1149 FMFSource: nullptr, Name: "hlsl.dot4add.i8packed");
1150 }
1151 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
1152 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1153 Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
1154 Value *Acc = EmitScalarExpr(E: E->getArg(Arg: 2));
1155
1156 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
1157 // Note that the argument order disagrees between the builtin and the
1158 // intrinsic here.
1159 return Builder.CreateIntrinsic(
1160 /*ReturnType=*/RetTy: Acc->getType(), ID, Args: ArrayRef<Value *>{Acc, X, Y},
1161 FMFSource: nullptr, Name: "hlsl.dot4add.u8packed");
1162 }
1163 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
1164 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1165
1166 return Builder.CreateIntrinsic(
1167 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1168 ID: getFirstBitHighIntrinsic(RT&: CGM.getHLSLRuntime(), QT: E->getArg(Arg: 0)->getType()),
1169 Args: ArrayRef<Value *>{X}, FMFSource: nullptr, Name: "hlsl.firstbithigh");
1170 }
1171 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
1172 Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
1173
1174 return Builder.CreateIntrinsic(
1175 /*ReturnType=*/RetTy: ConvertType(T: E->getType()),
1176 ID: CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), Args: ArrayRef<Value *>{X},
1177 FMFSource: nullptr, Name: "hlsl.firstbitlow");
1178 }
1179 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {
1180 return handleElementwiseF16ToF32(CGF&: *this, E);
1181 }
1182 case Builtin::BI__builtin_hlsl_elementwise_f32tof16: {
1183 return handleElementwiseF32ToF16(CGF&: *this, E);
1184 }
1185 case Builtin::BI__builtin_hlsl_elementwise_frac: {
1186 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1187 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1188 llvm_unreachable("frac operand must have a float representation");
1189 return Builder.CreateIntrinsic(
1190 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getFracIntrinsic(),
1191 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.frac");
1192 }
1193 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
1194 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1195 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1196 llvm_unreachable("isinf operand must have a float representation");
1197 llvm::Type *retType = getAggregateType(
1198 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1199 return Builder.CreateIntrinsic(
1200 RetTy: retType, ID: CGM.getHLSLRuntime().getIsInfIntrinsic(),
1201 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isinf");
1202 }
1203 case Builtin::BI__builtin_hlsl_elementwise_isnan: {
1204 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1205 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1206 llvm_unreachable("isnan operand must have a float representation");
1207 llvm::Type *retType = getAggregateType(
1208 ScalarTy: llvm::Type::getInt1Ty(C&: getLLVMContext()), ArgTy: E->getArg(Arg: 0)->getType());
1209 return Builder.CreateIntrinsic(
1210 RetTy: retType, ID: CGM.getHLSLRuntime().getIsNaNIntrinsic(),
1211 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.isnan");
1212 }
1213 case Builtin::BI__builtin_hlsl_mad: {
1214 Value *M = EmitScalarExpr(E: E->getArg(Arg: 0));
1215 Value *A = EmitScalarExpr(E: E->getArg(Arg: 1));
1216 Value *B = EmitScalarExpr(E: E->getArg(Arg: 2));
1217 if (E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1218 return Builder.CreateIntrinsic(
1219 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::fmuladd,
1220 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "hlsl.fmad");
1221
1222 if (E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()) {
1223 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1224 return Builder.CreateIntrinsic(
1225 /*ReturnType*/ RetTy: M->getType(), ID: Intrinsic::dx_imad,
1226 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.imad");
1227
1228 Value *Mul = Builder.CreateNSWMul(LHS: M, RHS: A);
1229 return Builder.CreateNSWAdd(LHS: Mul, RHS: B);
1230 }
1231 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
1232 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
1233 return Builder.CreateIntrinsic(
1234 /*ReturnType=*/RetTy: M->getType(), ID: Intrinsic::dx_umad,
1235 Args: ArrayRef<Value *>{M, A, B}, FMFSource: nullptr, Name: "dx.umad");
1236
1237 Value *Mul = Builder.CreateNUWMul(LHS: M, RHS: A);
1238 return Builder.CreateNUWAdd(LHS: Mul, RHS: B);
1239 }
1240 case Builtin::BI__builtin_hlsl_mul: {
1241 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1242 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1243 QualType QTy0 = E->getArg(Arg: 0)->getType();
1244 QualType QTy1 = E->getArg(Arg: 1)->getType();
1245
1246 bool IsVec0 = QTy0->isVectorType();
1247 bool IsVec1 = QTy1->isVectorType();
1248 bool IsMat0 = QTy0->isConstantMatrixType();
1249 bool IsMat1 = QTy1->isConstantMatrixType();
1250
1251 // The matrix multiply intrinsic only operates on column-major order
1252 // matrices. Therefore matrix memory layout transforms must be inserted
1253 // before and after matrix multiply intrinsics.
1254 // Use whichever operand is a matrix to discover its declared layout.
1255 bool IsRowMajorMat0 = IsMat0 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy0);
1256 bool IsRowMajorMat1 = IsMat1 && isMatrixRowMajor(LangOpts: getLangOpts(), T: QTy1);
1257
1258 llvm::MatrixBuilder MB(Builder);
1259 if (IsVec0 && IsMat1) {
1260 unsigned N = QTy0->castAs<VectorType>()->getNumElements();
1261 auto *MatTy = QTy1->castAs<ConstantMatrixType>();
1262 unsigned Rows = MatTy->getNumRows();
1263 unsigned Cols = MatTy->getNumColumns();
1264 assert(N == Rows && "vector length must match matrix row count");
1265 if (IsRowMajorMat1)
1266 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows, Columns: Cols);
1267 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: 1, LHSColumns: N, RHSColumns: Cols, Name: "hlsl.mul");
1268 }
1269 if (IsMat0 && IsVec1) {
1270 auto *MatTy = QTy0->castAs<ConstantMatrixType>();
1271 unsigned Rows = MatTy->getNumRows();
1272 unsigned Cols = MatTy->getNumColumns();
1273 assert(QTy1->castAs<VectorType>()->getNumElements() == Cols &&
1274 "vector length must match matrix column count");
1275 if (IsRowMajorMat0)
1276 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows, Columns: Cols);
1277 return MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows, LHSColumns: Cols, RHSColumns: 1, Name: "hlsl.mul");
1278 }
1279 assert(IsMat0 && IsMat1);
1280 auto *MatTy0 = QTy0->castAs<ConstantMatrixType>();
1281 auto *MatTy1 = QTy1->castAs<ConstantMatrixType>();
1282 unsigned Rows0 = MatTy0->getNumRows();
1283 unsigned Rows1 = MatTy1->getNumRows();
1284 unsigned Cols0 = MatTy0->getNumColumns();
1285 unsigned Cols1 = MatTy1->getNumColumns();
1286 assert(Cols0 == Rows1 &&
1287 "inner matrix dimensions must match for multiplication");
1288 if (IsRowMajorMat0)
1289 Op0 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op0, Rows: Rows0, Columns: Cols0);
1290 if (IsRowMajorMat1)
1291 Op1 = MB.CreateRowMajorToColumnMajorTransform(Matrix: Op1, Rows: Rows1, Columns: Cols1);
1292
1293 Value *Result =
1294 MB.CreateMatrixMultiply(LHS: Op0, RHS: Op1, LHSRows: Rows0, LHSColumns: Cols0, RHSColumns: Cols1, Name: "hlsl.mul");
1295
1296 bool IsResultRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1297 if (IsResultRowMajor)
1298 Result = MB.CreateColumnMajorToRowMajorTransform(Matrix: Result, Rows: Rows0, Columns: Cols1);
1299 return Result;
1300 }
1301 case Builtin::BI__builtin_hlsl_transpose: {
1302 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1303 auto *MatTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
1304 unsigned Rows = MatTy->getNumRows();
1305 unsigned Cols = MatTy->getNumColumns();
1306 llvm::MatrixBuilder MB(Builder);
1307 // The correct lowering of a transpose depends on both the source layout
1308 // and the result layout.
1309 bool SrcRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getArg(Arg: 0)->getType());
1310 bool DstRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getType());
1311 // When the source & result layouts differ, the operand already holds the
1312 // transposed result, ie transpose is a no-op on the underlying vector.
1313 if (SrcRowMajor != DstRowMajor)
1314 return Op0;
1315 // When the source and result share a layout, emit a transpose.
1316 if (SrcRowMajor)
1317 // For row-major operands the dimensions are swapped
1318 return MB.CreateMatrixTranspose(Matrix: Op0, Rows: Cols, Columns: Rows);
1319 return MB.CreateMatrixTranspose(Matrix: Op0, Rows, Columns: Cols);
1320 }
1321 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
1322 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1323 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1324 llvm_unreachable("rcp operand must have a float representation");
1325 llvm::Type *Ty = Op0->getType();
1326 llvm::Type *EltTy = Ty->getScalarType();
1327 Constant *One = Ty->isVectorTy()
1328 ? ConstantVector::getSplat(
1329 EC: ElementCount::getFixed(
1330 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
1331 Elt: ConstantFP::get(Ty: EltTy, V: 1.0))
1332 : ConstantFP::get(Ty: EltTy, V: 1.0);
1333 return Builder.CreateFDiv(L: One, R: Op0, Name: "hlsl.rcp");
1334 }
1335 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
1336 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1337 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1338 llvm_unreachable("rsqrt operand must have a float representation");
1339 return Builder.CreateIntrinsic(
1340 /*ReturnType=*/RetTy: Op0->getType(), ID: CGM.getHLSLRuntime().getRsqrtIntrinsic(),
1341 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.rsqrt");
1342 }
1343 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
1344 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1345 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1346 "saturate operand must have a float representation");
1347 return Builder.CreateIntrinsic(
1348 /*ReturnType=*/RetTy: Op0->getType(),
1349 ID: CGM.getHLSLRuntime().getSaturateIntrinsic(), Args: ArrayRef<Value *>{Op0},
1350 FMFSource: nullptr, Name: "hlsl.saturate");
1351 }
1352 case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
1353 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1354 assert(Op->getType()->isIntegerTy(1) &&
1355 "WavePrefixBitCount operand must be a boolean type");
1356
1357 Intrinsic::ID IID =
1358 getPrefixCountBitsIntrinsic(Arch: getTarget().getTriple().getArch());
1359
1360 return EmitIntrinsicCall(ID: IID, Args: ArrayRef{Op}, Name: "hlsl.wave.prefix.bit.count");
1361 }
1362 case Builtin::BI__builtin_hlsl_select: {
1363 Value *OpCond = EmitScalarExpr(E: E->getArg(Arg: 0));
1364 RValue RValTrue = EmitAnyExpr(E: E->getArg(Arg: 1));
1365 Value *OpTrue =
1366 RValTrue.isScalar()
1367 ? RValTrue.getScalarVal()
1368 : Builder.CreateLoad(Addr: RValTrue.getAggregateAddress(), Name: "true_val");
1369 RValue RValFalse = EmitAnyExpr(E: E->getArg(Arg: 2));
1370 Value *OpFalse =
1371 RValFalse.isScalar()
1372 ? RValFalse.getScalarVal()
1373 : Builder.CreateLoad(Addr: RValFalse.getAggregateAddress(), Name: "false_val");
1374 if (auto *VTy = E->getType()->getAs<VectorType>()) {
1375 if (!OpTrue->getType()->isVectorTy())
1376 OpTrue =
1377 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpTrue, Name: "splat");
1378 if (!OpFalse->getType()->isVectorTy())
1379 OpFalse =
1380 Builder.CreateVectorSplat(NumElts: VTy->getNumElements(), V: OpFalse, Name: "splat");
1381 }
1382
1383 Value *SelectVal =
1384 Builder.CreateSelect(C: OpCond, True: OpTrue, False: OpFalse, Name: "hlsl.select");
1385 if (!RValTrue.isScalar())
1386 Builder.CreateStore(Val: SelectVal, Addr: ReturnValue.getAddress(),
1387 IsVolatile: ReturnValue.isVolatile());
1388
1389 return SelectVal;
1390 }
1391 case Builtin::BI__builtin_hlsl_wave_active_all_equal: {
1392 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1393
1394 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllEqualIntrinsic();
1395 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: {Op});
1396 }
1397 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
1398 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1399 assert(Op->getType()->isIntegerTy(1) &&
1400 "Intrinsic WaveActiveAllTrue operand must be a bool");
1401
1402 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
1403 return EmitIntrinsicCall(ID, Args: {Op});
1404 }
1405 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
1406 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1407 assert(Op->getType()->isIntegerTy(1) &&
1408 "Intrinsic WaveActiveAnyTrue operand must be a bool");
1409
1410 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
1411 return EmitIntrinsicCall(ID, Args: {Op});
1412 }
1413 case Builtin::BI__builtin_hlsl_wave_active_bit_or: {
1414 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1415 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1416 "Intrinsic WaveActiveBitOr operand must have an unsigned integer "
1417 "representation");
1418
1419 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitOrIntrinsic();
1420 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1421 Name: "hlsl.wave.active.bit.or");
1422 }
1423 case Builtin::BI__builtin_hlsl_wave_active_bit_xor: {
1424 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1425 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1426 "Intrinsic WaveActiveBitXor operand must have an unsigned integer "
1427 "representation");
1428
1429 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitXorIntrinsic();
1430 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1431 Name: "hlsl.wave.active.bit.xor");
1432 }
1433 case Builtin::BI__builtin_hlsl_wave_active_bit_and: {
1434 Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1435 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
1436 "Intrinsic WaveActiveBitAnd operand must have an unsigned integer "
1437 "representation");
1438
1439 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveBitAndIntrinsic();
1440 return EmitIntrinsicCall(ID, Types: {Op->getType()}, Args: ArrayRef{Op},
1441 Name: "hlsl.wave.active.bit.and");
1442 }
1443 case Builtin::BI__builtin_hlsl_interlocked_add: {
1444 // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
1445 // up the raw instruction (DXIL routes it via `dx.resource.atomic.binop`
1446 // in DXILResourceAccess for resource pointers, SPIR-V lowers via
1447 // selectAtomicRMW). No intermediate intrinsic.
1448 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Add);
1449 }
1450 case Builtin::BI__builtin_hlsl_interlocked_min: {
1451 llvm::AtomicRMWInst::BinOp Op =
1452 E->getArg(Arg: 0)->getType()->hasSignedIntegerRepresentation()
1453 ? llvm::AtomicRMWInst::Min
1454 : llvm::AtomicRMWInst::UMin;
1455 return handleInterlockedOp(CGF&: *this, E, Op);
1456 }
1457 case Builtin::BI__builtin_hlsl_interlocked_or: {
1458 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Or);
1459 }
1460 case Builtin::BI__builtin_hlsl_interlocked_xor: {
1461 return handleInterlockedOp(CGF&: *this, E, Op: llvm::AtomicRMWInst::Xor);
1462 }
1463 case Builtin::BI__builtin_hlsl_wave_active_ballot: {
1464 [[maybe_unused]] Value *Op = EmitScalarExpr(E: E->getArg(Arg: 0));
1465 assert(Op->getType()->isIntegerTy(1) &&
1466 "Intrinsic WaveActiveBallot operand must be a bool");
1467
1468 return handleHlslWaveActiveBallot(CGF&: *this, E);
1469 }
1470 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
1471 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1472 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
1473 return EmitIntrinsicCall(ID, Args: ArrayRef{OpExpr});
1474 }
1475 case Builtin::BI__builtin_hlsl_wave_active_sum: {
1476 // Due to the use of variadic arguments, explicitly retrieve argument
1477 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1478 Intrinsic::ID IID = getWaveActiveSumIntrinsic(
1479 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1480
1481 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1482 Name: "hlsl.wave.active.sum");
1483 }
1484 case Builtin::BI__builtin_hlsl_wave_active_product: {
1485 // Due to the use of variadic arguments, explicitly retrieve argument
1486 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1487 Intrinsic::ID IID = getWaveActiveProductIntrinsic(
1488 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1489
1490 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1491 Name: "hlsl.wave.active.product");
1492 }
1493 case Builtin::BI__builtin_hlsl_wave_active_max: {
1494 // Due to the use of variadic arguments, explicitly retrieve argument
1495 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1496 QualType QT = E->getArg(Arg: 0)->getType();
1497 Intrinsic::ID IID;
1498 if (QT->isUnsignedIntegerType())
1499 IID = CGM.getHLSLRuntime().getWaveActiveUMaxIntrinsic();
1500 else
1501 IID = CGM.getHLSLRuntime().getWaveActiveMaxIntrinsic();
1502
1503 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1504 Name: "hlsl.wave.active.max");
1505 }
1506 case Builtin::BI__builtin_hlsl_wave_active_min: {
1507 // Due to the use of variadic arguments, explicitly retrieve argument
1508 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1509 QualType QT = E->getArg(Arg: 0)->getType();
1510 Intrinsic::ID IID;
1511 if (QT->isUnsignedIntegerType())
1512 IID = CGM.getHLSLRuntime().getWaveActiveUMinIntrinsic();
1513 else
1514 IID = CGM.getHLSLRuntime().getWaveActiveMinIntrinsic();
1515
1516 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1517 Name: "hlsl.wave.active.min");
1518 }
1519 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
1520 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
1521 // defined in SPIRVBuiltins.td. So instead we manually get the matching name
1522 // for the DirectX intrinsic and the demangled builtin name
1523 switch (CGM.getTarget().getTriple().getArch()) {
1524 case llvm::Triple::dxil:
1525 return EmitIntrinsicCall(ID: Intrinsic::dx_wave_getlaneindex);
1526 case llvm::Triple::spirv:
1527 return EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(
1528 Ty: llvm::FunctionType::get(Result: IntTy, Params: {}, isVarArg: false),
1529 Name: "__hlsl_wave_get_lane_index", ExtraAttrs: {}, Local: false, AssumeConvergent: true));
1530 default:
1531 llvm_unreachable(
1532 "Intrinsic WaveGetLaneIndex not supported by target architecture");
1533 }
1534 }
1535 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
1536 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
1537 return EmitIntrinsicCall(ID);
1538 }
1539 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
1540 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
1541 return EmitIntrinsicCall(ID);
1542 }
1543 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
1544 // Due to the use of variadic arguments we must explicitly retrieve them and
1545 // create our function type.
1546 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1547 Value *OpIndex = EmitScalarExpr(E: E->getArg(Arg: 1));
1548 return EmitIntrinsicCall(ID: CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
1549 Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr, OpIndex},
1550 Name: "hlsl.wave.readlane");
1551 }
1552 case Builtin::BI__builtin_hlsl_wave_prefix_sum: {
1553 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1554 Intrinsic::ID IID = getWavePrefixSumIntrinsic(
1555 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1556 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1557 Name: "hlsl.wave.prefix.sum");
1558 }
1559 case Builtin::BI__builtin_hlsl_wave_prefix_product: {
1560 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1561 Intrinsic::ID IID = getWavePrefixProductIntrinsic(
1562 Arch: getTarget().getTriple().getArch(), QT: E->getArg(Arg: 0)->getType());
1563 return EmitIntrinsicCall(ID: IID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1564 Name: "hlsl.wave.prefix.product");
1565 }
1566 case Builtin::BI__builtin_hlsl_quad_read_across_x: {
1567 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1568 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossXIntrinsic();
1569 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1570 Name: "hlsl.quad.read.across.x");
1571 }
1572 case Builtin::BI__builtin_hlsl_quad_read_across_y: {
1573 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1574 Intrinsic::ID ID = CGM.getHLSLRuntime().getQuadReadAcrossYIntrinsic();
1575 return EmitIntrinsicCall(ID, Types: {OpExpr->getType()}, Args: ArrayRef{OpExpr},
1576 Name: "hlsl.quad.read.across.y");
1577 }
1578 case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
1579 Value *OpExpr = EmitScalarExpr(E: E->getArg(Arg: 0));
1580 Intrinsic::ID ID =
1581 CGM.getHLSLRuntime().getQuadReadAcrossDiagonalIntrinsic();
1582 return EmitRuntimeCall(callee: Intrinsic::getOrInsertDeclaration(
1583 M: &CGM.getModule(), id: ID, OverloadTys: {OpExpr->getType()}),
1584 args: ArrayRef{OpExpr}, name: "hlsl.quad.read.across.diagonal");
1585 }
1586 case Builtin::BI__builtin_hlsl_elementwise_sign: {
1587 auto *Arg0 = E->getArg(Arg: 0);
1588 Value *Op0 = EmitScalarExpr(E: Arg0);
1589 llvm::Type *Xty = Op0->getType();
1590 llvm::Type *retType = llvm::Type::getInt32Ty(C&: this->getLLVMContext());
1591 if (Xty->isVectorTy()) {
1592 auto *XVecTy = Arg0->getType()->castAs<VectorType>();
1593 retType = llvm::VectorType::get(
1594 ElementType: retType, EC: ElementCount::getFixed(MinVal: XVecTy->getNumElements()));
1595 }
1596 assert((Arg0->getType()->hasFloatingRepresentation() ||
1597 Arg0->getType()->hasIntegerRepresentation()) &&
1598 "sign operand must have a float or int representation");
1599
1600 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
1601 Value *Cmp = Builder.CreateICmpEQ(LHS: Op0, RHS: ConstantInt::get(Ty: Xty, V: 0));
1602 return Builder.CreateSelect(C: Cmp, True: ConstantInt::get(Ty: retType, V: 0),
1603 False: ConstantInt::get(Ty: retType, V: 1), Name: "hlsl.sign");
1604 }
1605
1606 return Builder.CreateIntrinsic(
1607 RetTy: retType, ID: CGM.getHLSLRuntime().getSignIntrinsic(),
1608 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr, Name: "hlsl.sign");
1609 }
1610 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
1611 Value *ResHandle = EmitScalarExpr(E: E->getArg(Arg: 0));
1612 Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 1));
1613 Value *OffsetI8 = Builder.CreateIntCast(V: Offset, DestTy: Int8Ty, isSigned: true);
1614 return Builder.CreateIntrinsic(
1615 /*ReturnType=*/RetTy: Offset->getType(),
1616 ID: CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
1617 Args: ArrayRef<Value *>{ResHandle, OffsetI8}, FMFSource: nullptr);
1618 }
1619 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
1620
1621 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
1622 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
1623 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
1624 "asuint operands types mismatch");
1625 return handleHlslSplitdouble(E, CGF: this);
1626 }
1627 case Builtin::BI__builtin_hlsl_elementwise_clip:
1628 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
1629 "clip operands types mismatch");
1630 return handleHlslClip(E, CGF: this);
1631 case Builtin::BI__builtin_hlsl_all_memory_barrier: {
1632 Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
1633 return EmitIntrinsicCall(ID);
1634 }
1635 case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
1636 Intrinsic::ID ID =
1637 CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
1638 return EmitIntrinsicCall(ID);
1639 }
1640 case Builtin::BI__builtin_hlsl_device_memory_barrier: {
1641 Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
1642 return EmitIntrinsicCall(ID);
1643 }
1644 case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
1645 Intrinsic::ID ID =
1646 CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
1647 return EmitIntrinsicCall(ID);
1648 }
1649 case Builtin::BI__builtin_hlsl_group_memory_barrier: {
1650 Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
1651 return EmitIntrinsicCall(ID);
1652 }
1653 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
1654 Intrinsic::ID ID =
1655 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
1656 return EmitIntrinsicCall(ID);
1657 }
1658 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {
1659 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1660 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1661 llvm_unreachable("ddx_coarse operand must have a float representation");
1662 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();
1663 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1664 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1665 Name: "hlsl.ddx.coarse");
1666 }
1667 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {
1668 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1669 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1670 llvm_unreachable("ddy_coarse operand must have a float representation");
1671 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();
1672 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1673 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1674 Name: "hlsl.ddy.coarse");
1675 }
1676 case Builtin::BI__builtin_hlsl_elementwise_ddx_fine: {
1677 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1678 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1679 llvm_unreachable("ddx_fine operand must have a float representation");
1680 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxFineIntrinsic();
1681 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1682 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1683 Name: "hlsl.ddx.fine");
1684 }
1685 case Builtin::BI__builtin_hlsl_elementwise_ddy_fine: {
1686 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1687 if (!E->getArg(Arg: 0)->getType()->hasFloatingRepresentation())
1688 llvm_unreachable("ddy_fine operand must have a float representation");
1689 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyFineIntrinsic();
1690 return Builder.CreateIntrinsic(/*ReturnType=*/RetTy: Op0->getType(), ID,
1691 Args: ArrayRef<Value *>{Op0}, FMFSource: nullptr,
1692 Name: "hlsl.ddy.fine");
1693 }
1694 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
1695 case Builtin::BI__builtin_get_spirv_spec_constant_short:
1696 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
1697 case Builtin::BI__builtin_get_spirv_spec_constant_int:
1698 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
1699 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
1700 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
1701 case Builtin::BI__builtin_get_spirv_spec_constant_half:
1702 case Builtin::BI__builtin_get_spirv_spec_constant_float:
1703 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
1704 llvm::Function *SpecConstantFn = getSpecConstantFunction(SpecConstantType: E->getType());
1705 llvm::Value *SpecId = EmitScalarExpr(E: E->getArg(Arg: 0));
1706 llvm::Value *DefaultVal = EmitScalarExpr(E: E->getArg(Arg: 1));
1707 llvm::Value *Args[] = {SpecId, DefaultVal};
1708 return Builder.CreateCall(Callee: SpecConstantFn, Args);
1709 }
1710 }
1711 return nullptr;
1712}
1713
1714llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
1715 const clang::QualType &SpecConstantType) {
1716
1717 // Find or create the declaration for the function.
1718 llvm::Module *M = &CGM.getModule();
1719 std::string MangledName =
1720 getSpecConstantFunctionName(SpecConstantType, Context&: getContext());
1721 llvm::Function *SpecConstantFn = M->getFunction(Name: MangledName);
1722
1723 if (!SpecConstantFn) {
1724 llvm::Type *IntType = ConvertType(T: getContext().IntTy);
1725 llvm::Type *RetTy = ConvertType(T: SpecConstantType);
1726 llvm::Type *ArgTypes[] = {IntType, RetTy};
1727 llvm::FunctionType *FnTy = llvm::FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
1728 SpecConstantFn = llvm::Function::Create(
1729 Ty: FnTy, Linkage: llvm::GlobalValue::ExternalLinkage, N: MangledName, M);
1730 }
1731 return SpecConstantFn;
1732}
1733