1//===------- AMDCPU.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "CGDebugInfo.h"
15#include "CodeGenFunction.h"
16#include "TargetInfo.h"
17#include "clang/Basic/DiagnosticFrontend.h"
18#include "clang/Basic/SyncScope.h"
19#include "clang/Basic/TargetBuiltins.h"
20#include "llvm/Analysis/ValueTracking.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/IntrinsicsR600.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
26#include "llvm/Support/AMDGPUAddrSpace.h"
27#include "llvm/Support/AtomicOrdering.h"
28#include "llvm/TargetParser/AtomicScope.h"
29
30using namespace clang;
31using namespace CodeGen;
32using namespace llvm;
33
34namespace {
35
36static Value *emitAMDGPUSBufferLoadBuiltin(CodeGenFunction &CGF,
37 const CallExpr *E) {
38 llvm::Type *RetTy = CGF.ConvertType(T: E->getType());
39 Function *F =
40 CGF.CGM.getIntrinsic(IID: Intrinsic::amdgcn_ptr_s_buffer_load, Tys: RetTy);
41
42 Value *RsrcPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
43 CallInst *Call =
44 CGF.Builder.CreateCall(Callee: F, Args: {RsrcPtr, CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
45 CGF.EmitScalarExpr(E: E->getArg(Arg: 2))});
46 Call->setMetadata(KindID: llvm::LLVMContext::MD_invariant_load,
47 Node: llvm::MDNode::get(Context&: CGF.Builder.getContext(), MDs: {}));
48 return Call;
49}
50
51// Has second type mangled argument.
52static Value *
53emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
54 Intrinsic::ID IntrinsicID,
55 Intrinsic::ID ConstrainedIntrinsicID) {
56 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
57 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
58
59 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
60 if (CGF.Builder.getIsFPConstrained()) {
61 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
62 Tys: {Src0->getType(), Src1->getType()});
63 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
64 }
65
66 Function *F =
67 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
68 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
69}
70
71// If \p E is not null pointer, insert address space cast to match return
72// type of \p E if necessary.
73Value *EmitAMDGPUDispatchPtr(CodeGenFunction &CGF,
74 const CallExpr *E = nullptr) {
75 auto *F = CGF.CGM.getIntrinsic(IID: Intrinsic::amdgcn_dispatch_ptr);
76 auto *Call = CGF.Builder.CreateCall(Callee: F);
77 if (!E)
78 return Call;
79 QualType BuiltinRetType = E->getType();
80 auto *RetTy = cast<llvm::PointerType>(Val: CGF.ConvertType(T: BuiltinRetType));
81 if (RetTy == Call->getType())
82 return Call;
83 return CGF.Builder.CreateAddrSpaceCast(V: Call, DestTy: RetTy);
84}
85
86Value *EmitAMDGPUImplicitArgPtr(CodeGenFunction &CGF) {
87 auto *F = CGF.CGM.getIntrinsic(IID: Intrinsic::amdgcn_implicitarg_ptr);
88 auto *Call = CGF.Builder.CreateCall(Callee: F);
89 Call->addRetAttr(
90 Attr: Attribute::getWithDereferenceableBytes(Context&: Call->getContext(), Bytes: 256));
91 Call->addRetAttr(Attr: Attribute::getWithAlignment(Context&: Call->getContext(), Alignment: Align(8)));
92 return Call;
93}
94
95static llvm::Intrinsic::ID getAMDGPUWorkGroupID(CodeGenFunction &CGF,
96 unsigned Index) {
97 switch (Index) {
98 case 0:
99 return llvm::Intrinsic::amdgcn_workgroup_id_x;
100 case 1:
101 return llvm::Intrinsic::amdgcn_workgroup_id_y;
102 case 2:
103 return llvm::Intrinsic::amdgcn_workgroup_id_z;
104 default:
105 llvm_unreachable("unhandled index");
106 }
107}
108
109static void setNoundefInvariantLoad(llvm::LoadInst *Ld) {
110 Ld->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
111 Node: llvm::MDNode::get(Context&: Ld->getContext(), MDs: {}));
112 Ld->setMetadata(KindID: llvm::LLVMContext::MD_invariant_load,
113 Node: llvm::MDNode::get(Context&: Ld->getContext(), MDs: {}));
114}
115
116static void addMaxWorkGroupSizeRangeMetadata(CodeGenFunction &CGF,
117 llvm::LoadInst *GroupSize) {
118 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
119 llvm::MDNode *RNode = MDHelper.createRange(
120 Lo: APInt(16, 1), Hi: APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1));
121 GroupSize->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RNode);
122 setNoundefInvariantLoad(GroupSize);
123}
124
125static Value *emitAMDGPUWorkGroupSizeV5(CodeGenFunction &CGF, unsigned Index) {
126 llvm::Value *ImplicitArgPtr = EmitAMDGPUImplicitArgPtr(CGF);
127
128 // offsetof(amdhsa_implicit_kernarg_v5, block_count[Index])
129 unsigned BlockCountOffset = 0 + Index * 4;
130 // offsetof(amdhsa_implicit_kernarg_v5, group_size[Index])
131 unsigned GroupSizeOffset = 12 + Index * 2;
132 // offsetof(amdhsa_implicit_kernarg_v5, remainder[Index])
133 unsigned RemainderOffset = 18 + Index * 2;
134
135 if (CGF.CGM.getLangOpts().OffloadUniformBlock) {
136 // Indexing the implicit kernarg segment.
137 llvm::Value *GroupSizeGEP = CGF.Builder.CreateConstInBoundsGEP1_64(
138 Ty: CGF.Int8Ty, Ptr: ImplicitArgPtr, Idx0: GroupSizeOffset);
139 llvm::LoadInst *GroupSize = CGF.Builder.CreateLoad(
140 Addr: Address(GroupSizeGEP, CGF.Int16Ty, CharUnits::fromQuantity(Quantity: 2)));
141
142 addMaxWorkGroupSizeRangeMetadata(CGF, GroupSize);
143
144 return CGF.Builder.CreateZExt(V: GroupSize, DestTy: CGF.Int32Ty);
145 }
146
147 llvm::Value *BlockCountGEP = CGF.Builder.CreateConstGEP1_64(
148 Ty: CGF.Int8Ty, Ptr: ImplicitArgPtr, Idx0: BlockCountOffset);
149 llvm::LoadInst *BlockCount = CGF.Builder.CreateLoad(
150 Addr: Address(BlockCountGEP, CGF.Int32Ty, CharUnits::fromQuantity(Quantity: 4)));
151 setNoundefInvariantLoad(BlockCount);
152
153 llvm::Value *WorkgroupID =
154 CGF.Builder.CreateIntrinsic(ID: getAMDGPUWorkGroupID(CGF, Index), Args: {});
155 llvm::Value *IsFull = CGF.Builder.CreateICmpULT(LHS: WorkgroupID, RHS: BlockCount);
156
157 llvm::Value *StructOffset = CGF.Builder.CreateSelect(
158 C: IsFull, True: ConstantInt::get(Ty: CGF.Int32Ty, V: GroupSizeOffset),
159 False: ConstantInt::get(Ty: CGF.Int32Ty, V: RemainderOffset));
160
161 llvm::Value *SizeGEP =
162 CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ImplicitArgPtr, IdxList: StructOffset);
163 llvm::LoadInst *Size = CGF.Builder.CreateLoad(
164 Addr: Address(SizeGEP, CGF.Int16Ty, CharUnits::fromQuantity(Quantity: 2)));
165 addMaxWorkGroupSizeRangeMetadata(CGF, GroupSize: Size);
166 setNoundefInvariantLoad(Size);
167
168 return CGF.Builder.CreateZExt(V: Size, DestTy: CGF.Int32Ty);
169}
170
171static Value *emitAMDGPUWorkGroupSizeV4(CodeGenFunction &CGF, unsigned Index) {
172 llvm::Value *DispatchPtr = EmitAMDGPUDispatchPtr(CGF);
173
174 // Indexing the HSA kernel_dispatch_packet struct.
175 llvm::Value *GroupSizeGEP = CGF.Builder.CreateConstInBoundsGEP1_64(
176 Ty: CGF.Int8Ty, Ptr: DispatchPtr, Idx0: 4 + Index * 2);
177 llvm::LoadInst *GroupSizeLD = CGF.Builder.CreateLoad(
178 Addr: Address(GroupSizeGEP, CGF.Int16Ty, CharUnits::fromQuantity(Quantity: 2)));
179
180 addMaxWorkGroupSizeRangeMetadata(CGF, GroupSize: GroupSizeLD);
181
182 llvm::Value *GroupSize = CGF.Builder.CreateZExt(V: GroupSizeLD, DestTy: CGF.Int32Ty);
183
184 if (CGF.CGM.getLangOpts().OffloadUniformBlock)
185 return GroupSize;
186
187 llvm::Value *WorkgroupID =
188 CGF.Builder.CreateIntrinsic(ID: getAMDGPUWorkGroupID(CGF, Index), Args: {});
189
190 llvm::Value *GridSizeGEP = CGF.Builder.CreateConstInBoundsGEP1_64(
191 Ty: CGF.Int8Ty, Ptr: DispatchPtr, Idx0: 12 + Index * 4);
192 llvm::LoadInst *GridSize = CGF.Builder.CreateLoad(
193 Addr: Address(GridSizeGEP, CGF.Int32Ty, CharUnits::fromQuantity(Quantity: 4)));
194
195 llvm::MDBuilder MDB(CGF.getLLVMContext());
196
197 // Known non-zero.
198 GridSize->setMetadata(KindID: llvm::LLVMContext::MD_range,
199 Node: MDB.createRange(Lo: APInt(32, 1), Hi: APInt::getZero(numBits: 32)));
200 GridSize->setMetadata(KindID: llvm::LLVMContext::MD_invariant_load,
201 Node: llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: {}));
202
203 llvm::Value *Mul = CGF.Builder.CreateMul(LHS: WorkgroupID, RHS: GroupSize);
204 llvm::Value *Remainder = CGF.Builder.CreateSub(LHS: GridSize, RHS: Mul);
205
206 llvm::Value *IsPartial = CGF.Builder.CreateICmpULT(LHS: Remainder, RHS: GroupSize);
207
208 return CGF.Builder.CreateSelect(C: IsPartial, True: Remainder, False: GroupSize);
209}
210
211// \p Index is 0, 1, and 2 for x, y, and z dimension, respectively.
212/// Emit code based on Code Object ABI version.
213/// COV_4 : Emit code to use dispatch ptr
214/// COV_5+ : Emit code to use implicitarg ptr
215/// COV_NONE : Emit code to load a global variable "__oclc_ABI_version"
216/// and use its value for COV_4 or COV_5+ approach. It is used for
217/// compiling device libraries in an ABI-agnostic way.
218Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
219 auto Cov = CGF.getTarget().getTargetOpts().CodeObjectVersion;
220
221 // Do not emit __oclc_ABI_version references with non-empt environment.
222 if (Cov == CodeObjectVersionKind::COV_None &&
223 CGF.getTarget().getTriple().hasEnvironment())
224 Cov = CodeObjectVersionKind::COV_6;
225
226 if (Cov == CodeObjectVersionKind::COV_None) {
227 StringRef Name = "__oclc_ABI_version";
228 auto *ABIVersionC = CGF.CGM.getModule().getNamedGlobal(Name);
229 if (!ABIVersionC)
230 ABIVersionC = new llvm::GlobalVariable(
231 CGF.CGM.getModule(), CGF.Int32Ty, false,
232 llvm::GlobalValue::ExternalLinkage, nullptr, Name, nullptr,
233 llvm::GlobalVariable::NotThreadLocal,
234 CGF.CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_constant));
235
236 // This load will be eliminated by the IPSCCP because it is constant
237 // weak_odr without externally_initialized. Either changing it to weak or
238 // adding externally_initialized will keep the load.
239 Value *ABIVersion = CGF.Builder.CreateAlignedLoad(Ty: CGF.Int32Ty, Addr: ABIVersionC,
240 Align: CGF.CGM.getIntAlign());
241
242 Value *IsCOV5 = CGF.Builder.CreateICmpSGE(
243 LHS: ABIVersion,
244 RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: CodeObjectVersionKind::COV_5));
245
246 llvm::Value *V5Impl = emitAMDGPUWorkGroupSizeV5(CGF, Index);
247 llvm::Value *V4Impl = emitAMDGPUWorkGroupSizeV4(CGF, Index);
248 return CGF.Builder.CreateSelect(C: IsCOV5, True: V5Impl, False: V4Impl);
249 }
250
251 return Cov >= CodeObjectVersionKind::COV_5
252 ? emitAMDGPUWorkGroupSizeV5(CGF, Index)
253 : emitAMDGPUWorkGroupSizeV4(CGF, Index);
254}
255
256// \p Index is 0, 1, and 2 for x, y, and z dimension, respectively.
257Value *EmitAMDGPUGridSize(CodeGenFunction &CGF, unsigned Index) {
258 const unsigned XOffset = 12;
259 auto *DP = EmitAMDGPUDispatchPtr(CGF);
260 // Indexing the HSA kernel_dispatch_packet struct.
261 auto *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: XOffset + Index * 4);
262 auto *GEP = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: DP, IdxList: Offset);
263 auto *LD = CGF.Builder.CreateLoad(
264 Addr: Address(GEP, CGF.Int32Ty, CharUnits::fromQuantity(Quantity: 4)));
265
266 llvm::MDBuilder MDB(CGF.getLLVMContext());
267
268 // Known non-zero.
269 LD->setMetadata(KindID: llvm::LLVMContext::MD_range,
270 Node: MDB.createRange(Lo: APInt(32, 1), Hi: APInt::getZero(numBits: 32)));
271 LD->setMetadata(KindID: llvm::LLVMContext::MD_invariant_load,
272 Node: llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: {}));
273 return LD;
274}
275} // namespace
276
277// Generates the IR for __builtin_read_exec_*.
278// Lowers the builtin to amdgcn_ballot intrinsic.
279//
280// The ballot must be taken at the wavefront width: a ballot narrower than the
281// wave size cannot represent one bit per lane and fails to select. Request the
282// mask at the wave width and narrow it afterwards for the _lo and _hi halves.
283static Value *EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E,
284 llvm::Type *RegisterType,
285 llvm::Type *ValueType, bool isExecHi) {
286 CodeGen::CGBuilderTy &Builder = CGF.Builder;
287 CodeGen::CodeGenModule &CGM = CGF.CGM;
288
289 unsigned WaveSize = CGF.getTarget().getGridValue().GV_Warp_Size;
290 unsigned BallotSize = std::max(a: WaveSize, b: RegisterType->getIntegerBitWidth());
291 llvm::Type *BallotType = Builder.getIntNTy(N: BallotSize);
292
293 Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_ballot, Tys: {BallotType});
294 llvm::Value *Call = Builder.CreateCall(Callee: F, Args: {Builder.getInt1(V: true)});
295
296 if (isExecHi) {
297 Value *Rt2 = Builder.CreateLShr(LHS: Call, RHS: 32);
298 Rt2 = Builder.CreateTrunc(V: Rt2, DestTy: CGF.Int32Ty);
299 return Rt2;
300 }
301
302 return Builder.CreateTrunc(V: Call, DestTy: ValueType);
303}
304
305static llvm::Value *loadTextureDescPtorAsVec8I32(CodeGenFunction &CGF,
306 llvm::Value *RsrcPtr) {
307 auto &B = CGF.Builder;
308 auto *VecTy = llvm::FixedVectorType::get(ElementType: B.getInt32Ty(), NumElts: 8);
309
310 if (RsrcPtr->getType() == VecTy)
311 return RsrcPtr;
312
313 if (RsrcPtr->getType()->isIntegerTy(BitWidth: 32)) {
314 llvm::PointerType *VecPtrTy =
315 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: 8);
316 llvm::Value *Ptr = B.CreateIntToPtr(V: RsrcPtr, DestTy: VecPtrTy, Name: "tex.rsrc.from.int");
317 return B.CreateAlignedLoad(Ty: VecTy, Ptr, Align: llvm::Align(32), Name: "tex.rsrc.val");
318 }
319
320 if (RsrcPtr->getType()->isPointerTy()) {
321 auto *VecPtrTy = llvm::PointerType::get(
322 C&: CGF.getLLVMContext(), AddressSpace: RsrcPtr->getType()->getPointerAddressSpace());
323 llvm::Value *Typed = B.CreateBitCast(V: RsrcPtr, DestTy: VecPtrTy, Name: "tex.rsrc.typed");
324 return B.CreateAlignedLoad(Ty: VecTy, Ptr: Typed, Align: llvm::Align(32), Name: "tex.rsrc.val");
325 }
326
327 const auto &DL = CGF.CGM.getDataLayout();
328 if (DL.getTypeSizeInBits(Ty: RsrcPtr->getType()) == 256)
329 return B.CreateBitCast(V: RsrcPtr, DestTy: VecTy, Name: "tex.rsrc.val");
330
331 llvm::report_fatal_error(reason: "Unexpected texture resource argument form");
332}
333
334llvm::CallInst *
335emitAMDGCNImageOverloadedReturnType(clang::CodeGen::CodeGenFunction &CGF,
336 const clang::CallExpr *E,
337 unsigned IntrinsicID, bool IsImageStore) {
338 auto findTextureDescIndex = [&CGF](const CallExpr *E) -> unsigned {
339 QualType TexQT = CGF.getContext().AMDGPUTextureTy;
340 for (unsigned I = 0, N = E->getNumArgs(); I < N; ++I) {
341 QualType ArgTy = E->getArg(Arg: I)->getType();
342 if (ArgTy == TexQT) {
343 return I;
344 }
345
346 if (ArgTy.getCanonicalType() == TexQT.getCanonicalType()) {
347 return I;
348 }
349 }
350
351 return ~0U;
352 };
353
354 clang::SmallVector<llvm::Value *, 10> Args;
355 unsigned RsrcIndex = findTextureDescIndex(E);
356
357 if (RsrcIndex == ~0U) {
358 llvm::report_fatal_error(reason: "Invalid argument count for image builtin");
359 }
360
361 for (unsigned I = 0; I < E->getNumArgs(); ++I) {
362 llvm::Value *V = CGF.EmitScalarExpr(E: E->getArg(Arg: I));
363 if (I == RsrcIndex)
364 V = loadTextureDescPtorAsVec8I32(CGF, RsrcPtr: V);
365 Args.push_back(Elt: V);
366 }
367
368 llvm::Type *RetTy = IsImageStore ? CGF.VoidTy : CGF.ConvertType(T: E->getType());
369 llvm::CallInst *Call =
370 CGF.Builder.CreateIntrinsicWithoutFolding(RetTy, ID: IntrinsicID, Args);
371 return Call;
372}
373
374// Emit an intrinsic that has 1 float or double operand, and 1 integer.
375static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
376 const CallExpr *E,
377 unsigned IntrinsicID) {
378 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
379 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
380
381 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
382 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
383}
384
385// When the target is SPIR-V (spirv64-amd-amdhsa) re-spell the scope for that
386// target by parsing as AMDGPU and re-emitting it.
387static inline StringRef mapScopeToSPIRV(const llvm::Triple &TargetTriple,
388 StringRef AMDGCNScope) {
389 if (auto Parsed = llvm::parseAMDGPUAtomicScopeIRString(Name: AMDGCNScope)) {
390 auto [Scope, IsSingleAddressSpace] = *Parsed;
391 if (auto Str = llvm::getAtomicScopeIRString(T: TargetTriple, S: Scope,
392 IsSingleAddressSpace))
393 return *Str;
394 }
395 return AMDGCNScope;
396}
397
398static llvm::AtomicOrdering mapCABIAtomicOrdering(unsigned AO) {
399 // Map C11/C++11 memory ordering to LLVM memory ordering
400 assert(llvm::isValidAtomicOrderingCABI(AO));
401 switch (static_cast<llvm::AtomicOrderingCABI>(AO)) {
402 case llvm::AtomicOrderingCABI::acquire:
403 case llvm::AtomicOrderingCABI::consume:
404 return llvm::AtomicOrdering::Acquire;
405 case llvm::AtomicOrderingCABI::release:
406 return llvm::AtomicOrdering::Release;
407 case llvm::AtomicOrderingCABI::acq_rel:
408 return llvm::AtomicOrdering::AcquireRelease;
409 case llvm::AtomicOrderingCABI::seq_cst:
410 return llvm::AtomicOrdering::SequentiallyConsistent;
411 case llvm::AtomicOrderingCABI::relaxed:
412 return llvm::AtomicOrdering::Monotonic;
413 }
414 llvm_unreachable("Unknown AtomicOrderingCABI enum");
415}
416
417// Map a __MEMORY_SCOPE_* integer constant to the AMDGPU-specific syncscope.
418// Invalid scope values are mapped to system scope (empty string).
419static StringRef getAMDGPUSyncScopeStr(CodeGenModule &CGM, unsigned ScopeInt,
420 llvm::AtomicOrdering AO) {
421 AtomicScopeGenericModel ScopeModel;
422 if (!ScopeModel.isValid(S: ScopeInt))
423 return "";
424 clang::SyncScope Scope = ScopeModel.map(S: ScopeInt);
425 return CGM.getTargetCodeGenInfo().getLLVMSyncScopeStr(LangOpts: CGM.getLangOpts(),
426 Scope, Ordering: AO);
427}
428
429/// Convert a __MEMORY_SCOPE_* integer constant to a metadata node containing
430/// the target-specific sync scope string.
431static llvm::MetadataAsValue *emitScopeMD(
432 CodeGenFunction &CGF, unsigned ScopeInt,
433 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent) {
434 StringRef ScopeStr = getAMDGPUSyncScopeStr(CGM&: CGF.CGM, ScopeInt, AO);
435 llvm::LLVMContext &Ctx = CGF.CGM.getLLVMContext();
436 llvm::MDNode *MD =
437 llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: ScopeStr)});
438 return llvm::MetadataAsValue::get(Context&: Ctx, MD);
439}
440
441// For processing memory ordering and memory scope arguments of various
442// amdgcn builtins.
443// \p Order takes a C++11 compatible memory-ordering specifier and converts
444// it into LLVM's memory ordering specifier using atomic C ABI, and writes
445// to \p AO. \p Scope takes a const char * and converts it into AMDGCN
446// specific SyncScopeID and writes it to \p SSID.
447void CodeGenFunction::ProcessOrderScopeAMDGCN(Value *Order, Value *Scope,
448 llvm::AtomicOrdering &AO,
449 llvm::SyncScope::ID &SSID) {
450 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
451
452 // Map C11/C++11 memory ordering to LLVM memory ordering
453 AO = mapCABIAtomicOrdering(AO: ord);
454
455 // Some of the atomic builtins take the scope as a string name.
456 const llvm::Triple &TargetTriple = getTarget().getTriple();
457 StringRef scp;
458 if (llvm::getConstantStringInfo(V: Scope, Str&: scp)) {
459 if (TargetTriple.isSPIRV())
460 scp = mapScopeToSPIRV(TargetTriple, AMDGCNScope: scp);
461 SSID = getLLVMContext().getOrInsertSyncScopeID(SSN: scp);
462 return;
463 }
464
465 // Older builtins had an enum argument for the memory scope.
466 unsigned scope = cast<llvm::ConstantInt>(Val: Scope)->getZExtValue();
467 StringRef SSN = getAMDGPUSyncScopeStr(CGM, ScopeInt: scope, AO);
468 SSID = getLLVMContext().getOrInsertSyncScopeID(SSN);
469}
470
471void CodeGenFunction::AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
472 const CallExpr *E) {
473 constexpr const char *Tag = "amdgpu-synchronize-as";
474
475 SmallVector<MMRAMetadata::TagT, 3> MMRAs;
476 for (unsigned K = 2; K < E->getNumArgs(); ++K) {
477 llvm::Value *V = EmitScalarExpr(E: E->getArg(Arg: K));
478 StringRef AS;
479 if (llvm::getConstantStringInfo(V, Str&: AS)) {
480 MMRAs.push_back(Elt: {Tag, AS});
481 // TODO: Delete the resulting unused constant?
482 continue;
483 }
484 CGM.Error(loc: E->getExprLoc(),
485 error: "expected an address space name as a string literal");
486 }
487
488 MMRAMetadata::appendTags(I&: *Inst, Tags: MMRAs);
489}
490
491void CodeGenFunction::AddAMDGPUAvailableVisibleMMRA(llvm::Instruction *Inst) {
492 if (AMDGPUAvailableVisibleMode.empty())
493 return;
494
495 constexpr const char *Tag = "amdgcn-av";
496 MMRAMetadata::appendTags(I&: *Inst, Tags: {{Tag, AMDGPUAvailableVisibleMode}});
497}
498
499static Value *GetAMDGPUPredicate(CodeGenFunction &CGF, Twine Name) {
500 Constant *SpecId = ConstantInt::getAllOnesValue(Ty: CGF.Int32Ty);
501
502 LLVMContext &Ctx = CGF.getLLVMContext();
503 MDNode *Predicate = MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: Name.str()));
504 std::vector<Value *> Args = {SpecId, ConstantInt::getFalse(Context&: Ctx),
505 MetadataAsValue::get(Context&: Ctx, MD: Predicate)};
506 Value *Call = CGF.Builder.CreateIntrinsic(
507 ID: Intrinsic::spv_named_boolean_spec_constant, Args);
508
509 return Call;
510}
511
512static Intrinsic::ID getIntrinsicIDforWaveReduction(unsigned BuiltinID) {
513 switch (BuiltinID) {
514 default:
515 llvm_unreachable("Unknown BuiltinID for wave reduction");
516 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u32:
517 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u64:
518 return Intrinsic::amdgcn_wave_reduce_add;
519 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fadd_f32:
520 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fadd_f64:
521 return Intrinsic::amdgcn_wave_reduce_fadd;
522 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u32:
523 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u64:
524 return Intrinsic::amdgcn_wave_reduce_sub;
525 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fsub_f32:
526 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fsub_f64:
527 return Intrinsic::amdgcn_wave_reduce_fsub;
528 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i32:
529 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i64:
530 return Intrinsic::amdgcn_wave_reduce_min;
531 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fmin_f32:
532 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fmin_f64:
533 return Intrinsic::amdgcn_wave_reduce_fmin;
534 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u32:
535 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u64:
536 return Intrinsic::amdgcn_wave_reduce_umin;
537 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i32:
538 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i64:
539 return Intrinsic::amdgcn_wave_reduce_max;
540 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fmax_f32:
541 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_fmax_f64:
542 return Intrinsic::amdgcn_wave_reduce_fmax;
543 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u32:
544 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u64:
545 return Intrinsic::amdgcn_wave_reduce_umax;
546 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b32:
547 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b64:
548 return Intrinsic::amdgcn_wave_reduce_and;
549 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b32:
550 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b64:
551 return Intrinsic::amdgcn_wave_reduce_or;
552 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b32:
553 case clang::AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b64:
554 return Intrinsic::amdgcn_wave_reduce_xor;
555 }
556}
557
558static void emitBuiltinCheck(CodeGenFunction &CGF, const CallExpr *E,
559 CodeGenFunction::BuiltinCheckKind Kind,
560 llvm::function_ref<Value *()> EmitCond) {
561 if (!CGF.SanOpts.has(K: SanitizerKind::Builtin))
562 return;
563
564 auto CheckOrdinal = SanitizerKind::SO_Builtin;
565 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
566 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
567 CGF.EmitCheck(
568 Checked: std::make_pair(x: EmitCond(), y&: CheckOrdinal), Check: CheckHandler,
569 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getExprLoc()), CGF.Builder.getInt8(C: Kind)},
570 DynamicArgs: {});
571}
572
573Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
574 const CallExpr *E) {
575 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
576 llvm::SyncScope::ID SSID;
577 switch (BuiltinID) {
578 case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u32:
579 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fadd_f32:
580 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fadd_f64:
581 case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u32:
582 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fsub_f32:
583 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fsub_f64:
584 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i32:
585 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u32:
586 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fmin_f32:
587 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fmin_f64:
588 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i32:
589 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u32:
590 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fmax_f32:
591 case AMDGPU::BI__builtin_amdgcn_wave_reduce_fmax_f64:
592 case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b32:
593 case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b32:
594 case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b32:
595 case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u64:
596 case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u64:
597 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i64:
598 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u64:
599 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i64:
600 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u64:
601 case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b64:
602 case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b64:
603 case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b64: {
604 Intrinsic::ID IID = getIntrinsicIDforWaveReduction(BuiltinID);
605 llvm::Value *Value = EmitScalarExpr(E: E->getArg(Arg: 0));
606 llvm::Value *Strategy = EmitScalarExpr(E: E->getArg(Arg: 1));
607 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {Value->getType()});
608 return Builder.CreateCall(Callee: F, Args: {Value, Strategy});
609 }
610 case AMDGPU::BI__builtin_amdgcn_div_scale:
611 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
612 // Translate from the intrinsics's struct return to the builtin's out
613 // argument.
614
615 Address FlagOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
616
617 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
618 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
619 llvm::Value *Z = EmitScalarExpr(E: E->getArg(Arg: 2));
620
621 llvm::Function *Callee = CGM.getIntrinsic(IID: Intrinsic::amdgcn_div_scale,
622 Tys: X->getType());
623
624 llvm::Value *Tmp = Builder.CreateCall(Callee, Args: {X, Y, Z});
625
626 llvm::Value *Result = Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
627 llvm::Value *Flag = Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
628
629 llvm::Type *RealFlagType = FlagOutPtr.getElementType();
630
631 llvm::Value *FlagExt = Builder.CreateZExt(V: Flag, DestTy: RealFlagType);
632 Builder.CreateStore(Val: FlagExt, Addr: FlagOutPtr);
633 return Result;
634 }
635 case AMDGPU::BI__builtin_amdgcn_div_fmas:
636 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
637 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
638 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
639 llvm::Value *Src2 = EmitScalarExpr(E: E->getArg(Arg: 2));
640 llvm::Value *Src3 = EmitScalarExpr(E: E->getArg(Arg: 3));
641
642 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_div_fmas,
643 Tys: Src0->getType());
644 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Arg: Src3);
645 return Builder.CreateCall(Callee: F, Args: {Src0, Src1, Src2, Src3ToBool});
646 }
647
648 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
649 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
650 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
651 llvm::SmallVector<llvm::Value *, 6> Args;
652 // Find out if any arguments are required to be integer constant
653 // expressions.
654 unsigned ICEArguments = 0;
655 ASTContext::GetBuiltinTypeError Error;
656 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
657 assert(Error == ASTContext::GE_None && "Should not codegen an error");
658 llvm::Type *DataTy = ConvertType(T: E->getArg(Arg: 0)->getType());
659 unsigned Size = DataTy->getPrimitiveSizeInBits();
660 llvm::Type *IntTy =
661 llvm::IntegerType::get(C&: Builder.getContext(), NumBits: std::max(a: Size, b: 32u));
662 Function *F =
663 CGM.getIntrinsic(IID: BuiltinID == AMDGPU::BI__builtin_amdgcn_mov_dpp8
664 ? Intrinsic::amdgcn_mov_dpp8
665 : Intrinsic::amdgcn_update_dpp,
666 Tys: IntTy);
667 assert(E->getNumArgs() == 5 || E->getNumArgs() == 6 ||
668 E->getNumArgs() == 2);
669 bool InsertOld = BuiltinID == AMDGPU::BI__builtin_amdgcn_mov_dpp;
670 if (InsertOld)
671 Args.push_back(Elt: llvm::PoisonValue::get(T: IntTy));
672 for (unsigned I = 0; I != E->getNumArgs(); ++I) {
673 llvm::Value *V = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: I, E);
674 if (I < (BuiltinID == AMDGPU::BI__builtin_amdgcn_update_dpp ? 2u : 1u) &&
675 Size < 32) {
676 if (!DataTy->isIntegerTy())
677 V = Builder.CreateBitCast(
678 V, DestTy: llvm::IntegerType::get(C&: Builder.getContext(), NumBits: Size));
679 V = Builder.CreateZExtOrBitCast(V, DestTy: IntTy);
680 }
681 llvm::Type *ExpTy =
682 F->getFunctionType()->getFunctionParamType(i: I + InsertOld);
683 Args.push_back(Elt: Builder.CreateTruncOrBitCast(V, DestTy: ExpTy));
684 }
685 Value *V = Builder.CreateCall(Callee: F, Args);
686 if (Size < 32 && !DataTy->isIntegerTy())
687 V = Builder.CreateTrunc(
688 V, DestTy: llvm::IntegerType::get(C&: Builder.getContext(), NumBits: Size));
689 return Builder.CreateTruncOrBitCast(V, DestTy: DataTy);
690 }
691 case AMDGPU::BI__builtin_amdgcn_permlane16:
692 case AMDGPU::BI__builtin_amdgcn_permlanex16:
693 return emitBuiltinWithOneOverloadedType<6>(
694 CGF&: *this, E,
695 IntrinsicID: BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16
696 ? Intrinsic::amdgcn_permlane16
697 : Intrinsic::amdgcn_permlanex16);
698 case AMDGPU::BI__builtin_amdgcn_permlane64:
699 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
700 IntrinsicID: Intrinsic::amdgcn_permlane64);
701 case AMDGPU::BI__builtin_amdgcn_readlane: {
702 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
703 Value *Lane = EmitScalarExpr(E: E->getArg(Arg: 1));
704 // The lane index must be uniform across the wavefront.
705 emitBuiltinCheck(CGF&: *this, E, Kind: BCK_NonUniformLane, EmitCond: [&] {
706 // EmitIntrinsicCall attaches a convergence token when they are enabled.
707 Value *First = EmitIntrinsicCall(ID: Intrinsic::amdgcn_readfirstlane,
708 Types: {Lane->getType()}, Args: {Lane});
709 Value *Diff = Builder.CreateICmpNE(LHS: Lane, RHS: First);
710 Value *Mask =
711 EmitIntrinsicCall(ID: Intrinsic::amdgcn_ballot, Types: {Int64Ty}, Args: {Diff});
712 return Builder.CreateICmpEQ(LHS: Mask, RHS: Builder.getInt64(C: 0));
713 });
714 return EmitIntrinsicCall(ID: Intrinsic::amdgcn_readlane, Types: {Src->getType()},
715 Args: {Src, Lane});
716 }
717 case AMDGPU::BI__builtin_amdgcn_wave_shuffle: {
718 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
719 Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
720 // The source lane, wrapped to the wavefront size, must be active.
721 emitBuiltinCheck(CGF&: *this, E, Kind: BCK_InactiveLane, EmitCond: [&] {
722 Value *Active = EmitIntrinsicCall(ID: Intrinsic::amdgcn_ballot, Types: {Int64Ty},
723 Args: {Builder.getTrue()});
724 Value *Size = EmitIntrinsicCall(ID: Intrinsic::amdgcn_wavefrontsize);
725 Value *Lane =
726 Builder.CreateAnd(LHS: Idx, RHS: Builder.CreateSub(LHS: Size, RHS: Builder.getInt32(C: 1)));
727 Value *Bit =
728 Builder.CreateLShr(LHS: Active, RHS: Builder.CreateZExt(V: Lane, DestTy: Int64Ty));
729 return Builder.CreateTrunc(V: Bit, DestTy: Builder.getInt1Ty());
730 });
731 return EmitIntrinsicCall(ID: Intrinsic::amdgcn_wave_shuffle, Types: {Src->getType()},
732 Args: {Src, Idx});
733 }
734 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
735 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
736 IntrinsicID: Intrinsic::amdgcn_readfirstlane);
737 case AMDGPU::BI__builtin_amdgcn_div_fixup:
738 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
739 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
740 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
741 IntrinsicID: Intrinsic::amdgcn_div_fixup);
742 case AMDGPU::BI__builtin_amdgcn_trig_preop:
743 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
744 return emitFPIntBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_trig_preop);
745 case AMDGPU::BI__builtin_amdgcn_rcp:
746 case AMDGPU::BI__builtin_amdgcn_rcpf:
747 case AMDGPU::BI__builtin_amdgcn_rcph:
748 case AMDGPU::BI__builtin_amdgcn_rcp_bf16:
749 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_rcp);
750 case AMDGPU::BI__builtin_amdgcn_sqrt:
751 case AMDGPU::BI__builtin_amdgcn_sqrtf:
752 case AMDGPU::BI__builtin_amdgcn_sqrth:
753 case AMDGPU::BI__builtin_amdgcn_sqrt_bf16:
754 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
755 IntrinsicID: Intrinsic::amdgcn_sqrt);
756 case AMDGPU::BI__builtin_amdgcn_rsq:
757 case AMDGPU::BI__builtin_amdgcn_rsqf:
758 case AMDGPU::BI__builtin_amdgcn_rsqh:
759 case AMDGPU::BI__builtin_amdgcn_rsq_bf16:
760 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_rsq);
761 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
762 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
763 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
764 IntrinsicID: Intrinsic::amdgcn_rsq_clamp);
765 case AMDGPU::BI__builtin_amdgcn_sinf:
766 case AMDGPU::BI__builtin_amdgcn_sinh:
767 case AMDGPU::BI__builtin_amdgcn_sin_bf16:
768 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_sin);
769 case AMDGPU::BI__builtin_amdgcn_cosf:
770 case AMDGPU::BI__builtin_amdgcn_cosh:
771 case AMDGPU::BI__builtin_amdgcn_cos_bf16:
772 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_cos);
773 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
774 return EmitAMDGPUDispatchPtr(CGF&: *this, E);
775 case AMDGPU::BI__builtin_amdgcn_logf:
776 case AMDGPU::BI__builtin_amdgcn_log_bf16:
777 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_log);
778 case AMDGPU::BI__builtin_amdgcn_exp2f:
779 case AMDGPU::BI__builtin_amdgcn_exp2_bf16:
780 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
781 IntrinsicID: Intrinsic::amdgcn_exp2);
782 case AMDGPU::BI__builtin_amdgcn_log_clampf:
783 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
784 IntrinsicID: Intrinsic::amdgcn_log_clamp);
785 case AMDGPU::BI__builtin_amdgcn_ldexp:
786 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
787 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
788 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
789 llvm::Function *F =
790 CGM.getIntrinsic(IID: Intrinsic::ldexp, Tys: {Src0->getType(), Src1->getType()});
791 return Builder.CreateCall(Callee: F, Args: {Src0, Src1});
792 }
793 case AMDGPU::BI__builtin_amdgcn_ldexph: {
794 // The raw instruction has a different behavior for out of bounds exponent
795 // values (implicit truncation instead of saturate to short_min/short_max).
796 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
797 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
798 llvm::Function *F =
799 CGM.getIntrinsic(IID: Intrinsic::ldexp, Tys: {Src0->getType(), Int16Ty});
800 return Builder.CreateCall(Callee: F, Args: {Src0, Builder.CreateTrunc(V: Src1, DestTy: Int16Ty)});
801 }
802 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
803 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
804 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
805 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
806 IntrinsicID: Intrinsic::amdgcn_frexp_mant);
807 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
808 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
809 Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
810 Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_frexp_exp,
811 Tys: { Builder.getInt32Ty(), Src0->getType() });
812 return Builder.CreateCall(Callee: F, Args: Src0);
813 }
814 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
815 Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
816 Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_frexp_exp,
817 Tys: { Builder.getInt16Ty(), Src0->getType() });
818 return Builder.CreateCall(Callee: F, Args: Src0);
819 }
820 case AMDGPU::BI__builtin_amdgcn_fract:
821 case AMDGPU::BI__builtin_amdgcn_fractf:
822 case AMDGPU::BI__builtin_amdgcn_fracth:
823 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
824 IntrinsicID: Intrinsic::amdgcn_fract);
825 case AMDGPU::BI__builtin_amdgcn_ubfe:
826 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
827 IntrinsicID: Intrinsic::amdgcn_ubfe);
828 case AMDGPU::BI__builtin_amdgcn_sbfe:
829 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
830 IntrinsicID: Intrinsic::amdgcn_sbfe);
831 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
832 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
833 llvm::Type *ResultType = ConvertType(T: E->getType());
834 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
835 Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_ballot, Tys: {ResultType});
836 return Builder.CreateCall(Callee: F, Args: {Src});
837 }
838 case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w32:
839 case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w64: {
840 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
841 Function *F =
842 CGM.getIntrinsic(IID: Intrinsic::amdgcn_inverse_ballot, Tys: {Src->getType()});
843 return Builder.CreateCall(Callee: F, Args: {Src});
844 }
845 case AMDGPU::BI__builtin_amdgcn_tanhf:
846 case AMDGPU::BI__builtin_amdgcn_tanhh:
847 case AMDGPU::BI__builtin_amdgcn_tanh_bf16:
848 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
849 IntrinsicID: Intrinsic::amdgcn_tanh);
850 case AMDGPU::BI__builtin_amdgcn_uicmp:
851 case AMDGPU::BI__builtin_amdgcn_uicmpl:
852 case AMDGPU::BI__builtin_amdgcn_sicmp:
853 case AMDGPU::BI__builtin_amdgcn_sicmpl:
854 case AMDGPU::BI__builtin_amdgcn_fcmp:
855 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
856 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
857 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
858 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
859 cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 2)))->getZExtValue());
860
861 // FIXME-GFX10: How should 32 bit mask be handled?
862 return Builder.CreateIntrinsic(RetTy: Builder.getInt64Ty(),
863 ID: Intrinsic::amdgcn_ballot,
864 Args: Builder.CreateCmp(Pred, LHS, RHS));
865 }
866 case AMDGPU::BI__builtin_amdgcn_class:
867 case AMDGPU::BI__builtin_amdgcn_classf:
868 case AMDGPU::BI__builtin_amdgcn_classh:
869 return emitFPIntBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_class);
870 case AMDGPU::BI__builtin_amdgcn_fmed3f:
871 case AMDGPU::BI__builtin_amdgcn_fmed3h:
872 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
873 IntrinsicID: Intrinsic::amdgcn_fmed3);
874 case AMDGPU::BI__builtin_amdgcn_ds_append:
875 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
876 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
877 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
878 Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
879 Function *F = CGM.getIntrinsic(IID: Intrin, Tys: { Src0->getType() });
880 return Builder.CreateCall(Callee: F, Args: { Src0, Builder.getFalse() });
881 }
882 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
883 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
884 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
885 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
886 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
887 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
888 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
889 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
890 case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
891 case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
892 case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
893 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
894 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
895 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
896 case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
897 case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
898 case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
899 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
900 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
901 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
902 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
903 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
904 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
905 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
906 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
907 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: {
908 Intrinsic::ID IID;
909 switch (BuiltinID) {
910 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
911 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
912 case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
913 IID = Intrinsic::amdgcn_global_load_tr_b64;
914 break;
915 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
916 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
917 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
918 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
919 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
920 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
921 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
922 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
923 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
924 IID = Intrinsic::amdgcn_global_load_tr_b128;
925 break;
926 case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
927 IID = Intrinsic::amdgcn_global_load_tr4_b64;
928 break;
929 case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
930 IID = Intrinsic::amdgcn_global_load_tr6_b96;
931 break;
932 case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
933 IID = Intrinsic::amdgcn_ds_load_tr4_b64;
934 break;
935 case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
936 IID = Intrinsic::amdgcn_ds_load_tr6_b96;
937 break;
938 case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
939 IID = Intrinsic::amdgcn_ds_load_tr8_b64;
940 break;
941 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
942 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
943 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
944 IID = Intrinsic::amdgcn_ds_load_tr16_b128;
945 break;
946 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
947 IID = Intrinsic::amdgcn_ds_read_tr4_b64;
948 break;
949 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
950 IID = Intrinsic::amdgcn_ds_read_tr8_b64;
951 break;
952 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
953 IID = Intrinsic::amdgcn_ds_read_tr6_b96;
954 break;
955 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
956 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
957 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
958 IID = Intrinsic::amdgcn_ds_read_tr16_b64;
959 break;
960 }
961 llvm::Type *LoadTy = ConvertType(T: E->getType());
962 llvm::Value *Addr = EmitScalarExpr(E: E->getArg(Arg: 0));
963 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {LoadTy});
964 return Builder.CreateCall(Callee: F, Args: {Addr});
965 }
966 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b32:
967 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b64:
968 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b128:
969 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b32:
970 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b64:
971 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b128: {
972
973 Intrinsic::ID IID;
974 switch (BuiltinID) {
975 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b32:
976 IID = Intrinsic::amdgcn_global_load_monitor_b32;
977 break;
978 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b64:
979 IID = Intrinsic::amdgcn_global_load_monitor_b64;
980 break;
981 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b128:
982 IID = Intrinsic::amdgcn_global_load_monitor_b128;
983 break;
984 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b32:
985 IID = Intrinsic::amdgcn_flat_load_monitor_b32;
986 break;
987 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b64:
988 IID = Intrinsic::amdgcn_flat_load_monitor_b64;
989 break;
990 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b128:
991 IID = Intrinsic::amdgcn_flat_load_monitor_b128;
992 break;
993 }
994
995 llvm::Type *LoadTy = ConvertType(T: E->getType());
996 llvm::Value *Addr = EmitScalarExpr(E: E->getArg(Arg: 0));
997
998 auto *AOExpr = cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 1)));
999 auto *ScopeExpr = cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 2)));
1000 llvm::AtomicOrdering AO = mapCABIAtomicOrdering(AO: AOExpr->getZExtValue());
1001
1002 llvm::Value *ScopeMD = emitScopeMD(CGF&: *this, ScopeInt: ScopeExpr->getZExtValue(), AO);
1003 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {LoadTy});
1004 return Builder.CreateCall(Callee: F, Args: {Addr, AOExpr, ScopeMD});
1005 }
1006 case AMDGPU::BI__builtin_amdgcn_cluster_load_b32:
1007 case AMDGPU::BI__builtin_amdgcn_cluster_load_b64:
1008 case AMDGPU::BI__builtin_amdgcn_cluster_load_b128: {
1009 Intrinsic::ID IID;
1010 switch (BuiltinID) {
1011 case AMDGPU::BI__builtin_amdgcn_cluster_load_b32:
1012 IID = Intrinsic::amdgcn_cluster_load_b32;
1013 break;
1014 case AMDGPU::BI__builtin_amdgcn_cluster_load_b64:
1015 IID = Intrinsic::amdgcn_cluster_load_b64;
1016 break;
1017 case AMDGPU::BI__builtin_amdgcn_cluster_load_b128:
1018 IID = Intrinsic::amdgcn_cluster_load_b128;
1019 break;
1020 }
1021 SmallVector<Value *, 3> Args;
1022 for (int i = 0, e = E->getNumArgs(); i != e; ++i)
1023 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: i)));
1024 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {ConvertType(T: E->getType())});
1025 return Builder.CreateCall(Callee: F, Args: {Args});
1026 }
1027 case AMDGPU::BI__builtin_amdgcn_load_to_lds: {
1028 // Should this have asan instrumentation?
1029 return emitBuiltinWithOneOverloadedType<5>(CGF&: *this, E,
1030 IntrinsicID: Intrinsic::amdgcn_load_to_lds);
1031 }
1032 case AMDGPU::BI__builtin_amdgcn_load_async_to_lds: {
1033 // Should this have asan instrumentation?
1034 return emitBuiltinWithOneOverloadedType<5>(
1035 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_load_async_to_lds);
1036 }
1037 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_32x4B:
1038 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_32x4B:
1039 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_16x8B:
1040 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_16x8B:
1041 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_8x16B:
1042 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_8x16B: {
1043 Intrinsic::ID IID;
1044 switch (BuiltinID) {
1045 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_32x4B:
1046 IID = Intrinsic::amdgcn_cooperative_atomic_load_32x4B;
1047 break;
1048 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_32x4B:
1049 IID = Intrinsic::amdgcn_cooperative_atomic_store_32x4B;
1050 break;
1051 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_16x8B:
1052 IID = Intrinsic::amdgcn_cooperative_atomic_load_16x8B;
1053 break;
1054 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_16x8B:
1055 IID = Intrinsic::amdgcn_cooperative_atomic_store_16x8B;
1056 break;
1057 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_8x16B:
1058 IID = Intrinsic::amdgcn_cooperative_atomic_load_8x16B;
1059 break;
1060 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_8x16B:
1061 IID = Intrinsic::amdgcn_cooperative_atomic_store_8x16B;
1062 break;
1063 }
1064
1065 LLVMContext &Ctx = CGM.getLLVMContext();
1066 SmallVector<Value *, 5> Args;
1067 // last argument is a MD string
1068 const unsigned ScopeArg = E->getNumArgs() - 1;
1069 for (unsigned i = 0; i != ScopeArg; ++i)
1070 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: i)));
1071 StringRef Arg = cast<StringLiteral>(Val: E->getArg(Arg: ScopeArg)->IgnoreParenCasts())
1072 ->getString();
1073 llvm::MDNode *MD = llvm::MDNode::get(Context&: Ctx, MDs: {llvm::MDString::get(Context&: Ctx, Str: Arg)});
1074 Args.push_back(Elt: llvm::MetadataAsValue::get(Context&: Ctx, MD));
1075 // Intrinsic is typed based on the pointer AS. Pointer is always the first
1076 // argument.
1077 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {Args[0]->getType()});
1078 return Builder.CreateCall(Callee: F, Args: {Args});
1079 }
1080 case AMDGPU::BI__builtin_amdgcn_av_load_b8:
1081 case AMDGPU::BI__builtin_amdgcn_av_load_b16:
1082 case AMDGPU::BI__builtin_amdgcn_av_load_b32:
1083 case AMDGPU::BI__builtin_amdgcn_av_load_b64:
1084 case AMDGPU::BI__builtin_amdgcn_av_load_b128:
1085 case AMDGPU::BI__builtin_amdgcn_av_store_b8:
1086 case AMDGPU::BI__builtin_amdgcn_av_store_b16:
1087 case AMDGPU::BI__builtin_amdgcn_av_store_b32:
1088 case AMDGPU::BI__builtin_amdgcn_av_store_b64:
1089 case AMDGPU::BI__builtin_amdgcn_av_store_b128: {
1090 Intrinsic::ID IID;
1091 switch (BuiltinID) {
1092 case AMDGPU::BI__builtin_amdgcn_av_load_b8:
1093 IID = Intrinsic::amdgcn_av_load_b8;
1094 break;
1095 case AMDGPU::BI__builtin_amdgcn_av_load_b16:
1096 IID = Intrinsic::amdgcn_av_load_b16;
1097 break;
1098 case AMDGPU::BI__builtin_amdgcn_av_load_b32:
1099 IID = Intrinsic::amdgcn_av_load_b32;
1100 break;
1101 case AMDGPU::BI__builtin_amdgcn_av_load_b64:
1102 IID = Intrinsic::amdgcn_av_load_b64;
1103 break;
1104 case AMDGPU::BI__builtin_amdgcn_av_load_b128:
1105 IID = Intrinsic::amdgcn_av_load_b128;
1106 break;
1107 case AMDGPU::BI__builtin_amdgcn_av_store_b8:
1108 IID = Intrinsic::amdgcn_av_store_b8;
1109 break;
1110 case AMDGPU::BI__builtin_amdgcn_av_store_b16:
1111 IID = Intrinsic::amdgcn_av_store_b16;
1112 break;
1113 case AMDGPU::BI__builtin_amdgcn_av_store_b32:
1114 IID = Intrinsic::amdgcn_av_store_b32;
1115 break;
1116 case AMDGPU::BI__builtin_amdgcn_av_store_b64:
1117 IID = Intrinsic::amdgcn_av_store_b64;
1118 break;
1119 case AMDGPU::BI__builtin_amdgcn_av_store_b128:
1120 IID = Intrinsic::amdgcn_av_store_b128;
1121 break;
1122 }
1123 const bool IsStore = E->getType()->isVoidType();
1124 SmallVector<Value *, 5> Args = {EmitScalarExpr(E: E->getArg(Arg: 0))}; // addr
1125 if (IsStore)
1126 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: 1))); // data
1127 const unsigned ScopeIdx = E->getNumArgs() - 1;
1128 auto *ScopeExpr =
1129 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: ScopeIdx)));
1130 Args.push_back(Elt: emitScopeMD(CGF&: *this, ScopeInt: ScopeExpr->getZExtValue()));
1131 llvm::Function *F = CGM.getIntrinsic(IID, Tys: {Args[0]->getType()});
1132 return Builder.CreateCall(Callee: F, Args);
1133 }
1134 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
1135 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_fpenv,
1136 Tys: {llvm::Type::getInt64Ty(C&: getLLVMContext())});
1137 return Builder.CreateCall(Callee: F);
1138 }
1139 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
1140 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_fpenv,
1141 Tys: {llvm::Type::getInt64Ty(C&: getLLVMContext())});
1142 llvm::Value *Env = EmitScalarExpr(E: E->getArg(Arg: 0));
1143 return Builder.CreateCall(Callee: F, Args: {Env});
1144 }
1145 case AMDGPU::BI__builtin_amdgcn_processor_is: {
1146 assert(CGM.getTriple().isSPIRV() &&
1147 "__builtin_amdgcn_processor_is should never reach CodeGen for "
1148 "concrete targets!");
1149 StringRef Proc = cast<clang::StringLiteral>(Val: E->getArg(Arg: 0))->getString();
1150 return GetAMDGPUPredicate(CGF&: *this, Name: "is." + Proc);
1151 }
1152 case AMDGPU::BI__builtin_amdgcn_is_invocable: {
1153 assert(CGM.getTriple().isSPIRV() &&
1154 "__builtin_amdgcn_is_invocable should never reach CodeGen for "
1155 "concrete targets!");
1156 auto *FD = cast<FunctionDecl>(
1157 Val: cast<DeclRefExpr>(Val: E->getArg(Arg: 0))->getReferencedDeclOfCallee());
1158 StringRef RF =
1159 getContext().BuiltinInfo.getRequiredFeatures(ID: FD->getBuiltinID());
1160 return GetAMDGPUPredicate(CGF&: *this, Name: "has." + RF);
1161 }
1162 case AMDGPU::BI__builtin_amdgcn_read_exec:
1163 return EmitAMDGCNBallotForExec(CGF&: *this, E, RegisterType: Int64Ty, ValueType: Int64Ty, isExecHi: false);
1164 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
1165 return EmitAMDGCNBallotForExec(CGF&: *this, E, RegisterType: Int32Ty, ValueType: Int32Ty, isExecHi: false);
1166 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
1167 return EmitAMDGCNBallotForExec(CGF&: *this, E, RegisterType: Int64Ty, ValueType: Int64Ty, isExecHi: true);
1168 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
1169 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
1170 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
1171 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
1172 llvm::Value *NodePtr = EmitScalarExpr(E: E->getArg(Arg: 0));
1173 llvm::Value *RayExtent = EmitScalarExpr(E: E->getArg(Arg: 1));
1174 llvm::Value *RayOrigin = EmitScalarExpr(E: E->getArg(Arg: 2));
1175 llvm::Value *RayDir = EmitScalarExpr(E: E->getArg(Arg: 3));
1176 llvm::Value *RayInverseDir = EmitScalarExpr(E: E->getArg(Arg: 4));
1177 llvm::Value *TextureDescr = EmitScalarExpr(E: E->getArg(Arg: 5));
1178
1179 // The builtins take these arguments as vec4 where the last element is
1180 // ignored. The intrinsic takes them as vec3.
1181 RayOrigin = Builder.CreateShuffleVector(V1: RayOrigin, V2: RayOrigin,
1182 Mask: {0, 1, 2});
1183 RayDir =
1184 Builder.CreateShuffleVector(V1: RayDir, V2: RayDir, Mask: {0, 1, 2});
1185 RayInverseDir = Builder.CreateShuffleVector(V1: RayInverseDir, V2: RayInverseDir,
1186 Mask: {0, 1, 2});
1187
1188 Function *F = CGM.getIntrinsic(IID: Intrinsic::amdgcn_image_bvh_intersect_ray,
1189 Tys: {NodePtr->getType(), RayDir->getType()});
1190 return Builder.CreateCall(Callee: F, Args: {NodePtr, RayExtent, RayOrigin, RayDir,
1191 RayInverseDir, TextureDescr});
1192 }
1193 case AMDGPU::BI__builtin_amdgcn_image_bvh8_intersect_ray:
1194 case AMDGPU::BI__builtin_amdgcn_image_bvh_dual_intersect_ray: {
1195 Intrinsic::ID IID;
1196 switch (BuiltinID) {
1197 case AMDGPU::BI__builtin_amdgcn_image_bvh8_intersect_ray:
1198 IID = Intrinsic::amdgcn_image_bvh8_intersect_ray;
1199 break;
1200 case AMDGPU::BI__builtin_amdgcn_image_bvh_dual_intersect_ray:
1201 IID = Intrinsic::amdgcn_image_bvh_dual_intersect_ray;
1202 break;
1203 }
1204 llvm::Value *NodePtr = EmitScalarExpr(E: E->getArg(Arg: 0));
1205 llvm::Value *RayExtent = EmitScalarExpr(E: E->getArg(Arg: 1));
1206 llvm::Value *InstanceMask = EmitScalarExpr(E: E->getArg(Arg: 2));
1207 llvm::Value *RayOrigin = EmitScalarExpr(E: E->getArg(Arg: 3));
1208 llvm::Value *RayDir = EmitScalarExpr(E: E->getArg(Arg: 4));
1209 llvm::Value *Offset = EmitScalarExpr(E: E->getArg(Arg: 5));
1210 llvm::Value *TextureDescr = EmitScalarExpr(E: E->getArg(Arg: 6));
1211
1212 Address RetRayOriginPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 7));
1213 Address RetRayDirPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 8));
1214
1215 llvm::Function *IntrinsicFunc = CGM.getIntrinsic(IID);
1216
1217 llvm::CallInst *CI = Builder.CreateCall(
1218 Callee: IntrinsicFunc, Args: {NodePtr, RayExtent, InstanceMask, RayOrigin, RayDir,
1219 Offset, TextureDescr});
1220
1221 llvm::Value *RetVData = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
1222 llvm::Value *RetRayOrigin = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
1223 llvm::Value *RetRayDir = Builder.CreateExtractValue(Agg: CI, Idxs: 2);
1224
1225 Builder.CreateStore(Val: RetRayOrigin, Addr: RetRayOriginPtr);
1226 Builder.CreateStore(Val: RetRayDir, Addr: RetRayDirPtr);
1227
1228 return RetVData;
1229 }
1230
1231 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn:
1232 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn:
1233 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn:
1234 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1235 Intrinsic::ID IID;
1236 switch (BuiltinID) {
1237 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn:
1238 IID = Intrinsic::amdgcn_ds_bvh_stack_rtn;
1239 break;
1240 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn:
1241 IID = Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn;
1242 break;
1243 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn:
1244 IID = Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn;
1245 break;
1246 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn:
1247 IID = Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn;
1248 break;
1249 }
1250
1251 SmallVector<Value *, 4> Args;
1252 for (int i = 0, e = E->getNumArgs(); i != e; ++i)
1253 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: i)));
1254
1255 Function *F = CGM.getIntrinsic(IID);
1256 Value *Call = Builder.CreateCall(Callee: F, Args);
1257 Value *Rtn = Builder.CreateExtractValue(Agg: Call, Idxs: 0);
1258 Value *A = Builder.CreateExtractValue(Agg: Call, Idxs: 1);
1259 llvm::Type *RetTy = ConvertType(T: E->getType());
1260 Value *I0 = Builder.CreateInsertElement(Vec: PoisonValue::get(T: RetTy), NewElt: Rtn,
1261 Idx: (uint64_t)0);
1262 // ds_bvh_stack_push8_pop2_rtn returns {i64, i32} but the builtin returns
1263 // <2 x i64>, zext the second value.
1264 if (A->getType()->getPrimitiveSizeInBits() <
1265 RetTy->getScalarType()->getPrimitiveSizeInBits())
1266 A = Builder.CreateZExt(V: A, DestTy: RetTy->getScalarType());
1267
1268 return Builder.CreateInsertElement(Vec: I0, NewElt: A, Idx: 1);
1269 }
1270 case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f32_i32:
1271 case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f16_i32:
1272 return emitAMDGCNImageOverloadedReturnType(
1273 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_1d, IsImageStore: false);
1274 case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f32_i32:
1275 case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f16_i32:
1276 return emitAMDGCNImageOverloadedReturnType(
1277 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_1darray, IsImageStore: false);
1278 case AMDGPU::BI__builtin_amdgcn_image_load_2d_f32_i32:
1279 case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f32_i32:
1280 case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f16_i32:
1281 return emitAMDGCNImageOverloadedReturnType(
1282 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_2d, IsImageStore: false);
1283 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_f32_i32:
1284 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f32_i32:
1285 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f16_i32:
1286 return emitAMDGCNImageOverloadedReturnType(
1287 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_2darray, IsImageStore: false);
1288 case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f32_i32:
1289 case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f16_i32:
1290 return emitAMDGCNImageOverloadedReturnType(
1291 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_3d, IsImageStore: false);
1292 case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f32_i32:
1293 case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f16_i32:
1294 return emitAMDGCNImageOverloadedReturnType(
1295 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_cube, IsImageStore: false);
1296 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f32_i32:
1297 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f16_i32:
1298 return emitAMDGCNImageOverloadedReturnType(
1299 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_1d, IsImageStore: false);
1300 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f32_i32:
1301 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f16_i32:
1302 return emitAMDGCNImageOverloadedReturnType(
1303 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_1darray, IsImageStore: false);
1304 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_f32_i32:
1305 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f32_i32:
1306 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f16_i32:
1307 return emitAMDGCNImageOverloadedReturnType(
1308 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_2d, IsImageStore: false);
1309 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_f32_i32:
1310 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f32_i32:
1311 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f16_i32:
1312 return emitAMDGCNImageOverloadedReturnType(
1313 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_2darray, IsImageStore: false);
1314 case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f32_i32:
1315 case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f16_i32:
1316 return emitAMDGCNImageOverloadedReturnType(
1317 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_3d, IsImageStore: false);
1318 case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f32_i32:
1319 case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f16_i32:
1320 return emitAMDGCNImageOverloadedReturnType(
1321 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_load_mip_cube, IsImageStore: false);
1322 case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f32_i32:
1323 case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f16_i32:
1324 return emitAMDGCNImageOverloadedReturnType(
1325 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_1d, IsImageStore: true);
1326 case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f32_i32:
1327 case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f16_i32:
1328 return emitAMDGCNImageOverloadedReturnType(
1329 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_1darray, IsImageStore: true);
1330 case AMDGPU::BI__builtin_amdgcn_image_store_2d_f32_i32:
1331 case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f32_i32:
1332 case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f16_i32:
1333 return emitAMDGCNImageOverloadedReturnType(
1334 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_2d, IsImageStore: true);
1335 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_f32_i32:
1336 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f32_i32:
1337 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f16_i32:
1338 return emitAMDGCNImageOverloadedReturnType(
1339 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_2darray, IsImageStore: true);
1340 case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f32_i32:
1341 case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f16_i32:
1342 return emitAMDGCNImageOverloadedReturnType(
1343 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_3d, IsImageStore: true);
1344 case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f32_i32:
1345 case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f16_i32:
1346 return emitAMDGCNImageOverloadedReturnType(
1347 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_cube, IsImageStore: true);
1348 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f32_i32:
1349 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f16_i32:
1350 return emitAMDGCNImageOverloadedReturnType(
1351 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_1d, IsImageStore: true);
1352 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f32_i32:
1353 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f16_i32:
1354 return emitAMDGCNImageOverloadedReturnType(
1355 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_1darray, IsImageStore: true);
1356 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_f32_i32:
1357 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f32_i32:
1358 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f16_i32:
1359 return emitAMDGCNImageOverloadedReturnType(
1360 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_2d, IsImageStore: true);
1361 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_f32_i32:
1362 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f32_i32:
1363 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f16_i32:
1364 return emitAMDGCNImageOverloadedReturnType(
1365 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_2darray, IsImageStore: true);
1366 case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f32_i32:
1367 case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f16_i32:
1368 return emitAMDGCNImageOverloadedReturnType(
1369 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_3d, IsImageStore: true);
1370 case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f32_i32:
1371 case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f16_i32:
1372 return emitAMDGCNImageOverloadedReturnType(
1373 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_store_mip_cube, IsImageStore: true);
1374 case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f32_f32:
1375 case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f16_f32:
1376 return emitAMDGCNImageOverloadedReturnType(
1377 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_1d, IsImageStore: false);
1378 case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f32_f32:
1379 case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f16_f32:
1380 return emitAMDGCNImageOverloadedReturnType(
1381 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_1darray, IsImageStore: false);
1382 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_f32_f32:
1383 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f32_f32:
1384 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f16_f32:
1385 return emitAMDGCNImageOverloadedReturnType(
1386 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_2d, IsImageStore: false);
1387 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_f32_f32:
1388 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f32_f32:
1389 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f16_f32:
1390 return emitAMDGCNImageOverloadedReturnType(
1391 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_2darray, IsImageStore: false);
1392 case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f32_f32:
1393 case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f16_f32:
1394 return emitAMDGCNImageOverloadedReturnType(
1395 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_3d, IsImageStore: false);
1396 case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f32_f32:
1397 case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f16_f32:
1398 return emitAMDGCNImageOverloadedReturnType(
1399 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_cube, IsImageStore: false);
1400 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f32_f32:
1401 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f16_f32:
1402 return emitAMDGCNImageOverloadedReturnType(
1403 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_1d, IsImageStore: false);
1404 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f32_f32:
1405 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f16_f32:
1406 return emitAMDGCNImageOverloadedReturnType(
1407 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_1d, IsImageStore: false);
1408 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f32_f32:
1409 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f16_f32:
1410 return emitAMDGCNImageOverloadedReturnType(
1411 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_d_1d, IsImageStore: false);
1412 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f32_f32:
1413 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f16_f32:
1414 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_f32_f32:
1415 return emitAMDGCNImageOverloadedReturnType(
1416 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_2d, IsImageStore: false);
1417 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f32_f32:
1418 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f16_f32:
1419 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_f32_f32:
1420 return emitAMDGCNImageOverloadedReturnType(
1421 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_2d, IsImageStore: false);
1422 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f32_f32:
1423 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f16_f32:
1424 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_f32_f32:
1425 return emitAMDGCNImageOverloadedReturnType(
1426 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_d_2d, IsImageStore: false);
1427 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f32_f32:
1428 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f16_f32:
1429 return emitAMDGCNImageOverloadedReturnType(
1430 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_3d, IsImageStore: false);
1431 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f32_f32:
1432 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f16_f32:
1433 return emitAMDGCNImageOverloadedReturnType(
1434 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_3d, IsImageStore: false);
1435 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f32_f32:
1436 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f16_f32:
1437 return emitAMDGCNImageOverloadedReturnType(
1438 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_d_3d, IsImageStore: false);
1439 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f32_f32:
1440 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f16_f32:
1441 return emitAMDGCNImageOverloadedReturnType(
1442 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_cube, IsImageStore: false);
1443 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f32_f32:
1444 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f16_f32:
1445 return emitAMDGCNImageOverloadedReturnType(
1446 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_cube, IsImageStore: false);
1447 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f32_f32:
1448 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f16_f32:
1449 return emitAMDGCNImageOverloadedReturnType(
1450 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_1darray, IsImageStore: false);
1451 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f32_f32:
1452 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f16_f32:
1453 return emitAMDGCNImageOverloadedReturnType(
1454 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_1darray, IsImageStore: false);
1455 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f32_f32:
1456 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f16_f32:
1457 return emitAMDGCNImageOverloadedReturnType(
1458 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_d_1darray, IsImageStore: false);
1459 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f32_f32:
1460 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f16_f32:
1461 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_f32_f32:
1462 return emitAMDGCNImageOverloadedReturnType(
1463 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_lz_2darray, IsImageStore: false);
1464 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f32_f32:
1465 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f16_f32:
1466 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_f32_f32:
1467 return emitAMDGCNImageOverloadedReturnType(
1468 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_l_2darray, IsImageStore: false);
1469 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f32_f32:
1470 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f16_f32:
1471 case clang::AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_f32_f32:
1472 return emitAMDGCNImageOverloadedReturnType(
1473 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_sample_d_2darray, IsImageStore: false);
1474 case clang::AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f32_f32:
1475 case clang::AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f16_f32:
1476 return emitAMDGCNImageOverloadedReturnType(
1477 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_image_gather4_lz_2d, IsImageStore: false);
1478 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
1479 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
1480 llvm::FixedVectorType *VT = FixedVectorType::get(ElementType: Builder.getInt32Ty(), NumElts: 8);
1481 Function *F = CGM.getIntrinsic(
1482 IID: BuiltinID == AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4
1483 ? Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4
1484 : Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4,
1485 Tys: {VT, VT});
1486
1487 SmallVector<Value *, 9> Args;
1488 for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
1489 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: I)));
1490 return Builder.CreateCall(Callee: F, Args);
1491 }
1492 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
1493 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
1494 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
1495 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
1496 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
1497 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
1498 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
1499 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
1500 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
1501 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
1502 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
1503 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
1504 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
1505 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
1506 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
1507 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
1508 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
1509 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
1510 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
1511 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
1512 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
1513 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
1514 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
1515 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
1516 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
1517 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
1518 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
1519 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
1520 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
1521 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
1522 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
1523 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
1524 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
1525 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
1526 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
1527 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
1528 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
1529 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
1530 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
1531 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
1532 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
1533 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
1534 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
1535 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
1536 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
1537 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
1538 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
1539 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
1540 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
1541 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
1542 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
1543 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
1544 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
1545 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
1546 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
1547 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
1548 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
1549 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
1550 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
1551 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
1552 // GFX1250 WMMA builtins
1553 case AMDGPU::BI__builtin_amdgcn_wmma_f64_16x16x4_f64:
1554 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x4_f32:
1555 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_bf16:
1556 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_f16:
1557 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x32_f16:
1558 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x32_bf16:
1559 case AMDGPU::BI__builtin_amdgcn_wmma_bf16f32_16x16x32_bf16:
1560 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8:
1561 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8:
1562 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8:
1563 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8:
1564 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8:
1565 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8:
1566 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8:
1567 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8:
1568 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8:
1569 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8:
1570 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8:
1571 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8:
1572 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8:
1573 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8:
1574 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8:
1575 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8:
1576 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x64_iu8:
1577 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_f8f6f4:
1578 case AMDGPU::BI__builtin_amdgcn_wmma_f32_32x16x128_f4:
1579 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
1580 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
1581 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_32x16x128_f4:
1582 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_32x16x128_f4:
1583 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_f16:
1584 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_bf16:
1585 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x64_f16:
1586 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x64_bf16:
1587 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16:
1588 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8:
1589 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8:
1590 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8:
1591 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8:
1592 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8:
1593 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8:
1594 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8:
1595 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8:
1596 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x128_iu8: {
1597
1598 // These operations perform a matrix multiplication and accumulation of
1599 // the form:
1600 // D = A * B + C
1601 // We need to specify one type for matrices AB and one for matrices CD.
1602 // Sparse matrix operations can have different types for A and B as well as
1603 // an additional type for sparsity index.
1604 // Destination type should be put before types used for source operands.
1605 SmallVector<unsigned, 2> ArgsForMatchingMatrixTypes;
1606 // On GFX12, the intrinsics with 16-bit accumulator use a packed layout.
1607 // There is no need for the variable opsel argument, so always set it to
1608 // "false".
1609 bool AppendFalseForOpselArg = false;
1610 unsigned BuiltinWMMAOp;
1611 // Need return type when D and C are of different types.
1612 bool NeedReturnType = false;
1613 // Need to remove unused neg modifiers.
1614 bool RemoveABNeg = false;
1615
1616 switch (BuiltinID) {
1617 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
1618 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
1619 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
1620 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
1621 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1622 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
1623 break;
1624 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
1625 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
1626 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
1627 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
1628 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1629 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
1630 break;
1631 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
1632 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
1633 AppendFalseForOpselArg = true;
1634 [[fallthrough]];
1635 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
1636 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
1637 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1638 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
1639 break;
1640 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
1641 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
1642 AppendFalseForOpselArg = true;
1643 [[fallthrough]];
1644 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
1645 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
1646 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1647 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
1648 break;
1649 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
1650 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
1651 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1652 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
1653 break;
1654 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
1655 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
1656 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1657 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
1658 break;
1659 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
1660 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
1661 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
1662 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
1663 ArgsForMatchingMatrixTypes = {4, 1}; // CD, AB
1664 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
1665 break;
1666 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
1667 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
1668 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
1669 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
1670 ArgsForMatchingMatrixTypes = {4, 1}; // CD, AB
1671 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
1672 break;
1673 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
1674 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
1675 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1676 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
1677 break;
1678 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
1679 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
1680 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1681 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
1682 break;
1683 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
1684 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
1685 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1686 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
1687 break;
1688 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
1689 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
1690 ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB
1691 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
1692 break;
1693 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
1694 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
1695 ArgsForMatchingMatrixTypes = {4, 1}; // CD, AB
1696 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
1697 break;
1698 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
1699 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
1700 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1701 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
1702 break;
1703 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
1704 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
1705 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1706 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
1707 break;
1708 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
1709 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
1710 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1711 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
1712 break;
1713 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
1714 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
1715 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1716 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
1717 break;
1718 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
1719 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
1720 ArgsForMatchingMatrixTypes = {4, 1, 3, 5}; // CD, A, B, Index
1721 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
1722 break;
1723 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
1724 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
1725 ArgsForMatchingMatrixTypes = {4, 1, 3, 5}; // CD, A, B, Index
1726 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
1727 break;
1728 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
1729 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
1730 ArgsForMatchingMatrixTypes = {4, 1, 3, 5}; // CD, A, B, Index
1731 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
1732 break;
1733 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
1734 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
1735 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1736 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
1737 break;
1738 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
1739 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
1740 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1741 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
1742 break;
1743 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
1744 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
1745 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1746 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
1747 break;
1748 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
1749 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
1750 ArgsForMatchingMatrixTypes = {2, 0, 1, 3}; // CD, A, B, Index
1751 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
1752 break;
1753 // GFX1250 WMMA builtins
1754 case AMDGPU::BI__builtin_amdgcn_wmma_f64_16x16x4_f64:
1755 ArgsForMatchingMatrixTypes = {5, 1};
1756 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f64_16x16x4_f64;
1757 break;
1758 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x4_f32:
1759 ArgsForMatchingMatrixTypes = {3, 0};
1760 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x4_f32;
1761 RemoveABNeg = true;
1762 break;
1763 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_bf16:
1764 ArgsForMatchingMatrixTypes = {3, 0};
1765 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x32_bf16;
1766 RemoveABNeg = true;
1767 break;
1768 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_f16:
1769 ArgsForMatchingMatrixTypes = {3, 0};
1770 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x32_f16;
1771 RemoveABNeg = true;
1772 break;
1773 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x32_f16:
1774 ArgsForMatchingMatrixTypes = {3, 0};
1775 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x32_f16;
1776 RemoveABNeg = true;
1777 break;
1778 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x32_bf16:
1779 ArgsForMatchingMatrixTypes = {3, 0};
1780 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16;
1781 RemoveABNeg = true;
1782 break;
1783 case AMDGPU::BI__builtin_amdgcn_wmma_bf16f32_16x16x32_bf16:
1784 NeedReturnType = true;
1785 ArgsForMatchingMatrixTypes = {0, 3};
1786 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16;
1787 RemoveABNeg = true;
1788 break;
1789 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8:
1790 ArgsForMatchingMatrixTypes = {3, 0};
1791 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_fp8;
1792 break;
1793 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8:
1794 ArgsForMatchingMatrixTypes = {3, 0};
1795 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_bf8;
1796 break;
1797 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8:
1798 ArgsForMatchingMatrixTypes = {3, 0};
1799 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_fp8;
1800 break;
1801 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8:
1802 ArgsForMatchingMatrixTypes = {3, 0};
1803 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_bf8;
1804 break;
1805 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8:
1806 ArgsForMatchingMatrixTypes = {3, 0};
1807 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_fp8;
1808 break;
1809 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8:
1810 ArgsForMatchingMatrixTypes = {3, 0};
1811 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_bf8;
1812 break;
1813 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8:
1814 ArgsForMatchingMatrixTypes = {3, 0};
1815 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_fp8;
1816 break;
1817 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8:
1818 ArgsForMatchingMatrixTypes = {3, 0};
1819 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_bf8;
1820 break;
1821 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8:
1822 ArgsForMatchingMatrixTypes = {3, 0};
1823 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_fp8;
1824 break;
1825 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8:
1826 ArgsForMatchingMatrixTypes = {3, 0};
1827 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_bf8;
1828 break;
1829 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8:
1830 ArgsForMatchingMatrixTypes = {3, 0};
1831 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_fp8;
1832 break;
1833 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8:
1834 ArgsForMatchingMatrixTypes = {3, 0};
1835 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_bf8;
1836 break;
1837 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8:
1838 ArgsForMatchingMatrixTypes = {3, 0};
1839 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_fp8;
1840 break;
1841 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8:
1842 ArgsForMatchingMatrixTypes = {3, 0};
1843 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_bf8;
1844 break;
1845 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8:
1846 ArgsForMatchingMatrixTypes = {3, 0};
1847 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_fp8;
1848 break;
1849 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8:
1850 ArgsForMatchingMatrixTypes = {3, 0};
1851 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_bf8;
1852 break;
1853 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x64_iu8:
1854 ArgsForMatchingMatrixTypes = {4, 1};
1855 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x64_iu8;
1856 break;
1857 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_f8f6f4:
1858 ArgsForMatchingMatrixTypes = {5, 1, 3};
1859 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4;
1860 break;
1861 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
1862 ArgsForMatchingMatrixTypes = {5, 1, 3};
1863 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4;
1864 break;
1865 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
1866 ArgsForMatchingMatrixTypes = {5, 1, 3};
1867 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4;
1868 break;
1869 case AMDGPU::BI__builtin_amdgcn_wmma_f32_32x16x128_f4:
1870 ArgsForMatchingMatrixTypes = {3, 0, 1};
1871 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_32x16x128_f4;
1872 break;
1873 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_32x16x128_f4:
1874 ArgsForMatchingMatrixTypes = {3, 0, 1};
1875 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_scale_f32_32x16x128_f4;
1876 break;
1877 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_32x16x128_f4:
1878 ArgsForMatchingMatrixTypes = {3, 0, 1};
1879 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_scale16_f32_32x16x128_f4;
1880 break;
1881 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_f16:
1882 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1883 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x64_f16;
1884 break;
1885 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_bf16:
1886 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1887 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16;
1888 break;
1889 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x64_f16:
1890 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1891 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x64_f16;
1892 break;
1893 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x64_bf16:
1894 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1895 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16;
1896 break;
1897 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16:
1898 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1899 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16;
1900 break;
1901 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8:
1902 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1903 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8;
1904 break;
1905 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8:
1906 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1907 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8;
1908 break;
1909 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8:
1910 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1911 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8;
1912 break;
1913 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8:
1914 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1915 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8;
1916 break;
1917 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8:
1918 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1919 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8;
1920 break;
1921 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8:
1922 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1923 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8;
1924 break;
1925 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8:
1926 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1927 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8;
1928 break;
1929 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8:
1930 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
1931 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8;
1932 break;
1933 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x128_iu8:
1934 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
1935 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8;
1936 break;
1937 }
1938
1939 SmallVector<Value *, 6> Args;
1940 for (int i = 0, e = E->getNumArgs(); i != e; ++i) {
1941 // Remove unused neg modifiers.
1942 if (RemoveABNeg && (i == 0 || i == 2))
1943 continue;
1944 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: i)));
1945 }
1946 if (AppendFalseForOpselArg)
1947 Args.push_back(Elt: Builder.getFalse());
1948
1949 // Handle the optional clamp argument of the following two builtins.
1950 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x64_iu8) {
1951 if (Args.size() == 7)
1952 Args.push_back(Elt: Builder.getFalse());
1953 assert(Args.size() == 8 && "Expected 8 arguments");
1954 Args[7] = Builder.CreateZExtOrTrunc(V: Args[7], DestTy: Builder.getInt1Ty());
1955 } else if (BuiltinID ==
1956 AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x128_iu8) {
1957 if (Args.size() == 8)
1958 Args.push_back(Elt: Builder.getFalse());
1959 assert(Args.size() == 9 && "Expected 9 arguments");
1960 Args[8] = Builder.CreateZExtOrTrunc(V: Args[8], DestTy: Builder.getInt1Ty());
1961 }
1962
1963 SmallVector<llvm::Type *, 6> ArgTypes;
1964 if (NeedReturnType)
1965 ArgTypes.push_back(Elt: ConvertType(T: E->getType()));
1966 for (auto ArgIdx : ArgsForMatchingMatrixTypes)
1967 ArgTypes.push_back(Elt: Args[ArgIdx]->getType());
1968
1969 Function *F = CGM.getIntrinsic(IID: BuiltinWMMAOp, Tys: ArgTypes);
1970 return Builder.CreateCall(Callee: F, Args);
1971 }
1972 // amdgcn workgroup size
1973 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
1974 return EmitAMDGPUWorkGroupSize(CGF&: *this, Index: 0);
1975 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
1976 return EmitAMDGPUWorkGroupSize(CGF&: *this, Index: 1);
1977 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
1978 return EmitAMDGPUWorkGroupSize(CGF&: *this, Index: 2);
1979
1980 // amdgcn grid size
1981 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
1982 return EmitAMDGPUGridSize(CGF&: *this, Index: 0);
1983 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
1984 return EmitAMDGPUGridSize(CGF&: *this, Index: 1);
1985 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
1986 return EmitAMDGPUGridSize(CGF&: *this, Index: 2);
1987
1988 // r600 intrinsics
1989 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
1990 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
1991 return emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E,
1992 IntrinsicID: Intrinsic::r600_recipsqrt_ieee);
1993 case AMDGPU::BI__builtin_amdgcn_alignbit: {
1994 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
1995 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
1996 llvm::Value *Src2 = EmitScalarExpr(E: E->getArg(Arg: 2));
1997 Function *F = CGM.getIntrinsic(IID: Intrinsic::fshr, Tys: Src0->getType());
1998 return Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
1999 }
2000 case AMDGPU::BI__builtin_amdgcn_fence: {
2001 ProcessOrderScopeAMDGCN(Order: EmitScalarExpr(E: E->getArg(Arg: 0)),
2002 Scope: EmitScalarExpr(E: E->getArg(Arg: 1)), AO, SSID);
2003 FenceInst *Fence = Builder.CreateFence(Ordering: AO, SSID);
2004 if (E->getNumArgs() > 2)
2005 AddAMDGPUFenceAddressSpaceMMRA(Inst: Fence, E);
2006 getTargetHooks().setTargetAtomicMetadata(CGF&: *this, AtomicInst&: *Fence);
2007 return Fence;
2008 }
2009 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
2010 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
2011 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
2012 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
2013 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
2014 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
2015 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
2016 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
2017 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
2018 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
2019 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
2020 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
2021 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
2022 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
2023 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
2024 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
2025 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
2026 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
2027 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
2028 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
2029 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
2030 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
2031 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: {
2032 llvm::AtomicRMWInst::BinOp BinOp;
2033 switch (BuiltinID) {
2034 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
2035 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
2036 BinOp = llvm::AtomicRMWInst::UIncWrap;
2037 break;
2038 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
2039 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
2040 BinOp = llvm::AtomicRMWInst::UDecWrap;
2041 break;
2042 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
2043 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
2044 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
2045 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
2046 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
2047 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
2048 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
2049 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
2050 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
2051 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
2052 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
2053 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
2054 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
2055 BinOp = llvm::AtomicRMWInst::FAdd;
2056 break;
2057 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
2058 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
2059 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
2060 BinOp = llvm::AtomicRMWInst::FMin;
2061 break;
2062 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
2063 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
2064 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
2065 BinOp = llvm::AtomicRMWInst::FMax;
2066 break;
2067 }
2068
2069 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
2070 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
2071 llvm::Type *OrigTy = Val->getType();
2072 QualType PtrTy = E->getArg(Arg: 0)->IgnoreImpCasts()->getType();
2073
2074 bool Volatile;
2075
2076 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf ||
2077 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf ||
2078 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) {
2079 // __builtin_amdgcn_ds_faddf/fminf/fmaxf has an explicit volatile argument
2080 Volatile =
2081 cast<ConstantInt>(Val: EmitScalarExpr(E: E->getArg(Arg: 4)))->getZExtValue();
2082 } else {
2083 // Infer volatile from the passed type.
2084 Volatile =
2085 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
2086 }
2087
2088 if (E->getNumArgs() >= 4) {
2089 // Some of the builtins have explicit ordering and scope arguments.
2090 ProcessOrderScopeAMDGCN(Order: EmitScalarExpr(E: E->getArg(Arg: 2)),
2091 Scope: EmitScalarExpr(E: E->getArg(Arg: 3)), AO, SSID);
2092 } else {
2093 // Most of the builtins do not have syncscope/order arguments. For DS
2094 // atomics the scope doesn't really matter, as they implicitly operate at
2095 // workgroup scope.
2096 //
2097 // The global/flat cases need to use agent scope to consistently produce
2098 // the native instruction instead of a cmpxchg expansion.
2099 SSID =
2100 getLLVMContext().getOrInsertSyncScopeID(SSN: *llvm::getAtomicScopeIRString(
2101 T: getTarget().getTriple(), S: llvm::AtomicScope::Device));
2102 AO = AtomicOrdering::Monotonic;
2103
2104 // The v2bf16 builtin uses i16 instead of a natural bfloat type.
2105 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16 ||
2106 BuiltinID == AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16 ||
2107 BuiltinID == AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16) {
2108 llvm::Type *V2BF16Ty = FixedVectorType::get(
2109 ElementType: llvm::Type::getBFloatTy(C&: Builder.getContext()), NumElts: 2);
2110 Val = Builder.CreateBitCast(V: Val, DestTy: V2BF16Ty);
2111 }
2112 }
2113
2114 llvm::AtomicRMWInst *RMW =
2115 Builder.CreateAtomicRMW(Op: BinOp, Addr: Ptr, Val, Ordering: AO, SSID);
2116 if (Volatile)
2117 RMW->setVolatile(true);
2118 AddAMDGPUAvailableVisibleMMRA(Inst: RMW);
2119
2120 unsigned AddrSpace = Ptr.getType()->getAddressSpace();
2121 if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
2122 // Most targets require "amdgpu.no.fine.grained.memory" to emit the native
2123 // instruction for flat and global operations.
2124 llvm::MDTuple *EmptyMD = MDNode::get(Context&: getLLVMContext(), MDs: {});
2125 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
2126
2127 // Most targets require "atomic.ignore.denormal.mode" to emit the native
2128 // instruction, but this only matters for float fadd.
2129 addAtomicIgnoreDenormalModeMetadata(CGF&: *this, AtomicInst&: *RMW);
2130 }
2131
2132 return Builder.CreateBitCast(V: RMW, DestTy: OrigTy);
2133 }
2134 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
2135 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
2136 llvm::Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
2137 llvm::Type *ResultType = ConvertType(T: E->getType());
2138 // s_sendmsg_rtn is mangled using return type only.
2139 Function *F =
2140 CGM.getIntrinsic(IID: Intrinsic::amdgcn_s_sendmsg_rtn, Tys: {ResultType});
2141 return Builder.CreateCall(Callee: F, Args: {Arg});
2142 }
2143 case AMDGPU::BI__builtin_amdgcn_permlane16_swap:
2144 case AMDGPU::BI__builtin_amdgcn_permlane32_swap: {
2145 // Because builtin types are limited, and the intrinsic uses a struct/pair
2146 // output, marshal the pair-of-i32 to <2 x i32>.
2147 Value *VDstOld = EmitScalarExpr(E: E->getArg(Arg: 0));
2148 Value *VSrcOld = EmitScalarExpr(E: E->getArg(Arg: 1));
2149 Value *FI = EmitScalarExpr(E: E->getArg(Arg: 2));
2150 Value *BoundCtrl = EmitScalarExpr(E: E->getArg(Arg: 3));
2151 Function *F =
2152 CGM.getIntrinsic(IID: BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16_swap
2153 ? Intrinsic::amdgcn_permlane16_swap
2154 : Intrinsic::amdgcn_permlane32_swap);
2155 llvm::CallInst *Call =
2156 Builder.CreateCall(Callee: F, Args: {VDstOld, VSrcOld, FI, BoundCtrl});
2157
2158 llvm::Value *Elt0 = Builder.CreateExtractValue(Agg: Call, Idxs: 0);
2159 llvm::Value *Elt1 = Builder.CreateExtractValue(Agg: Call, Idxs: 1);
2160
2161 llvm::Type *ResultType = ConvertType(T: E->getType());
2162
2163 llvm::Value *Insert0 = Builder.CreateInsertElement(
2164 Vec: llvm::PoisonValue::get(T: ResultType), NewElt: Elt0, UINT64_C(0));
2165 llvm::Value *AsVector =
2166 Builder.CreateInsertElement(Vec: Insert0, NewElt: Elt1, UINT64_C(1));
2167 return AsVector;
2168 }
2169 case AMDGPU::BI__builtin_amdgcn_bitop3_b32:
2170 case AMDGPU::BI__builtin_amdgcn_bitop3_b16:
2171 return emitBuiltinWithOneOverloadedType<4>(CGF&: *this, E,
2172 IntrinsicID: Intrinsic::amdgcn_bitop3);
2173 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc: {
2174 // TODO: LLVM has this overloaded to allow for fat pointers, but since
2175 // those haven't been plumbed through to Clang yet, default to creating the
2176 // resource type.
2177 SmallVector<Value *, 4> Args;
2178 for (unsigned I = 0; I < 4; ++I)
2179 Args.push_back(Elt: EmitScalarExpr(E: E->getArg(Arg: I)));
2180 llvm::PointerType *RetTy = llvm::PointerType::get(
2181 C&: Builder.getContext(), AddressSpace: llvm::AMDGPUAS::BUFFER_RESOURCE);
2182 Function *F =
2183 CGM.getIntrinsic(IID: Intrinsic::amdgcn_make_buffer_rsrc,
2184 Tys: {RetTy, Args[0]->getType(), Args[2]->getType()});
2185 return Builder.CreateCall(Callee: F, Args);
2186 }
2187 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
2188 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
2189 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
2190 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
2191 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
2192 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
2193 return emitBuiltinWithOneOverloadedType<5>(
2194 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_store);
2195 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32:
2196 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16:
2197 return emitBuiltinWithOneOverloadedType<5>(
2198 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_store_format);
2199 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
2200 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
2201 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
2202 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
2203 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
2204 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
2205 llvm::Type *RetTy = nullptr;
2206 switch (BuiltinID) {
2207 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
2208 RetTy = Int8Ty;
2209 break;
2210 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
2211 RetTy = Int16Ty;
2212 break;
2213 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
2214 RetTy = Int32Ty;
2215 break;
2216 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
2217 RetTy = llvm::FixedVectorType::get(ElementType: Int32Ty, /*NumElements=*/NumElts: 2);
2218 break;
2219 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
2220 RetTy = llvm::FixedVectorType::get(ElementType: Int32Ty, /*NumElements=*/NumElts: 3);
2221 break;
2222 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
2223 RetTy = llvm::FixedVectorType::get(ElementType: Int32Ty, /*NumElements=*/NumElts: 4);
2224 break;
2225 }
2226 Function *F =
2227 CGM.getIntrinsic(IID: Intrinsic::amdgcn_raw_ptr_buffer_load, Tys: RetTy);
2228 return Builder.CreateCall(
2229 Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1)),
2230 EmitScalarExpr(E: E->getArg(Arg: 2)), EmitScalarExpr(E: E->getArg(Arg: 3))});
2231 }
2232 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f32:
2233 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f16: {
2234 llvm::Type *RetTy = ConvertType(T: E->getType());
2235 Function *F =
2236 CGM.getIntrinsic(IID: Intrinsic::amdgcn_raw_ptr_buffer_load_format, Tys: {RetTy});
2237
2238 return Builder.CreateCall(
2239 Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1)),
2240 EmitScalarExpr(E: E->getArg(Arg: 2)), EmitScalarExpr(E: E->getArg(Arg: 3))});
2241 }
2242 case AMDGPU::BI__builtin_amdgcn_struct_buffer_store_format_v4f32:
2243 case AMDGPU::BI__builtin_amdgcn_struct_buffer_store_format_v4f16:
2244 return emitBuiltinWithOneOverloadedType<6>(
2245 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_struct_ptr_buffer_store_format);
2246 case AMDGPU::BI__builtin_amdgcn_struct_buffer_load_format_v4f32:
2247 case AMDGPU::BI__builtin_amdgcn_struct_buffer_load_format_v4f16: {
2248 llvm::Type *RetTy = ConvertType(T: E->getType());
2249 Function *F = CGM.getIntrinsic(
2250 IID: Intrinsic::amdgcn_struct_ptr_buffer_load_format, Tys: {RetTy});
2251
2252 return Builder.CreateCall(
2253 Callee: F, Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1)),
2254 EmitScalarExpr(E: E->getArg(Arg: 2)), EmitScalarExpr(E: E->getArg(Arg: 3)),
2255 EmitScalarExpr(E: E->getArg(Arg: 4))});
2256 }
2257 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32:
2258 return emitBuiltinWithOneOverloadedType<5>(
2259 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_atomic_add);
2260 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32:
2261 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f64:
2262 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16:
2263 return emitBuiltinWithOneOverloadedType<5>(
2264 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd);
2265 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32:
2266 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64:
2267 return emitBuiltinWithOneOverloadedType<5>(
2268 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin);
2269 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32:
2270 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64:
2271 return emitBuiltinWithOneOverloadedType<5>(
2272 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax);
2273 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i32:
2274 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2i32:
2275 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v3i32:
2276 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v4i32:
2277 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v8i32:
2278 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v16i32:
2279 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_f32:
2280 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2f32:
2281 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v3f32:
2282 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v4f32:
2283 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v8f32:
2284 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v16f32:
2285 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i8:
2286 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_u8:
2287 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_i16:
2288 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_u16:
2289 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2i8:
2290 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v3i8:
2291 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v4i8:
2292 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_f16:
2293 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v2f16:
2294 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v3f16:
2295 case AMDGPU::BI__builtin_amdgcn_s_buffer_load_v4f16:
2296 return emitAMDGPUSBufferLoadBuiltin(CGF&: *this, E);
2297 case AMDGPU::BI__builtin_amdgcn_s_prefetch_data:
2298 return emitBuiltinWithOneOverloadedType<2>(
2299 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_s_prefetch_data);
2300 case AMDGPU::BI__builtin_amdgcn_s_prefetch_inst:
2301 return emitBuiltinWithOneOverloadedType<2>(
2302 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_s_prefetch_inst);
2303 case Builtin::BIlogbf:
2304 case Builtin::BI__builtin_logbf: {
2305 Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
2306 Function *FrExpFunc = CGM.getIntrinsic(
2307 IID: Intrinsic::frexp, Tys: {Src0->getType(), Builder.getInt32Ty()});
2308 CallInst *FrExp = Builder.CreateCall(Callee: FrExpFunc, Args: Src0);
2309 Value *Exp = Builder.CreateExtractValue(Agg: FrExp, Idxs: 1);
2310 Value *Add = Builder.CreateAdd(
2311 LHS: Exp, RHS: ConstantInt::getSigned(Ty: Exp->getType(), V: -1), Name: "", HasNUW: false, HasNSW: true);
2312 Value *SIToFP = Builder.CreateSIToFP(V: Add, DestTy: Builder.getFloatTy());
2313 Value *Fabs =
2314 emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs);
2315 Value *FCmpONE = Builder.CreateFCmpONE(
2316 LHS: Fabs, RHS: ConstantFP::getInfinity(Ty: Builder.getFloatTy()));
2317 Value *Sel1 = Builder.CreateSelect(C: FCmpONE, True: SIToFP, False: Fabs);
2318 Value *FCmpOEQ =
2319 Builder.CreateFCmpOEQ(LHS: Src0, RHS: ConstantFP::getZero(Ty: Builder.getFloatTy()));
2320 Value *Sel2 = Builder.CreateSelect(
2321 C: FCmpOEQ,
2322 True: ConstantFP::getInfinity(Ty: Builder.getFloatTy(), /*Negative=*/true), False: Sel1);
2323 return Sel2;
2324 }
2325 case Builtin::BIlogb:
2326 case Builtin::BI__builtin_logb: {
2327 Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
2328 Function *FrExpFunc = CGM.getIntrinsic(
2329 IID: Intrinsic::frexp, Tys: {Src0->getType(), Builder.getInt32Ty()});
2330 CallInst *FrExp = Builder.CreateCall(Callee: FrExpFunc, Args: Src0);
2331 Value *Exp = Builder.CreateExtractValue(Agg: FrExp, Idxs: 1);
2332 Value *Add = Builder.CreateAdd(
2333 LHS: Exp, RHS: ConstantInt::getSigned(Ty: Exp->getType(), V: -1), Name: "", HasNUW: false, HasNSW: true);
2334 Value *SIToFP = Builder.CreateSIToFP(V: Add, DestTy: Builder.getDoubleTy());
2335 Value *Fabs =
2336 emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs);
2337 Value *FCmpONE = Builder.CreateFCmpONE(
2338 LHS: Fabs, RHS: ConstantFP::getInfinity(Ty: Builder.getDoubleTy()));
2339 Value *Sel1 = Builder.CreateSelect(C: FCmpONE, True: SIToFP, False: Fabs);
2340 Value *FCmpOEQ =
2341 Builder.CreateFCmpOEQ(LHS: Src0, RHS: ConstantFP::getZero(Ty: Builder.getDoubleTy()));
2342 Value *Sel2 = Builder.CreateSelect(
2343 C: FCmpOEQ,
2344 True: ConstantFP::getInfinity(Ty: Builder.getDoubleTy(), /*Negative=*/true),
2345 False: Sel1);
2346 return Sel2;
2347 }
2348 case Builtin::BIscalbnf:
2349 case Builtin::BI__builtin_scalbnf:
2350 case Builtin::BIscalbn:
2351 case Builtin::BI__builtin_scalbn:
2352 return emitBinaryExpMaybeConstrainedFPBuiltin(
2353 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp);
2354 case AMDGPU::BI__builtin_amdgcn_permlane_bcast:
2355 return emitBuiltinWithOneOverloadedType<3>(
2356 CGF&: *this, E, IntrinsicID: Intrinsic::amdgcn_permlane_bcast);
2357 case AMDGPU::BI__builtin_amdgcn_permlane_up:
2358 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
2359 IntrinsicID: Intrinsic::amdgcn_permlane_up);
2360 case AMDGPU::BI__builtin_amdgcn_permlane_down:
2361 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
2362 IntrinsicID: Intrinsic::amdgcn_permlane_down);
2363 case AMDGPU::BI__builtin_amdgcn_permlane_xor:
2364 return emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E,
2365 IntrinsicID: Intrinsic::amdgcn_permlane_xor);
2366 default:
2367 return nullptr;
2368 }
2369}
2370