1//===---- CIRGenBuiltinAMDGPU.cpp - Emit CIR for AMDGPU builtins ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit AMDGPU Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
16#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
17#include "mlir/IR/Value.h"
18#include "clang/Basic/TargetBuiltins.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/Support/AMDGPUAddrSpace.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/TargetParser/AtomicScope.h"
23
24using namespace clang;
25using namespace clang::CIRGen;
26using namespace cir;
27
28// Emit a no-operand intrinsic returning a pointer in the constant AS.
29static mlir::Value emitAMDGPUConstantPtrIntrinsic(CIRGenFunction &cgf,
30 mlir::Location loc,
31 llvm::StringRef name) {
32 CIRGenBuilderTy &builder = cgf.getBuilder();
33 mlir::Type retTy = builder.getVoidPtrTy(as: cir::TargetAddressSpaceAttr::get(
34 context: builder.getContext(), value: llvm::AMDGPUAS::CONSTANT_ADDRESS));
35 return builder.emitIntrinsicCallOp(loc, str: name, resTy: retTy, op: mlir::ValueRange{});
36}
37
38/// Map a constant integral to memory order.
39static cir::MemOrder decodeAtomicOrder(const Expr *arg, ASTContext &ctx) {
40 Expr::EvalResult orderRes;
41 if (!arg->EvaluateAsInt(Result&: orderRes, Ctx: ctx))
42 return cir::MemOrder::SequentiallyConsistent;
43 switch (orderRes.Val.getInt().getZExtValue()) {
44 case 0:
45 return cir::MemOrder::Relaxed;
46 case 1: // consume -> acquire
47 case 2:
48 return cir::MemOrder::Acquire;
49 case 3:
50 return cir::MemOrder::Release;
51 case 4:
52 return cir::MemOrder::AcquireRelease;
53 default:
54 return cir::MemOrder::SequentiallyConsistent;
55 }
56}
57
58/// Map an AMDGPU sync-scope string-literal argument onto cir::SyncScopeKind.
59/// An absent or unrecognized scope is system scope, which is the conservative
60/// choice and matches what an empty syncscope string means in LLVM.
61static cir::SyncScopeKind decodeAMDGPUSyncScope(const Expr *arg) {
62 const auto *sl =
63 llvm::dyn_cast<clang::StringLiteral>(Val: arg->IgnoreParenCasts());
64 if (!sl)
65 return cir::SyncScopeKind::System;
66 return llvm::StringSwitch<cir::SyncScopeKind>(sl->getString())
67 .Case(S: "singlethread", Value: cir::SyncScopeKind::SingleThread)
68 .Case(S: "wavefront", Value: cir::SyncScopeKind::Wavefront)
69 .Case(S: "workgroup", Value: cir::SyncScopeKind::Workgroup)
70 .Case(S: "agent", Value: cir::SyncScopeKind::Device)
71 .Default(Value: cir::SyncScopeKind::System);
72}
73
74static cir::SyncScopeKind convertAtomicScopeToCIR(llvm::AtomicScope scope) {
75 switch (scope) {
76 case llvm::AtomicScope::Single:
77 return cir::SyncScopeKind::SingleThread;
78 case llvm::AtomicScope::Wavefront:
79 return cir::SyncScopeKind::Wavefront;
80 case llvm::AtomicScope::Workgroup:
81 return cir::SyncScopeKind::Workgroup;
82 case llvm::AtomicScope::Cluster:
83 return cir::SyncScopeKind::Cluster;
84 case llvm::AtomicScope::Device:
85 return cir::SyncScopeKind::Device;
86 case llvm::AtomicScope::System:
87 return cir::SyncScopeKind::System;
88 }
89 llvm_unreachable("unknown atomic scope");
90}
91
92/// Emit one of the AMDGPU raw hardware atomic builtins as a cir.atomic.fetch.
93static mlir::Value emitAMDGPUAtomicRMW(CIRGenFunction &cgf,
94 const CallExpr *expr,
95 cir::AtomicFetchKind binOp,
96 bool hasVolatileArg) {
97 CIRGenBuilderTy &builder = cgf.getBuilder();
98 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
99
100 Address ptr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
101 mlir::Value val = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
102
103 bool isVolatile;
104 if (hasVolatileArg) {
105 assert(expr->getNumArgs() >= 5);
106 // ds_faddf/fminf/fmaxf spell the volatile flag out as a constant argument.
107 Expr::EvalResult volRes;
108 isVolatile = expr->getArg(Arg: 4)->EvaluateAsInt(Result&: volRes, Ctx: cgf.getContext()) &&
109 volRes.Val.getInt().getBoolValue();
110 } else {
111 // Everything else infers it from the pointee type.
112 QualType argTy = expr->getArg(Arg: 0)->IgnoreImpCasts()->getType();
113 isVolatile = argTy->castAs<clang::PointerType>()
114 ->getPointeeType()
115 .isVolatileQualified();
116 }
117
118 // Some of these builtins spell out the ordering and scope; the rest take the
119 // monotonic/agent default described above.
120 cir::MemOrder order = cir::MemOrder::Relaxed;
121 cir::SyncScopeKind scope = cir::SyncScopeKind::Device;
122 if (expr->getNumArgs() >= 4) {
123 order = decodeAtomicOrder(arg: expr->getArg(Arg: 2), ctx&: cgf.getContext());
124 scope = decodeAMDGPUSyncScope(arg: expr->getArg(Arg: 3));
125 }
126
127 mlir::Type origTy = val.getType();
128 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: origTy);
129 vecTy && mlir::isa<cir::IntType>(Val: vecTy.getElementType())) {
130 auto bf16VecTy =
131 cir::VectorType::get(element_type: builder.getBfloat16Ty(), size: vecTy.getSize());
132 val = builder.createBitcast(loc, src: val, newTy: bf16VecTy);
133 ptr = ptr.withElementType(builder, ElemTy: bf16VecTy);
134 }
135
136 auto rmw = cir::AtomicFetchOp::create(builder, location: loc, ptr: ptr.emitRawPointer(), val,
137 binop: binOp, mem_order: order, sync_scope: scope, is_volatile: isVolatile,
138 /*fetch_first=*/true);
139 rmw->setAttr(name: cir::CIRDialect::getAMDGPURawAtomicAttrName(),
140 value: builder.getUnitAttr());
141 return builder.createBitcast(loc, src: rmw->getResult(idx: 0), newTy: origTy);
142}
143
144// Emit the `amdgcn.dispatch.ptr` intrinsic, address-space-casting the
145// result to match \p e's return type when needed.
146// If \p e is null, returns the raw AS-4 pointer.
147static mlir::Value emitAMDGPUDispatchPtr(CIRGenFunction &cgf,
148 mlir::Location loc,
149 const CallExpr *e = nullptr) {
150 CIRGenBuilderTy &builder = cgf.getBuilder();
151 mlir::Value call =
152 emitAMDGPUConstantPtrIntrinsic(cgf, loc, name: "amdgcn.dispatch.ptr");
153 if (!e)
154 return call;
155 // Only cast when the caller-visible AS differs from the intrinsic's AS;
156 auto expectedPtrTy =
157 mlir::cast<cir::PointerType>(Val: cgf.convertType(t: e->getType()));
158 auto callPtrTy = mlir::cast<cir::PointerType>(Val: call.getType());
159 if (expectedPtrTy.getAddrSpace() == callPtrTy.getAddrSpace())
160 return call;
161 return builder.createAddrSpaceCast(loc, src: call, newTy: expectedPtrTy);
162}
163
164static mlir::Value emitAMDGPUImplicitArgPtr(CIRGenFunction &cgf,
165 mlir::Location loc) {
166 // OG also marks the result align 8 dereferenceable(256).
167 assert(!cir::MissingFeatures::opCallAttrs());
168 return emitAMDGPUConstantPtrIntrinsic(cgf, loc, name: "amdgcn.implicitarg.ptr");
169}
170
171static mlir::Value emitAMDGPUWorkGroupID(CIRGenFunction &cgf,
172 mlir::Location loc, unsigned index) {
173 static constexpr llvm::StringLiteral names[] = {"amdgcn.workgroup.id.x",
174 "amdgcn.workgroup.id.y",
175 "amdgcn.workgroup.id.z"};
176 CIRGenBuilderTy &builder = cgf.getBuilder();
177 return builder.emitIntrinsicCallOp(loc, str: names[index], resTy: builder.getUInt32Ty(),
178 op: mlir::ValueRange{});
179}
180
181// Load a \p ty value at byte \p offset from \p base as an invariant load.
182static mlir::Value emitAMDGPUInvariantLoad(CIRGenFunction &cgf,
183 mlir::Location loc, mlir::Value base,
184 mlir::Value offset, mlir::Type ty,
185 CharUnits align) {
186 CIRGenBuilderTy &builder = cgf.getBuilder();
187 mlir::Value addr = builder.createPtrStride(loc, base, stride: offset);
188 cir::LoadOp load = builder.createAlignedLoad(loc, ty, ptr: addr, align);
189 load.setInvariant(true);
190 assert(!cir::MissingFeatures::opLoadRangeNoundefMetadata());
191 return load;
192}
193
194static mlir::Value emitAMDGPUGridSize(CIRGenFunction &cgf, mlir::Location loc,
195 unsigned index) {
196 CIRGenBuilderTy &builder = cgf.getBuilder();
197 const unsigned xOffset = 12;
198 mlir::Value dispatchPtr = emitAMDGPUDispatchPtr(cgf, loc);
199 // Indexing the HSA kernel_dispatch_packet struct.
200 return emitAMDGPUInvariantLoad(
201 cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: xOffset + index * 4, loc),
202 ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4));
203}
204
205static mlir::Value emitAMDGPUWorkGroupSizeV5(CIRGenFunction &cgf,
206 mlir::Location loc,
207 unsigned index) {
208 CIRGenBuilderTy &builder = cgf.getBuilder();
209 mlir::Value implicitArgPtr = emitAMDGPUImplicitArgPtr(cgf, loc);
210
211 // offsetof(amdhsa_implicit_kernarg_v5, block_count[index])
212 unsigned blockCountOffset = 0 + index * 4;
213 // offsetof(amdhsa_implicit_kernarg_v5, group_size[index])
214 unsigned groupSizeOffset = 12 + index * 2;
215 // offsetof(amdhsa_implicit_kernarg_v5, remainder[index])
216 unsigned remainderOffset = 18 + index * 2;
217
218 mlir::Value sizeOffset;
219 if (cgf.getLangOpts().OffloadUniformBlock) {
220 sizeOffset = builder.getUInt64(c: groupSizeOffset, loc);
221 } else {
222 mlir::Value blockCount = emitAMDGPUInvariantLoad(
223 cgf, loc, base: implicitArgPtr, offset: builder.getUInt64(c: blockCountOffset, loc),
224 ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4));
225 mlir::Value workgroupID = emitAMDGPUWorkGroupID(cgf, loc, index);
226 mlir::Value isFull =
227 builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: workgroupID, rhs: blockCount);
228 sizeOffset = builder.createSelect(loc, condition: isFull,
229 trueValue: builder.getUInt64(c: groupSizeOffset, loc),
230 falseValue: builder.getUInt64(c: remainderOffset, loc));
231 }
232
233 mlir::Value size = emitAMDGPUInvariantLoad(cgf, loc, base: implicitArgPtr,
234 offset: sizeOffset, ty: builder.getUInt16Ty(),
235 align: CharUnits::fromQuantity(Quantity: 2));
236 return builder.createIntCast(src: size, newTy: builder.getUInt32Ty());
237}
238
239static mlir::Value emitAMDGPUWorkGroupSizeV4(CIRGenFunction &cgf,
240 mlir::Location loc,
241 unsigned index) {
242 CIRGenBuilderTy &builder = cgf.getBuilder();
243 mlir::Value dispatchPtr = emitAMDGPUDispatchPtr(cgf, loc);
244
245 // Indexing the HSA kernel_dispatch_packet struct.
246 mlir::Value groupSizeLd = emitAMDGPUInvariantLoad(
247 cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: 4 + index * 2, loc),
248 ty: builder.getUInt16Ty(), align: CharUnits::fromQuantity(Quantity: 2));
249 mlir::Value groupSize =
250 builder.createIntCast(src: groupSizeLd, newTy: builder.getUInt32Ty());
251
252 if (cgf.getLangOpts().OffloadUniformBlock)
253 return groupSize;
254
255 mlir::Value workgroupID = emitAMDGPUWorkGroupID(cgf, loc, index);
256 mlir::Value gridSize = emitAMDGPUInvariantLoad(
257 cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: 12 + index * 4, loc),
258 ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4));
259 mlir::Value mul = builder.createMul(loc, lhs: workgroupID, rhs: groupSize);
260 mlir::Value remainder = builder.createSub(loc, lhs: gridSize, rhs: mul);
261 mlir::Value isPartial =
262 builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: remainder, rhs: groupSize);
263 return builder.createSelect(loc, condition: isPartial, trueValue: remainder, falseValue: groupSize);
264}
265
266// \p index is 0, 1, and 2 for x, y, and z dimension, respectively.
267// Emit code based on Code Object ABI version.
268// COV_4 : Emit code to use dispatch ptr
269// COV_5+ : Emit code to use implicitarg ptr
270// COV_NONE : Emit code to load a global variable "__oclc_ABI_version"
271// and use its value for COV_4 or COV_5+ approach. It is used for
272// compiling device libraries in an ABI-agnostic way.
273static mlir::Value emitAMDGPUWorkGroupSize(CIRGenFunction &cgf,
274 mlir::Location loc, unsigned index) {
275 llvm::CodeObjectVersionKind cov =
276 cgf.getTarget().getTargetOpts().CodeObjectVersion;
277
278 // Do not emit __oclc_ABI_version references with non-empty environment.
279 if (cov == llvm::CodeObjectVersionKind::COV_None &&
280 cgf.getTarget().getTriple().hasEnvironment())
281 cov = llvm::CodeObjectVersionKind::COV_6;
282
283 if (cov != llvm::CodeObjectVersionKind::COV_None)
284 return cov >= llvm::CodeObjectVersionKind::COV_5
285 ? emitAMDGPUWorkGroupSizeV5(cgf, loc, index)
286 : emitAMDGPUWorkGroupSizeV4(cgf, loc, index);
287
288 CIRGenBuilderTy &builder = cgf.getBuilder();
289 CIRGenModule &cgm = cgf.cgm;
290 StringRef name = "__oclc_ABI_version";
291 mlir::Type int32Ty = builder.getSInt32Ty();
292 auto abiVersionGV =
293 mlir::dyn_cast_or_null<cir::GlobalOp>(Val: cgm.getGlobalValue(ref: name));
294 if (!abiVersionGV)
295 abiVersionGV =
296 cgm.createGlobalOp(loc, name, t: int32Ty, /*isConstant=*/false,
297 addrSpace: cir::toCIRAddressSpaceAttr(ctx&: cgm.getMLIRContext(),
298 langAS: LangAS::opencl_constant));
299 mlir::Value abiVersionPtr = builder.createGetGlobal(loc, global: abiVersionGV);
300 mlir::Value abiVersion = builder.createAlignedLoad(
301 loc, ty: int32Ty, ptr: abiVersionPtr,
302 align: cgf.getContext().getTypeAlignInChars(T: cgf.getContext().IntTy));
303 mlir::Value isCOV5 = builder.createCompare(
304 loc, kind: cir::CmpOpKind::ge, lhs: abiVersion,
305 rhs: builder.getConstantInt(loc, ty: int32Ty, value: llvm::CodeObjectVersionKind::COV_5));
306 mlir::Value v5Impl = emitAMDGPUWorkGroupSizeV5(cgf, loc, index);
307 mlir::Value v4Impl = emitAMDGPUWorkGroupSizeV4(cgf, loc, index);
308 return builder.createSelect(loc, condition: isCOV5, trueValue: v5Impl, falseValue: v4Impl);
309}
310
311static mlir::Value emitBinaryExpMaybeConstrainedFPBuiltin(
312 CIRGenFunction &cgf, const CallExpr *e, llvm::StringRef intrinsicName,
313 llvm::StringRef constrainedIntrinsicName) {
314 mlir::Value src0 = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
315 mlir::Value src1 = cgf.emitScalarExpr(e: e->getArg(Arg: 1));
316 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
317
318 CIRGenBuilderTy &builder = cgf.getBuilder();
319
320 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, e);
321
322 if (builder.getIsFPConstrained()) {
323 cgf.cgm.errorNYI(e->getSourceRange(),
324 "constrained FP intrinsic support is NYI.");
325 }
326
327 return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: src0.getType(),
328 op: mlir::ValueRange{src0, src1});
329}
330
331static mlir::Value emitLogbBuiltin(CIRGenFunction &cgf, const CallExpr *e,
332 const llvm::fltSemantics &fSem) {
333 CIRGenBuilderTy &builder = cgf.getBuilder();
334 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
335
336 mlir::Value src0 = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
337 mlir::Type srcTy = src0.getType();
338 mlir::Type int32Ty = builder.getSInt32Ty();
339
340 mlir::Type frExpResMembers[] = {srcTy, int32Ty};
341 cir::RecordType frExpResTy = builder.getAnonRecordTy(
342 members: frExpResMembers, /*packed=*/false,
343 memberKinds: cir::RecordType::getAllDataKinds(members: frExpResMembers));
344
345 mlir::Value frExpResult = builder.emitIntrinsicCallOp(
346 loc, str: "frexp", resTy: frExpResTy, op: mlir::ValueRange{src0});
347
348 mlir::Value exp =
349 cir::ExtractMemberOp::create(builder, location: loc, result: int32Ty, record: frExpResult, index: 1);
350
351 mlir::Value negativeOne =
352 builder.getConstant(loc, attr: cir::IntAttr::get(type: int32Ty, value: -1));
353 mlir::Value expMinus1 = builder.createNSWAdd(loc, lhs: exp, rhs: negativeOne);
354
355 mlir::Value siToFp = cir::CastOp::create(
356 builder, location: loc, result: srcTy, kind: cir::CastKind::int_to_float, src: expMinus1);
357
358 mlir::Value fabs = cir::FAbsOp::create(builder, location: loc, src: src0);
359
360 llvm::APFloat infVal = llvm::APFloat::getInf(Sem: fSem);
361 mlir::Value inf = builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: infVal));
362
363 mlir::Value fabsNegInf =
364 builder.createCompare(loc, kind: cir::CmpOpKind::one, lhs: fabs, rhs: inf);
365
366 mlir::Value sel = builder.createSelect(loc, condition: fabsNegInf, trueValue: siToFp, falseValue: fabs);
367
368 llvm::APFloat zeroValue = llvm::APFloat::getZero(Sem: fSem);
369 mlir::Value zero =
370 builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: zeroValue));
371
372 mlir::Value srcEqZero =
373 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: src0, rhs: zero);
374
375 llvm::APFloat negInfVal = llvm::APFloat::getInf(Sem: fSem, Negative: true);
376 mlir::Value negInf =
377 builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: negInfVal));
378
379 mlir::Value res = builder.createSelect(loc, condition: srcEqZero, trueValue: negInf, falseValue: sel);
380
381 return res;
382}
383
384static mlir::Value
385emitAMDGCNImageOverloadedReturnType(CIRGenFunction &cgf, const CallExpr *e,
386 llvm::StringRef intrinsicName,
387 bool isImageStore) {
388 auto &builder = cgf.getBuilder();
389
390 llvm::SmallVector<mlir::Value, 10> args;
391 for (unsigned i = 0, n = e->getNumArgs(); i < n; ++i)
392 args.push_back(Elt: cgf.emitScalarExpr(e: e->getArg(Arg: i)));
393
394 mlir::Type retTy = isImageStore ? cir::VoidType::get(ctx: builder.getContext())
395 : cgf.convertType(t: e->getType());
396
397 auto callOp = cir::LLVMIntrinsicCallOp::create(
398 builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
399 intrinsic_name: builder.getStringAttr(bytes: intrinsicName), resType: retTy, operands: args);
400
401 return callOp.getResult();
402}
403
404static mlir::Value emitAMDGPUPredicate(CIRGenFunction &cgf, const CallExpr *e,
405 const llvm::Twine &name) {
406 CIRGenBuilderTy &builder = cgf.getBuilder();
407 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
408 mlir::Value result = cir::AMDGPUFeaturePredicateOp::create(
409 builder, location: loc, result: builder.getBoolTy(), predicate: builder.getStringAttr(bytes: name));
410 // Sema retypes the call to the logical operation type, which is int in C.
411 return builder.createBoolToInt(src: result, newTy: cgf.convertType(t: e->getType()));
412}
413
414std::optional<mlir::Value>
415CIRGenFunction::emitAMDGPUBuiltinExpr(unsigned builtinId,
416 const CallExpr *expr) {
417 switch (builtinId) {
418 case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u32:
419 case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u32:
420 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i32:
421 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u32:
422 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i32:
423 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u32:
424 case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b32:
425 case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b32:
426 case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b32:
427 case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u64:
428 case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u64:
429 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i64:
430 case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u64:
431 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i64:
432 case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u64:
433 case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b64:
434 case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b64:
435 case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b64: {
436 cgm.errorNYI(expr->getSourceRange(),
437 std::string("unimplemented AMDGPU builtin call: ") +
438 getContext().BuiltinInfo.getName(ID: builtinId));
439 return mlir::Value{};
440 }
441 case AMDGPU::BI__builtin_amdgcn_div_scale:
442 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
443 Address flagOutPtr = emitPointerWithAlignment(expr: expr->getArg(Arg: 3));
444 llvm::StringRef intrinsicName = "amdgcn.div.scale";
445 mlir::Value x = emitScalarExpr(e: expr->getArg(Arg: 0));
446 mlir::Value y = emitScalarExpr(e: expr->getArg(Arg: 1));
447 mlir::Value z = emitScalarExpr(e: expr->getArg(Arg: 2));
448
449 auto i1Ty = builder.getUIntNTy(n: 1);
450 mlir::Type resMembers[] = {x.getType(), i1Ty};
451 cir::RecordType resTy =
452 builder.getAnonRecordTy(members: resMembers, /*packed=*/false,
453 memberKinds: cir::RecordType::getAllDataKinds(members: resMembers));
454
455 mlir::Value structResult =
456 cir::LLVMIntrinsicCallOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
457 intrinsic_name: builder.getStringAttr(bytes: intrinsicName),
458 resType: resTy, operands: {x, y, z})
459 .getResult();
460
461 mlir::Value result = cir::ExtractMemberOp::create(
462 builder, location: getLoc(srcLoc: expr->getExprLoc()), result: x.getType(), record: structResult, index: 0);
463 mlir::Value flag = cir::ExtractMemberOp::create(
464 builder, location: getLoc(srcLoc: expr->getExprLoc()), result: i1Ty, record: structResult, index: 1);
465
466 mlir::Type flagType = flagOutPtr.getElementType();
467 mlir::Value flagToStore =
468 cir::CastOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), result: flagType,
469 kind: cir::CastKind::int_to_bool, src: flag);
470 builder.createStore(loc: getLoc(srcLoc: expr->getExprLoc()), val: flagToStore, dst: flagOutPtr);
471 return result;
472 }
473 case AMDGPU::BI__builtin_amdgcn_div_fmas:
474 case AMDGPU::BI__builtin_amdgcn_div_fmasf:
475 return emitBuiltinWithOneOverloadedType<4>(e: expr, intrinName: "amdgcn.div.fmas")
476 .getValue();
477 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
478 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
479 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
480 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
481 unsigned iceArguments = 0;
482 ASTContext::GetBuiltinTypeError error;
483 getContext().GetBuiltinType(ID: builtinId, Error&: error, IntegerConstantArgs: &iceArguments);
484 assert(error == ASTContext::GE_None && "Should not codegen an error");
485 assert(expr->getNumArgs() == 5 || expr->getNumArgs() == 6 ||
486 expr->getNumArgs() == 2);
487
488 mlir::Type dataTy = convertType(t: expr->getArg(Arg: 0)->getType());
489 unsigned size = cgm.getDataLayout().getTypeSizeInBits(ty: dataTy);
490 cir::IntType intTy = builder.getUIntNTy(n: std::max(a: size, b: 32u));
491
492 bool isMovDpp8 = builtinId == AMDGPU::BI__builtin_amdgcn_mov_dpp8;
493 bool isMovDpp = builtinId == AMDGPU::BI__builtin_amdgcn_mov_dpp;
494 bool isUpdateDpp = builtinId == AMDGPU::BI__builtin_amdgcn_update_dpp;
495 llvm::StringRef intrinsicName =
496 isMovDpp8 ? "amdgcn.mov.dpp8" : "amdgcn.update.dpp";
497 cir::FuncType intrinsicTy =
498 getIntrinsicType(id: isMovDpp8 ? llvm::Intrinsic::amdgcn_mov_dpp8
499 : llvm::Intrinsic::amdgcn_update_dpp,
500 overloadTys: {intTy});
501
502 auto coerceTo = [&](mlir::Value from, mlir::Type to) -> mlir::Value {
503 if (from.getType() == to)
504 return from;
505 if (mlir::isa<cir::IntType>(Val: from.getType()) &&
506 mlir::isa<cir::IntType>(Val: to))
507 return builder.createIntCast(src: from, newTy: to);
508 return builder.createBitcast(src: from, newTy: to);
509 };
510
511 llvm::SmallVector<mlir::Value, 6> args;
512 // __builtin_amdgcn_mov_dpp has no "old" operand at the source level, but
513 // the real intrinsic it lowers to requires one, so we synthesize a poison
514 // value for it since it is never meaningfully read.
515 if (isMovDpp)
516 args.push_back(Elt: builder.getConstant(loc, attr: cir::PoisonAttr::get(type: intTy)));
517
518 // Number of builtin-level leading args that need zero-extend promotion when
519 // the data type is narrower than 32 bits.
520 unsigned numPromotedArgs = isUpdateDpp ? 2u : 1u;
521 for (unsigned i = 0; i != expr->getNumArgs(); ++i) {
522 mlir::Value v =
523 emitScalarOrConstFoldImmArg(iceArguments, idx: i, argExpr: expr->getArg(Arg: i));
524 if (i < numPromotedArgs && size < 32) {
525 mlir::Type sameWidthUTy = builder.getUIntNTy(n: size);
526 if (v.getType() != sameWidthUTy)
527 v = builder.createBitcast(src: v, newTy: sameWidthUTy);
528 v = builder.createIntCast(src: v, newTy: intTy);
529 }
530 args.push_back(Elt: coerceTo(v, intrinsicTy.getInput(i: i + unsigned(isMovDpp))));
531 }
532
533 mlir::Value result = builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: intTy,
534 op: mlir::ValueRange(args));
535 if (size < 32 && !mlir::isa<cir::IntType>(Val: dataTy))
536 result = builder.createIntCast(src: result, newTy: builder.getUIntNTy(n: size));
537 return coerceTo(result, dataTy);
538 }
539 case AMDGPU::BI__builtin_amdgcn_permlane16:
540 case AMDGPU::BI__builtin_amdgcn_permlanex16: {
541 llvm::StringRef intrinsicName =
542 builtinId == AMDGPU::BI__builtin_amdgcn_permlane16
543 ? "amdgcn.permlane16"
544 : "amdgcn.permlanex16";
545 return emitBuiltinWithOneOverloadedType<6>(e: expr, intrinName: intrinsicName).getValue();
546 }
547 case AMDGPU::BI__builtin_amdgcn_permlane64:
548 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.permlane64")
549 .getValue();
550 case AMDGPU::BI__builtin_amdgcn_permlane_bcast:
551 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.bcast")
552 .getValue();
553 case AMDGPU::BI__builtin_amdgcn_permlane_up:
554 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.up")
555 .getValue();
556 case AMDGPU::BI__builtin_amdgcn_permlane_down:
557 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.down")
558 .getValue();
559 case AMDGPU::BI__builtin_amdgcn_permlane_xor:
560 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.xor")
561 .getValue();
562 case AMDGPU::BI__builtin_amdgcn_readlane:
563 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.readlane")
564 .getValue();
565 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
566 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.readfirstlane")
567 .getValue();
568 case AMDGPU::BI__builtin_amdgcn_wave_shuffle:
569 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.wave.shuffle")
570 .getValue();
571 case AMDGPU::BI__builtin_amdgcn_div_fixup:
572 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
573 case AMDGPU::BI__builtin_amdgcn_div_fixuph: {
574 mlir::Value src0 = emitScalarExpr(e: expr->getArg(Arg: 0));
575 mlir::Value src1 = emitScalarExpr(e: expr->getArg(Arg: 1));
576 mlir::Value src2 = emitScalarExpr(e: expr->getArg(Arg: 2));
577 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
578 str: "amdgcn.div.fixup", resTy: src0.getType(),
579 op: mlir::ValueRange{src0, src1, src2});
580 }
581 case AMDGPU::BI__builtin_amdgcn_trig_preop:
582 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
583 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.trig.preop")
584 .getValue();
585 case AMDGPU::BI__builtin_amdgcn_rcp:
586 case AMDGPU::BI__builtin_amdgcn_rcpf:
587 case AMDGPU::BI__builtin_amdgcn_rcph:
588 case AMDGPU::BI__builtin_amdgcn_rcp_bf16: {
589 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rcp").getValue();
590 }
591 case AMDGPU::BI__builtin_amdgcn_sqrt:
592 case AMDGPU::BI__builtin_amdgcn_sqrtf:
593 case AMDGPU::BI__builtin_amdgcn_sqrth:
594 case AMDGPU::BI__builtin_amdgcn_sqrt_bf16: {
595 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.sqrt").getValue();
596 }
597 case AMDGPU::BI__builtin_amdgcn_rsq:
598 case AMDGPU::BI__builtin_amdgcn_rsqf:
599 case AMDGPU::BI__builtin_amdgcn_rsqh:
600 case AMDGPU::BI__builtin_amdgcn_rsq_bf16: {
601 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rsq").getValue();
602 }
603 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
604 case AMDGPU::BI__builtin_amdgcn_rsq_clampf: {
605 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rsq.clamp")
606 .getValue();
607 }
608 case AMDGPU::BI__builtin_amdgcn_sinf:
609 case AMDGPU::BI__builtin_amdgcn_sinh:
610 case AMDGPU::BI__builtin_amdgcn_sin_bf16: {
611 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.sin").getValue();
612 }
613 case AMDGPU::BI__builtin_amdgcn_cosf:
614 case AMDGPU::BI__builtin_amdgcn_cosh:
615 case AMDGPU::BI__builtin_amdgcn_cos_bf16: {
616 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.cos").getValue();
617 }
618 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
619 return emitAMDGPUDispatchPtr(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), e: expr);
620 case AMDGPU::BI__builtin_amdgcn_logf:
621 case AMDGPU::BI__builtin_amdgcn_log_bf16: {
622 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.log").getValue();
623 }
624 case AMDGPU::BI__builtin_amdgcn_exp2f:
625 case AMDGPU::BI__builtin_amdgcn_exp2_bf16: {
626 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.exp2").getValue();
627 }
628 case AMDGPU::BI__builtin_amdgcn_log_clampf: {
629 cgm.errorNYI(expr->getSourceRange(),
630 std::string("unimplemented AMDGPU builtin call: ") +
631 getContext().BuiltinInfo.getName(ID: builtinId));
632 return mlir::Value{};
633 }
634 case AMDGPU::BI__builtin_amdgcn_ldexp:
635 case AMDGPU::BI__builtin_amdgcn_ldexpf:
636 case AMDGPU::BI__builtin_amdgcn_ldexph: {
637 mlir::Value src0 = emitScalarExpr(e: expr->getArg(Arg: 0));
638 mlir::Value src1 = emitScalarExpr(e: expr->getArg(Arg: 1));
639 // For ldexph, the raw instruction has different behavior for out-of-bounds
640 // exponent values (implicit truncation instead of saturate to
641 // short_min/short_max), so truncate the exponent to i16 first.
642 mlir::Value exp =
643 builtinId == AMDGPU::BI__builtin_amdgcn_ldexph
644 ? cir::CastOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()),
645 result: builder.getSInt16Ty(),
646 kind: cir::CastKind::integral, src: src1)
647 : src1;
648 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "ldexp",
649 resTy: src0.getType(),
650 op: mlir::ValueRange{src0, exp});
651 }
652 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
653 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
654 case AMDGPU::BI__builtin_amdgcn_frexp_manth: {
655 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.frexp.mant")
656 .getValue();
657 }
658 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
659 case AMDGPU::BI__builtin_amdgcn_frexp_expf:
660 case AMDGPU::BI__builtin_amdgcn_frexp_exph:
661 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.frexp.exp",
662 resultType: convertType(t: expr->getType()))
663 .getValue();
664 case AMDGPU::BI__builtin_amdgcn_fract:
665 case AMDGPU::BI__builtin_amdgcn_fractf:
666 case AMDGPU::BI__builtin_amdgcn_fracth:
667 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.fract").getValue();
668 case AMDGPU::BI__builtin_amdgcn_lerp: {
669 cgm.errorNYI(expr->getSourceRange(),
670 std::string("unimplemented AMDGPU builtin call: ") +
671 getContext().BuiltinInfo.getName(ID: builtinId));
672 return mlir::Value{};
673 }
674 case AMDGPU::BI__builtin_amdgcn_ubfe:
675 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.ubfe").getValue();
676 case AMDGPU::BI__builtin_amdgcn_sbfe:
677 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.sbfe").getValue();
678 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
679 case AMDGPU::BI__builtin_amdgcn_ballot_w64:
680 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ballot",
681 resultType: convertType(t: expr->getType()))
682 .getValue();
683 case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w32:
684 case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w64:
685 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.inverse.ballot",
686 resultType: convertType(t: expr->getType()))
687 .getValue();
688 case AMDGPU::BI__builtin_amdgcn_tanhf:
689 case AMDGPU::BI__builtin_amdgcn_tanhh:
690 case AMDGPU::BI__builtin_amdgcn_tanh_bf16: {
691 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.tanh").getValue();
692 }
693 case AMDGPU::BI__builtin_amdgcn_uicmp:
694 case AMDGPU::BI__builtin_amdgcn_uicmpl:
695 case AMDGPU::BI__builtin_amdgcn_sicmp:
696 case AMDGPU::BI__builtin_amdgcn_sicmpl:
697 case AMDGPU::BI__builtin_amdgcn_fcmp:
698 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
699 mlir::Value lhs = emitScalarExpr(e: expr->getArg(Arg: 0));
700 mlir::Value rhs = emitScalarExpr(e: expr->getArg(Arg: 1));
701
702 uint64_t imm =
703 expr->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
704
705 cir::CmpOpKind pred;
706 switch (imm) {
707 case 0x1: // FCMP_OEQ
708 case 0x20: // ICMP_EQ
709 pred = cir::CmpOpKind::eq;
710 break;
711 case 0xe: // FCMP_UNE
712 case 0x21: // ICMP_NE
713 pred = cir::CmpOpKind::ne;
714 break;
715 case 0x2: // FCMP_OGT
716 case 0x22: // ICMP_UGT
717 case 0x26: // ICMP_SGT
718 pred = cir::CmpOpKind::gt;
719 break;
720 case 0x3: // FCMP_OGE
721 case 0x23: // ICMP_UGE
722 case 0x27: // ICMP_SGE
723 pred = cir::CmpOpKind::ge;
724 break;
725 case 0x4: // FCMP_OLT
726 case 0x24: // ICMP_ULT
727 case 0x28: // ICMP_SLT
728 pred = cir::CmpOpKind::lt;
729 break;
730 case 0x5: // FCMP_OLE
731 case 0x25: // ICMP_ULE
732 case 0x29: // ICMP_SLE
733 pred = cir::CmpOpKind::le;
734 break;
735 case 0x6: // FCMP_ONE
736 pred = cir::CmpOpKind::one;
737 break;
738 case 0x8: // FCMP_UNO
739 pred = cir::CmpOpKind::uno;
740 break;
741 default:
742 cgm.errorNYI(expr->getSourceRange(),
743 "amdgcn compare with unsupported predicate");
744 return mlir::Value{};
745 }
746
747 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
748 mlir::Value cmp = builder.createCompare(loc, kind: pred, lhs, rhs);
749 return builder.emitIntrinsicCallOp(loc, str: "amdgcn.ballot",
750 resTy: convertType(t: expr->getType()), op&: cmp);
751 }
752 case AMDGPU::BI__builtin_amdgcn_class:
753 case AMDGPU::BI__builtin_amdgcn_classf:
754 case AMDGPU::BI__builtin_amdgcn_classh:
755 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.class",
756 resultType: convertType(t: expr->getType()))
757 .getValue();
758 case AMDGPU::BI__builtin_amdgcn_fmed3f:
759 case AMDGPU::BI__builtin_amdgcn_fmed3h:
760 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.fmed3").getValue();
761 case AMDGPU::BI__builtin_amdgcn_ds_append:
762 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
763 cgm.errorNYI(expr->getSourceRange(),
764 std::string("unimplemented AMDGPU builtin call: ") +
765 getContext().BuiltinInfo.getName(ID: builtinId));
766 return mlir::Value{};
767 }
768 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
769 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
770 case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
771 return emitBuiltinWithOneOverloadedType<1>(
772 e: expr, intrinName: "amdgcn.global.load.tr.b64", resultType: convertType(t: expr->getType()))
773 .getValue();
774 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
775 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
776 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
777 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
778 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
779 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
780 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
781 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
782 case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
783 return emitBuiltinWithOneOverloadedType<1>(
784 e: expr, intrinName: "amdgcn.global.load.tr.b128", resultType: convertType(t: expr->getType()))
785 .getValue();
786 case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
787 return emitBuiltinWithOneOverloadedType<1>(
788 e: expr, intrinName: "amdgcn.global.load.tr4.b64", resultType: convertType(t: expr->getType()))
789 .getValue();
790 case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
791 return emitBuiltinWithOneOverloadedType<1>(
792 e: expr, intrinName: "amdgcn.global.load.tr6.b96", resultType: convertType(t: expr->getType()))
793 .getValue();
794 case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
795 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr4.b64",
796 resultType: convertType(t: expr->getType()))
797 .getValue();
798 case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
799 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr6.b96",
800 resultType: convertType(t: expr->getType()))
801 .getValue();
802 case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
803 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr8.b64",
804 resultType: convertType(t: expr->getType()))
805 .getValue();
806 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
807 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
808 case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
809 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr16.b128",
810 resultType: convertType(t: expr->getType()))
811 .getValue();
812 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
813 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr4.b64",
814 resultType: convertType(t: expr->getType()))
815 .getValue();
816 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
817 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr8.b64",
818 resultType: convertType(t: expr->getType()))
819 .getValue();
820 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
821 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr6.b96",
822 resultType: convertType(t: expr->getType()))
823 .getValue();
824 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
825 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
826 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
827 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr16.b64",
828 resultType: convertType(t: expr->getType()))
829 .getValue();
830 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b32:
831 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b64:
832 case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b128:
833 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b32:
834 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b64:
835 case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b128: {
836 cgm.errorNYI(expr->getSourceRange(),
837 std::string("unimplemented AMDGPU builtin call: ") +
838 getContext().BuiltinInfo.getName(ID: builtinId));
839 return mlir::Value{};
840 }
841 case AMDGPU::BI__builtin_amdgcn_cluster_load_b32:
842 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b32",
843 resultType: convertType(t: expr->getType()))
844 .getValue();
845 case AMDGPU::BI__builtin_amdgcn_cluster_load_b64:
846 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b64",
847 resultType: convertType(t: expr->getType()))
848 .getValue();
849 case AMDGPU::BI__builtin_amdgcn_cluster_load_b128:
850 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b128",
851 resultType: convertType(t: expr->getType()))
852 .getValue();
853 case AMDGPU::BI__builtin_amdgcn_load_to_lds:
854 return emitBuiltinWithOneOverloadedType<5>(
855 e: expr, intrinName: "amdgcn.load.to.lds",
856 resultType: cir::VoidType::get(ctx: builder.getContext()))
857 .getValue();
858 case AMDGPU::BI__builtin_amdgcn_load_async_to_lds:
859 return emitBuiltinWithOneOverloadedType<5>(
860 e: expr, intrinName: "amdgcn.load.async.to.lds",
861 resultType: cir::VoidType::get(ctx: builder.getContext()))
862 .getValue();
863 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_32x4B:
864 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_32x4B:
865 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_16x8B:
866 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_16x8B:
867 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_8x16B:
868 case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_8x16B: {
869 cgm.errorNYI(expr->getSourceRange(),
870 std::string("unimplemented AMDGPU builtin call: ") +
871 getContext().BuiltinInfo.getName(ID: builtinId));
872 return mlir::Value{};
873 }
874 case AMDGPU::BI__builtin_amdgcn_get_fpenv:
875 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
876 cgm.errorNYI(expr->getSourceRange(),
877 std::string("unimplemented AMDGPU builtin call: ") +
878 getContext().BuiltinInfo.getName(ID: builtinId));
879 return mlir::Value{};
880 }
881 case AMDGPU::BI__builtin_amdgcn_processor_is: {
882 assert(cgm.getTriple().isSPIRV() &&
883 "__builtin_amdgcn_processor_is should never reach CodeGen for "
884 "concrete targets!");
885 StringRef proc = cast<clang::StringLiteral>(Val: expr->getArg(Arg: 0))->getString();
886 return emitAMDGPUPredicate(cgf&: *this, e: expr, name: "is." + proc);
887 }
888 case AMDGPU::BI__builtin_amdgcn_is_invocable: {
889 assert(cgm.getTriple().isSPIRV() &&
890 "__builtin_amdgcn_is_invocable should never reach CodeGen for "
891 "concrete targets!");
892 auto *fd = cast<FunctionDecl>(
893 Val: cast<DeclRefExpr>(Val: expr->getArg(Arg: 0))->getReferencedDeclOfCallee());
894 StringRef rf =
895 getContext().BuiltinInfo.getRequiredFeatures(ID: fd->getBuiltinID());
896 return emitAMDGPUPredicate(cgf&: *this, e: expr, name: "has." + rf);
897 }
898 case AMDGPU::BI__builtin_amdgcn_read_exec:
899 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
900 case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
901 // The exec mask is read as a ballot over an all-true predicate. The
902 // ballot is at least as wide as the wavefront, so that a wave64 target
903 // still reports both halves for the _lo/_hi forms.
904 mlir::Location loc = getLoc(srcLoc: expr->getExprLoc());
905 unsigned registerWidth =
906 builtinId == AMDGPU::BI__builtin_amdgcn_read_exec_lo ? 32 : 64;
907 unsigned ballotWidth =
908 std::max(a: getTarget().getGridValue().GV_Warp_Size, b: registerWidth);
909 cir::IntType ballotTy = builder.getUIntNTy(n: ballotWidth);
910
911 mlir::Value truePred = builder.getBool(state: true, loc).getResult();
912 mlir::Value result =
913 builder.emitIntrinsicCallOp(loc, str: "amdgcn.ballot", resTy: ballotTy, op&: truePred);
914
915 if (builtinId == AMDGPU::BI__builtin_amdgcn_read_exec_hi)
916 result = builder.createShiftRight(loc, lhs: result, bits: 32);
917 return builder.createIntCast(src: result, newTy: convertType(t: expr->getType()));
918 }
919 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
920 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
921 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
922 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
923 cgm.errorNYI(expr->getSourceRange(),
924 std::string("unimplemented AMDGPU builtin call: ") +
925 getContext().BuiltinInfo.getName(ID: builtinId));
926 return mlir::Value{};
927 }
928 case AMDGPU::BI__builtin_amdgcn_image_bvh8_intersect_ray:
929 case AMDGPU::BI__builtin_amdgcn_image_bvh_dual_intersect_ray: {
930 cgm.errorNYI(expr->getSourceRange(),
931 std::string("unimplemented AMDGPU builtin call: ") +
932 getContext().BuiltinInfo.getName(ID: builtinId));
933 return mlir::Value{};
934 }
935 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn:
936 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn:
937 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn:
938 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn: {
939 cgm.errorNYI(expr->getSourceRange(),
940 std::string("unimplemented AMDGPU builtin call: ") +
941 getContext().BuiltinInfo.getName(ID: builtinId));
942 return mlir::Value{};
943 }
944 case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f32_i32:
945 case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f16_i32:
946 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
947 intrinsicName: "amdgcn.image.load.1d", isImageStore: false);
948 case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f32_i32:
949 case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f16_i32:
950 return emitAMDGCNImageOverloadedReturnType(
951 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.1darray", isImageStore: false);
952 case AMDGPU::BI__builtin_amdgcn_image_load_2d_f32_i32:
953 case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f32_i32:
954 case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f16_i32:
955 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
956 intrinsicName: "amdgcn.image.load.2d", isImageStore: false);
957 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_f32_i32:
958 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f32_i32:
959 case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f16_i32:
960 return emitAMDGCNImageOverloadedReturnType(
961 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.2darray", isImageStore: false);
962 case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f32_i32:
963 case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f16_i32:
964 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
965 intrinsicName: "amdgcn.image.load.3d", isImageStore: false);
966 case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f32_i32:
967 case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f16_i32:
968 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
969 intrinsicName: "amdgcn.image.load.cube", isImageStore: false);
970 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f32_i32:
971 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f16_i32:
972 return emitAMDGCNImageOverloadedReturnType(
973 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.1d", isImageStore: false);
974 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f32_i32:
975 case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f16_i32:
976 return emitAMDGCNImageOverloadedReturnType(
977 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.1darray", isImageStore: false);
978 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_f32_i32:
979 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f32_i32:
980 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f16_i32:
981 return emitAMDGCNImageOverloadedReturnType(
982 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.2d", isImageStore: false);
983 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_f32_i32:
984 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f32_i32:
985 case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f16_i32:
986 return emitAMDGCNImageOverloadedReturnType(
987 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.2darray", isImageStore: false);
988 case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f32_i32:
989 case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f16_i32:
990 return emitAMDGCNImageOverloadedReturnType(
991 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.3d", isImageStore: false);
992 case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f32_i32:
993 case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f16_i32:
994 return emitAMDGCNImageOverloadedReturnType(
995 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.cube", isImageStore: false);
996 case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f32_i32:
997 case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f16_i32:
998 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
999 intrinsicName: "amdgcn.image.store.1d", isImageStore: true);
1000 case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f32_i32:
1001 case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f16_i32:
1002 return emitAMDGCNImageOverloadedReturnType(
1003 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.1darray", isImageStore: true);
1004 case AMDGPU::BI__builtin_amdgcn_image_store_2d_f32_i32:
1005 case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f32_i32:
1006 case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f16_i32:
1007 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1008 intrinsicName: "amdgcn.image.store.2d", isImageStore: true);
1009 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_f32_i32:
1010 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f32_i32:
1011 case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f16_i32:
1012 return emitAMDGCNImageOverloadedReturnType(
1013 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.2darray", isImageStore: true);
1014 case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f32_i32:
1015 case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f16_i32:
1016 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1017 intrinsicName: "amdgcn.image.store.3d", isImageStore: true);
1018 case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f32_i32:
1019 case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f16_i32:
1020 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1021 intrinsicName: "amdgcn.image.store.cube", isImageStore: true);
1022 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f32_i32:
1023 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f16_i32:
1024 return emitAMDGCNImageOverloadedReturnType(
1025 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.1d", isImageStore: true);
1026 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f32_i32:
1027 case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f16_i32:
1028 return emitAMDGCNImageOverloadedReturnType(
1029 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.1darray", isImageStore: true);
1030 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_f32_i32:
1031 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f32_i32:
1032 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f16_i32:
1033 return emitAMDGCNImageOverloadedReturnType(
1034 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.2d", isImageStore: true);
1035 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_f32_i32:
1036 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f32_i32:
1037 case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f16_i32:
1038 return emitAMDGCNImageOverloadedReturnType(
1039 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.2darray", isImageStore: true);
1040 case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f32_i32:
1041 case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f16_i32:
1042 return emitAMDGCNImageOverloadedReturnType(
1043 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.3d", isImageStore: true);
1044 case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f32_i32:
1045 case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f16_i32:
1046 return emitAMDGCNImageOverloadedReturnType(
1047 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.cube", isImageStore: true);
1048 case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f32_f32:
1049 case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f16_f32:
1050 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1051 intrinsicName: "amdgcn.image.sample.1d", isImageStore: false);
1052 case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f32_f32:
1053 case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f16_f32:
1054 return emitAMDGCNImageOverloadedReturnType(
1055 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.1darray", isImageStore: false);
1056 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_f32_f32:
1057 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f32_f32:
1058 case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f16_f32:
1059 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1060 intrinsicName: "amdgcn.image.sample.2d", isImageStore: false);
1061 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_f32_f32:
1062 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f32_f32:
1063 case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f16_f32:
1064 return emitAMDGCNImageOverloadedReturnType(
1065 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.2darray", isImageStore: false);
1066 case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f32_f32:
1067 case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f16_f32:
1068 return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr,
1069 intrinsicName: "amdgcn.image.sample.3d", isImageStore: false);
1070 case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f32_f32:
1071 case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f16_f32:
1072 return emitAMDGCNImageOverloadedReturnType(
1073 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.cube", isImageStore: false);
1074 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f32_f32:
1075 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f16_f32:
1076 return emitAMDGCNImageOverloadedReturnType(
1077 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.1d", isImageStore: false);
1078 case AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f32_f32:
1079 case AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f16_f32:
1080 return emitAMDGCNImageOverloadedReturnType(
1081 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.1d", isImageStore: false);
1082 case AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f32_f32:
1083 case AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f16_f32:
1084 return emitAMDGCNImageOverloadedReturnType(
1085 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.1d", isImageStore: false);
1086 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f32_f32:
1087 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f16_f32:
1088 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_f32_f32:
1089 return emitAMDGCNImageOverloadedReturnType(
1090 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.2d", isImageStore: false);
1091 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f32_f32:
1092 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f16_f32:
1093 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_f32_f32:
1094 return emitAMDGCNImageOverloadedReturnType(
1095 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.2d", isImageStore: false);
1096 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f32_f32:
1097 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f16_f32:
1098 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_f32_f32:
1099 return emitAMDGCNImageOverloadedReturnType(
1100 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.2d", isImageStore: false);
1101 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f32_f32:
1102 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f16_f32:
1103 return emitAMDGCNImageOverloadedReturnType(
1104 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.3d", isImageStore: false);
1105 case AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f32_f32:
1106 case AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f16_f32:
1107 return emitAMDGCNImageOverloadedReturnType(
1108 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.3d", isImageStore: false);
1109 case AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f32_f32:
1110 case AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f16_f32:
1111 return emitAMDGCNImageOverloadedReturnType(
1112 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.3d", isImageStore: false);
1113 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f32_f32:
1114 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f16_f32:
1115 return emitAMDGCNImageOverloadedReturnType(
1116 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.cube", isImageStore: false);
1117 case AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f32_f32:
1118 case AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f16_f32:
1119 return emitAMDGCNImageOverloadedReturnType(
1120 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.cube", isImageStore: false);
1121 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f32_f32:
1122 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f16_f32:
1123 return emitAMDGCNImageOverloadedReturnType(
1124 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.1darray", isImageStore: false);
1125 case AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f32_f32:
1126 case AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f16_f32:
1127 return emitAMDGCNImageOverloadedReturnType(
1128 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.1darray", isImageStore: false);
1129 case AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f32_f32:
1130 case AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f16_f32:
1131 return emitAMDGCNImageOverloadedReturnType(
1132 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.1darray", isImageStore: false);
1133 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f32_f32:
1134 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f16_f32:
1135 case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_f32_f32:
1136 return emitAMDGCNImageOverloadedReturnType(
1137 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.2darray", isImageStore: false);
1138 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f32_f32:
1139 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f16_f32:
1140 case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_f32_f32:
1141 return emitAMDGCNImageOverloadedReturnType(
1142 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.2darray", isImageStore: false);
1143 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f32_f32:
1144 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f16_f32:
1145 case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_f32_f32:
1146 return emitAMDGCNImageOverloadedReturnType(
1147 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.2darray", isImageStore: false);
1148 case AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f32_f32:
1149 case AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f16_f32:
1150 return emitAMDGCNImageOverloadedReturnType(
1151 cgf&: *this, e: expr, intrinsicName: "amdgcn.image.gather4.lz.2d", isImageStore: false);
1152 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
1153 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
1154 cgm.errorNYI(expr->getSourceRange(),
1155 std::string("unimplemented AMDGPU builtin call: ") +
1156 getContext().BuiltinInfo.getName(ID: builtinId));
1157 return mlir::Value{};
1158 }
1159 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
1160 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
1161 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
1162 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
1163 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
1164 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
1165 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
1166 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
1167 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
1168 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
1169 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
1170 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
1171 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
1172 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
1173 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
1174 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
1175 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
1176 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
1177 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
1178 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
1179 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
1180 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
1181 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
1182 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
1183 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
1184 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
1185 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
1186 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
1187 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
1188 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
1189 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
1190 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
1191 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
1192 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
1193 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
1194 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
1195 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
1196 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12: {
1197 cgm.errorNYI(expr->getSourceRange(),
1198 std::string("unimplemented AMDGPU builtin call: ") +
1199 getContext().BuiltinInfo.getName(ID: builtinId));
1200 return mlir::Value{};
1201 }
1202 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
1203 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
1204 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
1205 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
1206 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
1207 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
1208 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
1209 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
1210 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
1211 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
1212 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
1213 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
1214 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
1215 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
1216 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
1217 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
1218 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
1219 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
1220 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
1221 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
1222 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
1223 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
1224 cgm.errorNYI(expr->getSourceRange(),
1225 std::string("unimplemented AMDGPU builtin call: ") +
1226 getContext().BuiltinInfo.getName(ID: builtinId));
1227 return mlir::Value{};
1228 }
1229 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x4_f32:
1230 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_bf16:
1231 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_f16:
1232 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x32_f16:
1233 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x32_bf16:
1234 case AMDGPU::BI__builtin_amdgcn_wmma_bf16f32_16x16x32_bf16:
1235 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8:
1236 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8:
1237 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8:
1238 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8:
1239 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8:
1240 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8:
1241 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8:
1242 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8:
1243 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8:
1244 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8:
1245 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8:
1246 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8:
1247 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8:
1248 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8:
1249 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8:
1250 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8:
1251 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x64_iu8:
1252 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_f8f6f4:
1253 case AMDGPU::BI__builtin_amdgcn_wmma_f32_32x16x128_f4:
1254 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
1255 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
1256 case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_32x16x128_f4:
1257 case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_32x16x128_f4: {
1258 cgm.errorNYI(expr->getSourceRange(),
1259 std::string("unimplemented AMDGPU builtin call: ") +
1260 getContext().BuiltinInfo.getName(ID: builtinId));
1261 return mlir::Value{};
1262 }
1263 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_f16:
1264 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_bf16:
1265 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x64_f16:
1266 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x64_bf16:
1267 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16:
1268 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8:
1269 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8:
1270 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8:
1271 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8:
1272 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8:
1273 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8:
1274 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8:
1275 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8:
1276 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x128_iu8: {
1277 cgm.errorNYI(expr->getSourceRange(),
1278 std::string("unimplemented AMDGPU builtin call: ") +
1279 getContext().BuiltinInfo.getName(ID: builtinId));
1280 return mlir::Value{};
1281 }
1282 // amdgcn workgroup size
1283 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
1284 return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 0);
1285 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
1286 return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 1);
1287 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
1288 return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 2);
1289 // amdgcn grid size
1290 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
1291 return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 0);
1292 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
1293 return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 1);
1294 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
1295 return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 2);
1296 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
1297 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef: {
1298 cgm.errorNYI(expr->getSourceRange(),
1299 std::string("unimplemented AMDGPU builtin call: ") +
1300 getContext().BuiltinInfo.getName(ID: builtinId));
1301 return mlir::Value{};
1302 }
1303 case AMDGPU::BI__builtin_amdgcn_alignbit: {
1304 cgm.errorNYI(expr->getSourceRange(),
1305 std::string("unimplemented AMDGPU builtin call: ") +
1306 getContext().BuiltinInfo.getName(ID: builtinId));
1307 return mlir::Value{};
1308 }
1309 case AMDGPU::BI__builtin_amdgcn_fence: {
1310 // The scope uses AMDGPU spelling on every target, like mapScopeToSPIRV.
1311 const llvm::Triple &triple = getTarget().getTriple();
1312 std::optional<std::pair<llvm::AtomicScope, bool>> parsed;
1313 if (std::optional<std::string> name =
1314 expr->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext())) {
1315 parsed = llvm::parseAMDGPUAtomicScopeIRString(Name: *name);
1316 if (!parsed && triple.isSPIRV())
1317 parsed = llvm::parseAtomicScopeIRString(T: triple, Name: *name);
1318 }
1319 // SPIR-V drops one-as, AMDGPU needs a CIR sync scope for it.
1320 if (!parsed || (parsed->second && !triple.isSPIRV())) {
1321 cgm.errorNYI(expr->getSourceRange(), "__builtin_amdgcn_fence scope");
1322 return mlir::Value{};
1323 }
1324 // Sorted and unique, like MMRAMetadata::appendTags.
1325 llvm::SmallVector<std::string> addrSpaces;
1326 addrSpaces.reserve(N: expr->getNumArgs() - 2);
1327 for (unsigned i = 2; i < expr->getNumArgs(); ++i) {
1328 if (std::optional<std::string> as =
1329 expr->getArg(Arg: i)->tryEvaluateString(Ctx&: getContext()))
1330 addrSpaces.push_back(Elt: std::move(*as));
1331 else
1332 cgm.error(loc: expr->getExprLoc(),
1333 error: "expected an address space name as a string literal");
1334 }
1335 llvm::sort(C&: addrSpaces);
1336 addrSpaces.erase(CS: llvm::unique(R&: addrSpaces), CE: addrSpaces.end());
1337 llvm::SmallVector<mlir::Attribute> mmras;
1338 mmras.reserve(N: addrSpaces.size());
1339 for (const std::string &as : addrSpaces)
1340 mmras.push_back(Elt: mlir::LLVM::MMRATagAttr::get(
1341 context: &getMLIRContext(), prefix: "amdgpu-synchronize-as", suffix: as));
1342
1343 auto fence = cir::AtomicFenceOp::create(
1344 builder, location: getLoc(srcLoc: expr->getExprLoc()),
1345 ordering: decodeAtomicOrder(arg: expr->getArg(Arg: 0), ctx&: getContext()),
1346 syncscope: cir::SyncScopeKindAttr::get(context: &getMLIRContext(),
1347 value: convertAtomicScopeToCIR(scope: parsed->first)));
1348 if (!mmras.empty())
1349 fence->setAttr(name: mlir::LLVM::LLVMDialect::getMmraAttrName(),
1350 value: mmras.size() == 1 ? mmras.front()
1351 : builder.getArrayAttr(value: mmras));
1352 return mlir::Value{};
1353 }
1354 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
1355 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
1356 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::UIncWrap,
1357 /*hasVolatileArg=*/false);
1358 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
1359 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
1360 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::UDecWrap,
1361 /*hasVolatileArg=*/false);
1362 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
1363 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
1364 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
1365 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
1366 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
1367 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
1368 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
1369 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
1370 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
1371 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
1372 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
1373 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
1374 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Add,
1375 /*hasVolatileArg=*/false);
1376 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
1377 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
1378 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Min,
1379 /*hasVolatileArg=*/false);
1380 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
1381 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
1382 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Max,
1383 /*hasVolatileArg=*/false);
1384 // The ds_ float forms are the same operations with the volatile flag spelled
1385 // out as an argument.
1386 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
1387 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Add,
1388 /*hasVolatileArg=*/true);
1389 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
1390 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Min,
1391 /*hasVolatileArg=*/true);
1392 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
1393 return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Max,
1394 /*hasVolatileArg=*/true);
1395 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
1396 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl:
1397 return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.s.sendmsg.rtn",
1398 resultType: convertType(t: expr->getType()))
1399 .getValue();
1400 case AMDGPU::BI__builtin_amdgcn_permlane16_swap:
1401 case AMDGPU::BI__builtin_amdgcn_permlane32_swap: {
1402 cgm.errorNYI(expr->getSourceRange(),
1403 std::string("unimplemented AMDGPU builtin call: ") +
1404 getContext().BuiltinInfo.getName(ID: builtinId));
1405 return mlir::Value{};
1406 }
1407 case AMDGPU::BI__builtin_amdgcn_bitop3_b32:
1408 case AMDGPU::BI__builtin_amdgcn_bitop3_b16:
1409 return emitBuiltinWithOneOverloadedType<4>(e: expr, intrinName: "amdgcn.bitop3")
1410 .getValue();
1411 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc: {
1412 cgm.errorNYI(expr->getSourceRange(),
1413 std::string("unimplemented AMDGPU builtin call: ") +
1414 getContext().BuiltinInfo.getName(ID: builtinId));
1415 return mlir::Value{};
1416 }
1417 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
1418 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
1419 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
1420 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
1421 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
1422 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128: {
1423 cgm.errorNYI(expr->getSourceRange(),
1424 std::string("unimplemented AMDGPU builtin call: ") +
1425 getContext().BuiltinInfo.getName(ID: builtinId));
1426 return mlir::Value{};
1427 }
1428 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32:
1429 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16:
1430 return emitBuiltinWithOneOverloadedType<5>(
1431 e: expr, intrinName: "amdgcn.raw.ptr.buffer.store.format",
1432 resultType: cir::VoidType::get(ctx: builder.getContext()))
1433 .getValue();
1434 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f32:
1435 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f16:
1436 return emitBuiltinWithOneOverloadedType<4>(
1437 e: expr, intrinName: "amdgcn.raw.ptr.buffer.load.format",
1438 resultType: convertType(t: expr->getType()))
1439 .getValue();
1440 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
1441 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
1442 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
1443 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
1444 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
1445 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
1446 cgm.errorNYI(expr->getSourceRange(),
1447 std::string("unimplemented AMDGPU builtin call: ") +
1448 getContext().BuiltinInfo.getName(ID: builtinId));
1449 return mlir::Value{};
1450 }
1451 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32:
1452 return emitBuiltinWithOneOverloadedType<5>(
1453 e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.add")
1454 .getValue();
1455 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32:
1456 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f64:
1457 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16:
1458 return emitBuiltinWithOneOverloadedType<5>(
1459 e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fadd")
1460 .getValue();
1461 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32:
1462 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64:
1463 return emitBuiltinWithOneOverloadedType<5>(
1464 e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fmin")
1465 .getValue();
1466 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32:
1467 case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64:
1468 return emitBuiltinWithOneOverloadedType<5>(
1469 e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fmax")
1470 .getValue();
1471 case AMDGPU::BI__builtin_amdgcn_s_prefetch_data:
1472 return emitBuiltinWithOneOverloadedType<2>(
1473 e: expr, intrinName: "amdgcn.s.prefetch.data",
1474 resultType: cir::VoidType::get(ctx: builder.getContext()))
1475 .getValue();
1476 case AMDGPU::BI__builtin_amdgcn_s_prefetch_inst:
1477 return emitBuiltinWithOneOverloadedType<2>(
1478 e: expr, intrinName: "amdgcn.s.prefetch.inst",
1479 resultType: cir::VoidType::get(ctx: builder.getContext()))
1480 .getValue();
1481 case Builtin::BIlogbf:
1482 case Builtin::BI__builtin_logbf:
1483 return emitLogbBuiltin(cgf&: *this, e: expr, fSem: llvm::APFloat::IEEEsingle());
1484 case Builtin::BIlogb:
1485 case Builtin::BI__builtin_logb:
1486 return emitLogbBuiltin(cgf&: *this, e: expr, fSem: llvm::APFloat::IEEEdouble());
1487 case Builtin::BIscalbnf:
1488 case Builtin::BI__builtin_scalbnf:
1489 case Builtin::BIscalbn:
1490 case Builtin::BI__builtin_scalbn: {
1491 return emitBinaryExpMaybeConstrainedFPBuiltin(
1492 cgf&: *this, e: expr, intrinsicName: "ldexp", constrainedIntrinsicName: "experimental.constrained.ldexp");
1493 }
1494 default:
1495 return std::nullopt;
1496 }
1497}
1498