| 1 | //===---- CIRGenBuiltinAMDGPU.cpp - Emit CIR for AMDGPU builtins ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This contains code to emit AMDGPU Builtin calls. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "CIRGenFunction.h" |
| 14 | |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h" |
| 17 | #include "mlir/IR/Value.h" |
| 18 | #include "clang/Basic/TargetBuiltins.h" |
| 19 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 20 | #include "llvm/Support/AMDGPUAddrSpace.h" |
| 21 | #include "llvm/Support/ErrorHandling.h" |
| 22 | #include "llvm/TargetParser/AtomicScope.h" |
| 23 | |
| 24 | using namespace clang; |
| 25 | using namespace clang::CIRGen; |
| 26 | using namespace cir; |
| 27 | |
| 28 | // Emit a no-operand intrinsic returning a pointer in the constant AS. |
| 29 | static mlir::Value emitAMDGPUConstantPtrIntrinsic(CIRGenFunction &cgf, |
| 30 | mlir::Location loc, |
| 31 | llvm::StringRef name) { |
| 32 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 33 | mlir::Type retTy = builder.getVoidPtrTy(as: cir::TargetAddressSpaceAttr::get( |
| 34 | context: builder.getContext(), value: llvm::AMDGPUAS::CONSTANT_ADDRESS)); |
| 35 | return builder.emitIntrinsicCallOp(loc, str: name, resTy: retTy, op: mlir::ValueRange{}); |
| 36 | } |
| 37 | |
| 38 | /// Map a constant integral to memory order. |
| 39 | static cir::MemOrder decodeAtomicOrder(const Expr *arg, ASTContext &ctx) { |
| 40 | Expr::EvalResult orderRes; |
| 41 | if (!arg->EvaluateAsInt(Result&: orderRes, Ctx: ctx)) |
| 42 | return cir::MemOrder::SequentiallyConsistent; |
| 43 | switch (orderRes.Val.getInt().getZExtValue()) { |
| 44 | case 0: |
| 45 | return cir::MemOrder::Relaxed; |
| 46 | case 1: // consume -> acquire |
| 47 | case 2: |
| 48 | return cir::MemOrder::Acquire; |
| 49 | case 3: |
| 50 | return cir::MemOrder::Release; |
| 51 | case 4: |
| 52 | return cir::MemOrder::AcquireRelease; |
| 53 | default: |
| 54 | return cir::MemOrder::SequentiallyConsistent; |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | /// Map an AMDGPU sync-scope string-literal argument onto cir::SyncScopeKind. |
| 59 | /// An absent or unrecognized scope is system scope, which is the conservative |
| 60 | /// choice and matches what an empty syncscope string means in LLVM. |
| 61 | static cir::SyncScopeKind decodeAMDGPUSyncScope(const Expr *arg) { |
| 62 | const auto *sl = |
| 63 | llvm::dyn_cast<clang::StringLiteral>(Val: arg->IgnoreParenCasts()); |
| 64 | if (!sl) |
| 65 | return cir::SyncScopeKind::System; |
| 66 | return llvm::StringSwitch<cir::SyncScopeKind>(sl->getString()) |
| 67 | .Case(S: "singlethread" , Value: cir::SyncScopeKind::SingleThread) |
| 68 | .Case(S: "wavefront" , Value: cir::SyncScopeKind::Wavefront) |
| 69 | .Case(S: "workgroup" , Value: cir::SyncScopeKind::Workgroup) |
| 70 | .Case(S: "agent" , Value: cir::SyncScopeKind::Device) |
| 71 | .Default(Value: cir::SyncScopeKind::System); |
| 72 | } |
| 73 | |
| 74 | static cir::SyncScopeKind convertAtomicScopeToCIR(llvm::AtomicScope scope) { |
| 75 | switch (scope) { |
| 76 | case llvm::AtomicScope::Single: |
| 77 | return cir::SyncScopeKind::SingleThread; |
| 78 | case llvm::AtomicScope::Wavefront: |
| 79 | return cir::SyncScopeKind::Wavefront; |
| 80 | case llvm::AtomicScope::Workgroup: |
| 81 | return cir::SyncScopeKind::Workgroup; |
| 82 | case llvm::AtomicScope::Cluster: |
| 83 | return cir::SyncScopeKind::Cluster; |
| 84 | case llvm::AtomicScope::Device: |
| 85 | return cir::SyncScopeKind::Device; |
| 86 | case llvm::AtomicScope::System: |
| 87 | return cir::SyncScopeKind::System; |
| 88 | } |
| 89 | llvm_unreachable("unknown atomic scope" ); |
| 90 | } |
| 91 | |
| 92 | /// Emit one of the AMDGPU raw hardware atomic builtins as a cir.atomic.fetch. |
| 93 | static mlir::Value emitAMDGPUAtomicRMW(CIRGenFunction &cgf, |
| 94 | const CallExpr *expr, |
| 95 | cir::AtomicFetchKind binOp, |
| 96 | bool hasVolatileArg) { |
| 97 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 98 | mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc()); |
| 99 | |
| 100 | Address ptr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0)); |
| 101 | mlir::Value val = cgf.emitScalarExpr(e: expr->getArg(Arg: 1)); |
| 102 | |
| 103 | bool isVolatile; |
| 104 | if (hasVolatileArg) { |
| 105 | assert(expr->getNumArgs() >= 5); |
| 106 | // ds_faddf/fminf/fmaxf spell the volatile flag out as a constant argument. |
| 107 | Expr::EvalResult volRes; |
| 108 | isVolatile = expr->getArg(Arg: 4)->EvaluateAsInt(Result&: volRes, Ctx: cgf.getContext()) && |
| 109 | volRes.Val.getInt().getBoolValue(); |
| 110 | } else { |
| 111 | // Everything else infers it from the pointee type. |
| 112 | QualType argTy = expr->getArg(Arg: 0)->IgnoreImpCasts()->getType(); |
| 113 | isVolatile = argTy->castAs<clang::PointerType>() |
| 114 | ->getPointeeType() |
| 115 | .isVolatileQualified(); |
| 116 | } |
| 117 | |
| 118 | // Some of these builtins spell out the ordering and scope; the rest take the |
| 119 | // monotonic/agent default described above. |
| 120 | cir::MemOrder order = cir::MemOrder::Relaxed; |
| 121 | cir::SyncScopeKind scope = cir::SyncScopeKind::Device; |
| 122 | if (expr->getNumArgs() >= 4) { |
| 123 | order = decodeAtomicOrder(arg: expr->getArg(Arg: 2), ctx&: cgf.getContext()); |
| 124 | scope = decodeAMDGPUSyncScope(arg: expr->getArg(Arg: 3)); |
| 125 | } |
| 126 | |
| 127 | mlir::Type origTy = val.getType(); |
| 128 | if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: origTy); |
| 129 | vecTy && mlir::isa<cir::IntType>(Val: vecTy.getElementType())) { |
| 130 | auto bf16VecTy = |
| 131 | cir::VectorType::get(element_type: builder.getBfloat16Ty(), size: vecTy.getSize()); |
| 132 | val = builder.createBitcast(loc, src: val, newTy: bf16VecTy); |
| 133 | ptr = ptr.withElementType(builder, ElemTy: bf16VecTy); |
| 134 | } |
| 135 | |
| 136 | auto rmw = cir::AtomicFetchOp::create(builder, location: loc, ptr: ptr.emitRawPointer(), val, |
| 137 | binop: binOp, mem_order: order, sync_scope: scope, is_volatile: isVolatile, |
| 138 | /*fetch_first=*/true); |
| 139 | rmw->setAttr(name: cir::CIRDialect::getAMDGPURawAtomicAttrName(), |
| 140 | value: builder.getUnitAttr()); |
| 141 | return builder.createBitcast(loc, src: rmw->getResult(idx: 0), newTy: origTy); |
| 142 | } |
| 143 | |
| 144 | // Emit the `amdgcn.dispatch.ptr` intrinsic, address-space-casting the |
| 145 | // result to match \p e's return type when needed. |
| 146 | // If \p e is null, returns the raw AS-4 pointer. |
| 147 | static mlir::Value emitAMDGPUDispatchPtr(CIRGenFunction &cgf, |
| 148 | mlir::Location loc, |
| 149 | const CallExpr *e = nullptr) { |
| 150 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 151 | mlir::Value call = |
| 152 | emitAMDGPUConstantPtrIntrinsic(cgf, loc, name: "amdgcn.dispatch.ptr" ); |
| 153 | if (!e) |
| 154 | return call; |
| 155 | // Only cast when the caller-visible AS differs from the intrinsic's AS; |
| 156 | auto expectedPtrTy = |
| 157 | mlir::cast<cir::PointerType>(Val: cgf.convertType(t: e->getType())); |
| 158 | auto callPtrTy = mlir::cast<cir::PointerType>(Val: call.getType()); |
| 159 | if (expectedPtrTy.getAddrSpace() == callPtrTy.getAddrSpace()) |
| 160 | return call; |
| 161 | return builder.createAddrSpaceCast(loc, src: call, newTy: expectedPtrTy); |
| 162 | } |
| 163 | |
| 164 | static mlir::Value emitAMDGPUImplicitArgPtr(CIRGenFunction &cgf, |
| 165 | mlir::Location loc) { |
| 166 | // OG also marks the result align 8 dereferenceable(256). |
| 167 | assert(!cir::MissingFeatures::opCallAttrs()); |
| 168 | return emitAMDGPUConstantPtrIntrinsic(cgf, loc, name: "amdgcn.implicitarg.ptr" ); |
| 169 | } |
| 170 | |
| 171 | static mlir::Value emitAMDGPUWorkGroupID(CIRGenFunction &cgf, |
| 172 | mlir::Location loc, unsigned index) { |
| 173 | static constexpr llvm::StringLiteral names[] = {"amdgcn.workgroup.id.x" , |
| 174 | "amdgcn.workgroup.id.y" , |
| 175 | "amdgcn.workgroup.id.z" }; |
| 176 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 177 | return builder.emitIntrinsicCallOp(loc, str: names[index], resTy: builder.getUInt32Ty(), |
| 178 | op: mlir::ValueRange{}); |
| 179 | } |
| 180 | |
| 181 | // Load a \p ty value at byte \p offset from \p base as an invariant load. |
| 182 | static mlir::Value emitAMDGPUInvariantLoad(CIRGenFunction &cgf, |
| 183 | mlir::Location loc, mlir::Value base, |
| 184 | mlir::Value offset, mlir::Type ty, |
| 185 | CharUnits align) { |
| 186 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 187 | mlir::Value addr = builder.createPtrStride(loc, base, stride: offset); |
| 188 | cir::LoadOp load = builder.createAlignedLoad(loc, ty, ptr: addr, align); |
| 189 | load.setInvariant(true); |
| 190 | assert(!cir::MissingFeatures::opLoadRangeNoundefMetadata()); |
| 191 | return load; |
| 192 | } |
| 193 | |
| 194 | static mlir::Value emitAMDGPUGridSize(CIRGenFunction &cgf, mlir::Location loc, |
| 195 | unsigned index) { |
| 196 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 197 | const unsigned xOffset = 12; |
| 198 | mlir::Value dispatchPtr = emitAMDGPUDispatchPtr(cgf, loc); |
| 199 | // Indexing the HSA kernel_dispatch_packet struct. |
| 200 | return emitAMDGPUInvariantLoad( |
| 201 | cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: xOffset + index * 4, loc), |
| 202 | ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4)); |
| 203 | } |
| 204 | |
| 205 | static mlir::Value emitAMDGPUWorkGroupSizeV5(CIRGenFunction &cgf, |
| 206 | mlir::Location loc, |
| 207 | unsigned index) { |
| 208 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 209 | mlir::Value implicitArgPtr = emitAMDGPUImplicitArgPtr(cgf, loc); |
| 210 | |
| 211 | // offsetof(amdhsa_implicit_kernarg_v5, block_count[index]) |
| 212 | unsigned blockCountOffset = 0 + index * 4; |
| 213 | // offsetof(amdhsa_implicit_kernarg_v5, group_size[index]) |
| 214 | unsigned groupSizeOffset = 12 + index * 2; |
| 215 | // offsetof(amdhsa_implicit_kernarg_v5, remainder[index]) |
| 216 | unsigned remainderOffset = 18 + index * 2; |
| 217 | |
| 218 | mlir::Value sizeOffset; |
| 219 | if (cgf.getLangOpts().OffloadUniformBlock) { |
| 220 | sizeOffset = builder.getUInt64(c: groupSizeOffset, loc); |
| 221 | } else { |
| 222 | mlir::Value blockCount = emitAMDGPUInvariantLoad( |
| 223 | cgf, loc, base: implicitArgPtr, offset: builder.getUInt64(c: blockCountOffset, loc), |
| 224 | ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4)); |
| 225 | mlir::Value workgroupID = emitAMDGPUWorkGroupID(cgf, loc, index); |
| 226 | mlir::Value isFull = |
| 227 | builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: workgroupID, rhs: blockCount); |
| 228 | sizeOffset = builder.createSelect(loc, condition: isFull, |
| 229 | trueValue: builder.getUInt64(c: groupSizeOffset, loc), |
| 230 | falseValue: builder.getUInt64(c: remainderOffset, loc)); |
| 231 | } |
| 232 | |
| 233 | mlir::Value size = emitAMDGPUInvariantLoad(cgf, loc, base: implicitArgPtr, |
| 234 | offset: sizeOffset, ty: builder.getUInt16Ty(), |
| 235 | align: CharUnits::fromQuantity(Quantity: 2)); |
| 236 | return builder.createIntCast(src: size, newTy: builder.getUInt32Ty()); |
| 237 | } |
| 238 | |
| 239 | static mlir::Value emitAMDGPUWorkGroupSizeV4(CIRGenFunction &cgf, |
| 240 | mlir::Location loc, |
| 241 | unsigned index) { |
| 242 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 243 | mlir::Value dispatchPtr = emitAMDGPUDispatchPtr(cgf, loc); |
| 244 | |
| 245 | // Indexing the HSA kernel_dispatch_packet struct. |
| 246 | mlir::Value groupSizeLd = emitAMDGPUInvariantLoad( |
| 247 | cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: 4 + index * 2, loc), |
| 248 | ty: builder.getUInt16Ty(), align: CharUnits::fromQuantity(Quantity: 2)); |
| 249 | mlir::Value groupSize = |
| 250 | builder.createIntCast(src: groupSizeLd, newTy: builder.getUInt32Ty()); |
| 251 | |
| 252 | if (cgf.getLangOpts().OffloadUniformBlock) |
| 253 | return groupSize; |
| 254 | |
| 255 | mlir::Value workgroupID = emitAMDGPUWorkGroupID(cgf, loc, index); |
| 256 | mlir::Value gridSize = emitAMDGPUInvariantLoad( |
| 257 | cgf, loc, base: dispatchPtr, offset: builder.getUInt64(c: 12 + index * 4, loc), |
| 258 | ty: builder.getUInt32Ty(), align: CharUnits::fromQuantity(Quantity: 4)); |
| 259 | mlir::Value mul = builder.createMul(loc, lhs: workgroupID, rhs: groupSize); |
| 260 | mlir::Value remainder = builder.createSub(loc, lhs: gridSize, rhs: mul); |
| 261 | mlir::Value isPartial = |
| 262 | builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: remainder, rhs: groupSize); |
| 263 | return builder.createSelect(loc, condition: isPartial, trueValue: remainder, falseValue: groupSize); |
| 264 | } |
| 265 | |
| 266 | // \p index is 0, 1, and 2 for x, y, and z dimension, respectively. |
| 267 | // Emit code based on Code Object ABI version. |
| 268 | // COV_4 : Emit code to use dispatch ptr |
| 269 | // COV_5+ : Emit code to use implicitarg ptr |
| 270 | // COV_NONE : Emit code to load a global variable "__oclc_ABI_version" |
| 271 | // and use its value for COV_4 or COV_5+ approach. It is used for |
| 272 | // compiling device libraries in an ABI-agnostic way. |
| 273 | static mlir::Value emitAMDGPUWorkGroupSize(CIRGenFunction &cgf, |
| 274 | mlir::Location loc, unsigned index) { |
| 275 | llvm::CodeObjectVersionKind cov = |
| 276 | cgf.getTarget().getTargetOpts().CodeObjectVersion; |
| 277 | |
| 278 | // Do not emit __oclc_ABI_version references with non-empty environment. |
| 279 | if (cov == llvm::CodeObjectVersionKind::COV_None && |
| 280 | cgf.getTarget().getTriple().hasEnvironment()) |
| 281 | cov = llvm::CodeObjectVersionKind::COV_6; |
| 282 | |
| 283 | if (cov != llvm::CodeObjectVersionKind::COV_None) |
| 284 | return cov >= llvm::CodeObjectVersionKind::COV_5 |
| 285 | ? emitAMDGPUWorkGroupSizeV5(cgf, loc, index) |
| 286 | : emitAMDGPUWorkGroupSizeV4(cgf, loc, index); |
| 287 | |
| 288 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 289 | CIRGenModule &cgm = cgf.cgm; |
| 290 | StringRef name = "__oclc_ABI_version" ; |
| 291 | mlir::Type int32Ty = builder.getSInt32Ty(); |
| 292 | auto abiVersionGV = |
| 293 | mlir::dyn_cast_or_null<cir::GlobalOp>(Val: cgm.getGlobalValue(ref: name)); |
| 294 | if (!abiVersionGV) |
| 295 | abiVersionGV = |
| 296 | cgm.createGlobalOp(loc, name, t: int32Ty, /*isConstant=*/false, |
| 297 | addrSpace: cir::toCIRAddressSpaceAttr(ctx&: cgm.getMLIRContext(), |
| 298 | langAS: LangAS::opencl_constant)); |
| 299 | mlir::Value abiVersionPtr = builder.createGetGlobal(loc, global: abiVersionGV); |
| 300 | mlir::Value abiVersion = builder.createAlignedLoad( |
| 301 | loc, ty: int32Ty, ptr: abiVersionPtr, |
| 302 | align: cgf.getContext().getTypeAlignInChars(T: cgf.getContext().IntTy)); |
| 303 | mlir::Value isCOV5 = builder.createCompare( |
| 304 | loc, kind: cir::CmpOpKind::ge, lhs: abiVersion, |
| 305 | rhs: builder.getConstantInt(loc, ty: int32Ty, value: llvm::CodeObjectVersionKind::COV_5)); |
| 306 | mlir::Value v5Impl = emitAMDGPUWorkGroupSizeV5(cgf, loc, index); |
| 307 | mlir::Value v4Impl = emitAMDGPUWorkGroupSizeV4(cgf, loc, index); |
| 308 | return builder.createSelect(loc, condition: isCOV5, trueValue: v5Impl, falseValue: v4Impl); |
| 309 | } |
| 310 | |
| 311 | static mlir::Value emitBinaryExpMaybeConstrainedFPBuiltin( |
| 312 | CIRGenFunction &cgf, const CallExpr *e, llvm::StringRef intrinsicName, |
| 313 | llvm::StringRef constrainedIntrinsicName) { |
| 314 | mlir::Value src0 = cgf.emitScalarExpr(e: e->getArg(Arg: 0)); |
| 315 | mlir::Value src1 = cgf.emitScalarExpr(e: e->getArg(Arg: 1)); |
| 316 | mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc()); |
| 317 | |
| 318 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 319 | |
| 320 | CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, e); |
| 321 | |
| 322 | if (builder.getIsFPConstrained()) { |
| 323 | cgf.cgm.errorNYI(e->getSourceRange(), |
| 324 | "constrained FP intrinsic support is NYI." ); |
| 325 | } |
| 326 | |
| 327 | return builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: src0.getType(), |
| 328 | op: mlir::ValueRange{src0, src1}); |
| 329 | } |
| 330 | |
| 331 | static mlir::Value emitLogbBuiltin(CIRGenFunction &cgf, const CallExpr *e, |
| 332 | const llvm::fltSemantics &fSem) { |
| 333 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 334 | mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc()); |
| 335 | |
| 336 | mlir::Value src0 = cgf.emitScalarExpr(e: e->getArg(Arg: 0)); |
| 337 | mlir::Type srcTy = src0.getType(); |
| 338 | mlir::Type int32Ty = builder.getSInt32Ty(); |
| 339 | |
| 340 | mlir::Type frExpResMembers[] = {srcTy, int32Ty}; |
| 341 | cir::RecordType frExpResTy = builder.getAnonRecordTy( |
| 342 | members: frExpResMembers, /*packed=*/false, |
| 343 | memberKinds: cir::RecordType::getAllDataKinds(members: frExpResMembers)); |
| 344 | |
| 345 | mlir::Value frExpResult = builder.emitIntrinsicCallOp( |
| 346 | loc, str: "frexp" , resTy: frExpResTy, op: mlir::ValueRange{src0}); |
| 347 | |
| 348 | mlir::Value exp = |
| 349 | cir::ExtractMemberOp::create(builder, location: loc, result: int32Ty, record: frExpResult, index: 1); |
| 350 | |
| 351 | mlir::Value negativeOne = |
| 352 | builder.getConstant(loc, attr: cir::IntAttr::get(type: int32Ty, value: -1)); |
| 353 | mlir::Value expMinus1 = builder.createNSWAdd(loc, lhs: exp, rhs: negativeOne); |
| 354 | |
| 355 | mlir::Value siToFp = cir::CastOp::create( |
| 356 | builder, location: loc, result: srcTy, kind: cir::CastKind::int_to_float, src: expMinus1); |
| 357 | |
| 358 | mlir::Value fabs = cir::FAbsOp::create(builder, location: loc, src: src0); |
| 359 | |
| 360 | llvm::APFloat infVal = llvm::APFloat::getInf(Sem: fSem); |
| 361 | mlir::Value inf = builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: infVal)); |
| 362 | |
| 363 | mlir::Value fabsNegInf = |
| 364 | builder.createCompare(loc, kind: cir::CmpOpKind::one, lhs: fabs, rhs: inf); |
| 365 | |
| 366 | mlir::Value sel = builder.createSelect(loc, condition: fabsNegInf, trueValue: siToFp, falseValue: fabs); |
| 367 | |
| 368 | llvm::APFloat zeroValue = llvm::APFloat::getZero(Sem: fSem); |
| 369 | mlir::Value zero = |
| 370 | builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: zeroValue)); |
| 371 | |
| 372 | mlir::Value srcEqZero = |
| 373 | builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: src0, rhs: zero); |
| 374 | |
| 375 | llvm::APFloat negInfVal = llvm::APFloat::getInf(Sem: fSem, Negative: true); |
| 376 | mlir::Value negInf = |
| 377 | builder.getConstant(loc, attr: cir::FPAttr::get(type: srcTy, value: negInfVal)); |
| 378 | |
| 379 | mlir::Value res = builder.createSelect(loc, condition: srcEqZero, trueValue: negInf, falseValue: sel); |
| 380 | |
| 381 | return res; |
| 382 | } |
| 383 | |
| 384 | static mlir::Value |
| 385 | emitAMDGCNImageOverloadedReturnType(CIRGenFunction &cgf, const CallExpr *e, |
| 386 | llvm::StringRef intrinsicName, |
| 387 | bool isImageStore) { |
| 388 | auto &builder = cgf.getBuilder(); |
| 389 | |
| 390 | llvm::SmallVector<mlir::Value, 10> args; |
| 391 | for (unsigned i = 0, n = e->getNumArgs(); i < n; ++i) |
| 392 | args.push_back(Elt: cgf.emitScalarExpr(e: e->getArg(Arg: i))); |
| 393 | |
| 394 | mlir::Type retTy = isImageStore ? cir::VoidType::get(ctx: builder.getContext()) |
| 395 | : cgf.convertType(t: e->getType()); |
| 396 | |
| 397 | auto callOp = cir::LLVMIntrinsicCallOp::create( |
| 398 | builder, location: cgf.getLoc(srcLoc: e->getExprLoc()), |
| 399 | intrinsic_name: builder.getStringAttr(bytes: intrinsicName), resType: retTy, operands: args); |
| 400 | |
| 401 | return callOp.getResult(); |
| 402 | } |
| 403 | |
| 404 | static mlir::Value emitAMDGPUPredicate(CIRGenFunction &cgf, const CallExpr *e, |
| 405 | const llvm::Twine &name) { |
| 406 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 407 | mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc()); |
| 408 | mlir::Value result = cir::AMDGPUFeaturePredicateOp::create( |
| 409 | builder, location: loc, result: builder.getBoolTy(), predicate: builder.getStringAttr(bytes: name)); |
| 410 | // Sema retypes the call to the logical operation type, which is int in C. |
| 411 | return builder.createBoolToInt(src: result, newTy: cgf.convertType(t: e->getType())); |
| 412 | } |
| 413 | |
| 414 | std::optional<mlir::Value> |
| 415 | CIRGenFunction::emitAMDGPUBuiltinExpr(unsigned builtinId, |
| 416 | const CallExpr *expr) { |
| 417 | switch (builtinId) { |
| 418 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u32: |
| 419 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u32: |
| 420 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i32: |
| 421 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u32: |
| 422 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i32: |
| 423 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u32: |
| 424 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b32: |
| 425 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b32: |
| 426 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b32: |
| 427 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_add_u64: |
| 428 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_sub_u64: |
| 429 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_i64: |
| 430 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_min_u64: |
| 431 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_i64: |
| 432 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_max_u64: |
| 433 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_and_b64: |
| 434 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_or_b64: |
| 435 | case AMDGPU::BI__builtin_amdgcn_wave_reduce_xor_b64: { |
| 436 | cgm.errorNYI(expr->getSourceRange(), |
| 437 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 438 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 439 | return mlir::Value{}; |
| 440 | } |
| 441 | case AMDGPU::BI__builtin_amdgcn_div_scale: |
| 442 | case AMDGPU::BI__builtin_amdgcn_div_scalef: { |
| 443 | Address flagOutPtr = emitPointerWithAlignment(expr: expr->getArg(Arg: 3)); |
| 444 | llvm::StringRef intrinsicName = "amdgcn.div.scale" ; |
| 445 | mlir::Value x = emitScalarExpr(e: expr->getArg(Arg: 0)); |
| 446 | mlir::Value y = emitScalarExpr(e: expr->getArg(Arg: 1)); |
| 447 | mlir::Value z = emitScalarExpr(e: expr->getArg(Arg: 2)); |
| 448 | |
| 449 | auto i1Ty = builder.getUIntNTy(n: 1); |
| 450 | mlir::Type resMembers[] = {x.getType(), i1Ty}; |
| 451 | cir::RecordType resTy = |
| 452 | builder.getAnonRecordTy(members: resMembers, /*packed=*/false, |
| 453 | memberKinds: cir::RecordType::getAllDataKinds(members: resMembers)); |
| 454 | |
| 455 | mlir::Value structResult = |
| 456 | cir::LLVMIntrinsicCallOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), |
| 457 | intrinsic_name: builder.getStringAttr(bytes: intrinsicName), |
| 458 | resType: resTy, operands: {x, y, z}) |
| 459 | .getResult(); |
| 460 | |
| 461 | mlir::Value result = cir::ExtractMemberOp::create( |
| 462 | builder, location: getLoc(srcLoc: expr->getExprLoc()), result: x.getType(), record: structResult, index: 0); |
| 463 | mlir::Value flag = cir::ExtractMemberOp::create( |
| 464 | builder, location: getLoc(srcLoc: expr->getExprLoc()), result: i1Ty, record: structResult, index: 1); |
| 465 | |
| 466 | mlir::Type flagType = flagOutPtr.getElementType(); |
| 467 | mlir::Value flagToStore = |
| 468 | cir::CastOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), result: flagType, |
| 469 | kind: cir::CastKind::int_to_bool, src: flag); |
| 470 | builder.createStore(loc: getLoc(srcLoc: expr->getExprLoc()), val: flagToStore, dst: flagOutPtr); |
| 471 | return result; |
| 472 | } |
| 473 | case AMDGPU::BI__builtin_amdgcn_div_fmas: |
| 474 | case AMDGPU::BI__builtin_amdgcn_div_fmasf: |
| 475 | return emitBuiltinWithOneOverloadedType<4>(e: expr, intrinName: "amdgcn.div.fmas" ) |
| 476 | .getValue(); |
| 477 | case AMDGPU::BI__builtin_amdgcn_mov_dpp8: |
| 478 | case AMDGPU::BI__builtin_amdgcn_mov_dpp: |
| 479 | case AMDGPU::BI__builtin_amdgcn_update_dpp: { |
| 480 | mlir::Location loc = getLoc(srcLoc: expr->getExprLoc()); |
| 481 | unsigned iceArguments = 0; |
| 482 | ASTContext::GetBuiltinTypeError error; |
| 483 | getContext().GetBuiltinType(ID: builtinId, Error&: error, IntegerConstantArgs: &iceArguments); |
| 484 | assert(error == ASTContext::GE_None && "Should not codegen an error" ); |
| 485 | assert(expr->getNumArgs() == 5 || expr->getNumArgs() == 6 || |
| 486 | expr->getNumArgs() == 2); |
| 487 | |
| 488 | mlir::Type dataTy = convertType(t: expr->getArg(Arg: 0)->getType()); |
| 489 | unsigned size = cgm.getDataLayout().getTypeSizeInBits(ty: dataTy); |
| 490 | cir::IntType intTy = builder.getUIntNTy(n: std::max(a: size, b: 32u)); |
| 491 | |
| 492 | bool isMovDpp8 = builtinId == AMDGPU::BI__builtin_amdgcn_mov_dpp8; |
| 493 | bool isMovDpp = builtinId == AMDGPU::BI__builtin_amdgcn_mov_dpp; |
| 494 | bool isUpdateDpp = builtinId == AMDGPU::BI__builtin_amdgcn_update_dpp; |
| 495 | llvm::StringRef intrinsicName = |
| 496 | isMovDpp8 ? "amdgcn.mov.dpp8" : "amdgcn.update.dpp" ; |
| 497 | cir::FuncType intrinsicTy = |
| 498 | getIntrinsicType(id: isMovDpp8 ? llvm::Intrinsic::amdgcn_mov_dpp8 |
| 499 | : llvm::Intrinsic::amdgcn_update_dpp, |
| 500 | overloadTys: {intTy}); |
| 501 | |
| 502 | auto coerceTo = [&](mlir::Value from, mlir::Type to) -> mlir::Value { |
| 503 | if (from.getType() == to) |
| 504 | return from; |
| 505 | if (mlir::isa<cir::IntType>(Val: from.getType()) && |
| 506 | mlir::isa<cir::IntType>(Val: to)) |
| 507 | return builder.createIntCast(src: from, newTy: to); |
| 508 | return builder.createBitcast(src: from, newTy: to); |
| 509 | }; |
| 510 | |
| 511 | llvm::SmallVector<mlir::Value, 6> args; |
| 512 | // __builtin_amdgcn_mov_dpp has no "old" operand at the source level, but |
| 513 | // the real intrinsic it lowers to requires one, so we synthesize a poison |
| 514 | // value for it since it is never meaningfully read. |
| 515 | if (isMovDpp) |
| 516 | args.push_back(Elt: builder.getConstant(loc, attr: cir::PoisonAttr::get(type: intTy))); |
| 517 | |
| 518 | // Number of builtin-level leading args that need zero-extend promotion when |
| 519 | // the data type is narrower than 32 bits. |
| 520 | unsigned numPromotedArgs = isUpdateDpp ? 2u : 1u; |
| 521 | for (unsigned i = 0; i != expr->getNumArgs(); ++i) { |
| 522 | mlir::Value v = |
| 523 | emitScalarOrConstFoldImmArg(iceArguments, idx: i, argExpr: expr->getArg(Arg: i)); |
| 524 | if (i < numPromotedArgs && size < 32) { |
| 525 | mlir::Type sameWidthUTy = builder.getUIntNTy(n: size); |
| 526 | if (v.getType() != sameWidthUTy) |
| 527 | v = builder.createBitcast(src: v, newTy: sameWidthUTy); |
| 528 | v = builder.createIntCast(src: v, newTy: intTy); |
| 529 | } |
| 530 | args.push_back(Elt: coerceTo(v, intrinsicTy.getInput(i: i + unsigned(isMovDpp)))); |
| 531 | } |
| 532 | |
| 533 | mlir::Value result = builder.emitIntrinsicCallOp(loc, str: intrinsicName, resTy: intTy, |
| 534 | op: mlir::ValueRange(args)); |
| 535 | if (size < 32 && !mlir::isa<cir::IntType>(Val: dataTy)) |
| 536 | result = builder.createIntCast(src: result, newTy: builder.getUIntNTy(n: size)); |
| 537 | return coerceTo(result, dataTy); |
| 538 | } |
| 539 | case AMDGPU::BI__builtin_amdgcn_permlane16: |
| 540 | case AMDGPU::BI__builtin_amdgcn_permlanex16: { |
| 541 | llvm::StringRef intrinsicName = |
| 542 | builtinId == AMDGPU::BI__builtin_amdgcn_permlane16 |
| 543 | ? "amdgcn.permlane16" |
| 544 | : "amdgcn.permlanex16" ; |
| 545 | return emitBuiltinWithOneOverloadedType<6>(e: expr, intrinName: intrinsicName).getValue(); |
| 546 | } |
| 547 | case AMDGPU::BI__builtin_amdgcn_permlane64: |
| 548 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.permlane64" ) |
| 549 | .getValue(); |
| 550 | case AMDGPU::BI__builtin_amdgcn_permlane_bcast: |
| 551 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.bcast" ) |
| 552 | .getValue(); |
| 553 | case AMDGPU::BI__builtin_amdgcn_permlane_up: |
| 554 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.up" ) |
| 555 | .getValue(); |
| 556 | case AMDGPU::BI__builtin_amdgcn_permlane_down: |
| 557 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.down" ) |
| 558 | .getValue(); |
| 559 | case AMDGPU::BI__builtin_amdgcn_permlane_xor: |
| 560 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.permlane.xor" ) |
| 561 | .getValue(); |
| 562 | case AMDGPU::BI__builtin_amdgcn_readlane: |
| 563 | return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.readlane" ) |
| 564 | .getValue(); |
| 565 | case AMDGPU::BI__builtin_amdgcn_readfirstlane: |
| 566 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.readfirstlane" ) |
| 567 | .getValue(); |
| 568 | case AMDGPU::BI__builtin_amdgcn_wave_shuffle: |
| 569 | return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.wave.shuffle" ) |
| 570 | .getValue(); |
| 571 | case AMDGPU::BI__builtin_amdgcn_div_fixup: |
| 572 | case AMDGPU::BI__builtin_amdgcn_div_fixupf: |
| 573 | case AMDGPU::BI__builtin_amdgcn_div_fixuph: { |
| 574 | mlir::Value src0 = emitScalarExpr(e: expr->getArg(Arg: 0)); |
| 575 | mlir::Value src1 = emitScalarExpr(e: expr->getArg(Arg: 1)); |
| 576 | mlir::Value src2 = emitScalarExpr(e: expr->getArg(Arg: 2)); |
| 577 | return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), |
| 578 | str: "amdgcn.div.fixup" , resTy: src0.getType(), |
| 579 | op: mlir::ValueRange{src0, src1, src2}); |
| 580 | } |
| 581 | case AMDGPU::BI__builtin_amdgcn_trig_preop: |
| 582 | case AMDGPU::BI__builtin_amdgcn_trig_preopf: |
| 583 | return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.trig.preop" ) |
| 584 | .getValue(); |
| 585 | case AMDGPU::BI__builtin_amdgcn_rcp: |
| 586 | case AMDGPU::BI__builtin_amdgcn_rcpf: |
| 587 | case AMDGPU::BI__builtin_amdgcn_rcph: |
| 588 | case AMDGPU::BI__builtin_amdgcn_rcp_bf16: { |
| 589 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rcp" ).getValue(); |
| 590 | } |
| 591 | case AMDGPU::BI__builtin_amdgcn_sqrt: |
| 592 | case AMDGPU::BI__builtin_amdgcn_sqrtf: |
| 593 | case AMDGPU::BI__builtin_amdgcn_sqrth: |
| 594 | case AMDGPU::BI__builtin_amdgcn_sqrt_bf16: { |
| 595 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.sqrt" ).getValue(); |
| 596 | } |
| 597 | case AMDGPU::BI__builtin_amdgcn_rsq: |
| 598 | case AMDGPU::BI__builtin_amdgcn_rsqf: |
| 599 | case AMDGPU::BI__builtin_amdgcn_rsqh: |
| 600 | case AMDGPU::BI__builtin_amdgcn_rsq_bf16: { |
| 601 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rsq" ).getValue(); |
| 602 | } |
| 603 | case AMDGPU::BI__builtin_amdgcn_rsq_clamp: |
| 604 | case AMDGPU::BI__builtin_amdgcn_rsq_clampf: { |
| 605 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.rsq.clamp" ) |
| 606 | .getValue(); |
| 607 | } |
| 608 | case AMDGPU::BI__builtin_amdgcn_sinf: |
| 609 | case AMDGPU::BI__builtin_amdgcn_sinh: |
| 610 | case AMDGPU::BI__builtin_amdgcn_sin_bf16: { |
| 611 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.sin" ).getValue(); |
| 612 | } |
| 613 | case AMDGPU::BI__builtin_amdgcn_cosf: |
| 614 | case AMDGPU::BI__builtin_amdgcn_cosh: |
| 615 | case AMDGPU::BI__builtin_amdgcn_cos_bf16: { |
| 616 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.cos" ).getValue(); |
| 617 | } |
| 618 | case AMDGPU::BI__builtin_amdgcn_dispatch_ptr: |
| 619 | return emitAMDGPUDispatchPtr(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), e: expr); |
| 620 | case AMDGPU::BI__builtin_amdgcn_logf: |
| 621 | case AMDGPU::BI__builtin_amdgcn_log_bf16: { |
| 622 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.log" ).getValue(); |
| 623 | } |
| 624 | case AMDGPU::BI__builtin_amdgcn_exp2f: |
| 625 | case AMDGPU::BI__builtin_amdgcn_exp2_bf16: { |
| 626 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.exp2" ).getValue(); |
| 627 | } |
| 628 | case AMDGPU::BI__builtin_amdgcn_log_clampf: { |
| 629 | cgm.errorNYI(expr->getSourceRange(), |
| 630 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 631 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 632 | return mlir::Value{}; |
| 633 | } |
| 634 | case AMDGPU::BI__builtin_amdgcn_ldexp: |
| 635 | case AMDGPU::BI__builtin_amdgcn_ldexpf: |
| 636 | case AMDGPU::BI__builtin_amdgcn_ldexph: { |
| 637 | mlir::Value src0 = emitScalarExpr(e: expr->getArg(Arg: 0)); |
| 638 | mlir::Value src1 = emitScalarExpr(e: expr->getArg(Arg: 1)); |
| 639 | // For ldexph, the raw instruction has different behavior for out-of-bounds |
| 640 | // exponent values (implicit truncation instead of saturate to |
| 641 | // short_min/short_max), so truncate the exponent to i16 first. |
| 642 | mlir::Value exp = |
| 643 | builtinId == AMDGPU::BI__builtin_amdgcn_ldexph |
| 644 | ? cir::CastOp::create(builder, location: getLoc(srcLoc: expr->getExprLoc()), |
| 645 | result: builder.getSInt16Ty(), |
| 646 | kind: cir::CastKind::integral, src: src1) |
| 647 | : src1; |
| 648 | return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()), str: "ldexp" , |
| 649 | resTy: src0.getType(), |
| 650 | op: mlir::ValueRange{src0, exp}); |
| 651 | } |
| 652 | case AMDGPU::BI__builtin_amdgcn_frexp_mant: |
| 653 | case AMDGPU::BI__builtin_amdgcn_frexp_mantf: |
| 654 | case AMDGPU::BI__builtin_amdgcn_frexp_manth: { |
| 655 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.frexp.mant" ) |
| 656 | .getValue(); |
| 657 | } |
| 658 | case AMDGPU::BI__builtin_amdgcn_frexp_exp: |
| 659 | case AMDGPU::BI__builtin_amdgcn_frexp_expf: |
| 660 | case AMDGPU::BI__builtin_amdgcn_frexp_exph: |
| 661 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.frexp.exp" , |
| 662 | resultType: convertType(t: expr->getType())) |
| 663 | .getValue(); |
| 664 | case AMDGPU::BI__builtin_amdgcn_fract: |
| 665 | case AMDGPU::BI__builtin_amdgcn_fractf: |
| 666 | case AMDGPU::BI__builtin_amdgcn_fracth: |
| 667 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.fract" ).getValue(); |
| 668 | case AMDGPU::BI__builtin_amdgcn_lerp: { |
| 669 | cgm.errorNYI(expr->getSourceRange(), |
| 670 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 671 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 672 | return mlir::Value{}; |
| 673 | } |
| 674 | case AMDGPU::BI__builtin_amdgcn_ubfe: |
| 675 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.ubfe" ).getValue(); |
| 676 | case AMDGPU::BI__builtin_amdgcn_sbfe: |
| 677 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.sbfe" ).getValue(); |
| 678 | case AMDGPU::BI__builtin_amdgcn_ballot_w32: |
| 679 | case AMDGPU::BI__builtin_amdgcn_ballot_w64: |
| 680 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ballot" , |
| 681 | resultType: convertType(t: expr->getType())) |
| 682 | .getValue(); |
| 683 | case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w32: |
| 684 | case AMDGPU::BI__builtin_amdgcn_inverse_ballot_w64: |
| 685 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.inverse.ballot" , |
| 686 | resultType: convertType(t: expr->getType())) |
| 687 | .getValue(); |
| 688 | case AMDGPU::BI__builtin_amdgcn_tanhf: |
| 689 | case AMDGPU::BI__builtin_amdgcn_tanhh: |
| 690 | case AMDGPU::BI__builtin_amdgcn_tanh_bf16: { |
| 691 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.tanh" ).getValue(); |
| 692 | } |
| 693 | case AMDGPU::BI__builtin_amdgcn_uicmp: |
| 694 | case AMDGPU::BI__builtin_amdgcn_uicmpl: |
| 695 | case AMDGPU::BI__builtin_amdgcn_sicmp: |
| 696 | case AMDGPU::BI__builtin_amdgcn_sicmpl: |
| 697 | case AMDGPU::BI__builtin_amdgcn_fcmp: |
| 698 | case AMDGPU::BI__builtin_amdgcn_fcmpf: { |
| 699 | mlir::Value lhs = emitScalarExpr(e: expr->getArg(Arg: 0)); |
| 700 | mlir::Value rhs = emitScalarExpr(e: expr->getArg(Arg: 1)); |
| 701 | |
| 702 | uint64_t imm = |
| 703 | expr->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue(); |
| 704 | |
| 705 | cir::CmpOpKind pred; |
| 706 | switch (imm) { |
| 707 | case 0x1: // FCMP_OEQ |
| 708 | case 0x20: // ICMP_EQ |
| 709 | pred = cir::CmpOpKind::eq; |
| 710 | break; |
| 711 | case 0xe: // FCMP_UNE |
| 712 | case 0x21: // ICMP_NE |
| 713 | pred = cir::CmpOpKind::ne; |
| 714 | break; |
| 715 | case 0x2: // FCMP_OGT |
| 716 | case 0x22: // ICMP_UGT |
| 717 | case 0x26: // ICMP_SGT |
| 718 | pred = cir::CmpOpKind::gt; |
| 719 | break; |
| 720 | case 0x3: // FCMP_OGE |
| 721 | case 0x23: // ICMP_UGE |
| 722 | case 0x27: // ICMP_SGE |
| 723 | pred = cir::CmpOpKind::ge; |
| 724 | break; |
| 725 | case 0x4: // FCMP_OLT |
| 726 | case 0x24: // ICMP_ULT |
| 727 | case 0x28: // ICMP_SLT |
| 728 | pred = cir::CmpOpKind::lt; |
| 729 | break; |
| 730 | case 0x5: // FCMP_OLE |
| 731 | case 0x25: // ICMP_ULE |
| 732 | case 0x29: // ICMP_SLE |
| 733 | pred = cir::CmpOpKind::le; |
| 734 | break; |
| 735 | case 0x6: // FCMP_ONE |
| 736 | pred = cir::CmpOpKind::one; |
| 737 | break; |
| 738 | case 0x8: // FCMP_UNO |
| 739 | pred = cir::CmpOpKind::uno; |
| 740 | break; |
| 741 | default: |
| 742 | cgm.errorNYI(expr->getSourceRange(), |
| 743 | "amdgcn compare with unsupported predicate" ); |
| 744 | return mlir::Value{}; |
| 745 | } |
| 746 | |
| 747 | mlir::Location loc = getLoc(srcLoc: expr->getExprLoc()); |
| 748 | mlir::Value cmp = builder.createCompare(loc, kind: pred, lhs, rhs); |
| 749 | return builder.emitIntrinsicCallOp(loc, str: "amdgcn.ballot" , |
| 750 | resTy: convertType(t: expr->getType()), op&: cmp); |
| 751 | } |
| 752 | case AMDGPU::BI__builtin_amdgcn_class: |
| 753 | case AMDGPU::BI__builtin_amdgcn_classf: |
| 754 | case AMDGPU::BI__builtin_amdgcn_classh: |
| 755 | return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "amdgcn.class" , |
| 756 | resultType: convertType(t: expr->getType())) |
| 757 | .getValue(); |
| 758 | case AMDGPU::BI__builtin_amdgcn_fmed3f: |
| 759 | case AMDGPU::BI__builtin_amdgcn_fmed3h: |
| 760 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.fmed3" ).getValue(); |
| 761 | case AMDGPU::BI__builtin_amdgcn_ds_append: |
| 762 | case AMDGPU::BI__builtin_amdgcn_ds_consume: { |
| 763 | cgm.errorNYI(expr->getSourceRange(), |
| 764 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 765 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 766 | return mlir::Value{}; |
| 767 | } |
| 768 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: |
| 769 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: |
| 770 | case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32: |
| 771 | return emitBuiltinWithOneOverloadedType<1>( |
| 772 | e: expr, intrinName: "amdgcn.global.load.tr.b64" , resultType: convertType(t: expr->getType())) |
| 773 | .getValue(); |
| 774 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: |
| 775 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: |
| 776 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: |
| 777 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: |
| 778 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: |
| 779 | case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: |
| 780 | case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16: |
| 781 | case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16: |
| 782 | case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16: |
| 783 | return emitBuiltinWithOneOverloadedType<1>( |
| 784 | e: expr, intrinName: "amdgcn.global.load.tr.b128" , resultType: convertType(t: expr->getType())) |
| 785 | .getValue(); |
| 786 | case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32: |
| 787 | return emitBuiltinWithOneOverloadedType<1>( |
| 788 | e: expr, intrinName: "amdgcn.global.load.tr4.b64" , resultType: convertType(t: expr->getType())) |
| 789 | .getValue(); |
| 790 | case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32: |
| 791 | return emitBuiltinWithOneOverloadedType<1>( |
| 792 | e: expr, intrinName: "amdgcn.global.load.tr6.b96" , resultType: convertType(t: expr->getType())) |
| 793 | .getValue(); |
| 794 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32: |
| 795 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr4.b64" , |
| 796 | resultType: convertType(t: expr->getType())) |
| 797 | .getValue(); |
| 798 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32: |
| 799 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr6.b96" , |
| 800 | resultType: convertType(t: expr->getType())) |
| 801 | .getValue(); |
| 802 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32: |
| 803 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr8.b64" , |
| 804 | resultType: convertType(t: expr->getType())) |
| 805 | .getValue(); |
| 806 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16: |
| 807 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16: |
| 808 | case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16: |
| 809 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.load.tr16.b128" , |
| 810 | resultType: convertType(t: expr->getType())) |
| 811 | .getValue(); |
| 812 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32: |
| 813 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr4.b64" , |
| 814 | resultType: convertType(t: expr->getType())) |
| 815 | .getValue(); |
| 816 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32: |
| 817 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr8.b64" , |
| 818 | resultType: convertType(t: expr->getType())) |
| 819 | .getValue(); |
| 820 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32: |
| 821 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr6.b96" , |
| 822 | resultType: convertType(t: expr->getType())) |
| 823 | .getValue(); |
| 824 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: |
| 825 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16: |
| 826 | case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16: |
| 827 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.ds.read.tr16.b64" , |
| 828 | resultType: convertType(t: expr->getType())) |
| 829 | .getValue(); |
| 830 | case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b32: |
| 831 | case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b64: |
| 832 | case AMDGPU::BI__builtin_amdgcn_global_load_monitor_b128: |
| 833 | case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b32: |
| 834 | case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b64: |
| 835 | case AMDGPU::BI__builtin_amdgcn_flat_load_monitor_b128: { |
| 836 | cgm.errorNYI(expr->getSourceRange(), |
| 837 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 838 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 839 | return mlir::Value{}; |
| 840 | } |
| 841 | case AMDGPU::BI__builtin_amdgcn_cluster_load_b32: |
| 842 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b32" , |
| 843 | resultType: convertType(t: expr->getType())) |
| 844 | .getValue(); |
| 845 | case AMDGPU::BI__builtin_amdgcn_cluster_load_b64: |
| 846 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b64" , |
| 847 | resultType: convertType(t: expr->getType())) |
| 848 | .getValue(); |
| 849 | case AMDGPU::BI__builtin_amdgcn_cluster_load_b128: |
| 850 | return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "amdgcn.cluster.load.b128" , |
| 851 | resultType: convertType(t: expr->getType())) |
| 852 | .getValue(); |
| 853 | case AMDGPU::BI__builtin_amdgcn_load_to_lds: |
| 854 | return emitBuiltinWithOneOverloadedType<5>( |
| 855 | e: expr, intrinName: "amdgcn.load.to.lds" , |
| 856 | resultType: cir::VoidType::get(ctx: builder.getContext())) |
| 857 | .getValue(); |
| 858 | case AMDGPU::BI__builtin_amdgcn_load_async_to_lds: |
| 859 | return emitBuiltinWithOneOverloadedType<5>( |
| 860 | e: expr, intrinName: "amdgcn.load.async.to.lds" , |
| 861 | resultType: cir::VoidType::get(ctx: builder.getContext())) |
| 862 | .getValue(); |
| 863 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_32x4B: |
| 864 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_32x4B: |
| 865 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_16x8B: |
| 866 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_16x8B: |
| 867 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_8x16B: |
| 868 | case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_store_8x16B: { |
| 869 | cgm.errorNYI(expr->getSourceRange(), |
| 870 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 871 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 872 | return mlir::Value{}; |
| 873 | } |
| 874 | case AMDGPU::BI__builtin_amdgcn_get_fpenv: |
| 875 | case AMDGPU::BI__builtin_amdgcn_set_fpenv: { |
| 876 | cgm.errorNYI(expr->getSourceRange(), |
| 877 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 878 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 879 | return mlir::Value{}; |
| 880 | } |
| 881 | case AMDGPU::BI__builtin_amdgcn_processor_is: { |
| 882 | assert(cgm.getTriple().isSPIRV() && |
| 883 | "__builtin_amdgcn_processor_is should never reach CodeGen for " |
| 884 | "concrete targets!" ); |
| 885 | StringRef proc = cast<clang::StringLiteral>(Val: expr->getArg(Arg: 0))->getString(); |
| 886 | return emitAMDGPUPredicate(cgf&: *this, e: expr, name: "is." + proc); |
| 887 | } |
| 888 | case AMDGPU::BI__builtin_amdgcn_is_invocable: { |
| 889 | assert(cgm.getTriple().isSPIRV() && |
| 890 | "__builtin_amdgcn_is_invocable should never reach CodeGen for " |
| 891 | "concrete targets!" ); |
| 892 | auto *fd = cast<FunctionDecl>( |
| 893 | Val: cast<DeclRefExpr>(Val: expr->getArg(Arg: 0))->getReferencedDeclOfCallee()); |
| 894 | StringRef rf = |
| 895 | getContext().BuiltinInfo.getRequiredFeatures(ID: fd->getBuiltinID()); |
| 896 | return emitAMDGPUPredicate(cgf&: *this, e: expr, name: "has." + rf); |
| 897 | } |
| 898 | case AMDGPU::BI__builtin_amdgcn_read_exec: |
| 899 | case AMDGPU::BI__builtin_amdgcn_read_exec_lo: |
| 900 | case AMDGPU::BI__builtin_amdgcn_read_exec_hi: { |
| 901 | // The exec mask is read as a ballot over an all-true predicate. The |
| 902 | // ballot is at least as wide as the wavefront, so that a wave64 target |
| 903 | // still reports both halves for the _lo/_hi forms. |
| 904 | mlir::Location loc = getLoc(srcLoc: expr->getExprLoc()); |
| 905 | unsigned registerWidth = |
| 906 | builtinId == AMDGPU::BI__builtin_amdgcn_read_exec_lo ? 32 : 64; |
| 907 | unsigned ballotWidth = |
| 908 | std::max(a: getTarget().getGridValue().GV_Warp_Size, b: registerWidth); |
| 909 | cir::IntType ballotTy = builder.getUIntNTy(n: ballotWidth); |
| 910 | |
| 911 | mlir::Value truePred = builder.getBool(state: true, loc).getResult(); |
| 912 | mlir::Value result = |
| 913 | builder.emitIntrinsicCallOp(loc, str: "amdgcn.ballot" , resTy: ballotTy, op&: truePred); |
| 914 | |
| 915 | if (builtinId == AMDGPU::BI__builtin_amdgcn_read_exec_hi) |
| 916 | result = builder.createShiftRight(loc, lhs: result, bits: 32); |
| 917 | return builder.createIntCast(src: result, newTy: convertType(t: expr->getType())); |
| 918 | } |
| 919 | case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray: |
| 920 | case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h: |
| 921 | case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l: |
| 922 | case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: { |
| 923 | cgm.errorNYI(expr->getSourceRange(), |
| 924 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 925 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 926 | return mlir::Value{}; |
| 927 | } |
| 928 | case AMDGPU::BI__builtin_amdgcn_image_bvh8_intersect_ray: |
| 929 | case AMDGPU::BI__builtin_amdgcn_image_bvh_dual_intersect_ray: { |
| 930 | cgm.errorNYI(expr->getSourceRange(), |
| 931 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 932 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 933 | return mlir::Value{}; |
| 934 | } |
| 935 | case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: |
| 936 | case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn: |
| 937 | case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn: |
| 938 | case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn: { |
| 939 | cgm.errorNYI(expr->getSourceRange(), |
| 940 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 941 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 942 | return mlir::Value{}; |
| 943 | } |
| 944 | case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f32_i32: |
| 945 | case AMDGPU::BI__builtin_amdgcn_image_load_1d_v4f16_i32: |
| 946 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 947 | intrinsicName: "amdgcn.image.load.1d" , isImageStore: false); |
| 948 | case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f32_i32: |
| 949 | case AMDGPU::BI__builtin_amdgcn_image_load_1darray_v4f16_i32: |
| 950 | return emitAMDGCNImageOverloadedReturnType( |
| 951 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.1darray" , isImageStore: false); |
| 952 | case AMDGPU::BI__builtin_amdgcn_image_load_2d_f32_i32: |
| 953 | case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f32_i32: |
| 954 | case AMDGPU::BI__builtin_amdgcn_image_load_2d_v4f16_i32: |
| 955 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 956 | intrinsicName: "amdgcn.image.load.2d" , isImageStore: false); |
| 957 | case AMDGPU::BI__builtin_amdgcn_image_load_2darray_f32_i32: |
| 958 | case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f32_i32: |
| 959 | case AMDGPU::BI__builtin_amdgcn_image_load_2darray_v4f16_i32: |
| 960 | return emitAMDGCNImageOverloadedReturnType( |
| 961 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.2darray" , isImageStore: false); |
| 962 | case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f32_i32: |
| 963 | case AMDGPU::BI__builtin_amdgcn_image_load_3d_v4f16_i32: |
| 964 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 965 | intrinsicName: "amdgcn.image.load.3d" , isImageStore: false); |
| 966 | case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f32_i32: |
| 967 | case AMDGPU::BI__builtin_amdgcn_image_load_cube_v4f16_i32: |
| 968 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 969 | intrinsicName: "amdgcn.image.load.cube" , isImageStore: false); |
| 970 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f32_i32: |
| 971 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_1d_v4f16_i32: |
| 972 | return emitAMDGCNImageOverloadedReturnType( |
| 973 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.1d" , isImageStore: false); |
| 974 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f32_i32: |
| 975 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_1darray_v4f16_i32: |
| 976 | return emitAMDGCNImageOverloadedReturnType( |
| 977 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.1darray" , isImageStore: false); |
| 978 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_f32_i32: |
| 979 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f32_i32: |
| 980 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2d_v4f16_i32: |
| 981 | return emitAMDGCNImageOverloadedReturnType( |
| 982 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.2d" , isImageStore: false); |
| 983 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_f32_i32: |
| 984 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f32_i32: |
| 985 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_2darray_v4f16_i32: |
| 986 | return emitAMDGCNImageOverloadedReturnType( |
| 987 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.2darray" , isImageStore: false); |
| 988 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f32_i32: |
| 989 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_3d_v4f16_i32: |
| 990 | return emitAMDGCNImageOverloadedReturnType( |
| 991 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.3d" , isImageStore: false); |
| 992 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f32_i32: |
| 993 | case AMDGPU::BI__builtin_amdgcn_image_load_mip_cube_v4f16_i32: |
| 994 | return emitAMDGCNImageOverloadedReturnType( |
| 995 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.load.mip.cube" , isImageStore: false); |
| 996 | case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f32_i32: |
| 997 | case AMDGPU::BI__builtin_amdgcn_image_store_1d_v4f16_i32: |
| 998 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 999 | intrinsicName: "amdgcn.image.store.1d" , isImageStore: true); |
| 1000 | case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f32_i32: |
| 1001 | case AMDGPU::BI__builtin_amdgcn_image_store_1darray_v4f16_i32: |
| 1002 | return emitAMDGCNImageOverloadedReturnType( |
| 1003 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.1darray" , isImageStore: true); |
| 1004 | case AMDGPU::BI__builtin_amdgcn_image_store_2d_f32_i32: |
| 1005 | case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f32_i32: |
| 1006 | case AMDGPU::BI__builtin_amdgcn_image_store_2d_v4f16_i32: |
| 1007 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1008 | intrinsicName: "amdgcn.image.store.2d" , isImageStore: true); |
| 1009 | case AMDGPU::BI__builtin_amdgcn_image_store_2darray_f32_i32: |
| 1010 | case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f32_i32: |
| 1011 | case AMDGPU::BI__builtin_amdgcn_image_store_2darray_v4f16_i32: |
| 1012 | return emitAMDGCNImageOverloadedReturnType( |
| 1013 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.2darray" , isImageStore: true); |
| 1014 | case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f32_i32: |
| 1015 | case AMDGPU::BI__builtin_amdgcn_image_store_3d_v4f16_i32: |
| 1016 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1017 | intrinsicName: "amdgcn.image.store.3d" , isImageStore: true); |
| 1018 | case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f32_i32: |
| 1019 | case AMDGPU::BI__builtin_amdgcn_image_store_cube_v4f16_i32: |
| 1020 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1021 | intrinsicName: "amdgcn.image.store.cube" , isImageStore: true); |
| 1022 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f32_i32: |
| 1023 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_1d_v4f16_i32: |
| 1024 | return emitAMDGCNImageOverloadedReturnType( |
| 1025 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.1d" , isImageStore: true); |
| 1026 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f32_i32: |
| 1027 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_1darray_v4f16_i32: |
| 1028 | return emitAMDGCNImageOverloadedReturnType( |
| 1029 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.1darray" , isImageStore: true); |
| 1030 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_f32_i32: |
| 1031 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f32_i32: |
| 1032 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2d_v4f16_i32: |
| 1033 | return emitAMDGCNImageOverloadedReturnType( |
| 1034 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.2d" , isImageStore: true); |
| 1035 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_f32_i32: |
| 1036 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f32_i32: |
| 1037 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_2darray_v4f16_i32: |
| 1038 | return emitAMDGCNImageOverloadedReturnType( |
| 1039 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.2darray" , isImageStore: true); |
| 1040 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f32_i32: |
| 1041 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_3d_v4f16_i32: |
| 1042 | return emitAMDGCNImageOverloadedReturnType( |
| 1043 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.3d" , isImageStore: true); |
| 1044 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f32_i32: |
| 1045 | case AMDGPU::BI__builtin_amdgcn_image_store_mip_cube_v4f16_i32: |
| 1046 | return emitAMDGCNImageOverloadedReturnType( |
| 1047 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.store.mip.cube" , isImageStore: true); |
| 1048 | case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f32_f32: |
| 1049 | case AMDGPU::BI__builtin_amdgcn_image_sample_1d_v4f16_f32: |
| 1050 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1051 | intrinsicName: "amdgcn.image.sample.1d" , isImageStore: false); |
| 1052 | case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f32_f32: |
| 1053 | case AMDGPU::BI__builtin_amdgcn_image_sample_1darray_v4f16_f32: |
| 1054 | return emitAMDGCNImageOverloadedReturnType( |
| 1055 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.1darray" , isImageStore: false); |
| 1056 | case AMDGPU::BI__builtin_amdgcn_image_sample_2d_f32_f32: |
| 1057 | case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f32_f32: |
| 1058 | case AMDGPU::BI__builtin_amdgcn_image_sample_2d_v4f16_f32: |
| 1059 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1060 | intrinsicName: "amdgcn.image.sample.2d" , isImageStore: false); |
| 1061 | case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_f32_f32: |
| 1062 | case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f32_f32: |
| 1063 | case AMDGPU::BI__builtin_amdgcn_image_sample_2darray_v4f16_f32: |
| 1064 | return emitAMDGCNImageOverloadedReturnType( |
| 1065 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.2darray" , isImageStore: false); |
| 1066 | case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f32_f32: |
| 1067 | case AMDGPU::BI__builtin_amdgcn_image_sample_3d_v4f16_f32: |
| 1068 | return emitAMDGCNImageOverloadedReturnType(cgf&: *this, e: expr, |
| 1069 | intrinsicName: "amdgcn.image.sample.3d" , isImageStore: false); |
| 1070 | case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f32_f32: |
| 1071 | case AMDGPU::BI__builtin_amdgcn_image_sample_cube_v4f16_f32: |
| 1072 | return emitAMDGCNImageOverloadedReturnType( |
| 1073 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.cube" , isImageStore: false); |
| 1074 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f32_f32: |
| 1075 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1d_v4f16_f32: |
| 1076 | return emitAMDGCNImageOverloadedReturnType( |
| 1077 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.1d" , isImageStore: false); |
| 1078 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f32_f32: |
| 1079 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_1d_v4f16_f32: |
| 1080 | return emitAMDGCNImageOverloadedReturnType( |
| 1081 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.1d" , isImageStore: false); |
| 1082 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f32_f32: |
| 1083 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_1d_v4f16_f32: |
| 1084 | return emitAMDGCNImageOverloadedReturnType( |
| 1085 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.1d" , isImageStore: false); |
| 1086 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f32_f32: |
| 1087 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_v4f16_f32: |
| 1088 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2d_f32_f32: |
| 1089 | return emitAMDGCNImageOverloadedReturnType( |
| 1090 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.2d" , isImageStore: false); |
| 1091 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f32_f32: |
| 1092 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_v4f16_f32: |
| 1093 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2d_f32_f32: |
| 1094 | return emitAMDGCNImageOverloadedReturnType( |
| 1095 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.2d" , isImageStore: false); |
| 1096 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f32_f32: |
| 1097 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_v4f16_f32: |
| 1098 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2d_f32_f32: |
| 1099 | return emitAMDGCNImageOverloadedReturnType( |
| 1100 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.2d" , isImageStore: false); |
| 1101 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f32_f32: |
| 1102 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_3d_v4f16_f32: |
| 1103 | return emitAMDGCNImageOverloadedReturnType( |
| 1104 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.3d" , isImageStore: false); |
| 1105 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f32_f32: |
| 1106 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_3d_v4f16_f32: |
| 1107 | return emitAMDGCNImageOverloadedReturnType( |
| 1108 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.3d" , isImageStore: false); |
| 1109 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f32_f32: |
| 1110 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_3d_v4f16_f32: |
| 1111 | return emitAMDGCNImageOverloadedReturnType( |
| 1112 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.3d" , isImageStore: false); |
| 1113 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f32_f32: |
| 1114 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_cube_v4f16_f32: |
| 1115 | return emitAMDGCNImageOverloadedReturnType( |
| 1116 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.cube" , isImageStore: false); |
| 1117 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f32_f32: |
| 1118 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_cube_v4f16_f32: |
| 1119 | return emitAMDGCNImageOverloadedReturnType( |
| 1120 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.cube" , isImageStore: false); |
| 1121 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f32_f32: |
| 1122 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_1darray_v4f16_f32: |
| 1123 | return emitAMDGCNImageOverloadedReturnType( |
| 1124 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.1darray" , isImageStore: false); |
| 1125 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f32_f32: |
| 1126 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_1darray_v4f16_f32: |
| 1127 | return emitAMDGCNImageOverloadedReturnType( |
| 1128 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.1darray" , isImageStore: false); |
| 1129 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f32_f32: |
| 1130 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_1darray_v4f16_f32: |
| 1131 | return emitAMDGCNImageOverloadedReturnType( |
| 1132 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.1darray" , isImageStore: false); |
| 1133 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f32_f32: |
| 1134 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_v4f16_f32: |
| 1135 | case AMDGPU::BI__builtin_amdgcn_image_sample_lz_2darray_f32_f32: |
| 1136 | return emitAMDGCNImageOverloadedReturnType( |
| 1137 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.lz.2darray" , isImageStore: false); |
| 1138 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f32_f32: |
| 1139 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_v4f16_f32: |
| 1140 | case AMDGPU::BI__builtin_amdgcn_image_sample_l_2darray_f32_f32: |
| 1141 | return emitAMDGCNImageOverloadedReturnType( |
| 1142 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.l.2darray" , isImageStore: false); |
| 1143 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f32_f32: |
| 1144 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_v4f16_f32: |
| 1145 | case AMDGPU::BI__builtin_amdgcn_image_sample_d_2darray_f32_f32: |
| 1146 | return emitAMDGCNImageOverloadedReturnType( |
| 1147 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.sample.d.2darray" , isImageStore: false); |
| 1148 | case AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f32_f32: |
| 1149 | case AMDGPU::BI__builtin_amdgcn_image_gather4_lz_2d_v4f16_f32: |
| 1150 | return emitAMDGCNImageOverloadedReturnType( |
| 1151 | cgf&: *this, e: expr, intrinsicName: "amdgcn.image.gather4.lz.2d" , isImageStore: false); |
| 1152 | case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4: |
| 1153 | case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: { |
| 1154 | cgm.errorNYI(expr->getSourceRange(), |
| 1155 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1156 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1157 | return mlir::Value{}; |
| 1158 | } |
| 1159 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32: |
| 1160 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32: |
| 1161 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64: |
| 1162 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64: |
| 1163 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32: |
| 1164 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32: |
| 1165 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64: |
| 1166 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64: |
| 1167 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32: |
| 1168 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64: |
| 1169 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32: |
| 1170 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64: |
| 1171 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32: |
| 1172 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64: |
| 1173 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32: |
| 1174 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64: |
| 1175 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12: |
| 1176 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12: |
| 1177 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12: |
| 1178 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12: |
| 1179 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12: |
| 1180 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12: |
| 1181 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12: |
| 1182 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12: |
| 1183 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12: |
| 1184 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12: |
| 1185 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12: |
| 1186 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12: |
| 1187 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12: |
| 1188 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12: |
| 1189 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12: |
| 1190 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12: |
| 1191 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12: |
| 1192 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12: |
| 1193 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12: |
| 1194 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12: |
| 1195 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12: |
| 1196 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12: { |
| 1197 | cgm.errorNYI(expr->getSourceRange(), |
| 1198 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1199 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1200 | return mlir::Value{}; |
| 1201 | } |
| 1202 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32: |
| 1203 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64: |
| 1204 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32: |
| 1205 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64: |
| 1206 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32: |
| 1207 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64: |
| 1208 | case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32: |
| 1209 | case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64: |
| 1210 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32: |
| 1211 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64: |
| 1212 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32: |
| 1213 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64: |
| 1214 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32: |
| 1215 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64: |
| 1216 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32: |
| 1217 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64: |
| 1218 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32: |
| 1219 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64: |
| 1220 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32: |
| 1221 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64: |
| 1222 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32: |
| 1223 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: { |
| 1224 | cgm.errorNYI(expr->getSourceRange(), |
| 1225 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1226 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1227 | return mlir::Value{}; |
| 1228 | } |
| 1229 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x4_f32: |
| 1230 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_bf16: |
| 1231 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x32_f16: |
| 1232 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x32_f16: |
| 1233 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x32_bf16: |
| 1234 | case AMDGPU::BI__builtin_amdgcn_wmma_bf16f32_16x16x32_bf16: |
| 1235 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8: |
| 1236 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8: |
| 1237 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8: |
| 1238 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8: |
| 1239 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8: |
| 1240 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8: |
| 1241 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8: |
| 1242 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8: |
| 1243 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8: |
| 1244 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8: |
| 1245 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8: |
| 1246 | case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8: |
| 1247 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8: |
| 1248 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8: |
| 1249 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8: |
| 1250 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8: |
| 1251 | case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x64_iu8: |
| 1252 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x128_f8f6f4: |
| 1253 | case AMDGPU::BI__builtin_amdgcn_wmma_f32_32x16x128_f4: |
| 1254 | case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4: |
| 1255 | case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: |
| 1256 | case AMDGPU::BI__builtin_amdgcn_wmma_scale_f32_32x16x128_f4: |
| 1257 | case AMDGPU::BI__builtin_amdgcn_wmma_scale16_f32_32x16x128_f4: { |
| 1258 | cgm.errorNYI(expr->getSourceRange(), |
| 1259 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1260 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1261 | return mlir::Value{}; |
| 1262 | } |
| 1263 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_f16: |
| 1264 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x64_bf16: |
| 1265 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x64_f16: |
| 1266 | case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x64_bf16: |
| 1267 | case AMDGPU::BI__builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16: |
| 1268 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8: |
| 1269 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8: |
| 1270 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8: |
| 1271 | case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8: |
| 1272 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8: |
| 1273 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8: |
| 1274 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8: |
| 1275 | case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8: |
| 1276 | case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x128_iu8: { |
| 1277 | cgm.errorNYI(expr->getSourceRange(), |
| 1278 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1279 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1280 | return mlir::Value{}; |
| 1281 | } |
| 1282 | // amdgcn workgroup size |
| 1283 | case AMDGPU::BI__builtin_amdgcn_workgroup_size_x: |
| 1284 | return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 0); |
| 1285 | case AMDGPU::BI__builtin_amdgcn_workgroup_size_y: |
| 1286 | return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 1); |
| 1287 | case AMDGPU::BI__builtin_amdgcn_workgroup_size_z: |
| 1288 | return emitAMDGPUWorkGroupSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 2); |
| 1289 | // amdgcn grid size |
| 1290 | case AMDGPU::BI__builtin_amdgcn_grid_size_x: |
| 1291 | return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 0); |
| 1292 | case AMDGPU::BI__builtin_amdgcn_grid_size_y: |
| 1293 | return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 1); |
| 1294 | case AMDGPU::BI__builtin_amdgcn_grid_size_z: |
| 1295 | return emitAMDGPUGridSize(cgf&: *this, loc: getLoc(srcLoc: expr->getExprLoc()), index: 2); |
| 1296 | case AMDGPU::BI__builtin_r600_recipsqrt_ieee: |
| 1297 | case AMDGPU::BI__builtin_r600_recipsqrt_ieeef: { |
| 1298 | cgm.errorNYI(expr->getSourceRange(), |
| 1299 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1300 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1301 | return mlir::Value{}; |
| 1302 | } |
| 1303 | case AMDGPU::BI__builtin_amdgcn_alignbit: { |
| 1304 | cgm.errorNYI(expr->getSourceRange(), |
| 1305 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1306 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1307 | return mlir::Value{}; |
| 1308 | } |
| 1309 | case AMDGPU::BI__builtin_amdgcn_fence: { |
| 1310 | // The scope uses AMDGPU spelling on every target, like mapScopeToSPIRV. |
| 1311 | const llvm::Triple &triple = getTarget().getTriple(); |
| 1312 | std::optional<std::pair<llvm::AtomicScope, bool>> parsed; |
| 1313 | if (std::optional<std::string> name = |
| 1314 | expr->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext())) { |
| 1315 | parsed = llvm::parseAMDGPUAtomicScopeIRString(Name: *name); |
| 1316 | if (!parsed && triple.isSPIRV()) |
| 1317 | parsed = llvm::parseAtomicScopeIRString(T: triple, Name: *name); |
| 1318 | } |
| 1319 | // SPIR-V drops one-as, AMDGPU needs a CIR sync scope for it. |
| 1320 | if (!parsed || (parsed->second && !triple.isSPIRV())) { |
| 1321 | cgm.errorNYI(expr->getSourceRange(), "__builtin_amdgcn_fence scope" ); |
| 1322 | return mlir::Value{}; |
| 1323 | } |
| 1324 | // Sorted and unique, like MMRAMetadata::appendTags. |
| 1325 | llvm::SmallVector<std::string> addrSpaces; |
| 1326 | addrSpaces.reserve(N: expr->getNumArgs() - 2); |
| 1327 | for (unsigned i = 2; i < expr->getNumArgs(); ++i) { |
| 1328 | if (std::optional<std::string> as = |
| 1329 | expr->getArg(Arg: i)->tryEvaluateString(Ctx&: getContext())) |
| 1330 | addrSpaces.push_back(Elt: std::move(*as)); |
| 1331 | else |
| 1332 | cgm.error(loc: expr->getExprLoc(), |
| 1333 | error: "expected an address space name as a string literal" ); |
| 1334 | } |
| 1335 | llvm::sort(C&: addrSpaces); |
| 1336 | addrSpaces.erase(CS: llvm::unique(R&: addrSpaces), CE: addrSpaces.end()); |
| 1337 | llvm::SmallVector<mlir::Attribute> mmras; |
| 1338 | mmras.reserve(N: addrSpaces.size()); |
| 1339 | for (const std::string &as : addrSpaces) |
| 1340 | mmras.push_back(Elt: mlir::LLVM::MMRATagAttr::get( |
| 1341 | context: &getMLIRContext(), prefix: "amdgpu-synchronize-as" , suffix: as)); |
| 1342 | |
| 1343 | auto fence = cir::AtomicFenceOp::create( |
| 1344 | builder, location: getLoc(srcLoc: expr->getExprLoc()), |
| 1345 | ordering: decodeAtomicOrder(arg: expr->getArg(Arg: 0), ctx&: getContext()), |
| 1346 | syncscope: cir::SyncScopeKindAttr::get(context: &getMLIRContext(), |
| 1347 | value: convertAtomicScopeToCIR(scope: parsed->first))); |
| 1348 | if (!mmras.empty()) |
| 1349 | fence->setAttr(name: mlir::LLVM::LLVMDialect::getMmraAttrName(), |
| 1350 | value: mmras.size() == 1 ? mmras.front() |
| 1351 | : builder.getArrayAttr(value: mmras)); |
| 1352 | return mlir::Value{}; |
| 1353 | } |
| 1354 | case AMDGPU::BI__builtin_amdgcn_atomic_inc32: |
| 1355 | case AMDGPU::BI__builtin_amdgcn_atomic_inc64: |
| 1356 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::UIncWrap, |
| 1357 | /*hasVolatileArg=*/false); |
| 1358 | case AMDGPU::BI__builtin_amdgcn_atomic_dec32: |
| 1359 | case AMDGPU::BI__builtin_amdgcn_atomic_dec64: |
| 1360 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::UDecWrap, |
| 1361 | /*hasVolatileArg=*/false); |
| 1362 | case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64: |
| 1363 | case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: |
| 1364 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32: |
| 1365 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64: |
| 1366 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32: |
| 1367 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64: |
| 1368 | case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: |
| 1369 | case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16: |
| 1370 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16: |
| 1371 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: |
| 1372 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16: |
| 1373 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: |
| 1374 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Add, |
| 1375 | /*hasVolatileArg=*/false); |
| 1376 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64: |
| 1377 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64: |
| 1378 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Min, |
| 1379 | /*hasVolatileArg=*/false); |
| 1380 | case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64: |
| 1381 | case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: |
| 1382 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Max, |
| 1383 | /*hasVolatileArg=*/false); |
| 1384 | // The ds_ float forms are the same operations with the volatile flag spelled |
| 1385 | // out as an argument. |
| 1386 | case AMDGPU::BI__builtin_amdgcn_ds_faddf: |
| 1387 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Add, |
| 1388 | /*hasVolatileArg=*/true); |
| 1389 | case AMDGPU::BI__builtin_amdgcn_ds_fminf: |
| 1390 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Min, |
| 1391 | /*hasVolatileArg=*/true); |
| 1392 | case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: |
| 1393 | return emitAMDGPUAtomicRMW(cgf&: *this, expr, binOp: cir::AtomicFetchKind::Max, |
| 1394 | /*hasVolatileArg=*/true); |
| 1395 | case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn: |
| 1396 | case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: |
| 1397 | return emitBuiltinWithOneOverloadedType<1>(e: expr, intrinName: "amdgcn.s.sendmsg.rtn" , |
| 1398 | resultType: convertType(t: expr->getType())) |
| 1399 | .getValue(); |
| 1400 | case AMDGPU::BI__builtin_amdgcn_permlane16_swap: |
| 1401 | case AMDGPU::BI__builtin_amdgcn_permlane32_swap: { |
| 1402 | cgm.errorNYI(expr->getSourceRange(), |
| 1403 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1404 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1405 | return mlir::Value{}; |
| 1406 | } |
| 1407 | case AMDGPU::BI__builtin_amdgcn_bitop3_b32: |
| 1408 | case AMDGPU::BI__builtin_amdgcn_bitop3_b16: |
| 1409 | return emitBuiltinWithOneOverloadedType<4>(e: expr, intrinName: "amdgcn.bitop3" ) |
| 1410 | .getValue(); |
| 1411 | case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc: { |
| 1412 | cgm.errorNYI(expr->getSourceRange(), |
| 1413 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1414 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1415 | return mlir::Value{}; |
| 1416 | } |
| 1417 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8: |
| 1418 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16: |
| 1419 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32: |
| 1420 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64: |
| 1421 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96: |
| 1422 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128: { |
| 1423 | cgm.errorNYI(expr->getSourceRange(), |
| 1424 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1425 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1426 | return mlir::Value{}; |
| 1427 | } |
| 1428 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f32: |
| 1429 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_format_v4f16: |
| 1430 | return emitBuiltinWithOneOverloadedType<5>( |
| 1431 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.store.format" , |
| 1432 | resultType: cir::VoidType::get(ctx: builder.getContext())) |
| 1433 | .getValue(); |
| 1434 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f32: |
| 1435 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_format_v4f16: |
| 1436 | return emitBuiltinWithOneOverloadedType<4>( |
| 1437 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.load.format" , |
| 1438 | resultType: convertType(t: expr->getType())) |
| 1439 | .getValue(); |
| 1440 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8: |
| 1441 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16: |
| 1442 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32: |
| 1443 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64: |
| 1444 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96: |
| 1445 | case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: { |
| 1446 | cgm.errorNYI(expr->getSourceRange(), |
| 1447 | std::string("unimplemented AMDGPU builtin call: " ) + |
| 1448 | getContext().BuiltinInfo.getName(ID: builtinId)); |
| 1449 | return mlir::Value{}; |
| 1450 | } |
| 1451 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_add_i32: |
| 1452 | return emitBuiltinWithOneOverloadedType<5>( |
| 1453 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.add" ) |
| 1454 | .getValue(); |
| 1455 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f32: |
| 1456 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f64: |
| 1457 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16: |
| 1458 | return emitBuiltinWithOneOverloadedType<5>( |
| 1459 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fadd" ) |
| 1460 | .getValue(); |
| 1461 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32: |
| 1462 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f64: |
| 1463 | return emitBuiltinWithOneOverloadedType<5>( |
| 1464 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fmin" ) |
| 1465 | .getValue(); |
| 1466 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32: |
| 1467 | case AMDGPU::BI__builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f64: |
| 1468 | return emitBuiltinWithOneOverloadedType<5>( |
| 1469 | e: expr, intrinName: "amdgcn.raw.ptr.buffer.atomic.fmax" ) |
| 1470 | .getValue(); |
| 1471 | case AMDGPU::BI__builtin_amdgcn_s_prefetch_data: |
| 1472 | return emitBuiltinWithOneOverloadedType<2>( |
| 1473 | e: expr, intrinName: "amdgcn.s.prefetch.data" , |
| 1474 | resultType: cir::VoidType::get(ctx: builder.getContext())) |
| 1475 | .getValue(); |
| 1476 | case AMDGPU::BI__builtin_amdgcn_s_prefetch_inst: |
| 1477 | return emitBuiltinWithOneOverloadedType<2>( |
| 1478 | e: expr, intrinName: "amdgcn.s.prefetch.inst" , |
| 1479 | resultType: cir::VoidType::get(ctx: builder.getContext())) |
| 1480 | .getValue(); |
| 1481 | case Builtin::BIlogbf: |
| 1482 | case Builtin::BI__builtin_logbf: |
| 1483 | return emitLogbBuiltin(cgf&: *this, e: expr, fSem: llvm::APFloat::IEEEsingle()); |
| 1484 | case Builtin::BIlogb: |
| 1485 | case Builtin::BI__builtin_logb: |
| 1486 | return emitLogbBuiltin(cgf&: *this, e: expr, fSem: llvm::APFloat::IEEEdouble()); |
| 1487 | case Builtin::BIscalbnf: |
| 1488 | case Builtin::BI__builtin_scalbnf: |
| 1489 | case Builtin::BIscalbn: |
| 1490 | case Builtin::BI__builtin_scalbn: { |
| 1491 | return emitBinaryExpMaybeConstrainedFPBuiltin( |
| 1492 | cgf&: *this, e: expr, intrinsicName: "ldexp" , constrainedIntrinsicName: "experimental.constrained.ldexp" ); |
| 1493 | } |
| 1494 | default: |
| 1495 | return std::nullopt; |
| 1496 | } |
| 1497 | } |
| 1498 | |