1//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit NVPTX Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/IR/Value.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRDialect.h"
18#include "llvm/IR/Intrinsics.h"
19#include "llvm/IR/IntrinsicsNVPTX.h"
20#include "llvm/Support/NVPTXAddrSpace.h"
21
22#include <algorithm>
23
24using namespace clang;
25using namespace clang::CIRGen;
26
27static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr,
28 llvm::StringRef intrinsicName) {
29 auto &builder = cgf.getBuilder();
30 Address ptr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
31 QualType argType = expr->getArg(Arg: 0)->getType();
32 mlir::Type elemTy = cgf.convertTypeForMem(t: argType->getPointeeType());
33 clang::CharUnits align = ptr.getAlignment();
34 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
35 mlir::Value alignVal =
36 builder.getConstantInt(loc, ty: builder.getSInt32Ty(), value: align.getQuantity());
37 return cir::LLVMIntrinsicCallOp::create(
38 builder, location: loc, intrinsic_name: builder.getStringAttr(bytes: intrinsicName), resType: elemTy,
39 operands: {ptr.emitRawPointer(), alignVal})
40 .getResult();
41}
42
43static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr) {
44 auto &builder = cgf.getBuilder();
45 Address ptr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
46 QualType argType = expr->getArg(Arg: 0)->getType();
47 mlir::Type elemTy = cgf.convertTypeForMem(t: argType->getPointeeType());
48 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
49
50 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL.
51 mlir::Type globalPtrTy = cir::PointerType::get(
52 pointee: elemTy, addrSpace: cir::TargetAddressSpaceAttr::get(
53 context: builder.getContext(), value: llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL));
54 mlir::Value asc =
55 builder.createAddrSpaceCast(loc, src: ptr.getPointer(), newTy: globalPtrTy);
56 cir::LoadOp load =
57 builder.createAlignedLoad(loc, ty: elemTy, ptr: asc, align: ptr.getAlignment());
58 load.setInvariant(true);
59 return load.getResult();
60}
61
62/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
63/// The result type is inferred from the single argument.
64static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
65 const CallExpr *expr,
66 llvm::StringRef intrinsicName) {
67 auto &builder = cgf.getBuilder();
68 mlir::Value arg = cgf.emitScalarExpr(e: expr->getArg(Arg: 0));
69 return cir::LLVMIntrinsicCallOp::create(
70 builder, location: cgf.getLoc(srcLoc: expr->getExprLoc()),
71 intrinsic_name: builder.getStringAttr(bytes: intrinsicName), resType: arg.getType(), operands: {arg})
72 .getResult();
73}
74
75/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd/fmul intrinsic, which takes
76/// the rounding mode as a trailing operand.
77static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr,
78 llvm::StringRef intrinsicName,
79 llvm::APFloat::roundingMode rm) {
80 auto &builder = cgf.getBuilder();
81 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
82 mlir::Value lhs = cgf.emitScalarExpr(e: expr->getArg(Arg: 0));
83 mlir::Value rhs = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
84 mlir::Value rnd =
85 builder.getConstInt(loc, t: builder.getSInt32Ty(), c: static_cast<int>(rm));
86 return cir::LLVMIntrinsicCallOp::create(builder, location: loc,
87 intrinsic_name: builder.getStringAttr(bytes: intrinsicName),
88 resType: lhs.getType(), operands: {lhs, rhs, rnd})
89 .getResult();
90}
91
92/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fma intrinsic, which takes the
93/// rounding mode as a trailing operand.
94static mlir::Value emitNVVMFMA(CIRGenFunction &cgf, const CallExpr *expr,
95 llvm::StringRef intrinsicName,
96 llvm::APFloat::roundingMode rm) {
97 auto &builder = cgf.getBuilder();
98 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
99 mlir::Value a = cgf.emitScalarExpr(e: expr->getArg(Arg: 0));
100 mlir::Value b = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
101 mlir::Value c = cgf.emitScalarExpr(e: expr->getArg(Arg: 2));
102 mlir::Value rnd =
103 builder.getConstInt(loc, t: builder.getSInt32Ty(), c: static_cast<int>(rm));
104 return cir::LLVMIntrinsicCallOp::create(builder, location: loc,
105 intrinsic_name: builder.getStringAttr(bytes: intrinsicName),
106 resType: a.getType(), operands: {a, b, c, rnd})
107 .getResult();
108}
109
110static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
111 llvm::StringRef intrinsicName,
112 bool returnsPred) {
113 CIRGenBuilderTy &builder = cgf.getBuilder();
114 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
115 mlir::Type si32Ty = builder.getSInt32Ty();
116 mlir::Value zero = builder.getNullValue(ty: si32Ty, loc);
117 mlir::Value pred = builder.createCompare(
118 loc, kind: cir::CmpOpKind::ne, lhs: cgf.emitScalarExpr(e: expr->getArg(Arg: 0)), rhs: zero);
119 mlir::Type resultTy = returnsPred ? mlir::Type(builder.getBoolTy()) : si32Ty;
120 mlir::Value result = builder.emitIntrinsicCallOp(
121 loc, str: intrinsicName, resTy: resultTy, op: mlir::ValueRange{zero, pred});
122 if (returnsPred)
123 result = builder.createBoolToInt(src: result, newTy: si32Ty);
124 return result;
125}
126
127static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf,
128 const CallExpr *expr,
129 cir::AtomicFetchKind kind,
130 cir::SyncScopeKind scope) {
131 auto &builder = cgf.getBuilder();
132 Address destAddr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
133 mlir::Value destValue = destAddr.emitRawPointer();
134 mlir::Value val = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
135 auto rmwi = cir::AtomicFetchOp::create(
136 builder, location: cgf.getLoc(srcLoc: expr->getSourceRange()), ptr: destValue, val, binop: kind,
137 mem_order: cir::MemOrder::Relaxed, sync_scope: scope, /*is_volatile=*/false,
138 /*fetch_first=*/true);
139 return rmwi->getResult(idx: 0);
140}
141
142static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
143 const CallExpr *expr,
144 cir::SyncScopeKind scope) {
145 auto &builder = cgf.getBuilder();
146 Address destAddr = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
147 mlir::Value destValue = destAddr.emitRawPointer();
148 mlir::Value val = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
149 auto xchg = cir::AtomicXchgOp::create(
150 builder, location: cgf.getLoc(srcLoc: expr->getSourceRange()), ptr: destValue, val,
151 mem_order: cir::MemOrder::Relaxed, sync_scope: scope, /*is_volatile=*/false);
152 return xchg.getResult();
153}
154
155// Mirrors the WMMA load mappings in classic CodeGen.
156struct NVPTXMmaLdstInfo {
157 unsigned NumResults; // Number of elements to load/store
158 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
159 unsigned IID_col;
160 unsigned IID_row;
161};
162
163#define MMA_INTR(geom_op_type, layout) \
164 llvm::Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
165#define MMA_LDST(n, geom_op_type) \
166 {n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row)}
167
168static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
169 switch (BuiltinID) {
170 // FP MMA loads
171 case NVPTX::BI__hmma_m16n16k16_ld_a:
172 return MMA_LDST(8, m16n16k16_load_a_f16);
173 case NVPTX::BI__hmma_m16n16k16_ld_b:
174 return MMA_LDST(8, m16n16k16_load_b_f16);
175 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
176 return MMA_LDST(4, m16n16k16_load_c_f16);
177 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
178 return MMA_LDST(8, m16n16k16_load_c_f32);
179 case NVPTX::BI__hmma_m32n8k16_ld_a:
180 return MMA_LDST(8, m32n8k16_load_a_f16);
181 case NVPTX::BI__hmma_m32n8k16_ld_b:
182 return MMA_LDST(8, m32n8k16_load_b_f16);
183 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
184 return MMA_LDST(4, m32n8k16_load_c_f16);
185 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
186 return MMA_LDST(8, m32n8k16_load_c_f32);
187 case NVPTX::BI__hmma_m8n32k16_ld_a:
188 return MMA_LDST(8, m8n32k16_load_a_f16);
189 case NVPTX::BI__hmma_m8n32k16_ld_b:
190 return MMA_LDST(8, m8n32k16_load_b_f16);
191 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
192 return MMA_LDST(4, m8n32k16_load_c_f16);
193 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
194 return MMA_LDST(8, m8n32k16_load_c_f32);
195
196 // Integer MMA loads
197 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
198 return MMA_LDST(2, m16n16k16_load_a_s8);
199 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
200 return MMA_LDST(2, m16n16k16_load_a_u8);
201 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
202 return MMA_LDST(2, m16n16k16_load_b_s8);
203 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
204 return MMA_LDST(2, m16n16k16_load_b_u8);
205 case NVPTX::BI__imma_m16n16k16_ld_c:
206 return MMA_LDST(8, m16n16k16_load_c_s32);
207 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
208 return MMA_LDST(4, m32n8k16_load_a_s8);
209 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
210 return MMA_LDST(4, m32n8k16_load_a_u8);
211 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
212 return MMA_LDST(1, m32n8k16_load_b_s8);
213 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
214 return MMA_LDST(1, m32n8k16_load_b_u8);
215 case NVPTX::BI__imma_m32n8k16_ld_c:
216 return MMA_LDST(8, m32n8k16_load_c_s32);
217 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
218 return MMA_LDST(1, m8n32k16_load_a_s8);
219 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
220 return MMA_LDST(1, m8n32k16_load_a_u8);
221 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
222 return MMA_LDST(4, m8n32k16_load_b_s8);
223 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
224 return MMA_LDST(4, m8n32k16_load_b_u8);
225 case NVPTX::BI__imma_m8n32k16_ld_c:
226 return MMA_LDST(8, m8n32k16_load_c_s32);
227
228 // Sub-integer MMA loads.
229 // Only row/col layout is supported by A/B fragments.
230 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
231 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_s4, row)};
232 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
233 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_u4, row)};
234 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
235 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_s4, col), .IID_row: 0};
236 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
237 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_u4, col), .IID_row: 0};
238 case NVPTX::BI__imma_m8n8k32_ld_c:
239 return MMA_LDST(2, m8n8k32_load_c_s32);
240 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
241 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k128_load_a_b1, row)};
242 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
243 return {.NumResults: 1, MMA_INTR(m8n8k128_load_b_b1, col), .IID_row: 0};
244 case NVPTX::BI__bmma_m8n8k128_ld_c:
245 return MMA_LDST(2, m8n8k128_load_c_s32);
246
247 // Double MMA loads
248 case NVPTX::BI__dmma_m8n8k4_ld_a:
249 return MMA_LDST(1, m8n8k4_load_a_f64);
250 case NVPTX::BI__dmma_m8n8k4_ld_b:
251 return MMA_LDST(1, m8n8k4_load_b_f64);
252 case NVPTX::BI__dmma_m8n8k4_ld_c:
253 return MMA_LDST(2, m8n8k4_load_c_f64);
254
255 // Alternate float MMA loads
256 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
257 return MMA_LDST(4, m16n16k16_load_a_bf16);
258 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
259 return MMA_LDST(4, m16n16k16_load_b_bf16);
260 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
261 return MMA_LDST(2, m8n32k16_load_a_bf16);
262 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
263 return MMA_LDST(8, m8n32k16_load_b_bf16);
264 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
265 return MMA_LDST(8, m32n8k16_load_a_bf16);
266 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
267 return MMA_LDST(2, m32n8k16_load_b_bf16);
268 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
269 return MMA_LDST(4, m16n16k8_load_a_tf32);
270 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
271 return MMA_LDST(4, m16n16k8_load_b_tf32);
272 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
273 return MMA_LDST(8, m16n16k8_load_c_f32);
274
275 default:
276 llvm_unreachable("Unknown WMMA load builtin");
277 }
278}
279#undef MMA_LDST
280#undef MMA_INTR
281
282static void storeWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
283 mlir::Value result, Address dst,
284 unsigned numRegisters) {
285 CharUnits alignment =
286 std::min(a: dst.getAlignment(), b: CharUnits::fromQuantity(Quantity: 4));
287 if (numRegisters == 1) {
288 builder.createStore(loc, val: result, dst: dst.withAlignment(newAlignment: alignment));
289 return;
290 }
291
292 // FP16 intrinsic registers are <2 x half>, stored in i32 fragment slots.
293 for (unsigned i = 0; i < numRegisters; ++i) {
294 mlir::Value value = cir::ExtractMemberOp::create(builder, location: loc, record: result, index: i);
295 value = builder.createBitcast(loc, src: value, newTy: dst.getElementType());
296 mlir::Value ptr = builder.createPtrStride(loc, base: dst.emitRawPointer(),
297 stride: builder.getSInt32(c: i, loc));
298 builder.createStore(loc, val: value, dst: Address(ptr, alignment));
299 }
300}
301
302static std::optional<mlir::Value>
303emitWMMALoad(CIRGenFunction &cgf, const CallExpr *expr, unsigned builtinId) {
304 auto &builder = cgf.getBuilder();
305 mlir::Location loc = cgf.getLoc(srcLoc: expr->getExprLoc());
306 Address dst = cgf.emitPointerWithAlignment(expr: expr->getArg(Arg: 0));
307 mlir::Value src = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
308 mlir::Value ldm = cgf.emitScalarExpr(e: expr->getArg(Arg: 2));
309 std::optional<llvm::APSInt> isColMajorArg =
310 expr->getArg(Arg: 3)->getIntegerConstantExpr(Ctx: cgf.getContext());
311 if (!isColMajorArg)
312 return std::nullopt;
313 bool isColMajor = isColMajorArg->getSExtValue();
314
315 NVPTXMmaLdstInfo info = getNVPTXMmaLdstInfo(BuiltinID: builtinId);
316 unsigned intrinsicId = isColMajor ? info.IID_col : info.IID_row;
317 if (!intrinsicId)
318 return std::nullopt;
319
320 cir::FuncType intrinsicType =
321 cgf.getIntrinsicType(id: intrinsicId, overloadTys: {src.getType()});
322 llvm::StringRef name = llvm::Intrinsic::getBaseName(id: intrinsicId);
323 assert(name.starts_with("llvm.") && "Not an LLVM intrinsic!");
324 name = name.drop_front(N: 5);
325 mlir::Value result = builder.emitIntrinsicCallOp(
326 loc, str: name, resTy: intrinsicType.getReturnType(), op: mlir::ValueRange{src, ldm});
327 storeWMMAFragment(builder, loc, result, dst, numRegisters: info.NumResults);
328 return mlir::Value{};
329}
330
331std::optional<mlir::Value>
332CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
333 switch (builtinId) {
334 case NVPTX::BI__nvvm_atom_add_gen_i:
335 case NVPTX::BI__nvvm_atom_add_gen_l:
336 case NVPTX::BI__nvvm_atom_add_gen_ll:
337 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Add, expr,
338 /*originalArgType=*/nullptr,
339 /*emittedArgValue=*/nullptr,
340 ordering: cir::MemOrder::Relaxed);
341 case NVPTX::BI__nvvm_atom_sub_gen_i:
342 case NVPTX::BI__nvvm_atom_sub_gen_l:
343 case NVPTX::BI__nvvm_atom_sub_gen_ll:
344 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Sub, expr,
345 /*originalArgType=*/nullptr,
346 /*emittedArgValue=*/nullptr,
347 ordering: cir::MemOrder::Relaxed);
348 case NVPTX::BI__nvvm_atom_and_gen_i:
349 case NVPTX::BI__nvvm_atom_and_gen_l:
350 case NVPTX::BI__nvvm_atom_and_gen_ll:
351 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::And, expr,
352 /*originalArgType=*/nullptr,
353 /*emittedArgValue=*/nullptr,
354 ordering: cir::MemOrder::Relaxed);
355 case NVPTX::BI__nvvm_atom_or_gen_i:
356 case NVPTX::BI__nvvm_atom_or_gen_l:
357 case NVPTX::BI__nvvm_atom_or_gen_ll:
358 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Or, expr,
359 /*originalArgType=*/nullptr,
360 /*emittedArgValue=*/nullptr,
361 ordering: cir::MemOrder::Relaxed);
362 case NVPTX::BI__nvvm_atom_xor_gen_i:
363 case NVPTX::BI__nvvm_atom_xor_gen_l:
364 case NVPTX::BI__nvvm_atom_xor_gen_ll:
365 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Xor, expr,
366 /*originalArgType=*/nullptr,
367 /*emittedArgValue=*/nullptr,
368 ordering: cir::MemOrder::Relaxed);
369 case NVPTX::BI__nvvm_atom_xchg_gen_i:
370 case NVPTX::BI__nvvm_atom_xchg_gen_l:
371 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
372 return makeScopedAtomicXchg(cgf&: *this, expr, scope: cir::SyncScopeKind::System);
373 case NVPTX::BI__nvvm_atom_max_gen_i:
374 case NVPTX::BI__nvvm_atom_max_gen_l:
375 case NVPTX::BI__nvvm_atom_max_gen_ll:
376 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Max, expr,
377 /*originalArgType=*/nullptr,
378 /*emittedArgValue=*/nullptr,
379 ordering: cir::MemOrder::Relaxed);
380 case NVPTX::BI__nvvm_atom_max_gen_ui:
381 case NVPTX::BI__nvvm_atom_max_gen_ul:
382 case NVPTX::BI__nvvm_atom_max_gen_ull:
383 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Max, expr,
384 /*originalArgType=*/nullptr,
385 /*emittedArgValue=*/nullptr,
386 ordering: cir::MemOrder::Relaxed);
387 case NVPTX::BI__nvvm_atom_min_gen_i:
388 case NVPTX::BI__nvvm_atom_min_gen_l:
389 case NVPTX::BI__nvvm_atom_min_gen_ll:
390 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Min, expr,
391 /*originalArgType=*/nullptr,
392 /*emittedArgValue=*/nullptr,
393 ordering: cir::MemOrder::Relaxed);
394 case NVPTX::BI__nvvm_atom_min_gen_ui:
395 case NVPTX::BI__nvvm_atom_min_gen_ul:
396 case NVPTX::BI__nvvm_atom_min_gen_ull:
397 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::Min, expr,
398 /*originalArgType=*/nullptr,
399 /*emittedArgValue=*/nullptr,
400 ordering: cir::MemOrder::Relaxed);
401 case NVPTX::BI__nvvm_atom_cas_gen_us:
402 case NVPTX::BI__nvvm_atom_cas_gen_i:
403 case NVPTX::BI__nvvm_atom_cas_gen_l:
404 case NVPTX::BI__nvvm_atom_cas_gen_ll:
405 return emitAtomicCmpXchg(expr, /*returnBool=*/false, successOrder: cir::MemOrder::Relaxed,
406 failureOrder: cir::MemOrder::Relaxed,
407 scope: cir::SyncScopeKind::System);
408 case NVPTX::BI__nvvm_atom_add_gen_f:
409 case NVPTX::BI__nvvm_atom_add_gen_d:
410 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Add,
411 scope: cir::SyncScopeKind::System);
412 case NVPTX::BI__nvvm_atom_inc_gen_ui:
413 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::UIncWrap, expr,
414 /*originalArgType=*/nullptr,
415 /*emittedArgValue=*/nullptr,
416 ordering: cir::MemOrder::Relaxed);
417 case NVPTX::BI__nvvm_atom_dec_gen_ui:
418 return makeBinaryAtomicValue(kind: cir::AtomicFetchKind::UDecWrap, expr,
419 /*originalArgType=*/nullptr,
420 /*emittedArgValue=*/nullptr,
421 ordering: cir::MemOrder::Relaxed);
422 case NVPTX::BI__nvvm_ldg_c:
423 case NVPTX::BI__nvvm_ldg_sc:
424 case NVPTX::BI__nvvm_ldg_c2:
425 case NVPTX::BI__nvvm_ldg_sc2:
426 case NVPTX::BI__nvvm_ldg_c4:
427 case NVPTX::BI__nvvm_ldg_sc4:
428 case NVPTX::BI__nvvm_ldg_s:
429 case NVPTX::BI__nvvm_ldg_s2:
430 case NVPTX::BI__nvvm_ldg_s4:
431 case NVPTX::BI__nvvm_ldg_i:
432 case NVPTX::BI__nvvm_ldg_i2:
433 case NVPTX::BI__nvvm_ldg_i4:
434 case NVPTX::BI__nvvm_ldg_l:
435 case NVPTX::BI__nvvm_ldg_l2:
436 case NVPTX::BI__nvvm_ldg_ll:
437 case NVPTX::BI__nvvm_ldg_ll2:
438 case NVPTX::BI__nvvm_ldg_uc:
439 case NVPTX::BI__nvvm_ldg_uc2:
440 case NVPTX::BI__nvvm_ldg_uc4:
441 case NVPTX::BI__nvvm_ldg_us:
442 case NVPTX::BI__nvvm_ldg_us2:
443 case NVPTX::BI__nvvm_ldg_us4:
444 case NVPTX::BI__nvvm_ldg_ui:
445 case NVPTX::BI__nvvm_ldg_ui2:
446 case NVPTX::BI__nvvm_ldg_ui4:
447 case NVPTX::BI__nvvm_ldg_ul:
448 case NVPTX::BI__nvvm_ldg_ul2:
449 case NVPTX::BI__nvvm_ldg_ull:
450 case NVPTX::BI__nvvm_ldg_ull2:
451 case NVPTX::BI__nvvm_ldg_f:
452 case NVPTX::BI__nvvm_ldg_f2:
453 case NVPTX::BI__nvvm_ldg_f4:
454 case NVPTX::BI__nvvm_ldg_d:
455 case NVPTX::BI__nvvm_ldg_d2:
456 return makeLdg(cgf&: *this, expr);
457 case NVPTX::BI__nvvm_ldu_c:
458 case NVPTX::BI__nvvm_ldu_sc:
459 case NVPTX::BI__nvvm_ldu_c2:
460 case NVPTX::BI__nvvm_ldu_sc2:
461 case NVPTX::BI__nvvm_ldu_c4:
462 case NVPTX::BI__nvvm_ldu_sc4:
463 case NVPTX::BI__nvvm_ldu_s:
464 case NVPTX::BI__nvvm_ldu_s2:
465 case NVPTX::BI__nvvm_ldu_s4:
466 case NVPTX::BI__nvvm_ldu_i:
467 case NVPTX::BI__nvvm_ldu_i2:
468 case NVPTX::BI__nvvm_ldu_i4:
469 case NVPTX::BI__nvvm_ldu_l:
470 case NVPTX::BI__nvvm_ldu_l2:
471 case NVPTX::BI__nvvm_ldu_ll:
472 case NVPTX::BI__nvvm_ldu_ll2:
473 case NVPTX::BI__nvvm_ldu_uc:
474 case NVPTX::BI__nvvm_ldu_uc2:
475 case NVPTX::BI__nvvm_ldu_uc4:
476 case NVPTX::BI__nvvm_ldu_us:
477 case NVPTX::BI__nvvm_ldu_us2:
478 case NVPTX::BI__nvvm_ldu_us4:
479 case NVPTX::BI__nvvm_ldu_ui:
480 case NVPTX::BI__nvvm_ldu_ui2:
481 case NVPTX::BI__nvvm_ldu_ui4:
482 case NVPTX::BI__nvvm_ldu_ul:
483 case NVPTX::BI__nvvm_ldu_ul2:
484 case NVPTX::BI__nvvm_ldu_ull:
485 case NVPTX::BI__nvvm_ldu_ull2:
486 return makeLdu(cgf&: *this, expr, intrinsicName: "nvvm.ldu.global.i");
487 case NVPTX::BI__nvvm_ldu_f:
488 case NVPTX::BI__nvvm_ldu_f2:
489 case NVPTX::BI__nvvm_ldu_f4:
490 case NVPTX::BI__nvvm_ldu_d:
491 case NVPTX::BI__nvvm_ldu_d2:
492 return makeLdu(cgf&: *this, expr, intrinsicName: "nvvm.ldu.global.f");
493 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
494 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
495 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
496 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Add,
497 scope: cir::SyncScopeKind::Workgroup);
498 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
499 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
500 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
501 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Add,
502 scope: cir::SyncScopeKind::System);
503 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
504 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
505 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Add,
506 scope: cir::SyncScopeKind::Workgroup);
507 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
508 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
509 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Add,
510 scope: cir::SyncScopeKind::System);
511 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
512 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
513 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
514 return makeScopedAtomicXchg(cgf&: *this, expr, scope: cir::SyncScopeKind::Workgroup);
515 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
516 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
517 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
518 return makeScopedAtomicXchg(cgf&: *this, expr, scope: cir::SyncScopeKind::System);
519 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
520 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
521 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
522 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
523 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
524 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
525 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Max,
526 scope: cir::SyncScopeKind::Workgroup);
527 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
528 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
529 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
530 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
531 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
532 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
533 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Max,
534 scope: cir::SyncScopeKind::System);
535 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
536 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
537 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
538 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
539 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
540 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
541 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Min,
542 scope: cir::SyncScopeKind::Workgroup);
543 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
544 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
545 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
546 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
547 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
548 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
549 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Min,
550 scope: cir::SyncScopeKind::System);
551 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
552 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::UIncWrap,
553 scope: cir::SyncScopeKind::Workgroup);
554 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
555 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::UDecWrap,
556 scope: cir::SyncScopeKind::Workgroup);
557 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
558 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::UIncWrap,
559 scope: cir::SyncScopeKind::System);
560 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
561 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::UDecWrap,
562 scope: cir::SyncScopeKind::System);
563 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
564 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
565 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
566 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::And,
567 scope: cir::SyncScopeKind::Workgroup);
568 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
569 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
570 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
571 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::And,
572 scope: cir::SyncScopeKind::System);
573 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
574 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
575 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
576 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Or,
577 scope: cir::SyncScopeKind::Workgroup);
578 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
579 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
580 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
581 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Or,
582 scope: cir::SyncScopeKind::System);
583 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
584 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
585 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
586 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Xor,
587 scope: cir::SyncScopeKind::Workgroup);
588 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
589 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
590 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
591 return makeScopedAtomicRMW(cgf&: *this, expr, kind: cir::AtomicFetchKind::Xor,
592 scope: cir::SyncScopeKind::System);
593 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
594 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
595 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
596 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
597 return emitAtomicCmpXchg(expr, /*returnBool=*/false, successOrder: cir::MemOrder::Relaxed,
598 failureOrder: cir::MemOrder::Relaxed,
599 scope: cir::SyncScopeKind::Workgroup);
600 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
601 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
602 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
603 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
604 return emitAtomicCmpXchg(expr, /*returnBool=*/false, successOrder: cir::MemOrder::Relaxed,
605 failureOrder: cir::MemOrder::Relaxed,
606 scope: cir::SyncScopeKind::System);
607 case NVPTX::BI__nvvm_match_all_sync_i32p:
608 case NVPTX::BI__nvvm_match_all_sync_i64p:
609 cgm.errorNYI(expr->getSourceRange(),
610 std::string("unimplemented NVPTX builtin call: ") +
611 getContext().BuiltinInfo.getName(ID: builtinId));
612 return mlir::Value{};
613 // WMMA fragment loads.
614 case NVPTX::BI__hmma_m16n16k16_ld_a:
615 case NVPTX::BI__hmma_m16n16k16_ld_b:
616 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
617 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
618 case NVPTX::BI__hmma_m32n8k16_ld_a:
619 case NVPTX::BI__hmma_m32n8k16_ld_b:
620 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
621 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
622 case NVPTX::BI__hmma_m8n32k16_ld_a:
623 case NVPTX::BI__hmma_m8n32k16_ld_b:
624 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
625 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
626 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
627 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
628 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
629 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
630 case NVPTX::BI__imma_m16n16k16_ld_c:
631 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
632 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
633 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
634 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
635 case NVPTX::BI__imma_m32n8k16_ld_c:
636 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
637 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
638 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
639 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
640 case NVPTX::BI__imma_m8n32k16_ld_c:
641 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
642 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
643 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
644 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
645 case NVPTX::BI__imma_m8n8k32_ld_c:
646 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
647 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
648 case NVPTX::BI__bmma_m8n8k128_ld_c:
649 case NVPTX::BI__dmma_m8n8k4_ld_a:
650 case NVPTX::BI__dmma_m8n8k4_ld_b:
651 case NVPTX::BI__dmma_m8n8k4_ld_c:
652 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
653 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
654 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
655 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
656 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
657 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
658 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
659 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
660 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
661 return emitWMMALoad(cgf&: *this, expr, builtinId);
662 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
663 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
664 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
665 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
666 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
667 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
668 case NVPTX::BI__imma_m16n16k16_st_c_i32:
669 case NVPTX::BI__imma_m32n8k16_st_c_i32:
670 case NVPTX::BI__imma_m8n32k16_st_c_i32:
671 case NVPTX::BI__imma_m8n8k32_st_c_i32:
672 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
673 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
674 case NVPTX::BI__mma_m16n16k8_st_c_f32:
675 cgm.errorNYI(expr->getSourceRange(),
676 std::string("unimplemented NVPTX builtin call: ") +
677 getContext().BuiltinInfo.getName(ID: builtinId));
678 return mlir::Value{};
679 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
680 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
681 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
682 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
683 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
684 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
685 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
686 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
687 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
688 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
689 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
690 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
691 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
692 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
693 case NVPTX::BI__imma_m16n16k16_mma_s8:
694 case NVPTX::BI__imma_m16n16k16_mma_u8:
695 case NVPTX::BI__imma_m32n8k16_mma_s8:
696 case NVPTX::BI__imma_m32n8k16_mma_u8:
697 case NVPTX::BI__imma_m8n32k16_mma_s8:
698 case NVPTX::BI__imma_m8n32k16_mma_u8:
699 case NVPTX::BI__imma_m8n8k32_mma_s4:
700 case NVPTX::BI__imma_m8n8k32_mma_u4:
701 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
702 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
703 case NVPTX::BI__dmma_m8n8k4_mma_f64:
704 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
705 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
706 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
707 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
708 cgm.errorNYI(expr->getSourceRange(),
709 std::string("unimplemented NVPTX builtin call: ") +
710 getContext().BuiltinInfo.getName(ID: builtinId));
711 return mlir::Value{};
712 // The following builtins require half type support
713 case NVPTX::BI__nvvm_ff2f16x2_rn:
714 cgm.errorNYI(expr->getSourceRange(),
715 std::string("unimplemented NVPTX builtin call: ") +
716 getContext().BuiltinInfo.getName(ID: builtinId));
717 return mlir::Value{};
718 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
719 cgm.errorNYI(expr->getSourceRange(),
720 std::string("unimplemented NVPTX builtin call: ") +
721 getContext().BuiltinInfo.getName(ID: builtinId));
722 return mlir::Value{};
723 case NVPTX::BI__nvvm_ff2f16x2_rz:
724 cgm.errorNYI(expr->getSourceRange(),
725 std::string("unimplemented NVPTX builtin call: ") +
726 getContext().BuiltinInfo.getName(ID: builtinId));
727 return mlir::Value{};
728 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
729 cgm.errorNYI(expr->getSourceRange(),
730 std::string("unimplemented NVPTX builtin call: ") +
731 getContext().BuiltinInfo.getName(ID: builtinId));
732 return mlir::Value{};
733 case NVPTX::BI__nvvm_fma_rn_f16:
734 case NVPTX::BI__nvvm_fma_rn_f16x2:
735 case NVPTX::BI__nvvm_fma_rn_bf16:
736 case NVPTX::BI__nvvm_fma_rn_bf16x2:
737 case NVPTX::BI__nvvm_fma_rn_f:
738 case NVPTX::BI__nvvm_fma_rn_d:
739 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma",
740 rm: llvm::APFloat::rmNearestTiesToEven);
741 case NVPTX::BI__nvvm_fma_rz_f:
742 case NVPTX::BI__nvvm_fma_rz_d:
743 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma", rm: llvm::APFloat::rmTowardZero);
744 case NVPTX::BI__nvvm_fma_rm_f:
745 case NVPTX::BI__nvvm_fma_rm_d:
746 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma",
747 rm: llvm::APFloat::rmTowardNegative);
748 case NVPTX::BI__nvvm_fma_rp_f:
749 case NVPTX::BI__nvvm_fma_rp_d:
750 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma",
751 rm: llvm::APFloat::rmTowardPositive);
752 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
753 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
754 case NVPTX::BI__nvvm_fma_rn_ftz_f:
755 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz",
756 rm: llvm::APFloat::rmNearestTiesToEven);
757 case NVPTX::BI__nvvm_fma_rz_ftz_f:
758 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz",
759 rm: llvm::APFloat::rmTowardZero);
760 case NVPTX::BI__nvvm_fma_rm_ftz_f:
761 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz",
762 rm: llvm::APFloat::rmTowardNegative);
763 case NVPTX::BI__nvvm_fma_rp_ftz_f:
764 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz",
765 rm: llvm::APFloat::rmTowardPositive);
766 case NVPTX::BI__nvvm_fma_rn_sat_f16:
767 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
768 case NVPTX::BI__nvvm_fma_rn_sat_f:
769 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.sat",
770 rm: llvm::APFloat::rmNearestTiesToEven);
771 case NVPTX::BI__nvvm_fma_rz_sat_f:
772 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.sat",
773 rm: llvm::APFloat::rmTowardZero);
774 case NVPTX::BI__nvvm_fma_rm_sat_f:
775 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.sat",
776 rm: llvm::APFloat::rmTowardNegative);
777 case NVPTX::BI__nvvm_fma_rp_sat_f:
778 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.sat",
779 rm: llvm::APFloat::rmTowardPositive);
780 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
781 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
782 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f:
783 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz.sat",
784 rm: llvm::APFloat::rmNearestTiesToEven);
785 case NVPTX::BI__nvvm_fma_rz_ftz_sat_f:
786 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz.sat",
787 rm: llvm::APFloat::rmTowardZero);
788 case NVPTX::BI__nvvm_fma_rm_ftz_sat_f:
789 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz.sat",
790 rm: llvm::APFloat::rmTowardNegative);
791 case NVPTX::BI__nvvm_fma_rp_ftz_sat_f:
792 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz.sat",
793 rm: llvm::APFloat::rmTowardPositive);
794 case NVPTX::BI__nvvm_fma_rn_relu_f16:
795 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
796 case NVPTX::BI__nvvm_fma_rn_relu_bf16:
797 case NVPTX::BI__nvvm_fma_rn_relu_bf16x2:
798 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.relu",
799 rm: llvm::APFloat::rmNearestTiesToEven);
800 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
801 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
802 return emitNVVMFMA(cgf&: *this, expr, intrinsicName: "nvvm.fma.ftz.relu",
803 rm: llvm::APFloat::rmNearestTiesToEven);
804 case NVPTX::BI__nvvm_fma_rn_oob_f16:
805 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
806 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
807 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
808 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "nvvm.fma.rn.oob")
809 .getValue();
810 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
811 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
812 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
813 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
814 return emitBuiltinWithOneOverloadedType<3>(e: expr, intrinName: "nvvm.fma.rn.oob.relu")
815 .getValue();
816 case NVPTX::BI__nvvm_fmax_f16:
817 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.f16")
818 .getValue();
819 case NVPTX::BI__nvvm_fmax_f16x2:
820 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.f16x2")
821 .getValue();
822 case NVPTX::BI__nvvm_fmax_ftz_f16:
823 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.ftz.f16")
824 .getValue();
825 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
826 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.ftz.f16x2")
827 .getValue();
828 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
829 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.ftz.nan.f16")
830 .getValue();
831 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
832 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.ftz.nan.f16x2")
833 .getValue();
834 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
835 return emitBuiltinWithOneOverloadedType<2>(
836 e: expr, intrinName: "nvvm.fmax.ftz.nan.xorsign.abs.f16")
837 .getValue();
838 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
839 return emitBuiltinWithOneOverloadedType<2>(
840 e: expr, intrinName: "nvvm.fmax.ftz.nan.xorsign.abs.f16x2")
841 .getValue();
842 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
843 return emitBuiltinWithOneOverloadedType<2>(e: expr,
844 intrinName: "nvvm.fmax.ftz.xorsign.abs.f16")
845 .getValue();
846 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
847 return emitBuiltinWithOneOverloadedType<2>(
848 e: expr, intrinName: "nvvm.fmax.ftz.xorsign.abs.f16x2")
849 .getValue();
850 case NVPTX::BI__nvvm_fmax_nan_f16:
851 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.nan.f16")
852 .getValue();
853 case NVPTX::BI__nvvm_fmax_nan_f16x2:
854 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmax.nan.f16x2")
855 .getValue();
856 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
857 return emitBuiltinWithOneOverloadedType<2>(e: expr,
858 intrinName: "nvvm.fmax.nan.xorsign.abs.f16")
859 .getValue();
860 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
861 return emitBuiltinWithOneOverloadedType<2>(
862 e: expr, intrinName: "nvvm.fmax.nan.xorsign.abs.f16x2")
863 .getValue();
864 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
865 return emitBuiltinWithOneOverloadedType<2>(e: expr,
866 intrinName: "nvvm.fmax.xorsign.abs.f16")
867 .getValue();
868 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
869 return emitBuiltinWithOneOverloadedType<2>(e: expr,
870 intrinName: "nvvm.fmax.xorsign.abs.f16x2")
871 .getValue();
872 case NVPTX::BI__nvvm_fmin_f16:
873 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.f16")
874 .getValue();
875 case NVPTX::BI__nvvm_fmin_f16x2:
876 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.f16x2")
877 .getValue();
878 case NVPTX::BI__nvvm_fmin_ftz_f16:
879 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.ftz.f16")
880 .getValue();
881 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
882 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.ftz.f16x2")
883 .getValue();
884 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
885 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.ftz.nan.f16")
886 .getValue();
887 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
888 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.ftz.nan.f16x2")
889 .getValue();
890 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
891 return emitBuiltinWithOneOverloadedType<2>(
892 e: expr, intrinName: "nvvm.fmin.ftz.nan.xorsign.abs.f16")
893 .getValue();
894 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
895 return emitBuiltinWithOneOverloadedType<2>(
896 e: expr, intrinName: "nvvm.fmin.ftz.nan.xorsign.abs.f16x2")
897 .getValue();
898 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
899 return emitBuiltinWithOneOverloadedType<2>(e: expr,
900 intrinName: "nvvm.fmin.ftz.xorsign.abs.f16")
901 .getValue();
902 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
903 return emitBuiltinWithOneOverloadedType<2>(
904 e: expr, intrinName: "nvvm.fmin.ftz.xorsign.abs.f16x2")
905 .getValue();
906 case NVPTX::BI__nvvm_fmin_nan_f16:
907 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.nan.f16")
908 .getValue();
909 case NVPTX::BI__nvvm_fmin_nan_f16x2:
910 return emitBuiltinWithOneOverloadedType<2>(e: expr, intrinName: "nvvm.fmin.nan.f16x2")
911 .getValue();
912 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
913 return emitBuiltinWithOneOverloadedType<2>(e: expr,
914 intrinName: "nvvm.fmin.nan.xorsign.abs.f16")
915 .getValue();
916 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
917 return emitBuiltinWithOneOverloadedType<2>(
918 e: expr, intrinName: "nvvm.fmin.nan.xorsign.abs.f16x2")
919 .getValue();
920 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
921 return emitBuiltinWithOneOverloadedType<2>(e: expr,
922 intrinName: "nvvm.fmin.xorsign.abs.f16")
923 .getValue();
924 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
925 return emitBuiltinWithOneOverloadedType<2>(e: expr,
926 intrinName: "nvvm.fmin.xorsign.abs.f16x2")
927 .getValue();
928 case NVPTX::BI__nvvm_fabs_f:
929 case NVPTX::BI__nvvm_abs_bf16:
930 case NVPTX::BI__nvvm_abs_bf16x2:
931 case NVPTX::BI__nvvm_fabs_f16:
932 case NVPTX::BI__nvvm_fabs_f16x2:
933 return emitUnaryNVVMIntrinsic(cgf&: *this, expr, intrinsicName: "nvvm.fabs");
934 case NVPTX::BI__nvvm_fabs_ftz_f:
935 case NVPTX::BI__nvvm_fabs_ftz_f16:
936 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
937 return emitUnaryNVVMIntrinsic(cgf&: *this, expr, intrinsicName: "nvvm.fabs.ftz");
938 case NVPTX::BI__nvvm_fabs_d:
939 return emitUnaryNVVMIntrinsic(cgf&: *this, expr, intrinsicName: "fabs");
940 case NVPTX::BI__nvvm_ex2_approx_d:
941 case NVPTX::BI__nvvm_ex2_approx_f:
942 case NVPTX::BI__nvvm_ex2_approx_f16:
943 case NVPTX::BI__nvvm_ex2_approx_f16x2:
944 return emitUnaryNVVMIntrinsic(cgf&: *this, expr, intrinsicName: "nvvm.ex2.approx");
945 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
946 return emitUnaryNVVMIntrinsic(cgf&: *this, expr, intrinsicName: "nvvm.ex2.approx.ftz");
947 case NVPTX::BI__nvvm_add_rn_f:
948 case NVPTX::BI__nvvm_add_rn_d:
949 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd",
950 rm: llvm::APFloat::rmNearestTiesToEven);
951 case NVPTX::BI__nvvm_add_rz_f:
952 case NVPTX::BI__nvvm_add_rz_d:
953 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd",
954 rm: llvm::APFloat::rmTowardZero);
955 case NVPTX::BI__nvvm_add_rm_f:
956 case NVPTX::BI__nvvm_add_rm_d:
957 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd",
958 rm: llvm::APFloat::rmTowardNegative);
959 case NVPTX::BI__nvvm_add_rp_f:
960 case NVPTX::BI__nvvm_add_rp_d:
961 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd",
962 rm: llvm::APFloat::rmTowardPositive);
963 case NVPTX::BI__nvvm_add_rn_ftz_f:
964 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz",
965 rm: llvm::APFloat::rmNearestTiesToEven);
966 case NVPTX::BI__nvvm_add_rz_ftz_f:
967 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz",
968 rm: llvm::APFloat::rmTowardZero);
969 case NVPTX::BI__nvvm_add_rm_ftz_f:
970 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz",
971 rm: llvm::APFloat::rmTowardNegative);
972 case NVPTX::BI__nvvm_add_rp_ftz_f:
973 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz",
974 rm: llvm::APFloat::rmTowardPositive);
975 case NVPTX::BI__nvvm_add_rn_sat_f:
976 case NVPTX::BI__nvvm_add_rn_sat_f16:
977 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
978 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.sat",
979 rm: llvm::APFloat::rmNearestTiesToEven);
980 case NVPTX::BI__nvvm_add_rz_sat_f:
981 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.sat",
982 rm: llvm::APFloat::rmTowardZero);
983 case NVPTX::BI__nvvm_add_rm_sat_f:
984 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.sat",
985 rm: llvm::APFloat::rmTowardNegative);
986 case NVPTX::BI__nvvm_add_rp_sat_f:
987 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.sat",
988 rm: llvm::APFloat::rmTowardPositive);
989 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
990 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
991 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
992 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz.sat",
993 rm: llvm::APFloat::rmNearestTiesToEven);
994 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
995 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz.sat",
996 rm: llvm::APFloat::rmTowardZero);
997 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
998 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz.sat",
999 rm: llvm::APFloat::rmTowardNegative);
1000 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
1001 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fadd.ftz.sat",
1002 rm: llvm::APFloat::rmTowardPositive);
1003 case NVPTX::BI__nvvm_mul_rn_f:
1004 case NVPTX::BI__nvvm_mul_rn_d:
1005 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul",
1006 rm: llvm::APFloat::rmNearestTiesToEven);
1007 case NVPTX::BI__nvvm_mul_rz_f:
1008 case NVPTX::BI__nvvm_mul_rz_d:
1009 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul",
1010 rm: llvm::APFloat::rmTowardZero);
1011 case NVPTX::BI__nvvm_mul_rm_f:
1012 case NVPTX::BI__nvvm_mul_rm_d:
1013 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul",
1014 rm: llvm::APFloat::rmTowardNegative);
1015 case NVPTX::BI__nvvm_mul_rp_f:
1016 case NVPTX::BI__nvvm_mul_rp_d:
1017 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul",
1018 rm: llvm::APFloat::rmTowardPositive);
1019 case NVPTX::BI__nvvm_mul_rn_ftz_f:
1020 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.ftz",
1021 rm: llvm::APFloat::rmNearestTiesToEven);
1022 case NVPTX::BI__nvvm_mul_rz_ftz_f:
1023 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.ftz",
1024 rm: llvm::APFloat::rmTowardZero);
1025 case NVPTX::BI__nvvm_mul_rm_ftz_f:
1026 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.ftz",
1027 rm: llvm::APFloat::rmTowardNegative);
1028 case NVPTX::BI__nvvm_mul_rp_ftz_f:
1029 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.ftz",
1030 rm: llvm::APFloat::rmTowardPositive);
1031 case NVPTX::BI__nvvm_mul_rn_sat_f16:
1032 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
1033 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.sat",
1034 rm: llvm::APFloat::rmNearestTiesToEven);
1035 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
1036 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
1037 return emitNVVMFPArith(cgf&: *this, expr, intrinsicName: "nvvm.fmul.ftz.sat",
1038 rm: llvm::APFloat::rmNearestTiesToEven);
1039 case NVPTX::BI__nvvm_ldg_h:
1040 case NVPTX::BI__nvvm_ldg_h2:
1041 cgm.errorNYI(expr->getSourceRange(),
1042 std::string("unimplemented NVPTX builtin call: ") +
1043 getContext().BuiltinInfo.getName(ID: builtinId));
1044 return mlir::Value{};
1045 case NVPTX::BI__nvvm_ldu_h:
1046 case NVPTX::BI__nvvm_ldu_h2:
1047 return makeLdu(cgf&: *this, expr, intrinsicName: "nvvm.ldu.global.f");
1048 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
1049 cgm.errorNYI(expr->getSourceRange(),
1050 std::string("unimplemented NVPTX builtin call: ") +
1051 getContext().BuiltinInfo.getName(ID: builtinId));
1052 return mlir::Value{};
1053 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
1054 cgm.errorNYI(expr->getSourceRange(),
1055 std::string("unimplemented NVPTX builtin call: ") +
1056 getContext().BuiltinInfo.getName(ID: builtinId));
1057 return mlir::Value{};
1058 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
1059 cgm.errorNYI(expr->getSourceRange(),
1060 std::string("unimplemented NVPTX builtin call: ") +
1061 getContext().BuiltinInfo.getName(ID: builtinId));
1062 return mlir::Value{};
1063 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
1064 cgm.errorNYI(expr->getSourceRange(),
1065 std::string("unimplemented NVPTX builtin call: ") +
1066 getContext().BuiltinInfo.getName(ID: builtinId));
1067 return mlir::Value{};
1068 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
1069 cgm.errorNYI(expr->getSourceRange(),
1070 std::string("unimplemented NVPTX builtin call: ") +
1071 getContext().BuiltinInfo.getName(ID: builtinId));
1072 return mlir::Value{};
1073 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
1074 cgm.errorNYI(expr->getSourceRange(),
1075 std::string("unimplemented NVPTX builtin call: ") +
1076 getContext().BuiltinInfo.getName(ID: builtinId));
1077 return mlir::Value{};
1078 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
1079 cgm.errorNYI(expr->getSourceRange(),
1080 std::string("unimplemented NVPTX builtin call: ") +
1081 getContext().BuiltinInfo.getName(ID: builtinId));
1082 return mlir::Value{};
1083 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
1084 cgm.errorNYI(expr->getSourceRange(),
1085 std::string("unimplemented NVPTX builtin call: ") +
1086 getContext().BuiltinInfo.getName(ID: builtinId));
1087 return mlir::Value{};
1088 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
1089 cgm.errorNYI(expr->getSourceRange(),
1090 std::string("unimplemented NVPTX builtin call: ") +
1091 getContext().BuiltinInfo.getName(ID: builtinId));
1092 return mlir::Value{};
1093 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
1094 cgm.errorNYI(expr->getSourceRange(),
1095 std::string("unimplemented NVPTX builtin call: ") +
1096 getContext().BuiltinInfo.getName(ID: builtinId));
1097 return mlir::Value{};
1098 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
1099 cgm.errorNYI(expr->getSourceRange(),
1100 std::string("unimplemented NVPTX builtin call: ") +
1101 getContext().BuiltinInfo.getName(ID: builtinId));
1102 return mlir::Value{};
1103 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
1104 cgm.errorNYI(expr->getSourceRange(),
1105 std::string("unimplemented NVPTX builtin call: ") +
1106 getContext().BuiltinInfo.getName(ID: builtinId));
1107 return mlir::Value{};
1108 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
1109 cgm.errorNYI(expr->getSourceRange(),
1110 std::string("unimplemented NVPTX builtin call: ") +
1111 getContext().BuiltinInfo.getName(ID: builtinId));
1112 return mlir::Value{};
1113 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
1114 cgm.errorNYI(expr->getSourceRange(),
1115 std::string("unimplemented NVPTX builtin call: ") +
1116 getContext().BuiltinInfo.getName(ID: builtinId));
1117 return mlir::Value{};
1118 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
1119 cgm.errorNYI(expr->getSourceRange(),
1120 std::string("unimplemented NVPTX builtin call: ") +
1121 getContext().BuiltinInfo.getName(ID: builtinId));
1122 return mlir::Value{};
1123 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
1124 cgm.errorNYI(expr->getSourceRange(),
1125 std::string("unimplemented NVPTX builtin call: ") +
1126 getContext().BuiltinInfo.getName(ID: builtinId));
1127 return mlir::Value{};
1128 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
1129 cgm.errorNYI(expr->getSourceRange(),
1130 std::string("unimplemented NVPTX builtin call: ") +
1131 getContext().BuiltinInfo.getName(ID: builtinId));
1132 return mlir::Value{};
1133 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
1134 cgm.errorNYI(expr->getSourceRange(),
1135 std::string("unimplemented NVPTX builtin call: ") +
1136 getContext().BuiltinInfo.getName(ID: builtinId));
1137 return mlir::Value{};
1138 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
1139 cgm.errorNYI(expr->getSourceRange(),
1140 std::string("unimplemented NVPTX builtin call: ") +
1141 getContext().BuiltinInfo.getName(ID: builtinId));
1142 return mlir::Value{};
1143 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
1144 cgm.errorNYI(expr->getSourceRange(),
1145 std::string("unimplemented NVPTX builtin call: ") +
1146 getContext().BuiltinInfo.getName(ID: builtinId));
1147 return mlir::Value{};
1148 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
1149 cgm.errorNYI(expr->getSourceRange(),
1150 std::string("unimplemented NVPTX builtin call: ") +
1151 getContext().BuiltinInfo.getName(ID: builtinId));
1152 return mlir::Value{};
1153 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
1154 cgm.errorNYI(expr->getSourceRange(),
1155 std::string("unimplemented NVPTX builtin call: ") +
1156 getContext().BuiltinInfo.getName(ID: builtinId));
1157 return mlir::Value{};
1158 case NVPTX::BI__nvvm_is_explicit_cluster:
1159 cgm.errorNYI(expr->getSourceRange(),
1160 std::string("unimplemented NVPTX builtin call: ") +
1161 getContext().BuiltinInfo.getName(ID: builtinId));
1162 return mlir::Value{};
1163 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1164 cgm.errorNYI(expr->getSourceRange(),
1165 std::string("unimplemented NVPTX builtin call: ") +
1166 getContext().BuiltinInfo.getName(ID: builtinId));
1167 return mlir::Value{};
1168 case NVPTX::BI__nvvm_mapa:
1169 cgm.errorNYI(expr->getSourceRange(),
1170 std::string("unimplemented NVPTX builtin call: ") +
1171 getContext().BuiltinInfo.getName(ID: builtinId));
1172 return mlir::Value{};
1173 case NVPTX::BI__nvvm_mapa_shared_cluster:
1174 cgm.errorNYI(expr->getSourceRange(),
1175 std::string("unimplemented NVPTX builtin call: ") +
1176 getContext().BuiltinInfo.getName(ID: builtinId));
1177 return mlir::Value{};
1178 case NVPTX::BI__nvvm_getctarank:
1179 cgm.errorNYI(expr->getSourceRange(),
1180 std::string("unimplemented NVPTX builtin call: ") +
1181 getContext().BuiltinInfo.getName(ID: builtinId));
1182 return mlir::Value{};
1183 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1184 cgm.errorNYI(expr->getSourceRange(),
1185 std::string("unimplemented NVPTX builtin call: ") +
1186 getContext().BuiltinInfo.getName(ID: builtinId));
1187 return mlir::Value{};
1188 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1189 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1190 str: "nvvm.barrier.cluster.arrive",
1191 resTy: builder.getVoidTy());
1192 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1193 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1194 str: "nvvm.barrier.cluster.arrive.relaxed",
1195 resTy: builder.getVoidTy());
1196 case NVPTX::BI__nvvm_barrier_cluster_wait:
1197 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1198 str: "nvvm.barrier.cluster.wait",
1199 resTy: builder.getVoidTy());
1200 case NVPTX::BI__nvvm_fence_sc_cluster:
1201 return builder.emitIntrinsicCallOp(loc: getLoc(srcLoc: expr->getExprLoc()),
1202 str: "nvvm.fence.sc.cluster",
1203 resTy: builder.getVoidTy());
1204 case NVPTX::BI__nvvm_bar_sync:
1205 return builder.emitIntrinsicCallOp(
1206 loc: getLoc(srcLoc: expr->getExprLoc()), str: "nvvm.barrier.cta.sync.aligned.all",
1207 resTy: builder.getVoidTy(), op: mlir::ValueRange{emitScalarExpr(e: expr->getArg(Arg: 0))});
1208 case NVPTX::BI__syncthreads:
1209 return builder.emitIntrinsicCallOp(
1210 loc: getLoc(srcLoc: expr->getExprLoc()), str: "nvvm.barrier.cta.sync.aligned.all",
1211 resTy: builder.getVoidTy(),
1212 op: mlir::ValueRange{builder.getConstInt(loc: getLoc(srcLoc: expr->getExprLoc()),
1213 t: builder.getSInt32Ty(), c: 0)});
1214 case NVPTX::BI__nvvm_barrier_sync:
1215 return builder.emitIntrinsicCallOp(
1216 loc: getLoc(srcLoc: expr->getExprLoc()), str: "nvvm.barrier.cta.sync.all",
1217 resTy: builder.getVoidTy(), op: mlir::ValueRange{emitScalarExpr(e: expr->getArg(Arg: 0))});
1218 case NVPTX::BI__nvvm_barrier_sync_cnt:
1219 return builder.emitIntrinsicCallOp(
1220 loc: getLoc(srcLoc: expr->getExprLoc()), str: "nvvm.barrier.cta.sync.count",
1221 resTy: builder.getVoidTy(),
1222 op: mlir::ValueRange{emitScalarExpr(e: expr->getArg(Arg: 0)),
1223 emitScalarExpr(e: expr->getArg(Arg: 1))});
1224 case NVPTX::BI__nvvm_bar0_and:
1225 return emitBar0Reduction(cgf&: *this, expr,
1226 intrinsicName: "nvvm.barrier.cta.red.and.aligned.all",
1227 /*returnsPred=*/true);
1228 case NVPTX::BI__nvvm_bar0_or:
1229 return emitBar0Reduction(cgf&: *this, expr, intrinsicName: "nvvm.barrier.cta.red.or.aligned.all",
1230 /*returnsPred=*/true);
1231 case NVPTX::BI__nvvm_bar0_popc:
1232 return emitBar0Reduction(cgf&: *this, expr,
1233 intrinsicName: "nvvm.barrier.cta.red.popc.aligned.all",
1234 /*returnsPred=*/false);
1235
1236 default:
1237 return std::nullopt;
1238 }
1239}
1240