1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as CIR or a function call to be
10// later resolved.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CIRGenCall.h"
15#include "CIRGenFunction.h"
16#include "CIRGenModule.h"
17#include "CIRGenValue.h"
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/IR/Value.h"
20#include "mlir/Support/LLVM.h"
21#include "clang/AST/DeclBase.h"
22#include "clang/AST/Expr.h"
23#include "clang/AST/GlobalDecl.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticFrontend.h"
26#include "clang/Basic/OperatorKinds.h"
27#include "clang/CIR/Dialect/IR/CIRTypes.h"
28#include "clang/CIR/MissingFeatures.h"
29#include "clang/CodeGenUtils/FunctionUtils.h"
30#include "clang/CodeGenUtils/TargetUtils.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/IR/Intrinsics.h"
33#include "llvm/Support/ErrorHandling.h"
34
35using namespace clang;
36using namespace clang::CIRGen;
37using namespace llvm;
38
39static bool shouldEmitBuiltinAsIR(unsigned builtinID,
40 const Builtin::Context &bi,
41 const CIRGenFunction &cgf) {
42 if (!cgf.cgm.getLangOpts().MathErrno &&
43 cgf.curFPFeatures.getExceptionMode() ==
44 LangOptions::FPExceptionModeKind::FPE_Ignore &&
45 !cgf.cgm.getTargetCIRGenInfo().supportsLibCall()) {
46 switch (builtinID) {
47 default:
48 return false;
49 case Builtin::BIlogbf:
50 case Builtin::BI__builtin_logbf:
51 case Builtin::BIlogb:
52 case Builtin::BI__builtin_logb:
53 case Builtin::BIscalbnf:
54 case Builtin::BI__builtin_scalbnf:
55 case Builtin::BIscalbn:
56 case Builtin::BI__builtin_scalbn:
57 return true;
58 }
59 }
60 return false;
61}
62
63static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,
64 const CallExpr *e, mlir::Operation *calleeValue) {
65 CIRGenCallee callee = CIRGenCallee::forDirect(funcPtr: calleeValue, abstractInfo: GlobalDecl(fd));
66 return cgf.emitCall(calleeTy: e->getCallee()->getType(), callee, e, returnValue: ReturnValueSlot());
67}
68
69template <typename Op, typename... Args>
70static mlir::Value createBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
71 mlir::Value arg, Args... args) {
72 CIRGenBuilderTy &builder = cgf.getBuilder();
73 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
74 auto op = Op::create(builder, loc, arg, args...);
75 mlir::Value result = op.getResult();
76 mlir::Type resultTy = cgf.convertType(t: e->getType());
77 if (resultTy != result.getType())
78 result = builder.createIntCast(src: result, newTy: resultTy);
79 return result;
80}
81
82template <typename Op, typename... Args>
83static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
84 Args... args) {
85 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
86 return RValue::get(v: createBuiltinBitOp<Op>(cgf, e, arg, args...));
87}
88
89template <typename Op>
90static RValue emitBuiltinVectorReduction(CIRGenFunction &cgf,
91 const CallExpr *e) {
92 mlir::Value input = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
93 return RValue::get(
94 v: Op::create(cgf.getBuilder(), cgf.getLoc(srcLoc: e->getExprLoc()), input)
95 .getResult());
96}
97
98/// Emit a clz/ctz bit op with optional fallback for __builtin_c[lt]zg.
99/// When a fallback is present, the result is the fallback value if the input is
100/// zero, otherwise the bit count.
101template <typename Op>
102static RValue emitBuiltinBitOpWithFallback(CIRGenFunction &cgf,
103 const CallExpr *e) {
104 bool hasFallback = e->getNumArgs() > 1;
105 bool poisonZero = hasFallback || cgf.getTarget().isCLZForZeroUndef();
106
107 if (!hasFallback) {
108 assert(!cir::MissingFeatures::builtinCheckKind());
109 return emitBuiltinBitOp<Op>(cgf, e, poisonZero);
110 }
111
112 assert(!cir::MissingFeatures::builtinBitCountExpr());
113 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
114 mlir::Value result = createBuiltinBitOp<Op>(cgf, e, arg, poisonZero);
115
116 CIRGenBuilderTy &builder = cgf.getBuilder();
117 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
118 mlir::Value zero = builder.getNullValue(ty: arg.getType(), loc);
119 mlir::Value isZero =
120 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: arg, rhs: zero);
121 mlir::Value fallbackValue = cgf.emitScalarExpr(e: e->getArg(Arg: 1));
122 return RValue::get(v: builder.createSelect(loc, condition: isZero, trueValue: fallbackValue, falseValue: result));
123}
124
125static bool isStdcBitOpWidthSupported(cir::IntType intTy) {
126 switch (intTy.getWidth()) {
127 case 8:
128 case 16:
129 case 32:
130 case 64:
131 case 128:
132 return true;
133 default:
134 return false;
135 }
136}
137
138static RValue errorStdcBitOpWidthNYI(CIRGenFunction &cgf, const CallExpr *e) {
139 cgf.cgm.errorNYI(e->getSourceRange(),
140 "stdc bit builtin with unsupported argument integer width");
141 return cgf.getUndefRValue(ty: e->getType());
142}
143
144template <typename Op, typename... Args>
145static RValue emitStdcFirstBit(CIRGenFunction &cgf, const CallExpr *e,
146 bool invertArg, Args... args) {
147 CIRGenBuilderTy &builder = cgf.getBuilder();
148 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
149 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
150 auto argTy = mlir::cast<cir::IntType>(Val: arg.getType());
151 if (!isStdcBitOpWidthSupported(intTy: argTy))
152 return errorStdcBitOpWidthNYI(cgf, e);
153
154 mlir::Value actualArg = invertArg ? builder.createNot(loc, value: arg) : arg;
155 mlir::Value count = Op::create(builder, loc, actualArg, args...).getResult();
156
157 mlir::Value zero = builder.getNullValue(ty: argTy, loc);
158 mlir::Value one = builder.getConstInt(loc, t: argTy, c: 1);
159 mlir::Value countPlusOne = builder.createAdd(loc, lhs: count, rhs: one);
160 mlir::Value isZero =
161 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: actualArg, rhs: zero);
162 mlir::Value result = builder.createSelect(loc, condition: isZero, trueValue: zero, falseValue: countPlusOne);
163
164 mlir::Type resultTy = cgf.convertType(t: e->getType());
165 if (result.getType() != resultTy)
166 result = builder.createIntCast(src: result, newTy: resultTy);
167
168 return RValue::get(v: result);
169}
170
171static RValue emitStdcBitCeil(CIRGenFunction &cgf, const CallExpr *e) {
172 CIRGenBuilderTy &builder = cgf.getBuilder();
173 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
174 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
175 auto argTy = mlir::cast<cir::IntType>(Val: arg.getType());
176 if (!isStdcBitOpWidthSupported(intTy: argTy))
177 return errorStdcBitOpWidthNYI(cgf, e);
178
179 mlir::Value one = builder.getConstInt(loc, t: argTy, c: 1);
180 mlir::Value argMinusOne = builder.createSub(loc, lhs: arg, rhs: one);
181 mlir::Value lz =
182 cir::BitClzOp::create(builder, location: loc, input: argMinusOne, /*poisonZero=*/poison_zero: false)
183 .getResult();
184 mlir::Value widthMinusOne =
185 builder.getConstInt(loc, t: argTy, c: argTy.getWidth() - 1);
186 mlir::Value shiftAmt = builder.createSub(loc, lhs: widthMinusOne, rhs: lz);
187 mlir::Value isLeOne =
188 builder.createCompare(loc, kind: cir::CmpOpKind::le, lhs: arg, rhs: one);
189 mlir::Value two = builder.createShiftLeft(loc, lhs: one, rhs: one);
190 mlir::Value ceil = builder.createShiftLeft(loc, lhs: two, rhs: shiftAmt);
191 mlir::Value result = builder.createSelect(loc, condition: isLeOne, trueValue: one, falseValue: ceil);
192 return RValue::get(v: result);
193}
194
195// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
196// clz, ctz, or popcount. InvertArg flips the input to count the opposite bit
197// value.
198template <typename Op, typename... Args>
199static RValue emitStdcCountOp(CIRGenFunction &cgf, const CallExpr *e,
200 bool invertArg, Args... args) {
201 CIRGenBuilderTy &builder = cgf.getBuilder();
202 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
203 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
204 if (!isStdcBitOpWidthSupported(intTy: mlir::cast<cir::IntType>(Val: arg.getType())))
205 return errorStdcBitOpWidthNYI(cgf, e);
206 mlir::Value actualArg = invertArg ? builder.createNot(loc, value: arg) : arg;
207 mlir::Value result = Op::create(builder, loc, actualArg, args...).getResult();
208
209 mlir::Type resultTy = cgf.convertType(t: e->getType());
210 if (result.getType() != resultTy)
211 result = builder.createIntCast(src: result, newTy: resultTy);
212 return RValue::get(v: result);
213}
214
215// stdc_count_zeros (BitWidth - popcount) and stdc_bit_width (BitWidth - clz).
216// The subtract is performed in the argument type and cast once at the end.
217template <typename Op, typename... Args>
218static RValue emitStdcBitWidthMinus(CIRGenFunction &cgf, const CallExpr *e,
219 Args... args) {
220 CIRGenBuilderTy &builder = cgf.getBuilder();
221 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
222 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
223 auto argTy = mlir::cast<cir::IntType>(Val: arg.getType());
224 if (!isStdcBitOpWidthSupported(intTy: argTy))
225 return errorStdcBitOpWidthNYI(cgf, e);
226
227 mlir::Value cnt = Op::create(builder, loc, arg, args...).getResult();
228 mlir::Value width = builder.getConstInt(loc, t: argTy, c: argTy.getWidth());
229 mlir::Value result = builder.createSub(loc, lhs: width, rhs: cnt);
230
231 mlir::Type resultTy = cgf.convertType(t: e->getType());
232 if (result.getType() != resultTy)
233 result = builder.createIntCast(src: result, newTy: resultTy);
234
235 return RValue::get(v: result);
236}
237
238static RValue emitStdcHasSingleBit(CIRGenFunction &cgf, const CallExpr *e) {
239 CIRGenBuilderTy &builder = cgf.getBuilder();
240 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
241 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
242 if (!isStdcBitOpWidthSupported(intTy: mlir::cast<cir::IntType>(Val: arg.getType())))
243 return errorStdcBitOpWidthNYI(cgf, e);
244
245 mlir::Value popCount =
246 cir::BitPopcountOp::create(builder, location: loc, input: arg).getResult();
247 mlir::Value one = builder.getConstInt(loc, t: popCount.getType(), c: 1);
248 mlir::Value result =
249 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: popCount, rhs: one);
250 return RValue::get(v: result);
251}
252
253static RValue emitStdcBitFloor(CIRGenFunction &cgf, const CallExpr *e) {
254 CIRGenBuilderTy &builder = cgf.getBuilder();
255 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
256 mlir::Value arg = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
257 auto argTy = mlir::cast<cir::IntType>(Val: arg.getType());
258 if (!isStdcBitOpWidthSupported(intTy: argTy))
259 return errorStdcBitOpWidthNYI(cgf, e);
260
261 mlir::Value widthMinusOne =
262 builder.getConstInt(loc, t: argTy, c: argTy.getWidth() - 1);
263 mlir::Value one = builder.getConstInt(loc, t: argTy, c: 1);
264 mlir::Value lz =
265 cir::BitClzOp::create(builder, location: loc, input: arg, /*poisonZero=*/poison_zero: true).getResult();
266 mlir::Value shiftAmt = builder.createSub(loc, lhs: widthMinusOne, rhs: lz);
267 mlir::Value zero = builder.getNullValue(ty: argTy, loc);
268 mlir::Value isZero =
269 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: arg, rhs: zero);
270 mlir::Value floor = builder.createShiftLeft(loc, lhs: one, rhs: shiftAmt);
271 mlir::Value result = builder.createSelect(loc, condition: isZero, trueValue: zero, falseValue: floor);
272 return RValue::get(v: result);
273}
274
275/// Emit the conversions required to turn the given value into an
276/// integer of the given size.
277static mlir::Value emitToInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
278 cir::IntType intType) {
279 v = cgf.emitToMemory(value: v, ty: t);
280
281 if (mlir::isa<cir::PointerType>(Val: v.getType()))
282 return cgf.getBuilder().createPtrToInt(src: v, newTy: intType);
283
284 assert(v.getType() == intType);
285 return v;
286}
287
288static mlir::Value emitFromInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
289 mlir::Type resultType) {
290 v = cgf.emitFromMemory(value: v, ty: t);
291
292 if (mlir::isa<cir::PointerType>(Val: resultType))
293 return cgf.getBuilder().createIntToPtr(src: v, newTy: resultType);
294
295 assert(v.getType() == resultType);
296 return v;
297}
298
299static mlir::Value emitSignBit(mlir::Location loc, CIRGenFunction &cgf,
300 mlir::Value val) {
301 assert(!::cir::MissingFeatures::isPPC_FP128Ty());
302 cir::SignBitOp returnValue = cgf.getBuilder().createSignBit(loc, val);
303 return returnValue->getResult(idx: 0);
304}
305
306static Address checkAtomicAlignment(CIRGenFunction &cgf, const CallExpr *e) {
307 ASTContext &astContext = cgf.getContext();
308 Address ptr = cgf.emitPointerWithAlignment(expr: e->getArg(Arg: 0));
309 unsigned bytes =
310 mlir::isa<cir::PointerType>(Val: ptr.getElementType())
311 ? astContext.getTypeSizeInChars(T: astContext.VoidPtrTy).getQuantity()
312 : cgf.cgm.getDataLayout().getTypeSizeInBits(ty: ptr.getElementType()) /
313 cgf.cgm.getASTContext().getCharWidth();
314
315 unsigned align = ptr.getAlignment().getQuantity();
316 if (align % bytes != 0) {
317 DiagnosticsEngine &diags = cgf.cgm.getDiags();
318 diags.Report(Loc: e->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
319 // Force address to be at least naturally-aligned.
320 return ptr.withAlignment(newAlignment: CharUnits::fromQuantity(Quantity: bytes));
321 }
322 return ptr;
323}
324
325/// Utility to insert an atomic instruction based on Intrinsic::ID
326/// and the expression node.
327mlir::Value CIRGenFunction::makeBinaryAtomicValue(cir::AtomicFetchKind kind,
328 const CallExpr *expr,
329 mlir::Type *originalArgType,
330 mlir::Value *emittedArgValue,
331 cir::MemOrder ordering) {
332 CIRGenFunction &cgf = *this;
333
334 QualType type = expr->getType();
335 QualType ptrType = expr->getArg(Arg: 0)->getType();
336
337 assert(ptrType->isPointerType());
338 assert(
339 cgf.getContext().hasSameUnqualifiedType(type, ptrType->getPointeeType()));
340 assert(cgf.getContext().hasSameUnqualifiedType(type,
341 expr->getArg(1)->getType()));
342
343 Address destAddr = checkAtomicAlignment(cgf, e: expr);
344 CIRGenBuilderTy &builder = cgf.getBuilder();
345
346 mlir::Value val = cgf.emitScalarExpr(e: expr->getArg(Arg: 1));
347 mlir::Type valueType = val.getType();
348 mlir::Value destValue = destAddr.emitRawPointer();
349
350 if (ptrType->getPointeeType()->isPointerType()) {
351 // Pointer to pointer
352 // `cir.atomic.fetch` expects a pointer to an integer type, so we cast
353 // ptr<ptr<T>> to ptr<intPtrSize>
354 cir::IntType ptrSizeInt =
355 builder.getSIntNTy(n: cgf.getContext().getTypeSize(T: ptrType));
356 destValue =
357 builder.createBitcast(src: destValue, newTy: builder.getPointerTo(ty: ptrSizeInt));
358 val = emitToInt(cgf, v: val, t: type, intType: ptrSizeInt);
359 } else {
360 // Pointer to integer type
361 cir::IntType intType =
362 ptrType->getPointeeType()->isUnsignedIntegerType()
363 ? builder.getUIntNTy(n: cgf.getContext().getTypeSize(T: type))
364 : builder.getSIntNTy(n: cgf.getContext().getTypeSize(T: type));
365 val = emitToInt(cgf, v: val, t: type, intType);
366 }
367
368 // This output argument is needed for post atomic fetch operations
369 // that calculate the result of the operation as return value of
370 // <binop>_and_fetch builtins. The `AtomicFetch` operation only updates the
371 // memory location and returns the old value.
372 if (emittedArgValue) {
373 *emittedArgValue = val;
374 assert(originalArgType != nullptr &&
375 "originalArgType must be provided when emittedArgValue is set");
376 *originalArgType = valueType;
377 }
378
379 auto rmwi = cir::AtomicFetchOp::create(
380 builder, location: cgf.getLoc(srcLoc: expr->getSourceRange()), ptr: destValue, val, binop: kind,
381 mem_order: ordering, sync_scope: cir::SyncScopeKind::System, is_volatile: false, /* is volatile */
382 fetch_first: true); /* fetch first */
383 return rmwi->getResult(idx: 0);
384}
385
386static RValue emitBinaryAtomic(CIRGenFunction &cgf,
387 cir::AtomicFetchKind atomicOpkind,
388 const CallExpr *e) {
389 return RValue::get(v: cgf.makeBinaryAtomicValue(kind: atomicOpkind, expr: e));
390}
391
392template <typename BinOp>
393static RValue emitBinaryAtomicPost(CIRGenFunction &cgf,
394 cir::AtomicFetchKind atomicOpkind,
395 const CallExpr *e, bool invert = false) {
396 mlir::Value emittedArgValue;
397 mlir::Type originalArgType;
398 clang::QualType typ = e->getType();
399 mlir::Value result = cgf.makeBinaryAtomicValue(
400 kind: atomicOpkind, expr: e, originalArgType: &originalArgType, emittedArgValue: &emittedArgValue);
401 clang::CIRGen::CIRGenBuilderTy &builder = cgf.getBuilder();
402 result = BinOp::create(builder, result.getLoc(), result, emittedArgValue);
403
404 if (invert)
405 result = builder.createNot(value: result);
406
407 result = emitFromInt(cgf, v: result, t: typ, resultType: originalArgType);
408 return RValue::get(v: result);
409}
410
411mlir::Value CIRGenFunction::emitAtomicCmpXchg(const CallExpr *e,
412 bool returnBool,
413 cir::MemOrder successOrder,
414 cir::MemOrder failureOrder,
415 cir::SyncScopeKind scope) {
416 Address destAddr = checkAtomicAlignment(cgf&: *this, e);
417 CIRGenBuilderTy &builder = getBuilder();
418 mlir::Value destValue = destAddr.emitRawPointer();
419 mlir::Value expected = emitScalarExpr(e: e->getArg(Arg: 1));
420 mlir::Value desired = emitScalarExpr(e: e->getArg(Arg: 2));
421
422 auto cmpxchg = cir::AtomicCmpXchgOp::create(
423 builder, location: getLoc(srcLoc: e->getSourceRange()), ptr: destValue, expected, desired,
424 succ_order: successOrder, fail_order: failureOrder, sync_scope: scope,
425 /*alignment=*/nullptr, /*weak=*/false, /*is_volatile=*/false);
426
427 if (returnBool)
428 return cmpxchg.getSuccess();
429 return cmpxchg.getOld();
430}
431
432/// Emit a `cir.atomic.xchg` for __sync_swap_N and __sync_lock_test_and_set_N.
433static RValue emitAtomicXchg(CIRGenFunction &cgf, const CallExpr *e,
434 cir::MemOrder ordering) {
435 Address destAddr = checkAtomicAlignment(cgf, e);
436 CIRGenBuilderTy &builder = cgf.getBuilder();
437 mlir::Value destValue = destAddr.emitRawPointer();
438 mlir::Value val = cgf.emitScalarExpr(e: e->getArg(Arg: 1));
439
440 auto xchg = cir::AtomicXchgOp::create(
441 builder, location: cgf.getLoc(srcLoc: e->getSourceRange()), ptr: destValue, val, mem_order: ordering,
442 sync_scope: cir::SyncScopeKind::System, /*is_volatile=*/false);
443 return RValue::get(v: xchg.getResult());
444}
445
446/// Emit a release store of 0 for __sync_lock_release_N.
447static void emitAtomicLockRelease(CIRGenFunction &cgf, const CallExpr *e) {
448 CIRGenBuilderTy &builder = cgf.getBuilder();
449 Address destAddr = checkAtomicAlignment(cgf, e);
450 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
451 mlir::Type elemTy = destAddr.getElementType();
452 mlir::Value zero = builder.getConstant(loc, attr: builder.getZeroInitAttr(ty: elemTy));
453 auto orderAttr =
454 cir::MemOrderAttr::get(context: &cgf.getMLIRContext(), value: cir::MemOrder::Release);
455 auto scopeAttr = cir::SyncScopeKindAttr::get(context: &cgf.getMLIRContext(),
456 value: cir::SyncScopeKind::System);
457 builder.createStore(loc, val: zero, dst: destAddr, /*isVolatile=*/false,
458 /*isNontemporal=*/false,
459 /*align=*/mlir::IntegerAttr{}, scope: scopeAttr, order: orderAttr);
460}
461
462static void emitAtomicFenceOp(CIRGenFunction &cgf, const CallExpr *expr,
463 cir::SyncScopeKind syncScope) {
464 CIRGenBuilderTy &builder = cgf.getBuilder();
465 mlir::Location loc = cgf.getLoc(srcLoc: expr->getSourceRange());
466
467 auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
468 cir::AtomicFenceOp::create(
469 builder, location: loc, ordering: memOrder,
470 syncscope: cir::SyncScopeKindAttr::get(context: &cgf.getMLIRContext(), value: syncScope));
471 };
472
473 cgf.emitAtomicExprWithMemOrder(memOrder: expr->getArg(Arg: 0), /*isStore*/ false,
474 /*isLoad*/ false, /*isFence*/ true,
475 emitAtomicOp: emitAtomicOpCallBackFn);
476}
477
478// Emit a runtime call to bool __atomic_is_lock_free(size_t size, void *ptr).
479// For the __c11 builtin the pointer is null, since an _Atomic object is always
480// suitably aligned.
481static RValue emitAtomicIsLockFree(CIRGenFunction &cgf, const CallExpr *e,
482 unsigned builtinID) {
483 CIRGenBuilderTy &builder = cgf.getBuilder();
484 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
485
486 mlir::Type sizeTy = cgf.convertType(t: cgf.getContext().getSizeType());
487 mlir::Value size = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
488 mlir::Value ptr;
489 if (builtinID == Builtin::BI__atomic_is_lock_free)
490 ptr = builder.createBitcast(src: cgf.emitScalarExpr(e: e->getArg(Arg: 1)),
491 newTy: builder.getVoidPtrTy());
492 else
493 ptr = builder.getNullPtr(ty: builder.getVoidPtrTy(), loc);
494
495 cir::FuncOp func = cgf.cgm.createRuntimeFunction(
496 ty: cir::FuncType::get(inputs: {sizeTy, builder.getVoidPtrTy()}, returnType: builder.getBoolTy()),
497 name: "__atomic_is_lock_free");
498 // TODO(cir): set the runtime calling convention to this call.
499 assert(!cir::MissingFeatures::opFuncCallingConv());
500 return RValue::get(
501 v: builder.createCallOp(loc, callee: func, operands: mlir::ValueRange{size, ptr}).getResult());
502}
503
504namespace {
505struct WidthAndSignedness {
506 unsigned width;
507 bool isSigned;
508};
509} // namespace
510
511static WidthAndSignedness
512getIntegerWidthAndSignedness(const clang::ASTContext &astContext,
513 const clang::QualType type) {
514 assert(type->isIntegerType() && "Given type is not an integer.");
515 unsigned width = type->isBooleanType() ? 1
516 : type->isBitIntType() ? astContext.getIntWidth(T: type)
517 : astContext.getTypeInfo(T: type).Width;
518 bool isSigned = type->isSignedIntegerType();
519 return {.width: width, .isSigned: isSigned};
520}
521
522/// Create a checked overflow arithmetic op and return its result and overflow
523/// flag.
524template <typename OpTy>
525static std::pair<mlir::Value, mlir::Value>
526emitOverflowOp(CIRGenBuilderTy &builder, mlir::Location loc,
527 mlir::Type resultTy, mlir::Value lhs, mlir::Value rhs) {
528 auto op = OpTy::create(builder, loc, resultTy, lhs, rhs);
529 return {op.getResult(), op.getOverflow()};
530}
531
532// Given one or more integer types, this function produces an integer type that
533// encompasses them: any value in one of the given types could be expressed in
534// the encompassing type.
535static struct WidthAndSignedness
536EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> types) {
537 assert(types.size() > 0 && "Empty list of types.");
538
539 // If any of the given types is signed, we must return a signed type.
540 bool isSigned = llvm::any_of(Range&: types, P: [](const auto &t) { return t.isSigned; });
541
542 // The encompassing type must have a width greater than or equal to the width
543 // of the specified types. Additionally, if the encompassing type is signed,
544 // its width must be strictly greater than the width of any unsigned types
545 // given.
546 unsigned width = 0;
547 for (const auto &type : types)
548 width = std::max(a: width, b: type.width + (isSigned && !type.isSigned));
549
550 return {.width: width, .isSigned: isSigned};
551}
552
553RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) {
554 mlir::Value input = emitScalarExpr(e: e->getArg(Arg: 0));
555 mlir::Value amount = emitScalarExpr(e: e->getArg(Arg: 1));
556
557 if (amount.getType() != input.getType())
558 amount = builder.createIntCast(src: amount, newTy: input.getType());
559
560 auto r = cir::RotateOp::create(builder, location: getLoc(srcLoc: e->getSourceRange()), input,
561 amount, rotateLeft: isRotateLeft);
562 return RValue::get(v: r);
563}
564
565template <class Operation>
566static RValue emitUnaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
567 const CallExpr &e) {
568 mlir::Value arg = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
569
570 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
571
572 auto call = Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(),
573 arg, cgf.getBuilder().getConstrainedFPAttr());
574 return RValue::get(v: call->getResult(0));
575}
576
577template <class Operation>
578static RValue emitUnaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
579 mlir::Value arg = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
580 auto call = Operation::create(cgf.getBuilder(), arg.getLoc(), arg);
581 return RValue::get(v: call->getResult(0));
582}
583
584template <typename Op>
585static RValue emitUnaryMaybeConstrainedFPToIntBuiltin(CIRGenFunction &cgf,
586 const CallExpr &e) {
587 mlir::Type resultType = cgf.convertType(t: e.getType());
588 mlir::Value src = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
589
590 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
591
592 auto call = Op::create(cgf.getBuilder(), src.getLoc(), resultType, src,
593 cgf.getBuilder().getConstrainedFPAttr());
594 return RValue::get(v: call->getResult(0));
595}
596
597template <typename Op>
598static RValue emitBinaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
599 mlir::Value arg0 = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
600 mlir::Value arg1 = cgf.emitScalarExpr(e: e.getArg(Arg: 1));
601
602 mlir::Location loc = cgf.getLoc(srcLoc: e.getExprLoc());
603 auto call = Op::create(cgf.getBuilder(), loc, arg0, arg1);
604
605 return RValue::get(v: call->getResult(0));
606}
607
608template <typename Op>
609static RValue emitTernarySameTypeBuiltin(CIRGenFunction &cgf,
610 const CallExpr &e) {
611 mlir::Value arg0 = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
612 mlir::Value arg1 = cgf.emitScalarExpr(e: e.getArg(Arg: 1));
613 mlir::Value arg2 = cgf.emitScalarExpr(e: e.getArg(Arg: 2));
614
615 mlir::Location loc = cgf.getLoc(srcLoc: e.getExprLoc());
616 mlir::Type ty = cgf.convertType(t: e.getType());
617 auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2);
618
619 return RValue::get(v: call->getResult(0));
620}
621
622template <typename Op>
623static RValue emitTernaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
624 const CallExpr &e) {
625 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
626
627 mlir::Value arg0 = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
628 mlir::Value arg1 = cgf.emitScalarExpr(e: e.getArg(Arg: 1));
629 mlir::Value arg2 = cgf.emitScalarExpr(e: e.getArg(Arg: 2));
630
631 mlir::Location loc = cgf.getLoc(srcLoc: e.getExprLoc());
632 mlir::Type ty = cgf.convertType(t: e.getType());
633
634 auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2,
635 cgf.getBuilder().getConstrainedFPAttr(),
636 cgf.getBuilder().getFastMathFlagsAttr());
637 return RValue::get(v: call->getResult(0));
638}
639
640template <typename Op>
641static mlir::Value emitBinaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
642 const CallExpr &e) {
643 mlir::Value arg0 = cgf.emitScalarExpr(e: e.getArg(Arg: 0));
644 mlir::Value arg1 = cgf.emitScalarExpr(e: e.getArg(Arg: 1));
645
646 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
647
648 mlir::Location loc = cgf.getLoc(srcLoc: e.getExprLoc());
649 auto call = Op::create(cgf.getBuilder(), loc, arg0, arg1,
650 cgf.getBuilder().getConstrainedFPAttr());
651 return call->getResult(0);
652}
653
654static RValue errorBuiltinNYI(CIRGenFunction &cgf, const CallExpr *e,
655 unsigned builtinID) {
656
657 if (cgf.getContext().BuiltinInfo.isLibFunction(ID: builtinID)) {
658 cgf.cgm.errorNYI(
659 e->getSourceRange(),
660 std::string("unimplemented library function builtin call: ") +
661 cgf.getContext().BuiltinInfo.getName(ID: builtinID));
662 } else {
663 cgf.cgm.errorNYI(e->getSourceRange(),
664 std::string("unimplemented builtin call: ") +
665 cgf.getContext().BuiltinInfo.getName(ID: builtinID));
666 }
667
668 return cgf.getUndefRValue(ty: e->getType());
669}
670
671static RValue emitBuiltinAlloca(CIRGenFunction &cgf, const CallExpr *e,
672 unsigned builtinID) {
673 assert(builtinID == Builtin::BI__builtin_alloca ||
674 builtinID == Builtin::BI__builtin_alloca_uninitialized ||
675 builtinID == Builtin::BIalloca || builtinID == Builtin::BI_alloca);
676
677 // Get alloca size input
678 mlir::Value size = cgf.emitScalarExpr(e: e->getArg(Arg: 0));
679
680 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
681 const TargetInfo &ti = cgf.getContext().getTargetInfo();
682 const CharUnits suitableAlignmentInBytes =
683 cgf.getContext().toCharUnitsFromBits(BitSize: ti.getSuitableAlign());
684
685 // Emit the alloca op with type `u8 *` to match the semantics of
686 // `llvm.alloca`. We later bitcast the type to `void *` to match the
687 // semantics of C/C++
688 // FIXME(cir): It may make sense to allow AllocaOp of type `u8` to return a
689 // pointer of type `void *`. This will require a change to the allocaOp
690 // verifier.
691 CIRGenBuilderTy &builder = cgf.getBuilder();
692 mlir::Value allocaAddr = builder.createAlloca(
693 loc: cgf.getLoc(srcLoc: e->getSourceRange()), addrType: builder.getUInt8PtrTy(),
694 type: builder.getUInt8Ty(), name: "bi_alloca", alignment: suitableAlignmentInBytes, dynAllocSize: size);
695
696 // Initialize the allocated buffer if required.
697 if (builtinID != Builtin::BI__builtin_alloca_uninitialized) {
698 // Initialize the alloca with the given size and alignment according to
699 // the lang opts. Only the trivial non-initialization is supported for
700 // now.
701
702 switch (cgf.getLangOpts().getTrivialAutoVarInit()) {
703 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
704 // Nothing to initialize.
705 break;
706 case LangOptions::TrivialAutoVarInitKind::Zero:
707 case LangOptions::TrivialAutoVarInitKind::Pattern:
708 cgf.cgm.errorNYI(feature: "trivial auto var init");
709 break;
710 }
711 }
712
713 // An alloca will always return a pointer to the alloca (stack) address
714 // space. This address space need not be the same as the AST / Language
715 // default (e.g. in C / C++ auto vars are in the generic address space). At
716 // the AST level this is handled within CreateTempAlloca et al., but for the
717 // builtin / dynamic alloca we have to handle it here.
718
719 if (!cir::isMatchingAddressSpace(
720 cirAS: cgf.getCIRAllocaAddressSpace(),
721 as: e->getType()->getPointeeType().getAddressSpace())) {
722 cgf.cgm.errorNYI(e->getSourceRange(),
723 "Address Space Cast for builtin alloca");
724 }
725
726 // Bitcast the alloca to the expected type.
727 return RValue::get(v: builder.createBitcast(
728 src: allocaAddr, newTy: builder.getVoidPtrTy(as: cgf.getCIRAllocaAddressSpace())));
729}
730
731static bool shouldCIREmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
732 unsigned builtinID) {
733 std::optional<bool> errnoOverriden;
734 // ErrnoOverriden is true if math-errno is overriden via the
735 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
736 // which implies math-errno.
737 if (e->hasStoredFPFeatures()) {
738 FPOptionsOverride op = e->getFPFeatures();
739 if (op.hasMathErrnoOverride())
740 errnoOverriden = op.getMathErrnoOverride();
741 }
742 // True if 'attribute__((optnone))' is used. This attribute overrides
743 // fast-math which implies math-errno.
744 bool optNone =
745 cgf.curFuncDecl && cgf.curFuncDecl->hasAttr<OptimizeNoneAttr>();
746 bool isOptimizationEnabled = cgf.cgm.getCodeGenOpts().OptimizationLevel != 0;
747 bool generateFPMathIntrinsics =
748 cgf.getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
749 BuiltinID: builtinID, Trip: cgf.cgm.getTriple(), ErrnoOverwritten: errnoOverriden,
750 MathErrnoEnabled: cgf.getLangOpts().MathErrno, HasOptNoneAttr: optNone, IsOptimizationEnabled: isOptimizationEnabled);
751 return generateFPMathIntrinsics;
752}
753
754static RValue tryEmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
755 unsigned builtinID) {
756 assert(!cir::MissingFeatures::fastMathFlags());
757 switch (builtinID) {
758 case Builtin::BIacos:
759 case Builtin::BIacosf:
760 case Builtin::BIacosl:
761 case Builtin::BI__builtin_acos:
762 case Builtin::BI__builtin_acosf:
763 case Builtin::BI__builtin_acosf16:
764 case Builtin::BI__builtin_acosl:
765 case Builtin::BI__builtin_acosf128:
766 case Builtin::BI__builtin_elementwise_acos:
767 return emitUnaryMaybeConstrainedFPBuiltin<cir::ACosOp>(cgf, e: *e);
768 case Builtin::BIasin:
769 case Builtin::BIasinf:
770 case Builtin::BIasinl:
771 case Builtin::BI__builtin_asin:
772 case Builtin::BI__builtin_asinf:
773 case Builtin::BI__builtin_asinf16:
774 case Builtin::BI__builtin_asinl:
775 case Builtin::BI__builtin_asinf128:
776 case Builtin::BI__builtin_elementwise_asin:
777 return emitUnaryMaybeConstrainedFPBuiltin<cir::ASinOp>(cgf, e: *e);
778 case Builtin::BIatan:
779 case Builtin::BIatanf:
780 case Builtin::BIatanl:
781 case Builtin::BI__builtin_atan:
782 case Builtin::BI__builtin_atanf:
783 case Builtin::BI__builtin_atanf16:
784 case Builtin::BI__builtin_atanl:
785 case Builtin::BI__builtin_atanf128:
786 case Builtin::BI__builtin_elementwise_atan:
787 return emitUnaryMaybeConstrainedFPBuiltin<cir::ATanOp>(cgf, e: *e);
788 case Builtin::BIatan2:
789 case Builtin::BIatan2f:
790 case Builtin::BIatan2l:
791 case Builtin::BI__builtin_atan2:
792 case Builtin::BI__builtin_atan2f:
793 case Builtin::BI__builtin_atan2f16:
794 case Builtin::BI__builtin_atan2l:
795 case Builtin::BI__builtin_atan2f128:
796 case Builtin::BI__builtin_elementwise_atan2:
797 return RValue::get(
798 v: emitBinaryMaybeConstrainedFPBuiltin<cir::ATan2Op>(cgf, e: *e));
799 case Builtin::BIceil:
800 case Builtin::BIceilf:
801 case Builtin::BIceill:
802 case Builtin::BI__builtin_ceil:
803 case Builtin::BI__builtin_ceilf:
804 case Builtin::BI__builtin_ceilf16:
805 case Builtin::BI__builtin_ceill:
806 case Builtin::BI__builtin_ceilf128:
807 case Builtin::BI__builtin_elementwise_ceil:
808 return emitUnaryMaybeConstrainedFPBuiltin<cir::CeilOp>(cgf, e: *e);
809 case Builtin::BIcopysign:
810 case Builtin::BIcopysignf:
811 case Builtin::BIcopysignl:
812 case Builtin::BI__builtin_copysign:
813 case Builtin::BI__builtin_copysignf:
814 case Builtin::BI__builtin_copysignf16:
815 case Builtin::BI__builtin_copysignl:
816 case Builtin::BI__builtin_copysignf128:
817 return emitBinaryFPBuiltin<cir::CopysignOp>(cgf, e: *e);
818 case Builtin::BIcos:
819 case Builtin::BIcosf:
820 case Builtin::BIcosl:
821 case Builtin::BI__builtin_cos:
822 case Builtin::BI__builtin_cosf:
823 case Builtin::BI__builtin_cosf16:
824 case Builtin::BI__builtin_cosl:
825 case Builtin::BI__builtin_cosf128:
826 case Builtin::BI__builtin_elementwise_cos:
827 return emitUnaryMaybeConstrainedFPBuiltin<cir::CosOp>(cgf, e: *e);
828 case Builtin::BIcosh:
829 case Builtin::BIcoshf:
830 case Builtin::BIcoshl:
831 case Builtin::BI__builtin_cosh:
832 case Builtin::BI__builtin_coshf:
833 case Builtin::BI__builtin_coshf16:
834 case Builtin::BI__builtin_coshl:
835 case Builtin::BI__builtin_coshf128:
836 case Builtin::BI__builtin_elementwise_cosh:
837 return emitUnaryMaybeConstrainedFPBuiltin<cir::CoshOp>(cgf, e: *e);
838 case Builtin::BIexp:
839 case Builtin::BIexpf:
840 case Builtin::BIexpl:
841 case Builtin::BI__builtin_exp:
842 case Builtin::BI__builtin_expf:
843 case Builtin::BI__builtin_expf16:
844 case Builtin::BI__builtin_expl:
845 case Builtin::BI__builtin_expf128:
846 case Builtin::BI__builtin_elementwise_exp:
847 return emitUnaryMaybeConstrainedFPBuiltin<cir::ExpOp>(cgf, e: *e);
848 case Builtin::BIexp2:
849 case Builtin::BIexp2f:
850 case Builtin::BIexp2l:
851 case Builtin::BI__builtin_exp2:
852 case Builtin::BI__builtin_exp2f:
853 case Builtin::BI__builtin_exp2f16:
854 case Builtin::BI__builtin_exp2l:
855 case Builtin::BI__builtin_exp2f128:
856 case Builtin::BI__builtin_elementwise_exp2:
857 return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp2Op>(cgf, e: *e);
858 case Builtin::BI__builtin_exp10:
859 case Builtin::BI__builtin_exp10f:
860 case Builtin::BI__builtin_exp10f16:
861 case Builtin::BI__builtin_exp10l:
862 case Builtin::BI__builtin_exp10f128:
863 case Builtin::BI__builtin_elementwise_exp10:
864 return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp10Op>(cgf, e: *e);
865 case Builtin::BIfabs:
866 case Builtin::BIfabsf:
867 case Builtin::BIfabsl:
868 case Builtin::BI__builtin_fabs:
869 case Builtin::BI__builtin_fabsf:
870 case Builtin::BI__builtin_fabsf16:
871 case Builtin::BI__builtin_fabsl:
872 case Builtin::BI__builtin_fabsf128:
873 return emitUnaryMaybeConstrainedFPBuiltin<cir::FAbsOp>(cgf, e: *e);
874 case Builtin::BIfloor:
875 case Builtin::BIfloorf:
876 case Builtin::BIfloorl:
877 case Builtin::BI__builtin_floor:
878 case Builtin::BI__builtin_floorf:
879 case Builtin::BI__builtin_floorf16:
880 case Builtin::BI__builtin_floorl:
881 case Builtin::BI__builtin_floorf128:
882 case Builtin::BI__builtin_elementwise_floor:
883 return emitUnaryMaybeConstrainedFPBuiltin<cir::FloorOp>(cgf, e: *e);
884 case Builtin::BIfma:
885 case Builtin::BIfmaf:
886 case Builtin::BIfmal:
887 case Builtin::BI__builtin_fma:
888 case Builtin::BI__builtin_fmaf:
889 case Builtin::BI__builtin_fmaf16:
890 case Builtin::BI__builtin_fmal:
891 case Builtin::BI__builtin_fmaf128:
892 case Builtin::BI__builtin_elementwise_fma:
893 return emitTernaryMaybeConstrainedFPBuiltin<cir::FMAOp>(cgf, e: *e);
894 case Builtin::BIfmax:
895 case Builtin::BIfmaxf:
896 case Builtin::BIfmaxl:
897 case Builtin::BI__builtin_fmax:
898 case Builtin::BI__builtin_fmaxf:
899 case Builtin::BI__builtin_fmaxf16:
900 case Builtin::BI__builtin_fmaxl:
901 case Builtin::BI__builtin_fmaxf128:
902 return RValue::get(
903 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(cgf, e: *e));
904 case Builtin::BIfmin:
905 case Builtin::BIfminf:
906 case Builtin::BIfminl:
907 case Builtin::BI__builtin_fmin:
908 case Builtin::BI__builtin_fminf:
909 case Builtin::BI__builtin_fminf16:
910 case Builtin::BI__builtin_fminl:
911 case Builtin::BI__builtin_fminf128:
912 return RValue::get(
913 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(cgf, e: *e));
914 case Builtin::BIfmaximum_num:
915 case Builtin::BIfmaximum_numf:
916 case Builtin::BIfmaximum_numl:
917 case Builtin::BI__builtin_fmaximum_num:
918 case Builtin::BI__builtin_fmaximum_numf:
919 case Builtin::BI__builtin_fmaximum_numf16:
920 case Builtin::BI__builtin_fmaximum_numl:
921 case Builtin::BI__builtin_fmaximum_numf128:
922 case Builtin::BIfminimum_num:
923 case Builtin::BIfminimum_numf:
924 case Builtin::BIfminimum_numl:
925 case Builtin::BI__builtin_fminimum_num:
926 case Builtin::BI__builtin_fminimum_numf:
927 case Builtin::BI__builtin_fminimum_numf16:
928 case Builtin::BI__builtin_fminimum_numl:
929 case Builtin::BI__builtin_fminimum_numf128:
930 return errorBuiltinNYI(cgf, e, builtinID);
931 case Builtin::BIfmod:
932 case Builtin::BIfmodf:
933 case Builtin::BIfmodl:
934 case Builtin::BI__builtin_fmod:
935 case Builtin::BI__builtin_fmodf:
936 case Builtin::BI__builtin_fmodf16:
937 case Builtin::BI__builtin_fmodl:
938 case Builtin::BI__builtin_fmodf128:
939 case Builtin::BI__builtin_elementwise_fmod:
940 return RValue::get(
941 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FModOp>(cgf, e: *e));
942 case Builtin::BIlog:
943 case Builtin::BIlogf:
944 case Builtin::BIlogl:
945 case Builtin::BI__builtin_log:
946 case Builtin::BI__builtin_logf:
947 case Builtin::BI__builtin_logf16:
948 case Builtin::BI__builtin_logl:
949 case Builtin::BI__builtin_logf128:
950 case Builtin::BI__builtin_elementwise_log:
951 return emitUnaryMaybeConstrainedFPBuiltin<cir::LogOp>(cgf, e: *e);
952 case Builtin::BIlog10:
953 case Builtin::BIlog10f:
954 case Builtin::BIlog10l:
955 case Builtin::BI__builtin_log10:
956 case Builtin::BI__builtin_log10f:
957 case Builtin::BI__builtin_log10f16:
958 case Builtin::BI__builtin_log10l:
959 case Builtin::BI__builtin_log10f128:
960 case Builtin::BI__builtin_elementwise_log10:
961 return emitUnaryMaybeConstrainedFPBuiltin<cir::Log10Op>(cgf, e: *e);
962 case Builtin::BIlog2:
963 case Builtin::BIlog2f:
964 case Builtin::BIlog2l:
965 case Builtin::BI__builtin_log2:
966 case Builtin::BI__builtin_log2f:
967 case Builtin::BI__builtin_log2f16:
968 case Builtin::BI__builtin_log2l:
969 case Builtin::BI__builtin_log2f128:
970 case Builtin::BI__builtin_elementwise_log2:
971 return emitUnaryMaybeConstrainedFPBuiltin<cir::Log2Op>(cgf, e: *e);
972 case Builtin::BInearbyint:
973 case Builtin::BInearbyintf:
974 case Builtin::BInearbyintl:
975 case Builtin::BI__builtin_nearbyint:
976 case Builtin::BI__builtin_nearbyintf:
977 case Builtin::BI__builtin_nearbyintl:
978 case Builtin::BI__builtin_nearbyintf128:
979 case Builtin::BI__builtin_elementwise_nearbyint:
980 return emitUnaryMaybeConstrainedFPBuiltin<cir::NearbyintOp>(cgf, e: *e);
981 case Builtin::BIpow:
982 case Builtin::BIpowf:
983 case Builtin::BIpowl:
984 case Builtin::BI__builtin_pow:
985 case Builtin::BI__builtin_powf:
986 case Builtin::BI__builtin_powf16:
987 case Builtin::BI__builtin_powl:
988 case Builtin::BI__builtin_powf128:
989 return RValue::get(
990 v: emitBinaryMaybeConstrainedFPBuiltin<cir::PowOp>(cgf, e: *e));
991 case Builtin::BI__builtin_elementwise_pow:
992 return errorBuiltinNYI(cgf, e, builtinID);
993 case Builtin::BIrint:
994 case Builtin::BIrintf:
995 case Builtin::BIrintl:
996 case Builtin::BI__builtin_rint:
997 case Builtin::BI__builtin_rintf:
998 case Builtin::BI__builtin_rintf16:
999 case Builtin::BI__builtin_rintl:
1000 case Builtin::BI__builtin_rintf128:
1001 case Builtin::BI__builtin_elementwise_rint:
1002 return emitUnaryMaybeConstrainedFPBuiltin<cir::RintOp>(cgf, e: *e);
1003 case Builtin::BIround:
1004 case Builtin::BIroundf:
1005 case Builtin::BIroundl:
1006 case Builtin::BI__builtin_round:
1007 case Builtin::BI__builtin_roundf:
1008 case Builtin::BI__builtin_roundf16:
1009 case Builtin::BI__builtin_roundl:
1010 case Builtin::BI__builtin_roundf128:
1011 case Builtin::BI__builtin_elementwise_round:
1012 return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundOp>(cgf, e: *e);
1013 case Builtin::BIroundeven:
1014 case Builtin::BIroundevenf:
1015 case Builtin::BIroundevenl:
1016 case Builtin::BI__builtin_roundeven:
1017 case Builtin::BI__builtin_roundevenf:
1018 case Builtin::BI__builtin_roundevenf16:
1019 case Builtin::BI__builtin_roundevenl:
1020 case Builtin::BI__builtin_roundevenf128:
1021 case Builtin::BI__builtin_elementwise_roundeven:
1022 return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundEvenOp>(cgf, e: *e);
1023 case Builtin::BIsin:
1024 case Builtin::BIsinf:
1025 case Builtin::BIsinl:
1026 case Builtin::BI__builtin_sin:
1027 case Builtin::BI__builtin_sinf:
1028 case Builtin::BI__builtin_sinf16:
1029 case Builtin::BI__builtin_sinl:
1030 case Builtin::BI__builtin_sinf128:
1031 case Builtin::BI__builtin_elementwise_sin:
1032 return emitUnaryMaybeConstrainedFPBuiltin<cir::SinOp>(cgf, e: *e);
1033 case Builtin::BIsinh:
1034 case Builtin::BIsinhf:
1035 case Builtin::BIsinhl:
1036 case Builtin::BI__builtin_sinh:
1037 case Builtin::BI__builtin_sinhf:
1038 case Builtin::BI__builtin_sinhf16:
1039 case Builtin::BI__builtin_sinhl:
1040 case Builtin::BI__builtin_sinhf128:
1041 case Builtin::BI__builtin_elementwise_sinh:
1042 return emitUnaryMaybeConstrainedFPBuiltin<cir::SinhOp>(cgf, e: *e);
1043 case Builtin::BI__builtin_sincospi:
1044 case Builtin::BI__builtin_sincospif:
1045 case Builtin::BI__builtin_sincospil:
1046 case Builtin::BIsincos:
1047 case Builtin::BIsincosf:
1048 case Builtin::BIsincosl:
1049 case Builtin::BI__builtin_sincos:
1050 case Builtin::BI__builtin_sincosf:
1051 case Builtin::BI__builtin_sincosf16:
1052 case Builtin::BI__builtin_sincosl:
1053 case Builtin::BI__builtin_sincosf128:
1054 return errorBuiltinNYI(cgf, e, builtinID);
1055 case Builtin::BIsqrt:
1056 case Builtin::BIsqrtf:
1057 case Builtin::BIsqrtl:
1058 case Builtin::BI__builtin_sqrt:
1059 case Builtin::BI__builtin_sqrtf:
1060 case Builtin::BI__builtin_sqrtf16:
1061 case Builtin::BI__builtin_sqrtl:
1062 case Builtin::BI__builtin_sqrtf128:
1063 case Builtin::BI__builtin_elementwise_sqrt:
1064 return emitUnaryMaybeConstrainedFPBuiltin<cir::SqrtOp>(cgf, e: *e);
1065 case Builtin::BItan:
1066 case Builtin::BItanf:
1067 case Builtin::BItanl:
1068 case Builtin::BI__builtin_tan:
1069 case Builtin::BI__builtin_tanf:
1070 case Builtin::BI__builtin_tanf16:
1071 case Builtin::BI__builtin_tanl:
1072 case Builtin::BI__builtin_tanf128:
1073 case Builtin::BI__builtin_elementwise_tan:
1074 return emitUnaryMaybeConstrainedFPBuiltin<cir::TanOp>(cgf, e: *e);
1075 case Builtin::BItanh:
1076 case Builtin::BItanhf:
1077 case Builtin::BItanhl:
1078 case Builtin::BI__builtin_tanh:
1079 case Builtin::BI__builtin_tanhf:
1080 case Builtin::BI__builtin_tanhf16:
1081 case Builtin::BI__builtin_tanhl:
1082 case Builtin::BI__builtin_tanhf128:
1083 case Builtin::BI__builtin_elementwise_tanh:
1084 return emitUnaryMaybeConstrainedFPBuiltin<cir::TanhOp>(cgf, e: *e);
1085 case Builtin::BItrunc:
1086 case Builtin::BItruncf:
1087 case Builtin::BItruncl:
1088 case Builtin::BI__builtin_trunc:
1089 case Builtin::BI__builtin_truncf:
1090 case Builtin::BI__builtin_truncf16:
1091 case Builtin::BI__builtin_truncl:
1092 case Builtin::BI__builtin_truncf128:
1093 case Builtin::BI__builtin_elementwise_trunc:
1094 return emitUnaryMaybeConstrainedFPBuiltin<cir::TruncOp>(cgf, e: *e);
1095 case Builtin::BIlround:
1096 case Builtin::BIlroundf:
1097 case Builtin::BIlroundl:
1098 case Builtin::BI__builtin_lround:
1099 case Builtin::BI__builtin_lroundf:
1100 case Builtin::BI__builtin_lroundl:
1101 case Builtin::BI__builtin_lroundf128:
1102 return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LroundOp>(cgf, e: *e);
1103 case Builtin::BIllround:
1104 case Builtin::BIllroundf:
1105 case Builtin::BIllroundl:
1106 case Builtin::BI__builtin_llround:
1107 case Builtin::BI__builtin_llroundf:
1108 case Builtin::BI__builtin_llroundl:
1109 case Builtin::BI__builtin_llroundf128:
1110 return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlroundOp>(cgf, e: *e);
1111 case Builtin::BIlrint:
1112 case Builtin::BIlrintf:
1113 case Builtin::BIlrintl:
1114 case Builtin::BI__builtin_lrint:
1115 case Builtin::BI__builtin_lrintf:
1116 case Builtin::BI__builtin_lrintl:
1117 case Builtin::BI__builtin_lrintf128:
1118 return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LrintOp>(cgf, e: *e);
1119 case Builtin::BIllrint:
1120 case Builtin::BIllrintf:
1121 case Builtin::BIllrintl:
1122 case Builtin::BI__builtin_llrint:
1123 case Builtin::BI__builtin_llrintf:
1124 case Builtin::BI__builtin_llrintl:
1125 case Builtin::BI__builtin_llrintf128:
1126 return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlrintOp>(cgf, e: *e);
1127 case Builtin::BI__builtin_ldexp:
1128 case Builtin::BI__builtin_ldexpf:
1129 case Builtin::BI__builtin_ldexpl:
1130 case Builtin::BI__builtin_ldexpf16:
1131 case Builtin::BI__builtin_ldexpf128:
1132 case Builtin::BI__builtin_elementwise_ldexp:
1133 return RValue::get(
1134 v: emitBinaryMaybeConstrainedFPBuiltin<cir::LdexpOp>(cgf, e: *e));
1135 default:
1136 break;
1137 }
1138
1139 return RValue::getIgnored();
1140}
1141
1142// FIXME: Remove cgf parameter when all descriptor kinds are implemented
1143static mlir::Type
1144decodeFixedType(CIRGenFunction &cgf,
1145 ArrayRef<llvm::Intrinsic::IITDescriptor> &infos,
1146 ArrayRef<mlir::Type> overloadTys, mlir::MLIRContext *context) {
1147 using namespace llvm::Intrinsic;
1148
1149 IITDescriptor descriptor = infos.front();
1150 infos = infos.slice(N: 1);
1151
1152 switch (descriptor.Kind) {
1153 case IITDescriptor::Overloaded:
1154 case IITDescriptor::Match:
1155 if (descriptor.getOverloadIndex() < overloadTys.size())
1156 return overloadTys[descriptor.getOverloadIndex()];
1157 cgf.cgm.errorNYI(feature: "Overloaded intrinsic type without overload types");
1158 return cir::VoidType::get(ctx: context);
1159 case IITDescriptor::Void:
1160 return cir::VoidType::get(ctx: context);
1161 case IITDescriptor::Half:
1162 return cir::FP16Type::get(ctx: context);
1163 case IITDescriptor::BFloat:
1164 return cir::BF16Type::get(ctx: context);
1165 case IITDescriptor::Float:
1166 return cir::SingleType::get(ctx: context);
1167 case IITDescriptor::Double:
1168 return cir::DoubleType::get(ctx: context);
1169 case IITDescriptor::Quad:
1170 return cir::FP128Type::get(ctx: context);
1171 // If the intrinsic expects unsigned integers, the signedness is corrected in
1172 // correctIntegerSignedness()
1173 case IITDescriptor::Integer:
1174 return cir::IntType::get(context, width: descriptor.IntegerWidth,
1175 /*isSigned=*/is_signed: true);
1176 case IITDescriptor::Vector: {
1177 mlir::Type elementType = decodeFixedType(cgf, infos, overloadTys, context);
1178 unsigned numElements = descriptor.VectorWidth.getFixedValue();
1179 return cir::VectorType::getFixed(elementType, size: numElements);
1180 }
1181 case IITDescriptor::Struct: {
1182 SmallVector<mlir::Type, 8> members;
1183 for (unsigned i = 0; i < descriptor.StructNumElements; ++i)
1184 members.push_back(Elt: decodeFixedType(cgf, infos, overloadTys, context));
1185 return cgf.getBuilder().getAnonRecordTy(
1186 members, /*packed=*/false, memberKinds: cir::RecordType::getAllDataKinds(members));
1187 }
1188 case IITDescriptor::Pointer: {
1189 mlir::Builder builder(context);
1190 auto addrSpace = cir::TargetAddressSpaceAttr::get(
1191 context, value: descriptor.PointerAddressSpace);
1192 return cir::PointerType::get(pointee: cir::VoidType::get(ctx: context), addrSpace);
1193 }
1194 default:
1195 cgf.cgm.errorNYI(feature: "Unimplemented intrinsic type descriptor");
1196 return cir::VoidType::get(ctx: context);
1197 }
1198}
1199
1200/// Helper function to correct integer signedness for intrinsic arguments and
1201/// return type. IIT always returns signed integers, but the actual intrinsic
1202/// may expect unsigned integers based on the AST FunctionDecl parameter types.
1203static mlir::Type correctIntegerSignedness(mlir::Type iitType, QualType astType,
1204 mlir::MLIRContext *context) {
1205 auto intTy = dyn_cast<cir::IntType>(Val&: iitType);
1206 if (!intTy)
1207 return iitType;
1208
1209 if (astType->isUnsignedIntegerType())
1210 return cir::IntType::get(context, width: intTy.getWidth(), /*isSigned=*/is_signed: false);
1211
1212 return iitType;
1213}
1214
1215/// Helper function to correct the return type for intrinsic calls. This is
1216/// needed because the AST FunctionDecl may have a different return type than
1217/// the intrinsic's IIT descriptor. For example, builtins may need their
1218/// signedness corrected, or a builtin may return a bool while the intrinsic
1219/// returns an i1.
1220static mlir::Type correctReturnType(mlir::Type iitType,
1221 const FunctionDecl *funcDecl,
1222 mlir::MLIRContext *context) {
1223 if (!funcDecl)
1224 return iitType;
1225 QualType astType = funcDecl->getReturnType();
1226
1227 // Relabel the return type to cir.bool if the builtin returns a bool and
1228 // the intrinsic returns an i1.
1229 auto intTy = mlir::dyn_cast<cir::IntType>(Val&: iitType);
1230 if (intTy && intTy.getWidth() == 1 && astType->isBooleanType())
1231 return cir::BoolType::get(ctx: context);
1232
1233 return correctIntegerSignedness(iitType, astType, context);
1234}
1235
1236static mlir::Value getCorrectedPtr(mlir::Value argValue, mlir::Type expectedTy,
1237 CIRGenBuilderTy &builder) {
1238 auto ptrType = mlir::cast<cir::PointerType>(Val: argValue.getType());
1239
1240 auto expectedPtrType = mlir::cast<cir::PointerType>(Val&: expectedTy);
1241 assert(ptrType != expectedPtrType && "types should not match");
1242
1243 if (ptrType.getAddrSpace() != expectedPtrType.getAddrSpace()) {
1244 assert(!cir::MissingFeatures::addressSpace() &&
1245 "address space handling not yet implemented");
1246 auto newPtrType = cir::PointerType::get(pointee: ptrType.getPointee(),
1247 addrSpace: expectedPtrType.getAddrSpace());
1248 return builder.createAddrSpaceCast(src: argValue, newTy: newPtrType);
1249 }
1250
1251 return builder.createBitcast(src: argValue, newTy: expectedTy);
1252}
1253
1254cir::FuncType
1255CIRGenFunction::getIntrinsicType(llvm::Intrinsic::ID id,
1256 ArrayRef<mlir::Type> overloadTys) {
1257 using namespace llvm::Intrinsic;
1258
1259 mlir::MLIRContext *context = &getMLIRContext();
1260 SmallVector<IITDescriptor, 8> table;
1261 auto [tableRef, _, isVarArg] = getIntrinsicInfoTableEntries(id, T&: table);
1262
1263 mlir::Type resultTy = decodeFixedType(cgf&: *this, infos&: tableRef, overloadTys, context);
1264
1265 SmallVector<mlir::Type, 8> argTypes;
1266 while (!tableRef.empty())
1267 argTypes.push_back(Elt: decodeFixedType(cgf&: *this, infos&: tableRef, overloadTys, context));
1268
1269 // CIR convention: no explicit void return type
1270 if (isa<cir::VoidType>(Val: resultTy))
1271 return cir::FuncType::get(context, inputs: argTypes, /*optionalReturnType=*/nullptr,
1272 varArg: isVarArg);
1273
1274 return cir::FuncType::get(context, inputs: argTypes, optionalReturnType: resultTy, varArg: isVarArg);
1275}
1276
1277void CIRGenFunction::checkTargetFeatures(const CallExpr *e,
1278 const FunctionDecl *targetDecl) {
1279 const FunctionDecl *fd = dyn_cast_or_null<FunctionDecl>(Val: curCodeDecl);
1280 CodeGenUtils::checkTargetFeatures(Ctx&: getContext(), Diags&: cgm.getDiags(), LangOpts: getLangOpts(),
1281 E: e, Caller: fd, TargetDecl: targetDecl);
1282}
1283
1284void CIRGenFunction::checkTargetFeatures(SourceLocation loc,
1285 const FunctionDecl *targetDecl) {
1286 const FunctionDecl *fd = dyn_cast_or_null<FunctionDecl>(Val: curCodeDecl);
1287 CodeGenUtils::checkTargetFeatures(Ctx&: getContext(), Diags&: cgm.getDiags(), LangOpts: getLangOpts(),
1288 Loc: loc, Caller: fd, TargetDecl: targetDecl);
1289}
1290
1291RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
1292 const CallExpr *e,
1293 ReturnValueSlot returnValue) {
1294 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
1295
1296 // See if we can constant fold this builtin. If so, don't emit it at all.
1297 // TODO: Extend this handling to all builtin calls that we can constant-fold.
1298 // Do not constant-fold immediate (target-specific) builtins; their ASTs can
1299 // trigger the constant evaluator in cases it cannot safely handle.
1300 // Skip EvaluateAsRValue for those.
1301 Expr::EvalResult result;
1302 if (e->isPRValue() && !getContext().BuiltinInfo.isImmediate(ID: builtinID) &&
1303 e->EvaluateAsRValue(Result&: result, Ctx: cgm.getASTContext()) &&
1304 !result.hasSideEffects()) {
1305 if (result.Val.isInt()) {
1306 QualType type = e->getType();
1307 if (type->isBooleanType())
1308 return RValue::get(
1309 v: builder.getBool(state: result.Val.getInt().getBoolValue(), loc));
1310 return RValue::get(v: builder.getConstInt(loc, intVal: result.Val.getInt()));
1311 }
1312 if (result.Val.isFloat()) {
1313 // Note: we are using result type of CallExpr to determine the type of
1314 // the constant. Classic codegen uses the result value to determine the
1315 // type. We feel it should be Ok to use expression type because it is
1316 // hard to imagine a builtin function evaluates to a value that
1317 // over/underflows its own defined type.
1318 mlir::Type type = convertType(t: e->getType());
1319 return RValue::get(v: builder.getConstFP(loc, t: type, fpVal: result.Val.getFloat()));
1320 }
1321 }
1322
1323 const FunctionDecl *fd = gd.getDecl()->getAsFunction();
1324
1325 assert(!cir::MissingFeatures::builtinCallF128());
1326
1327 // If the builtin has been declared explicitly with an assembler label,
1328 // disable the specialized emitting below. Ideally we should communicate the
1329 // rename in IR, or at least avoid generating the intrinsic calls that are
1330 // likely to get lowered to the renamed library functions.
1331 unsigned builtinIDIfNoAsmLabel = fd->hasAttr<AsmLabelAttr>() ? 0 : builtinID;
1332
1333 bool generateFPMathIntrinsics =
1334 shouldCIREmitFPMathIntrinsic(cgf&: *this, e, builtinID);
1335
1336 if (generateFPMathIntrinsics) {
1337 // Try to match the builtinID with a floating point math builtin.
1338 RValue rv = tryEmitFPMathIntrinsic(cgf&: *this, e, builtinID: builtinIDIfNoAsmLabel);
1339
1340 // Return the result directly if a math intrinsic was generated.
1341 if (!rv.isIgnored()) {
1342 return rv;
1343 }
1344 }
1345
1346 assert(!cir::MissingFeatures::builtinCall());
1347
1348 switch (builtinIDIfNoAsmLabel) {
1349 default:
1350 break;
1351
1352 // C stdarg builtins.
1353 case Builtin::BI__builtin_stdarg_start:
1354 case Builtin::BI__builtin_va_start:
1355 case Builtin::BI__va_start:
1356 case Builtin::BI__builtin_c23_va_start: {
1357 mlir::Value vaList = builtinID == Builtin::BI__va_start
1358 ? emitScalarExpr(e: e->getArg(Arg: 0))
1359 : emitVAListRef(e: e->getArg(Arg: 0)).getPointer();
1360 emitVAStart(vaList);
1361 return {};
1362 }
1363
1364 case Builtin::BI__builtin_va_end:
1365 emitVAEnd(vaList: emitVAListRef(e: e->getArg(Arg: 0)).getPointer());
1366 return {};
1367 case Builtin::BI__builtin_va_copy: {
1368 mlir::Value dstPtr = emitVAListRef(e: e->getArg(Arg: 0)).getPointer();
1369 mlir::Value srcPtr = emitVAListRef(e: e->getArg(Arg: 1)).getPointer();
1370 cir::VACopyOp::create(builder, location: dstPtr.getLoc(), dst_list: dstPtr, src_list: srcPtr);
1371 return {};
1372 }
1373
1374 case Builtin::BIabs:
1375 case Builtin::BIlabs:
1376 case Builtin::BIllabs:
1377 case Builtin::BI__builtin_abs:
1378 case Builtin::BI__builtin_labs:
1379 case Builtin::BI__builtin_llabs: {
1380 bool sanitizeOverflow = sanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
1381 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
1382 mlir::Value result;
1383 switch (getLangOpts().getSignedOverflowBehavior()) {
1384 case LangOptions::SOB_Defined:
1385 result = cir::AbsOp::create(builder, location: loc, result: arg.getType(), src: arg,
1386 /*minIsPoison=*/min_is_poison: false);
1387 break;
1388 case LangOptions::SOB_Undefined:
1389 if (!sanitizeOverflow) {
1390 result = cir::AbsOp::create(builder, location: loc, result: arg.getType(), src: arg,
1391 /*minIsPoison=*/min_is_poison: true);
1392 break;
1393 }
1394 [[fallthrough]];
1395 case LangOptions::SOB_Trapping:
1396 cgm.errorNYI(e->getSourceRange(), "abs with overflow handling");
1397 return RValue::get(v: nullptr);
1398 }
1399 return RValue::get(v: result);
1400 }
1401
1402 case Builtin::BI__builtin_complex: {
1403 mlir::Value real = emitScalarExpr(e: e->getArg(Arg: 0));
1404 mlir::Value imag = emitScalarExpr(e: e->getArg(Arg: 1));
1405 mlir::Value complex = builder.createComplexCreate(loc, real, imag);
1406 return RValue::getComplex(v: complex);
1407 }
1408
1409 case Builtin::BI__builtin_conj:
1410 case Builtin::BI__builtin_conjf:
1411 case Builtin::BI__builtin_conjl:
1412 case Builtin::BIconj:
1413 case Builtin::BIconjf:
1414 case Builtin::BIconjl: {
1415 mlir::Value complex = emitComplexExpr(e: e->getArg(Arg: 0));
1416 mlir::Value conj = builder.createComplexConj(loc, operand: complex);
1417 return RValue::getComplex(v: conj);
1418 }
1419
1420 case Builtin::BI__builtin_creal:
1421 case Builtin::BI__builtin_crealf:
1422 case Builtin::BI__builtin_creall:
1423 case Builtin::BIcreal:
1424 case Builtin::BIcrealf:
1425 case Builtin::BIcreall: {
1426 mlir::Value complex = emitComplexExpr(e: e->getArg(Arg: 0));
1427 mlir::Value real = builder.createComplexReal(loc, operand: complex);
1428 return RValue::get(v: real);
1429 }
1430
1431 case Builtin::BI__builtin_cimag:
1432 case Builtin::BI__builtin_cimagf:
1433 case Builtin::BI__builtin_cimagl:
1434 case Builtin::BIcimag:
1435 case Builtin::BIcimagf:
1436 case Builtin::BIcimagl: {
1437 mlir::Value complex = emitComplexExpr(e: e->getArg(Arg: 0));
1438 mlir::Value imag = builder.createComplexImag(loc, operand: complex);
1439 return RValue::get(v: imag);
1440 }
1441
1442 case Builtin::BI__builtin_clrsb:
1443 case Builtin::BI__builtin_clrsbl:
1444 case Builtin::BI__builtin_clrsbll:
1445 return emitBuiltinBitOp<cir::BitClrsbOp>(cgf&: *this, e);
1446
1447 case Builtin::BI__builtin_ctzs:
1448 case Builtin::BI__builtin_ctz:
1449 case Builtin::BI__builtin_ctzl:
1450 case Builtin::BI__builtin_ctzll:
1451 assert(!cir::MissingFeatures::builtinCheckKind());
1452 return emitBuiltinBitOp<cir::BitCtzOp>(cgf&: *this, e,
1453 args: getTarget().isCLZForZeroUndef());
1454 case Builtin::BI__builtin_ctzg:
1455 return emitBuiltinBitOpWithFallback<cir::BitCtzOp>(cgf&: *this, e);
1456 case Builtin::BI__builtin_elementwise_ctzg:
1457 cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_ctzg");
1458 return RValue::get(v: nullptr);
1459
1460 case Builtin::BI__builtin_clzs:
1461 case Builtin::BI__builtin_clz:
1462 case Builtin::BI__builtin_clzl:
1463 case Builtin::BI__builtin_clzll:
1464 assert(!cir::MissingFeatures::builtinCheckKind());
1465 return emitBuiltinBitOp<cir::BitClzOp>(cgf&: *this, e,
1466 args: getTarget().isCLZForZeroUndef());
1467 case Builtin::BI__builtin_clzg:
1468 return emitBuiltinBitOpWithFallback<cir::BitClzOp>(cgf&: *this, e);
1469 case Builtin::BI__builtin_elementwise_clzg:
1470 cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_clzg");
1471 return RValue::get(v: nullptr);
1472
1473 case Builtin::BI__builtin_ffs:
1474 case Builtin::BI__builtin_ffsl:
1475 case Builtin::BI__builtin_ffsll:
1476 return emitBuiltinBitOp<cir::BitFfsOp>(cgf&: *this, e);
1477
1478 case Builtin::BI__builtin_parity:
1479 case Builtin::BI__builtin_parityl:
1480 case Builtin::BI__builtin_parityll:
1481 return emitBuiltinBitOp<cir::BitParityOp>(cgf&: *this, e);
1482
1483 case Builtin::BI__lzcnt16:
1484 case Builtin::BI__lzcnt:
1485 case Builtin::BI__lzcnt64:
1486 return emitBuiltinBitOp<cir::BitClzOp>(cgf&: *this, e);
1487
1488 case Builtin::BI__popcnt16:
1489 case Builtin::BI__popcnt:
1490 case Builtin::BI__popcnt64:
1491 case Builtin::BI__builtin_popcount:
1492 case Builtin::BI__builtin_popcountl:
1493 case Builtin::BI__builtin_popcountll:
1494 case Builtin::BI__builtin_popcountg:
1495 return emitBuiltinBitOp<cir::BitPopcountOp>(cgf&: *this, e);
1496
1497 // Always return the argument of __builtin_unpredictable. LLVM does not
1498 // have an intrinsic corresponding to this builtin. Metadata for this
1499 // builtin should be added directly to instructions such as branches or
1500 // switches that use it.
1501 case Builtin::BI__builtin_unpredictable: {
1502 return RValue::get(v: emitScalarExpr(e: e->getArg(Arg: 0)));
1503 }
1504
1505 case Builtin::BI__builtin_expect:
1506 case Builtin::BI__builtin_expect_with_probability: {
1507 mlir::Value argValue = emitScalarExpr(e: e->getArg(Arg: 0));
1508 if (cgm.getCodeGenOpts().OptimizationLevel == 0)
1509 return RValue::get(v: argValue);
1510
1511 mlir::Value expectedValue = emitScalarExpr(e: e->getArg(Arg: 1));
1512
1513 mlir::FloatAttr probAttr;
1514 if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {
1515 llvm::APFloat probability(0.0);
1516 const Expr *probArg = e->getArg(Arg: 2);
1517 [[maybe_unused]] bool evalSucceeded =
1518 probArg->EvaluateAsFloat(Result&: probability, Ctx: cgm.getASTContext());
1519 assert(evalSucceeded &&
1520 "probability should be able to evaluate as float");
1521 bool loseInfo = false; // ignored
1522 probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
1523 RM: llvm::RoundingMode::Dynamic, losesInfo: &loseInfo);
1524 probAttr = mlir::FloatAttr::get(type: mlir::Float64Type::get(context: &getMLIRContext()),
1525 value: probability);
1526 }
1527
1528 auto result = cir::ExpectOp::create(builder, location: loc, result: argValue.getType(),
1529 val: argValue, expected: expectedValue, prob: probAttr);
1530 return RValue::get(v: result);
1531 }
1532
1533 case Builtin::BI__builtin_assume_aligned: {
1534 const Expr *ptrExpr = e->getArg(Arg: 0);
1535 mlir::Value ptrValue = emitScalarExpr(e: ptrExpr);
1536 mlir::Value offsetValue =
1537 (e->getNumArgs() > 2) ? emitScalarExpr(e: e->getArg(Arg: 2)) : nullptr;
1538
1539 std::optional<llvm::APSInt> alignment =
1540 e->getArg(Arg: 1)->getIntegerConstantExpr(Ctx: getContext());
1541 assert(alignment.has_value() &&
1542 "the second argument to __builtin_assume_aligned must be an "
1543 "integral constant expression");
1544
1545 mlir::Value result =
1546 emitAlignmentAssumption(ptrValue, expr: ptrExpr, assumptionLoc: ptrExpr->getExprLoc(),
1547 alignment: alignment->getSExtValue(), offsetValue);
1548 return RValue::get(v: result);
1549 }
1550
1551 case Builtin::BI__builtin_assume_dereferenceable: {
1552 mlir::Value ptrValue = emitScalarExpr(e: e->getArg(Arg: 0));
1553 mlir::Value sizeValue = emitScalarExpr(e: e->getArg(Arg: 1));
1554 // The `dereferenceable` operand bundle expects a pointer-sized unsigned
1555 // integer; widen/narrow as needed.
1556 mlir::Type uintPtrTy = convertType(t: getContext().getUIntPtrType());
1557 if (sizeValue.getType() != uintPtrTy)
1558 sizeValue = builder.createIntCast(src: sizeValue, newTy: uintPtrTy);
1559 mlir::Value cond = builder.getBool(state: true, loc);
1560 cir::AssumeOp::create(builder, location: loc, predicate: cond,
1561 bundle_kind: cir::AssumeBundleKind::Dereferenceable,
1562 bundle_args: mlir::ValueRange{ptrValue, sizeValue});
1563 return RValue::get(v: nullptr);
1564 }
1565
1566 case Builtin::BI__builtin_annotation: {
1567 mlir::Value value = emitScalarExpr(e: e->getArg(Arg: 0));
1568 StringRef str =
1569 cast<StringLiteral>(Val: e->getArg(Arg: 1)->IgnoreParenCasts())->getString();
1570 auto annotation = cir::AnnotationAttr::get(
1571 context: &getMLIRContext(), name: builder.getStringAttr(bytes: str), /*args=*/{});
1572 return RValue::get(v: cir::AnnotationOp::create(
1573 builder, location: getLoc(srcLoc: e->getExprLoc()), value, annotation));
1574 }
1575
1576 case Builtin::BI__assume:
1577 case Builtin::BI__builtin_assume: {
1578 if (e->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
1579 return RValue::get(v: nullptr);
1580
1581 mlir::Value argValue = emitCheckedArgForAssume(e: e->getArg(Arg: 0));
1582 cir::AssumeOp::create(builder, location: loc, predicate: argValue, bundle_kind: cir::AssumeBundleKind::None,
1583 bundle_args: mlir::ValueRange{});
1584 return RValue::get(v: nullptr);
1585 }
1586
1587 case Builtin::BI__builtin_assume_separate_storage: {
1588 mlir::Value value0 = emitScalarExpr(e: e->getArg(Arg: 0));
1589 mlir::Value value1 = emitScalarExpr(e: e->getArg(Arg: 1));
1590 mlir::Value cond = builder.getBool(state: true, loc);
1591 cir::AssumeOp::create(builder, location: loc, predicate: cond,
1592 bundle_kind: cir::AssumeBundleKind::SeparateStorage,
1593 bundle_args: mlir::ValueRange{value0, value1});
1594 return RValue::get(v: nullptr);
1595 }
1596
1597 case Builtin::BI__arithmetic_fence: {
1598 assert(!cir::MissingFeatures::fastMathFlags());
1599 QualType argType = e->getArg(Arg: 0)->getType();
1600 FPOptions fpFeatures = e->getFPFeaturesInEffect(LO: getLangOpts());
1601 if (fpFeatures.getAllowFPReassociate() &&
1602 getContext().getTargetInfo().checkArithmeticFenceSupported()) {
1603 cgm.errorNYI(e->getSourceRange(), "__arithmetic_fence with reassocc");
1604 }
1605 if (argType->isComplexType())
1606 return RValue::getComplex(v: emitComplexExpr(e: e->getArg(Arg: 0)));
1607 return RValue::get(v: emitScalarExpr(e: e->getArg(Arg: 0)));
1608 }
1609
1610 case Builtin::BI__builtin_bswapg: {
1611 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
1612 // CIR models bool as cir.bool rather than an integer, so peel it off
1613 // before the cast below. Like classic codegen's i1 case, it byte-swaps
1614 // to itself.
1615 if (mlir::isa<cir::BoolType>(Val: arg.getType()))
1616 return RValue::get(v: arg);
1617 auto argTy = mlir::cast<cir::IntType>(Val: arg.getType());
1618 // A single bit or a single byte byte-swaps to itself.
1619 if (argTy.getWidth() == 1 || argTy.getWidth() == 8)
1620 return RValue::get(v: arg);
1621 assert(argTy.getWidth() % 16 == 0 &&
1622 "__builtin_bswapg requires a single byte or a multiple of 16 bits");
1623 // cir.byte_swap requires an unsigned operand. Reinterpret a signed
1624 // argument as unsigned of the same width; createBuiltinBitOp casts the
1625 // swapped result back to the builtin's (possibly signed) return type.
1626 if (argTy.isSigned())
1627 arg = builder.createIntCast(src: arg, newTy: builder.getUIntNTy(n: argTy.getWidth()));
1628 return RValue::get(v: createBuiltinBitOp<cir::ByteSwapOp>(cgf&: *this, e, arg));
1629 }
1630
1631 case Builtin::BI__builtin_bswap16:
1632 case Builtin::BI__builtin_bswap32:
1633 case Builtin::BI__builtin_bswap64:
1634 case Builtin::BI_byteswap_ushort:
1635 case Builtin::BI_byteswap_ulong:
1636 case Builtin::BI_byteswap_uint64: {
1637 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
1638 return RValue::get(v: cir::ByteSwapOp::create(builder, location: loc, input: arg));
1639 }
1640
1641 case Builtin::BI__builtin_bitreverse8:
1642 case Builtin::BI__builtin_bitreverse16:
1643 case Builtin::BI__builtin_bitreverse32:
1644 case Builtin::BI__builtin_bitreverse64: {
1645 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
1646 return RValue::get(v: cir::BitReverseOp::create(builder, location: loc, input: arg));
1647 }
1648
1649 case Builtin::BI__builtin_rotateleft8:
1650 case Builtin::BI__builtin_rotateleft16:
1651 case Builtin::BI__builtin_rotateleft32:
1652 case Builtin::BI__builtin_rotateleft64:
1653 case Builtin::BI__builtin_stdc_rotate_left:
1654 case Builtin::BIstdc_rotate_left_uc:
1655 case Builtin::BIstdc_rotate_left_us:
1656 case Builtin::BIstdc_rotate_left_ui:
1657 case Builtin::BIstdc_rotate_left_ul:
1658 case Builtin::BIstdc_rotate_left_ull:
1659 return emitRotate(e, /*isRotateLeft=*/true);
1660
1661 case Builtin::BI__builtin_rotateright8:
1662 case Builtin::BI__builtin_rotateright16:
1663 case Builtin::BI__builtin_rotateright32:
1664 case Builtin::BI__builtin_rotateright64:
1665 case Builtin::BI__builtin_stdc_rotate_right:
1666 case Builtin::BIstdc_rotate_right_uc:
1667 case Builtin::BIstdc_rotate_right_us:
1668 case Builtin::BIstdc_rotate_right_ui:
1669 case Builtin::BIstdc_rotate_right_ul:
1670 case Builtin::BIstdc_rotate_right_ull:
1671 return emitRotate(e, /*isRotateLeft=*/false);
1672
1673 case Builtin::BIstdc_leading_zeros_uc:
1674 case Builtin::BIstdc_leading_zeros_us:
1675 case Builtin::BIstdc_leading_zeros_ui:
1676 case Builtin::BIstdc_leading_zeros_ul:
1677 case Builtin::BIstdc_leading_zeros_ull:
1678 case Builtin::BI__builtin_stdc_leading_zeros:
1679 return emitStdcCountOp<cir::BitClzOp>(cgf&: *this, e, /*invertArg=*/false,
1680 /*poisonZero=*/args: false);
1681
1682 case Builtin::BIstdc_leading_ones_uc:
1683 case Builtin::BIstdc_leading_ones_us:
1684 case Builtin::BIstdc_leading_ones_ui:
1685 case Builtin::BIstdc_leading_ones_ul:
1686 case Builtin::BIstdc_leading_ones_ull:
1687 case Builtin::BI__builtin_stdc_leading_ones:
1688 return emitStdcCountOp<cir::BitClzOp>(cgf&: *this, e, /*invertArg=*/true,
1689 /*poisonZero=*/args: false);
1690
1691 case Builtin::BIstdc_trailing_zeros_uc:
1692 case Builtin::BIstdc_trailing_zeros_us:
1693 case Builtin::BIstdc_trailing_zeros_ui:
1694 case Builtin::BIstdc_trailing_zeros_ul:
1695 case Builtin::BIstdc_trailing_zeros_ull:
1696 case Builtin::BI__builtin_stdc_trailing_zeros:
1697 return emitStdcCountOp<cir::BitCtzOp>(cgf&: *this, e, /*invertArg=*/false,
1698 /*poisonZero=*/args: false);
1699
1700 case Builtin::BIstdc_trailing_ones_uc:
1701 case Builtin::BIstdc_trailing_ones_us:
1702 case Builtin::BIstdc_trailing_ones_ui:
1703 case Builtin::BIstdc_trailing_ones_ul:
1704 case Builtin::BIstdc_trailing_ones_ull:
1705 case Builtin::BI__builtin_stdc_trailing_ones:
1706 return emitStdcCountOp<cir::BitCtzOp>(cgf&: *this, e, /*invertArg=*/true,
1707 /*poisonZero=*/args: false);
1708
1709 case Builtin::BIstdc_first_leading_zero_uc:
1710 case Builtin::BIstdc_first_leading_zero_us:
1711 case Builtin::BIstdc_first_leading_zero_ui:
1712 case Builtin::BIstdc_first_leading_zero_ul:
1713 case Builtin::BIstdc_first_leading_zero_ull:
1714 case Builtin::BI__builtin_stdc_first_leading_zero:
1715 return emitStdcFirstBit<cir::BitClzOp>(cgf&: *this, e, /*invertArg=*/true,
1716 /*poisonZero=*/args: false);
1717
1718 case Builtin::BIstdc_first_leading_one_uc:
1719 case Builtin::BIstdc_first_leading_one_us:
1720 case Builtin::BIstdc_first_leading_one_ui:
1721 case Builtin::BIstdc_first_leading_one_ul:
1722 case Builtin::BIstdc_first_leading_one_ull:
1723 case Builtin::BI__builtin_stdc_first_leading_one:
1724 return emitStdcFirstBit<cir::BitClzOp>(cgf&: *this, e, /*invertArg=*/false,
1725 /*poisonZero=*/args: false);
1726
1727 case Builtin::BIstdc_first_trailing_zero_uc:
1728 case Builtin::BIstdc_first_trailing_zero_us:
1729 case Builtin::BIstdc_first_trailing_zero_ui:
1730 case Builtin::BIstdc_first_trailing_zero_ul:
1731 case Builtin::BIstdc_first_trailing_zero_ull:
1732 case Builtin::BI__builtin_stdc_first_trailing_zero:
1733 return emitStdcFirstBit<cir::BitCtzOp>(cgf&: *this, e, /*invertArg=*/true,
1734 /*poisonZero=*/args: false);
1735
1736 case Builtin::BIstdc_first_trailing_one_uc:
1737 case Builtin::BIstdc_first_trailing_one_us:
1738 case Builtin::BIstdc_first_trailing_one_ui:
1739 case Builtin::BIstdc_first_trailing_one_ul:
1740 case Builtin::BIstdc_first_trailing_one_ull:
1741 case Builtin::BI__builtin_stdc_first_trailing_one:
1742 return emitStdcFirstBit<cir::BitCtzOp>(cgf&: *this, e, /*invertArg=*/false,
1743 /*poisonZero=*/args: false);
1744
1745 case Builtin::BIstdc_count_zeros_uc:
1746 case Builtin::BIstdc_count_zeros_us:
1747 case Builtin::BIstdc_count_zeros_ui:
1748 case Builtin::BIstdc_count_zeros_ul:
1749 case Builtin::BIstdc_count_zeros_ull:
1750 case Builtin::BI__builtin_stdc_count_zeros:
1751 return emitStdcBitWidthMinus<cir::BitPopcountOp>(cgf&: *this, e);
1752
1753 case Builtin::BIstdc_count_ones_uc:
1754 case Builtin::BIstdc_count_ones_us:
1755 case Builtin::BIstdc_count_ones_ui:
1756 case Builtin::BIstdc_count_ones_ul:
1757 case Builtin::BIstdc_count_ones_ull:
1758 case Builtin::BI__builtin_stdc_count_ones:
1759 return emitStdcCountOp<cir::BitPopcountOp>(cgf&: *this, e,
1760 /*invertArg=*/false);
1761
1762 case Builtin::BIstdc_has_single_bit_uc:
1763 case Builtin::BIstdc_has_single_bit_us:
1764 case Builtin::BIstdc_has_single_bit_ui:
1765 case Builtin::BIstdc_has_single_bit_ul:
1766 case Builtin::BIstdc_has_single_bit_ull:
1767 case Builtin::BI__builtin_stdc_has_single_bit:
1768 return emitStdcHasSingleBit(cgf&: *this, e);
1769
1770 case Builtin::BIstdc_bit_width_uc:
1771 case Builtin::BIstdc_bit_width_us:
1772 case Builtin::BIstdc_bit_width_ui:
1773 case Builtin::BIstdc_bit_width_ul:
1774 case Builtin::BIstdc_bit_width_ull:
1775 case Builtin::BI__builtin_stdc_bit_width:
1776 return emitStdcBitWidthMinus<cir::BitClzOp>(cgf&: *this, e,
1777 /*poisonZero=*/args: false);
1778
1779 case Builtin::BIstdc_bit_floor_uc:
1780 case Builtin::BIstdc_bit_floor_us:
1781 case Builtin::BIstdc_bit_floor_ui:
1782 case Builtin::BIstdc_bit_floor_ul:
1783 case Builtin::BIstdc_bit_floor_ull:
1784 case Builtin::BI__builtin_stdc_bit_floor:
1785 return emitStdcBitFloor(cgf&: *this, e);
1786
1787 case Builtin::BIstdc_bit_ceil_uc:
1788 case Builtin::BIstdc_bit_ceil_us:
1789 case Builtin::BIstdc_bit_ceil_ui:
1790 case Builtin::BIstdc_bit_ceil_ul:
1791 case Builtin::BIstdc_bit_ceil_ull:
1792 case Builtin::BI__builtin_stdc_bit_ceil:
1793 return emitStdcBitCeil(cgf&: *this, e);
1794
1795 // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
1796 case Builtin::BIstdc_memreverse8u8:
1797 return RValue::get(v: emitScalarExpr(e: e->getArg(Arg: 0)));
1798 case Builtin::BIstdc_memreverse8u16:
1799 case Builtin::BIstdc_memreverse8u32:
1800 case Builtin::BIstdc_memreverse8u64: {
1801 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
1802 return RValue::get(v: cir::ByteSwapOp::create(builder, location: loc, input: arg));
1803 }
1804 case Builtin::BIstdc_memreverse8:
1805 case Builtin::BI__builtin_stdc_memreverse8: {
1806 Expr::EvalResult result;
1807 if (e->getArg(Arg: 0)->EvaluateAsInt(Result&: result, Ctx: getContext())) {
1808 uint64_t size = result.Val.getInt().getZExtValue();
1809 if (size <= 1) {
1810 emitIgnoredExpr(e: e->getArg(Arg: 1));
1811 return RValue::get(v: nullptr);
1812 }
1813 if (size == 2 || size == 4 || size == 8) {
1814 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
1815 Address ptrAddr = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
1816 mlir::Type intTy = builder.getUIntNTy(n: size * 8);
1817 Address addr = builder.createElementBitCast(loc, addr: ptrAddr, destType: intTy);
1818 mlir::Value val = builder.createLoad(loc, addr);
1819 mlir::Value swapped = cir::ByteSwapOp::create(builder, location: loc, input: val);
1820 builder.createStore(loc, val: swapped, dst: addr);
1821 return RValue::get(v: nullptr);
1822 }
1823 }
1824 // General case: fall back to the library function stdc_memreverse8.
1825 break;
1826 }
1827
1828 case Builtin::BI__builtin_constant_p: {
1829 mlir::Type resultType = convertType(t: e->getType());
1830
1831 const Expr *arg = e->getArg(Arg: 0);
1832 QualType argType = arg->getType();
1833 // FIXME: The allowance for Obj-C pointers and block pointers is historical
1834 // and likely a mistake.
1835 if (!argType->isIntegralOrEnumerationType() && !argType->isFloatingType() &&
1836 !argType->isObjCObjectPointerType() && !argType->isBlockPointerType()) {
1837 // Per the GCC documentation, only numeric constants are recognized after
1838 // inlining.
1839 return RValue::get(
1840 v: builder.getConstInt(loc: getLoc(srcLoc: e->getSourceRange()),
1841 t: mlir::cast<cir::IntType>(Val&: resultType), c: 0));
1842 }
1843
1844 if (arg->HasSideEffects(Ctx: getContext())) {
1845 // The argument is unevaluated, so be conservative if it might have
1846 // side-effects.
1847 return RValue::get(
1848 v: builder.getConstInt(loc: getLoc(srcLoc: e->getSourceRange()),
1849 t: mlir::cast<cir::IntType>(Val&: resultType), c: 0));
1850 }
1851
1852 mlir::Value argValue = emitScalarExpr(e: arg);
1853 if (argType->isObjCObjectPointerType()) {
1854 cgm.errorNYI(e->getSourceRange(),
1855 "__builtin_constant_p: Obj-C object pointer");
1856 return {};
1857 }
1858 argValue = builder.createBitcast(src: argValue, newTy: convertType(t: argType));
1859
1860 mlir::Value result = cir::IsConstantOp::create(
1861 builder, location: getLoc(srcLoc: e->getSourceRange()), val: argValue);
1862 // IsConstantOp returns a bool, but __builtin_constant_p returns an int.
1863 result = builder.createBoolToInt(src: result, newTy: resultType);
1864 return RValue::get(v: result);
1865 }
1866 case Builtin::BI__builtin_dynamic_object_size:
1867 case Builtin::BI__builtin_object_size: {
1868 unsigned type =
1869 e->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
1870 auto resType = mlir::cast<cir::IntType>(Val: convertType(t: e->getType()));
1871
1872 // We pass this builtin onto the optimizer so that it can figure out the
1873 // object size in more complex cases.
1874 bool isDynamic = builtinID == Builtin::BI__builtin_dynamic_object_size;
1875 return RValue::get(v: emitBuiltinObjectSize(e: e->getArg(Arg: 0), type, resType,
1876 /*EmittedE=*/emittedE: nullptr, isDynamic));
1877 }
1878
1879 case Builtin::BI__builtin_prefetch: {
1880 auto evaluateOperandAsInt = [&](const Expr *arg) {
1881 Expr::EvalResult res;
1882 [[maybe_unused]] bool evalSucceed =
1883 arg->EvaluateAsInt(Result&: res, Ctx: cgm.getASTContext());
1884 assert(evalSucceed && "expression should be able to evaluate as int");
1885 return res.Val.getInt().getZExtValue();
1886 };
1887
1888 bool isWrite = false;
1889 if (e->getNumArgs() > 1)
1890 isWrite = evaluateOperandAsInt(e->getArg(Arg: 1));
1891
1892 int locality = 3;
1893 if (e->getNumArgs() > 2)
1894 locality = evaluateOperandAsInt(e->getArg(Arg: 2));
1895
1896 mlir::Value address = emitScalarExpr(e: e->getArg(Arg: 0));
1897 cir::PrefetchOp::create(builder, location: loc, addr: address, locality, isWrite);
1898 return RValue::get(v: nullptr);
1899 }
1900 case Builtin::BI__builtin_readcyclecounter:
1901 return RValue::get(v: builder.emitIntrinsicCallOp(
1902 loc: getLoc(srcLoc: e->getExprLoc()), str: "readcyclecounter", resTy: builder.getUInt64Ty()));
1903 case Builtin::BI__builtin_readsteadycounter:
1904 return RValue::get(v: builder.emitIntrinsicCallOp(
1905 loc: getLoc(srcLoc: e->getExprLoc()), str: "readsteadycounter", resTy: builder.getUInt64Ty()));
1906 case Builtin::BI__builtin___clear_cache: {
1907 mlir::Value begin =
1908 builder.createPtrBitcast(src: emitScalarExpr(e: e->getArg(Arg: 0)), newPointeeTy: cgm.voidTy);
1909 mlir::Value end =
1910 builder.createPtrBitcast(src: emitScalarExpr(e: e->getArg(Arg: 1)), newPointeeTy: cgm.voidTy);
1911 cir::ClearCacheOp::create(builder, location: getLoc(srcLoc: e->getSourceRange()), begin, end);
1912 return RValue::get(v: nullptr);
1913 }
1914 case Builtin::BI__builtin_trap:
1915 emitTrap(loc, /*createNewBlock=*/true);
1916 return RValue::getIgnored();
1917 case Builtin::BI__builtin_verbose_trap:
1918 assert(!cir::MissingFeatures::generateDebugInfo());
1919 emitTrap(loc, /*createNewBlock=*/true);
1920 return RValue::getIgnored();
1921 case Builtin::BI__debugbreak:
1922 return errorBuiltinNYI(cgf&: *this, e, builtinID);
1923 case Builtin::BI__builtin_unreachable:
1924 emitUnreachable(loc: e->getExprLoc(), /*createNewBlock=*/true);
1925 return RValue::getIgnored();
1926 case Builtin::BI__builtin_powi:
1927 case Builtin::BI__builtin_powif:
1928 case Builtin::BI__builtin_powil: {
1929 mlir::Value src0 = emitScalarExpr(e: e->getArg(Arg: 0));
1930 mlir::Value src1 = emitScalarExpr(e: e->getArg(Arg: 1));
1931 return RValue::get(v: builder.emitIntrinsicCallOp(
1932 loc: getLoc(srcLoc: e->getExprLoc()), str: "powi", resTy: src0.getType(),
1933 op: mlir::ValueRange{src0, src1}));
1934 }
1935 case Builtin::BI__builtin_frexpl:
1936 case Builtin::BI__builtin_frexp:
1937 case Builtin::BI__builtin_frexpf:
1938 case Builtin::BI__builtin_frexpf128:
1939 case Builtin::BI__builtin_frexpf16: {
1940 mlir::Value val = emitScalarExpr(e: e->getArg(Arg: 0));
1941 mlir::Value ptr = emitScalarExpr(e: e->getArg(Arg: 1));
1942 mlir::Type fpTy = val.getType();
1943 QualType intQualTy = e->getArg(Arg: 1)->getType()->getPointeeType();
1944 mlir::Type intTy = convertType(t: intQualTy);
1945 mlir::Location callLoc = getLoc(srcLoc: e->getExprLoc());
1946 auto frexpOp = cir::FrexpOp::create(builder, location: callLoc, result: fpTy, exp: intTy, src: val);
1947 LValue lv = makeNaturalAlignAddrLValue(val: ptr, ty: intQualTy);
1948 emitStoreOfScalar(value: frexpOp.getExp(), lvalue: lv, /*isInit=*/false);
1949 return RValue::get(v: frexpOp.getResult());
1950 }
1951 case Builtin::BImodf:
1952 case Builtin::BImodff:
1953 case Builtin::BImodfl:
1954 case Builtin::BI__builtin_modf:
1955 case Builtin::BI__builtin_modff:
1956 case Builtin::BI__builtin_modfl: {
1957 mlir::Value val = emitScalarExpr(e: e->getArg(Arg: 0));
1958 mlir::Value ptr = emitScalarExpr(e: e->getArg(Arg: 1));
1959 mlir::Type fpTy = val.getType();
1960 mlir::Location callLoc = getLoc(srcLoc: e->getExprLoc());
1961 auto modfOp = cir::ModfOp::create(builder, location: callLoc, fractional: fpTy, integral: fpTy, src: val);
1962 QualType destPtrTy = e->getArg(Arg: 1)->getType()->getPointeeType();
1963 LValue lv = makeNaturalAlignAddrLValue(val: ptr, ty: destPtrTy);
1964 emitStoreOfScalar(value: modfOp.getIntegral(), lvalue: lv, /*isInit=*/false);
1965 return RValue::get(v: modfOp.getFractional());
1966 }
1967 case Builtin::BI__builtin_isgreater:
1968 case Builtin::BI__builtin_isgreaterequal:
1969 case Builtin::BI__builtin_isless:
1970 case Builtin::BI__builtin_islessequal:
1971 case Builtin::BI__builtin_islessgreater:
1972 case Builtin::BI__builtin_isunordered: {
1973 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
1974 mlir::Value lhs = emitScalarExpr(e: e->getArg(Arg: 0));
1975 mlir::Value rhs = emitScalarExpr(e: e->getArg(Arg: 1));
1976 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
1977 mlir::Type intTy = convertType(t: e->getType());
1978
1979 mlir::Value cmpResult;
1980 switch (builtinID) {
1981 case Builtin::BI__builtin_isgreater:
1982 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::gt, lhs, rhs);
1983 break;
1984 case Builtin::BI__builtin_isgreaterequal:
1985 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::ge, lhs, rhs);
1986 break;
1987 case Builtin::BI__builtin_isless:
1988 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs, rhs);
1989 break;
1990 case Builtin::BI__builtin_islessequal:
1991 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::le, lhs, rhs);
1992 break;
1993 case Builtin::BI__builtin_islessgreater:
1994 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::one, lhs, rhs);
1995 break;
1996 case Builtin::BI__builtin_isunordered:
1997 cmpResult = builder.createCompare(loc, kind: cir::CmpOpKind::uno, lhs, rhs);
1998 break;
1999 default:
2000 llvm_unreachable("Unknown ordered comparison");
2001 }
2002 return RValue::get(v: builder.createBoolToInt(src: cmpResult, newTy: intTy));
2003 }
2004 // From https://clang.llvm.org/docs/LanguageExtensions.html#builtin-isfpclass
2005 //
2006 // The `__builtin_isfpclass()` builtin is a generalization of functions
2007 // isnan, isinf, isfinite and some others defined by the C standard. It tests
2008 // if the floating-point value, specified by the first argument, falls into
2009 // any of data classes, specified by the second argument.
2010 case Builtin::BI__builtin_isnan: {
2011 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2012 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2013 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2014 return RValue::get(v: builder.createBoolToInt(
2015 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Nan),
2016 newTy: convertType(t: e->getType())));
2017 }
2018
2019 case Builtin::BI__builtin_issignaling: {
2020 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2021 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2022 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2023 return RValue::get(v: builder.createBoolToInt(
2024 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::SignalingNaN),
2025 newTy: convertType(t: e->getType())));
2026 }
2027
2028 case Builtin::BI__builtin_isinf: {
2029 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2030 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2031 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2032 return RValue::get(v: builder.createBoolToInt(
2033 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Infinity),
2034 newTy: convertType(t: e->getType())));
2035 }
2036 case Builtin::BIfinite:
2037 case Builtin::BI__finite:
2038 case Builtin::BIfinitef:
2039 case Builtin::BI__finitef:
2040 case Builtin::BIfinitel:
2041 case Builtin::BI__finitel:
2042 case Builtin::BI__builtin_isfinite: {
2043 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2044 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2045 return RValue::get(v: builder.createBoolToInt(
2046 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Finite),
2047 newTy: convertType(t: e->getType())));
2048 }
2049
2050 case Builtin::BI__builtin_isnormal: {
2051 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2052 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2053 return RValue::get(v: builder.createBoolToInt(
2054 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Normal),
2055 newTy: convertType(t: e->getType())));
2056 }
2057
2058 case Builtin::BI__builtin_issubnormal: {
2059 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2060 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2061 return RValue::get(v: builder.createBoolToInt(
2062 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Subnormal),
2063 newTy: convertType(t: e->getType())));
2064 }
2065
2066 case Builtin::BI__builtin_iszero: {
2067 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2068 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2069 return RValue::get(v: builder.createBoolToInt(
2070 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest::Zero),
2071 newTy: convertType(t: e->getType())));
2072 }
2073 case Builtin::BI__builtin_isfpclass: {
2074 Expr::EvalResult result;
2075 if (!e->getArg(Arg: 1)->EvaluateAsInt(Result&: result, Ctx: cgm.getASTContext()))
2076 break;
2077
2078 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2079 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2080 uint64_t test = result.Val.getInt().getLimitedValue();
2081 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2082 return RValue::get(v: builder.createBoolToInt(
2083 src: builder.createIsFPClass(loc, src: v, flags: cir::FPClassTest(test)),
2084 newTy: convertType(t: e->getType())));
2085 }
2086 case Builtin::BI__builtin_nondeterministic_value: {
2087 mlir::Type ty = convertType(t: e->getArg(Arg: 0)->getType());
2088 mlir::Value result =
2089 cir::ConstantOp::create(builder, location: loc, res: ty, value: cir::PoisonAttr::get(type: ty));
2090 result = cir::FreezeOp::create(builder, location: loc, input: result);
2091 return RValue::get(v: result);
2092 }
2093 case Builtin::BI__builtin_elementwise_abs: {
2094 mlir::Type cirTy = convertType(t: e->getArg(Arg: 0)->getType());
2095 bool isIntTy = cir::isIntOrVectorOfIntType(type: cirTy);
2096 if (!isIntTy)
2097 return emitUnaryFPBuiltin<cir::FAbsOp>(cgf&: *this, e: *e);
2098 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
2099 mlir::Value result = cir::AbsOp::create(builder, location: getLoc(srcLoc: e->getExprLoc()),
2100 result: arg.getType(), src: arg, min_is_poison: false);
2101 return RValue::get(v: result);
2102 }
2103 case Builtin::BI__builtin_elementwise_bitreverse:
2104 case Builtin::BI__builtin_elementwise_popcount:
2105 case Builtin::BI__builtin_elementwise_canonicalize:
2106 case Builtin::BI__builtin_elementwise_copysign:
2107 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2108 case Builtin::BI__builtin_elementwise_fshl:
2109 return emitTernarySameTypeBuiltin<cir::FshlOp>(cgf&: *this, e: *e);
2110 case Builtin::BI__builtin_elementwise_fshr:
2111 return emitTernarySameTypeBuiltin<cir::FshrOp>(cgf&: *this, e: *e);
2112 case Builtin::BI__builtin_elementwise_clmul:
2113 case Builtin::BI__builtin_elementwise_pext:
2114 case Builtin::BI__builtin_elementwise_pdep:
2115 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2116 case Builtin::BI__builtin_elementwise_add_sat:
2117 case Builtin::BI__builtin_elementwise_sub_sat: {
2118 // The saturating ops have no !cir.bool form, and emitScalarExpr on a bool
2119 // vector hits the NYI bool-vector load and returns null, so bail first.
2120 QualType argTy = e->getArg(Arg: 0)->getType();
2121 if (argTy->isBooleanType() || argTy->isExtVectorBoolType()) {
2122 cgm.errorNYI(e->getSourceRange(),
2123 "saturating add/sub on a boolean operand");
2124 return RValue::get(v: nullptr);
2125 }
2126 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2127 mlir::Value op0 = emitScalarExpr(e: e->getArg(Arg: 0));
2128 mlir::Value op1 = emitScalarExpr(e: e->getArg(Arg: 1));
2129 assert(cir::isIntOrVectorOfIntType(op0.getType()) &&
2130 "elementwise saturating add/sub requires integer operands");
2131 mlir::Value val =
2132 builtinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat
2133 ? builder.createAddSat(loc, lhs: op0, rhs: op1)
2134 : builder.createSubSat(loc, lhs: op0, rhs: op1);
2135 return RValue::get(v: val);
2136 }
2137 case Builtin::BI__builtin_elementwise_max: {
2138 if (cir::isIntOrVectorOfIntType(type: convertType(t: e->getArg(Arg: 0)->getType()))) {
2139 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2140 mlir::Value op0 = emitScalarExpr(e: e->getArg(Arg: 0));
2141 mlir::Value op1 = emitScalarExpr(e: e->getArg(Arg: 1));
2142 return RValue::get(v: builder.createMax(loc, lhs: op0, rhs: op1));
2143 }
2144 return RValue::get(
2145 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(cgf&: *this, e: *e));
2146 }
2147 case Builtin::BI__builtin_elementwise_min: {
2148 if (cir::isIntOrVectorOfIntType(type: convertType(t: e->getArg(Arg: 0)->getType()))) {
2149 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2150 mlir::Value op0 = emitScalarExpr(e: e->getArg(Arg: 0));
2151 mlir::Value op1 = emitScalarExpr(e: e->getArg(Arg: 1));
2152 return RValue::get(v: builder.createMin(loc, lhs: op0, rhs: op1));
2153 }
2154 return RValue::get(
2155 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(cgf&: *this, e: *e));
2156 }
2157 case Builtin::BI__builtin_elementwise_maxnum:
2158 return RValue::get(
2159 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(cgf&: *this, e: *e));
2160 case Builtin::BI__builtin_elementwise_minnum:
2161 return RValue::get(
2162 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(cgf&: *this, e: *e));
2163 case Builtin::BI__builtin_elementwise_maximum:
2164 return RValue::get(
2165 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMaximumOp>(cgf&: *this, e: *e));
2166 case Builtin::BI__builtin_elementwise_minimum:
2167 return RValue::get(
2168 v: emitBinaryMaybeConstrainedFPBuiltin<cir::FMinimumOp>(cgf&: *this, e: *e));
2169
2170 case Builtin::BI__builtin_elementwise_maximumnum:
2171 case Builtin::BI__builtin_elementwise_minimumnum:
2172 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2173 case Builtin::BI__builtin_reduce_max:
2174 case Builtin::BI__builtin_reduce_min: {
2175 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2176 QualType type = e->getArg(Arg: 0)->getType();
2177 if (const auto *vecTy = type->getAs<VectorType>())
2178 type = vecTy->getElementType();
2179 else if (type->isSizelessVectorType())
2180 type = type->getSizelessVectorEltType(Ctx: getContext());
2181
2182 mlir::Value input = emitScalarExpr(e: e->getArg(Arg: 0));
2183 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2184 const bool isMax = builtinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_max;
2185
2186 if (type->isSignedIntegerType()) {
2187 if (isMax)
2188 return RValue::get(
2189 v: cir::VecReduceSMaxOp::create(builder, location: loc, input).getResult());
2190 return RValue::get(
2191 v: cir::VecReduceSMinOp::create(builder, location: loc, input).getResult());
2192 }
2193
2194 if (type->isUnsignedIntegerType()) {
2195 if (isMax)
2196 return RValue::get(
2197 v: cir::VecReduceUMaxOp::create(builder, location: loc, input).getResult());
2198 return RValue::get(
2199 v: cir::VecReduceUMinOp::create(builder, location: loc, input).getResult());
2200 }
2201
2202 assert(type->isFloatingType() && "must have a float here");
2203 cir::FastMathFlagsAttr fastMath =
2204 getFastMathFlagsAttr(fastMathFlags: getCurrentFastMathFlags());
2205 if (isMax)
2206 return RValue::get(
2207 v: cir::VecReduceFMaxOp::create(builder, location: loc, input, fastmath_flags: fastMath)
2208 .getResult());
2209 return RValue::get(
2210 v: cir::VecReduceFMinOp::create(builder, location: loc, input, fastmath_flags: fastMath)
2211 .getResult());
2212 }
2213 case Builtin::BI__builtin_reduce_add:
2214 return emitBuiltinVectorReduction<cir::VecReduceAddOp>(cgf&: *this, e);
2215 case Builtin::BI__builtin_reduce_mul:
2216 return emitBuiltinVectorReduction<cir::VecReduceMulOp>(cgf&: *this, e);
2217 case Builtin::BI__builtin_reduce_xor:
2218 return emitBuiltinVectorReduction<cir::VecReduceXorOp>(cgf&: *this, e);
2219 case Builtin::BI__builtin_reduce_or:
2220 return emitBuiltinVectorReduction<cir::VecReduceOrOp>(cgf&: *this, e);
2221 case Builtin::BI__builtin_reduce_and:
2222 return emitBuiltinVectorReduction<cir::VecReduceAndOp>(cgf&: *this, e);
2223 case Builtin::BI__builtin_reduce_maximum:
2224 case Builtin::BI__builtin_reduce_minimum:
2225 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2226 case Builtin::BI__builtin_reduce_assoc_fadd:
2227 case Builtin::BI__builtin_reduce_in_order_fadd: {
2228 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2229 bool isAssociative =
2230 builtinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_assoc_fadd;
2231
2232 assert((isAssociative ? e->getNumArgs() == 1 || e->getNumArgs() == 2
2233 : e->getNumArgs() == 2) &&
2234 "invalid argument count for floating-point reduction");
2235 mlir::Value vector = emitScalarExpr(e: e->getArg(Arg: 0));
2236 auto vectorTy = cast<cir::VectorType>(Val: vector.getType());
2237 mlir::Type scalarTy = vectorTy.getElementType();
2238 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2239 mlir::Value startValue;
2240 if (e->getNumArgs() == 2) {
2241 startValue = emitScalarExpr(e: e->getArg(Arg: 1));
2242 if (startValue.getType() != scalarTy)
2243 startValue =
2244 builder.createCast(loc: getLoc(srcLoc: e->getArg(Arg: 1)->getExprLoc()),
2245 kind: cir::CastKind::floating, src: startValue, newTy: scalarTy);
2246 } else {
2247 auto fpTy = cast<cir::FPTypeInterface>(Val&: scalarTy);
2248 startValue = cir::ConstantOp::create(
2249 builder, location: loc,
2250 value: cir::FPAttr::get(type: scalarTy,
2251 value: llvm::APFloat::getZero(Sem: fpTy.getFloatSemantics(),
2252 /*Negative=*/true)));
2253 }
2254
2255 cir::FastMathFlags fastMathFlags = getCurrentFastMathFlags();
2256 if (isAssociative)
2257 fastMathFlags |= cir::FastMathFlags::reassoc;
2258 cir::FastMathFlagsAttr fastMath = getFastMathFlagsAttr(fastMathFlags);
2259
2260 auto reduction = cir::VecReduceFAddOp::create(builder, location: loc, accumulator: startValue,
2261 input: vector, fastmath_flags: fastMath);
2262 return RValue::get(v: reduction.getResult());
2263 }
2264 case Builtin::BI__builtin_matrix_transpose: {
2265 mlir::Value matrix = emitScalarExpr(e: e->getArg(Arg: 0));
2266 mlir::Value result = builder.createMatrixTranspose(loc, matrix);
2267 return RValue::get(v: result);
2268 }
2269 case Builtin::BI__builtin_matrix_column_major_load: {
2270 // Emit everything that isn't dependent on the first parameter type
2271 mlir::Value stride = emitScalarExpr(e: e->getArg(Arg: 3));
2272 const QualType resultTy = e->getType();
2273 mlir::Type resultType = convertType(t: resultTy);
2274 auto *ptrTy = e->getArg(Arg: 0)->getType()->getAs<PointerType>();
2275 assert(ptrTy && "arg0 must be of pointer type");
2276 bool isVolatile = ptrTy->getPointeeType().isVolatileQualified();
2277 Address src = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2278 emitNonNullArgCheck(rv: RValue::get(v: src.emitRawPointer()),
2279 argType: e->getArg(Arg: 0)->getType(), argLoc: e->getArg(Arg: 0)->getExprLoc(), ac: fd,
2280 paramNum: 0);
2281 mlir::Value dataPtr = src.emitRawPointer();
2282 mlir::Value result = builder.createMatrixColumnMajorLoad(
2283 loc, resultTy: resultType, value: dataPtr, stride, isVolatile);
2284 return RValue::get(v: result);
2285 }
2286 case Builtin::BI__builtin_matrix_column_major_store: {
2287 mlir::Value matrix = emitScalarExpr(e: e->getArg(Arg: 0));
2288 Address dst = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
2289 mlir::Value stride = emitScalarExpr(e: e->getArg(Arg: 2));
2290
2291 auto *ptrTy = e->getArg(Arg: 1)->getType()->getAs<PointerType>();
2292 assert(ptrTy && "arg1 must be of pointer type");
2293 bool isVolatile = ptrTy->getPointeeType().isVolatileQualified();
2294
2295 emitNonNullArgCheck(rv: RValue::get(v: dst.emitRawPointer()),
2296 argType: e->getArg(Arg: 1)->getType(), argLoc: e->getArg(Arg: 1)->getExprLoc(), ac: fd,
2297 paramNum: 0);
2298 builder.createMatrixColumnMajorStore(loc, matrix, data: dst.emitRawPointer(),
2299 stride, isVolatile);
2300 return RValue::get(v: nullptr);
2301 }
2302 case Builtin::BI__builtin_masked_load:
2303 case Builtin::BI__builtin_masked_expand_load:
2304 case Builtin::BI__builtin_masked_gather:
2305 case Builtin::BI__builtin_masked_store:
2306 case Builtin::BI__builtin_masked_compress_store:
2307 case Builtin::BI__builtin_masked_scatter:
2308 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2309 case Builtin::BI__builtin_isinf_sign: {
2310 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
2311 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2312 mlir::Value arg = emitScalarExpr(e: e->getArg(Arg: 0));
2313 mlir::Value isInf =
2314 builder.createIsFPClass(loc, src: arg, flags: cir::FPClassTest::Infinity);
2315 mlir::Value isNeg = emitSignBit(loc, cgf&: *this, val: arg);
2316 mlir::Type intTy = convertType(t: e->getType());
2317 cir::ConstantOp zero = builder.getNullValue(ty: intTy, loc);
2318 cir::ConstantOp one = builder.getConstant(loc, attr: cir::IntAttr::get(type: intTy, value: 1));
2319 cir::ConstantOp negativeOne =
2320 builder.getConstant(loc, attr: cir::IntAttr::get(type: intTy, value: -1));
2321 mlir::Value signResult = builder.createSelect(loc, condition: isNeg, trueValue: negativeOne, falseValue: one);
2322 mlir::Value result = builder.createSelect(loc, condition: isInf, trueValue: signResult, falseValue: zero);
2323 return RValue::get(v: result);
2324 }
2325 case Builtin::BI__builtin_flt_rounds: {
2326 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2327 mlir::Type resultType = convertType(t: e->getType());
2328 mlir::Value result =
2329 builder.emitIntrinsicCallOp(loc, str: "get.rounding", resTy: resultType);
2330 if (result.getType() != resultType)
2331 result =
2332 builder.createCast(loc, kind: cir::CastKind::integral, src: result, newTy: resultType);
2333 return RValue::get(v: result);
2334 }
2335 case Builtin::BI__builtin_set_flt_rounds: {
2336 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2337 mlir::Value v = emitScalarExpr(e: e->getArg(Arg: 0));
2338 builder.emitIntrinsicCallOp(loc, str: "set.rounding", resTy: builder.getVoidTy(),
2339 op: mlir::ValueRange{v});
2340 return RValue::get(v: nullptr);
2341 }
2342 case Builtin::BI__builtin_fpclassify: {
2343 CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
2344 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2345 mlir::Value value = emitScalarExpr(e: e->getArg(Arg: 5));
2346 mlir::Type resultTy = convertType(t: e->getType());
2347 // if isZero then
2348 // result = FP_ZERO
2349 // elseif isNan then
2350 // result = FP_NAN
2351 // elseif isInfinity then
2352 // result = FP_INFINITE
2353 // elseif isNormal then
2354 // result = FP_NORMAL
2355 // else
2356 // result = FP_SUBNORMAL
2357 auto isZero =
2358 cir::IsFPClassOp::create(builder, location: loc, src: value, flags: cir::FPClassTest::Zero);
2359 mlir::Value result =
2360 cir::TernaryOp::create(
2361 builder, location: loc, cond: isZero,
2362 /*thenBuilder=*/
2363 trueBuilder: [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
2364 mlir::Value zeroLiteral = emitScalarExpr(e: e->getArg(Arg: 4));
2365 cir::YieldOp::create(builder&: opBuilder, location, args: zeroLiteral);
2366 },
2367 /*elseBuilder=*/
2368 falseBuilder: [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
2369 auto isNan = cir::IsFPClassOp::create(builder&: opBuilder, location, src: value,
2370 flags: cir::FPClassTest::Nan);
2371 mlir::Value nanResult =
2372 cir::TernaryOp::create(
2373 builder&: opBuilder, location, cond: isNan,
2374 /*thenBuilder=*/
2375 trueBuilder: [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
2376 mlir::Value nanLiteral = emitScalarExpr(e: e->getArg(Arg: 0));
2377 cir::YieldOp::create(builder&: opBuilder, location, args: nanLiteral);
2378 },
2379 /*elseBuilder=*/
2380 falseBuilder: [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
2381 auto isInfinity = cir::IsFPClassOp::create(
2382 builder&: opBuilder, location, src: value,
2383 flags: cir::FPClassTest::Infinity);
2384 mlir::Value infResult =
2385 cir::TernaryOp::create(
2386 builder&: opBuilder, location, cond: isInfinity,
2387 /*thenBuilder=*/
2388 trueBuilder: [&](mlir::OpBuilder &opBuilder,
2389 mlir::Location location) {
2390 mlir::Value infinityLiteral =
2391 emitScalarExpr(e: e->getArg(Arg: 1));
2392 cir::YieldOp::create(builder&: opBuilder, location,
2393 args: infinityLiteral);
2394 },
2395 /*elseBuilder=*/
2396 falseBuilder: [&](mlir::OpBuilder &opBuilder,
2397 mlir::Location location) {
2398 auto isNormal = cir::IsFPClassOp::create(
2399 builder&: opBuilder, location, src: value,
2400 flags: cir::FPClassTest::Normal);
2401 mlir::Value fpNormal =
2402 emitScalarExpr(e: e->getArg(Arg: 2));
2403 mlir::Value fpSubnormal =
2404 emitScalarExpr(e: e->getArg(Arg: 3));
2405 mlir::Value returnValue =
2406 cir::SelectOp::create(
2407 builder&: opBuilder, location, result: resultTy,
2408 condition: isNormal, true_value: fpNormal, false_value: fpSubnormal);
2409 cir::YieldOp::create(builder&: opBuilder, location,
2410 args: returnValue);
2411 })
2412 .getResult();
2413 cir::YieldOp::create(builder&: opBuilder, location, args: infResult);
2414 })
2415 .getResult();
2416 cir::YieldOp::create(builder&: opBuilder, location, args: nanResult);
2417 })
2418 .getResult();
2419 return RValue::get(v: result);
2420 }
2421 case Builtin::BIalloca:
2422 case Builtin::BI_alloca:
2423 case Builtin::BI__builtin_alloca_uninitialized:
2424 case Builtin::BI__builtin_alloca:
2425 return emitBuiltinAlloca(cgf&: *this, e, builtinID);
2426 case Builtin::BI__builtin_alloca_with_align_uninitialized:
2427 case Builtin::BI__builtin_alloca_with_align:
2428 case Builtin::BI__builtin_infer_alloc_token:
2429 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2430 case Builtin::BIbzero:
2431 case Builtin::BI__builtin_bzero: {
2432 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2433 Address destPtr = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2434 Address destPtrCast = destPtr.withElementType(builder, ElemTy: cgm.voidTy);
2435 mlir::Value size = emitScalarExpr(e: e->getArg(Arg: 1));
2436 mlir::Value zero = builder.getNullValue(ty: builder.getUInt8Ty(), loc);
2437 assert(!cir::MissingFeatures::sanitizers());
2438 builder.createMemSet(loc, dst: destPtrCast, val: zero, len: size);
2439 assert(!cir::MissingFeatures::generateDebugInfo());
2440 return RValue::getIgnored();
2441 }
2442 case Builtin::BIbcopy:
2443 case Builtin::BI__builtin_bcopy: {
2444 Address src = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2445 Address dest = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
2446 mlir::Value sizeVal = emitScalarExpr(e: e->getArg(Arg: 2));
2447 emitNonNullArgCheck(rv: RValue::get(v: src.getPointer()), argType: e->getArg(Arg: 0)->getType(),
2448 argLoc: e->getArg(Arg: 0)->getExprLoc(), ac: fd, paramNum: 0);
2449 emitNonNullArgCheck(rv: RValue::get(v: dest.getPointer()), argType: e->getArg(Arg: 1)->getType(),
2450 argLoc: e->getArg(Arg: 1)->getExprLoc(), ac: fd, paramNum: 0);
2451 builder.createMemMove(loc: getLoc(srcLoc: e->getSourceRange()), dst: dest, src, len: sizeVal);
2452 return RValue::get(v: nullptr);
2453 }
2454 case Builtin::BImemcpy:
2455 case Builtin::BI__builtin_memcpy:
2456 case Builtin::BImempcpy:
2457 case Builtin::BI__builtin_mempcpy: {
2458 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2459 Address dest = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2460 Address src = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
2461 mlir::Value sizeVal = emitScalarExpr(e: e->getArg(Arg: 2));
2462 Address destCast = dest.withElementType(builder, ElemTy: cgm.voidTy);
2463 Address srcCast = src.withElementType(builder, ElemTy: cgm.voidTy);
2464 assert(!cir::MissingFeatures::sanitizers());
2465 builder.createMemCpy(loc, dst: destCast, src: srcCast, len: sizeVal);
2466 assert(!cir::MissingFeatures::generateDebugInfo());
2467 if (builtinID == Builtin::BImempcpy ||
2468 builtinID == Builtin::BI__builtin_mempcpy) {
2469 mlir::Value destPtr = destCast.getPointer();
2470 mlir::Value end = builder.createPtrStride(loc, base: destPtr, stride: sizeVal);
2471 return RValue::get(v: end);
2472 }
2473 return RValue::get(v: dest.getPointer());
2474 }
2475 case Builtin::BI__builtin_memcpy_inline:
2476 break; // Handled as library calls below.
2477 case Builtin::BI__builtin_char_memchr:
2478 case Builtin::BI__builtin_memchr: {
2479 Address srcPtr = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2480 mlir::Value src =
2481 builder.createBitcast(src: srcPtr.getPointer(), newTy: builder.getVoidPtrTy());
2482 mlir::Value pattern = emitScalarExpr(e: e->getArg(Arg: 1));
2483 mlir::Value len = emitScalarExpr(e: e->getArg(Arg: 2));
2484 mlir::Value res = cir::MemChrOp::create(builder, location: getLoc(srcLoc: e->getExprLoc()),
2485 src, pattern, len);
2486 // builtin_char_memchr needs its type converted to 'char', but MemChrOp is a
2487 // 'void' result type.
2488 res = builder.createBitcast(src: res, newTy: convertType(t: e->getType()));
2489 return RValue::get(v: res);
2490 }
2491 case Builtin::BI__builtin___memcpy_chk:
2492 case Builtin::BI__builtin_objc_memmove_collectable:
2493 case Builtin::BI__builtin___memmove_chk:
2494 break; // Handled as library calls below.
2495 case Builtin::BI__builtin_trivially_relocate:
2496 case Builtin::BImemmove:
2497 case Builtin::BI__builtin_memmove: {
2498 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2499 Address dest = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2500 Address src = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
2501 mlir::Value sizeVal = emitScalarExpr(e: e->getArg(Arg: 2));
2502 if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate) {
2503 QualType eltTy = e->getArg(Arg: 0)->getType()->getPointeeType();
2504 CharUnits eltSize = getContext().getTypeSizeInChars(T: eltTy);
2505 mlir::Value eltSizeVal =
2506 builder.getConstInt(loc, t: sizeVal.getType(), c: eltSize.getQuantity());
2507 sizeVal = builder.createMul(loc, lhs: sizeVal, rhs: eltSizeVal);
2508 }
2509 emitNonNullArgCheck(rv: RValue::get(v: dest.getPointer()), argType: e->getArg(Arg: 0)->getType(),
2510 argLoc: e->getArg(Arg: 0)->getExprLoc(), ac: fd, paramNum: 0);
2511 emitNonNullArgCheck(rv: RValue::get(v: src.getPointer()), argType: e->getArg(Arg: 1)->getType(),
2512 argLoc: e->getArg(Arg: 1)->getExprLoc(), ac: fd, paramNum: 1);
2513 assert(!cir::MissingFeatures::sanitizers());
2514 Address destCast = dest.withElementType(builder, ElemTy: cgm.voidTy);
2515 Address srcCast = src.withElementType(builder, ElemTy: cgm.voidTy);
2516 builder.createMemMove(loc, dst: destCast, src: srcCast, len: sizeVal);
2517 assert(!cir::MissingFeatures::generateDebugInfo());
2518 return RValue::get(v: dest.getPointer());
2519 }
2520 case Builtin::BImemset:
2521 case Builtin::BI__builtin_memset: {
2522 Address dest = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2523 mlir::Value byteVal = builder.createIntCast(src: emitScalarExpr(e: e->getArg(Arg: 1)),
2524 newTy: builder.getUInt8Ty());
2525 mlir::Value sizeVal = emitScalarExpr(e: e->getArg(Arg: 2));
2526 emitNonNullArgCheck(rv: RValue::get(v: dest.getPointer()), argType: e->getArg(Arg: 0)->getType(),
2527 argLoc: e->getArg(Arg: 0)->getExprLoc(), ac: fd, paramNum: 0);
2528 builder.createMemSet(loc: getLoc(srcLoc: e->getSourceRange()), dst: dest, val: byteVal, len: sizeVal);
2529 assert(!cir::MissingFeatures::generateDebugInfo());
2530 return RValue::get(v: dest.getPointer());
2531 }
2532 case Builtin::BI__builtin_memset_inline:
2533 case Builtin::BI__builtin___memset_chk:
2534 case Builtin::BI__builtin_wmemchr:
2535 case Builtin::BI__builtin_wmemcmp:
2536 break; // Handled as library calls below.
2537 case Builtin::BI__builtin_dwarf_cfa:
2538 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2539 case Builtin::BI__builtin_return_address: {
2540 llvm::APSInt level = e->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext());
2541 return RValue::get(v: cir::ReturnAddrOp::create(
2542 builder, location: getLoc(srcLoc: e->getExprLoc()),
2543 level: builder.getConstAPInt(loc, typ: builder.getUInt32Ty(), val: level)));
2544 }
2545 case Builtin::BI_ReturnAddress: {
2546 return RValue::get(v: cir::ReturnAddrOp::create(
2547 builder, location: getLoc(srcLoc: e->getExprLoc()),
2548 level: builder.getConstInt(loc, t: builder.getUInt32Ty(), c: 0)));
2549 }
2550 case Builtin::BI__builtin_frame_address: {
2551 llvm::APSInt level = e->getArg(Arg: 0)->EvaluateKnownConstInt(Ctx: getContext());
2552 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2553 mlir::Value addr = cir::FrameAddrOp::create(
2554 builder, location: loc, result: allocaInt8PtrTy,
2555 level: builder.getConstAPInt(loc, typ: builder.getUInt32Ty(), val: level));
2556 addr = builder.createBitcast(
2557 src: addr, newTy: builder.getVoidPtrTy(as: getCIRAllocaAddressSpace()));
2558 return RValue::get(v: performAddrSpaceCast(v: addr, destTy: convertType(t: e->getType())));
2559 }
2560 case Builtin::BI__builtin_extract_return_addr:
2561 case Builtin::BI__builtin_frob_return_addr:
2562 case Builtin::BI__builtin_dwarf_sp_column:
2563 case Builtin::BI__builtin_init_dwarf_reg_size_table:
2564 case Builtin::BI__builtin_eh_return:
2565 case Builtin::BI__builtin_unwind_init:
2566 case Builtin::BI__builtin_extend_pointer:
2567 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2568 case Builtin::BI__builtin_setjmp: {
2569 Address buf = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2570 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2571
2572 cir::PointerType voidPtrTy = builder.getVoidPtrTy();
2573 cir::PointerType ppTy = builder.getPointerTo(ty: voidPtrTy);
2574 Address castBuf = buf.withElementType(builder, ElemTy: voidPtrTy);
2575
2576 assert(!cir::MissingFeatures::emitCheckedInBoundsGEP());
2577 if (getTarget().getTriple().isSystemZ()) {
2578 cgm.errorNYI(e->getExprLoc(), "setjmp on SystemZ");
2579 return {};
2580 }
2581
2582 mlir::Value frameAddress =
2583 cir::FrameAddrOp::create(builder, location: loc, resultTypes: voidPtrTy,
2584 operands: mlir::ValueRange{builder.getUInt32(c: 0, loc)})
2585 .getResult();
2586
2587 builder.createStore(loc, val: frameAddress, dst: castBuf);
2588
2589 mlir::Value stacksave =
2590 cir::StackSaveOp::create(builder, location: loc, result: voidPtrTy).getResult();
2591 cir::PtrStrideOp stackSaveSlot = cir::PtrStrideOp::create(
2592 builder, location: loc, result: ppTy, base: castBuf.getPointer(), stride: builder.getSInt32(c: 2, loc));
2593 llvm::TypeSize voidPtrTySize =
2594 cgm.getDataLayout().getTypeAllocSize(ty: voidPtrTy);
2595 CharUnits slotAlign = castBuf.getAlignment().alignmentAtOffset(
2596 offset: CharUnits().fromQuantity(Quantity: 2 * voidPtrTySize));
2597 Address slotAddr = Address(stackSaveSlot, voidPtrTy, slotAlign);
2598 builder.createStore(loc, val: stacksave, dst: slotAddr);
2599 auto op = cir::EhSetjmpOp::create(builder, location: loc, env: castBuf.getPointer());
2600 return RValue::get(v: op);
2601 }
2602 case Builtin::BI__builtin_longjmp: {
2603 mlir::Value buf = emitScalarExpr(e: e->getArg(Arg: 0));
2604 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2605
2606 cir::EhLongjmpOp::create(builder, location: loc, env: buf);
2607 cir::UnreachableOp::create(builder, location: loc);
2608 return RValue::get(v: nullptr);
2609 }
2610 case Builtin::BI__builtin_launder: {
2611 const Expr *arg = e->getArg(Arg: 0);
2612 QualType argTy = arg->getType()->getPointeeType();
2613 mlir::Value ptr = emitScalarExpr(e: arg);
2614
2615 if (cgm.getCodeGenOpts().StrictVTablePointers &&
2616 argTy.requiresBuiltinLaunder(Context: cgm.getASTContext())) {
2617 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2618 ptr = cir::LaunderOp::create(builder, location: loc, arg: ptr).getResult();
2619 }
2620 return RValue::get(v: ptr);
2621 }
2622 case Builtin::BI__builtin_clear_padding: {
2623 Address addr = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2624 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
2625 QualType pointeeTy = e->getArg(Arg: 0)->getType()->getPointeeType();
2626
2627 llvm::ArrayRef<ASTContext::BitInterval> padding =
2628 cgm.getASTContext().getPaddingIntervals(Ty: pointeeTy);
2629
2630 llvm::SmallVector<mlir::Attribute> paddingLocs;
2631 for (const auto &interval : padding)
2632 paddingLocs.push_back(Elt: cir::OffsetPairAttr::get(
2633 context: &getMLIRContext(), start: interval.First, end: interval.Last));
2634
2635 cir::ClearPaddingOp::create(
2636 builder, location: loc, arg: addr.getPointer(),
2637 alignment: builder.getI64IntegerAttr(value: addr.getAlignment().getQuantity()),
2638 padding: mlir::ArrayAttr::get(context: &getMLIRContext(), value: paddingLocs));
2639 return RValue::get(v: nullptr);
2640 }
2641 case Builtin::BI__sync_fetch_and_add:
2642 case Builtin::BI__sync_fetch_and_sub:
2643 case Builtin::BI__sync_fetch_and_or:
2644 case Builtin::BI__sync_fetch_and_and:
2645 case Builtin::BI__sync_fetch_and_xor:
2646 case Builtin::BI__sync_fetch_and_nand:
2647 case Builtin::BI__sync_add_and_fetch:
2648 case Builtin::BI__sync_sub_and_fetch:
2649 case Builtin::BI__sync_and_and_fetch:
2650 case Builtin::BI__sync_or_and_fetch:
2651 case Builtin::BI__sync_xor_and_fetch:
2652 case Builtin::BI__sync_nand_and_fetch:
2653 case Builtin::BI__sync_val_compare_and_swap:
2654 case Builtin::BI__sync_bool_compare_and_swap:
2655 case Builtin::BI__sync_lock_test_and_set:
2656 case Builtin::BI__sync_lock_release:
2657 case Builtin::BI__sync_swap:
2658 llvm_unreachable("Shouldn't make it through sema");
2659 case Builtin::BI__sync_fetch_and_add_1:
2660 case Builtin::BI__sync_fetch_and_add_2:
2661 case Builtin::BI__sync_fetch_and_add_4:
2662 case Builtin::BI__sync_fetch_and_add_8:
2663 case Builtin::BI__sync_fetch_and_add_16:
2664 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Add, e);
2665 case Builtin::BI__sync_fetch_and_sub_1:
2666 case Builtin::BI__sync_fetch_and_sub_2:
2667 case Builtin::BI__sync_fetch_and_sub_4:
2668 case Builtin::BI__sync_fetch_and_sub_8:
2669 case Builtin::BI__sync_fetch_and_sub_16:
2670 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Sub, e);
2671 case Builtin::BI__sync_fetch_and_or_1:
2672 case Builtin::BI__sync_fetch_and_or_2:
2673 case Builtin::BI__sync_fetch_and_or_4:
2674 case Builtin::BI__sync_fetch_and_or_8:
2675 case Builtin::BI__sync_fetch_and_or_16:
2676 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Or, e);
2677 case Builtin::BI__sync_fetch_and_and_1:
2678 case Builtin::BI__sync_fetch_and_and_2:
2679 case Builtin::BI__sync_fetch_and_and_4:
2680 case Builtin::BI__sync_fetch_and_and_8:
2681 case Builtin::BI__sync_fetch_and_and_16:
2682 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::And, e);
2683 case Builtin::BI__sync_fetch_and_xor_1:
2684 case Builtin::BI__sync_fetch_and_xor_2:
2685 case Builtin::BI__sync_fetch_and_xor_4:
2686 case Builtin::BI__sync_fetch_and_xor_8:
2687 case Builtin::BI__sync_fetch_and_xor_16:
2688 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Xor, e);
2689 case Builtin::BI__sync_fetch_and_nand_1:
2690 case Builtin::BI__sync_fetch_and_nand_2:
2691 case Builtin::BI__sync_fetch_and_nand_4:
2692 case Builtin::BI__sync_fetch_and_nand_8:
2693 case Builtin::BI__sync_fetch_and_nand_16:
2694 return emitBinaryAtomic(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Nand, e);
2695 case Builtin::BI__sync_fetch_and_min:
2696 case Builtin::BI__sync_fetch_and_max:
2697 case Builtin::BI__sync_fetch_and_umin:
2698 case Builtin::BI__sync_fetch_and_umax:
2699 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2700 return getUndefRValue(ty: e->getType());
2701 case Builtin::BI__sync_add_and_fetch_1:
2702 case Builtin::BI__sync_add_and_fetch_2:
2703 case Builtin::BI__sync_add_and_fetch_4:
2704 case Builtin::BI__sync_add_and_fetch_8:
2705 case Builtin::BI__sync_add_and_fetch_16:
2706 return emitBinaryAtomicPost<cir::AddOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Add,
2707 e);
2708 case Builtin::BI__sync_sub_and_fetch_1:
2709 case Builtin::BI__sync_sub_and_fetch_2:
2710 case Builtin::BI__sync_sub_and_fetch_4:
2711 case Builtin::BI__sync_sub_and_fetch_8:
2712 case Builtin::BI__sync_sub_and_fetch_16:
2713 return emitBinaryAtomicPost<cir::SubOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Sub,
2714 e);
2715 case Builtin::BI__sync_and_and_fetch_1:
2716 case Builtin::BI__sync_and_and_fetch_2:
2717 case Builtin::BI__sync_and_and_fetch_4:
2718 case Builtin::BI__sync_and_and_fetch_8:
2719 case Builtin::BI__sync_and_and_fetch_16:
2720 return emitBinaryAtomicPost<cir::AndOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::And,
2721 e);
2722 case Builtin::BI__sync_or_and_fetch_1:
2723 case Builtin::BI__sync_or_and_fetch_2:
2724 case Builtin::BI__sync_or_and_fetch_4:
2725 case Builtin::BI__sync_or_and_fetch_8:
2726 case Builtin::BI__sync_or_and_fetch_16:
2727 return emitBinaryAtomicPost<cir::OrOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Or, e);
2728 case Builtin::BI__sync_xor_and_fetch_1:
2729 case Builtin::BI__sync_xor_and_fetch_2:
2730 case Builtin::BI__sync_xor_and_fetch_4:
2731 case Builtin::BI__sync_xor_and_fetch_8:
2732 case Builtin::BI__sync_xor_and_fetch_16:
2733 return emitBinaryAtomicPost<cir::XorOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Xor,
2734 e);
2735 case Builtin::BI__sync_nand_and_fetch_1:
2736 case Builtin::BI__sync_nand_and_fetch_2:
2737 case Builtin::BI__sync_nand_and_fetch_4:
2738 case Builtin::BI__sync_nand_and_fetch_8:
2739 case Builtin::BI__sync_nand_and_fetch_16:
2740 return emitBinaryAtomicPost<cir::AndOp>(cgf&: *this, atomicOpkind: cir::AtomicFetchKind::Nand,
2741 e, /*invert=*/true);
2742 case Builtin::BI__sync_val_compare_and_swap_1:
2743 case Builtin::BI__sync_val_compare_and_swap_2:
2744 case Builtin::BI__sync_val_compare_and_swap_4:
2745 case Builtin::BI__sync_val_compare_and_swap_8:
2746 return RValue::get(v: emitAtomicCmpXchg(e, /*returnBool=*/false));
2747 case Builtin::BI__sync_val_compare_and_swap_16:
2748 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2749 case Builtin::BI__sync_bool_compare_and_swap_1:
2750 case Builtin::BI__sync_bool_compare_and_swap_2:
2751 case Builtin::BI__sync_bool_compare_and_swap_4:
2752 case Builtin::BI__sync_bool_compare_and_swap_8:
2753 return RValue::get(v: emitAtomicCmpXchg(e, /*returnBool=*/true));
2754 case Builtin::BI__sync_bool_compare_and_swap_16:
2755 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2756 case Builtin::BI__sync_swap_1:
2757 case Builtin::BI__sync_swap_2:
2758 case Builtin::BI__sync_swap_4:
2759 case Builtin::BI__sync_swap_8:
2760 return emitAtomicXchg(cgf&: *this, e, ordering: cir::MemOrder::SequentiallyConsistent);
2761 case Builtin::BI__sync_swap_16:
2762 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2763 case Builtin::BI__sync_lock_test_and_set_1:
2764 case Builtin::BI__sync_lock_test_and_set_2:
2765 case Builtin::BI__sync_lock_test_and_set_4:
2766 case Builtin::BI__sync_lock_test_and_set_8:
2767 return emitAtomicXchg(cgf&: *this, e, ordering: cir::MemOrder::SequentiallyConsistent);
2768 case Builtin::BI__sync_lock_test_and_set_16:
2769 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2770 case Builtin::BI__sync_lock_release_1:
2771 case Builtin::BI__sync_lock_release_2:
2772 case Builtin::BI__sync_lock_release_4:
2773 case Builtin::BI__sync_lock_release_8:
2774 emitAtomicLockRelease(cgf&: *this, e);
2775 return RValue::get(v: nullptr);
2776 case Builtin::BI__sync_lock_release_16:
2777 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2778 case Builtin::BI__sync_synchronize: {
2779 // We assume this is supposed to correspond to a C++0x-style
2780 // sequentially-consistent fence (i.e. this is only usable for
2781 // synchronization, not device I/O or anything like that). This intrinsic
2782 // is really badly designed in the sense that in theory, there isn't
2783 // any way to safely use it... but in practice, it mostly works
2784 // to use it with non-atomic loads and stores to get acquire/release
2785 // semantics.
2786 cir::AtomicFenceOp::create(
2787 builder, location: getLoc(srcLoc: e->getSourceRange()),
2788 ordering: cir::MemOrder::SequentiallyConsistent,
2789 syncscope: cir::SyncScopeKindAttr::get(context: &getMLIRContext(),
2790 value: cir::SyncScopeKind::System));
2791 return RValue::get(v: nullptr);
2792 }
2793 case Builtin::BI__builtin_nontemporal_load: {
2794 Address addr = emitPointerWithAlignment(expr: e->getArg(Arg: 0));
2795 LValue lv = makeAddrLValue(addr, ty: e->getType(),
2796 baseInfo: LValueBaseInfo(AlignmentSource::Type));
2797 lv.setNontemporal(true);
2798 mlir::Value val = emitLoadOfScalar(lvalue: lv, loc: e->getExprLoc());
2799 return RValue::get(v: val);
2800 }
2801 case Builtin::BI__builtin_nontemporal_store: {
2802 mlir::Value val = emitScalarExpr(e: e->getArg(Arg: 0));
2803 Address addr = emitPointerWithAlignment(expr: e->getArg(Arg: 1));
2804 val = emitToMemory(value: val, ty: e->getArg(Arg: 0)->getType());
2805 LValue lv = makeAddrLValue(addr, ty: e->getArg(Arg: 0)->getType(),
2806 baseInfo: LValueBaseInfo(AlignmentSource::Type));
2807 lv.setNontemporal(true);
2808 emitStoreOfScalar(value: val, lvalue: lv, /*isInit=*/false);
2809 return RValue::get(v: nullptr);
2810 }
2811 case Builtin::BI__c11_atomic_is_lock_free:
2812 case Builtin::BI__atomic_is_lock_free:
2813 return emitAtomicIsLockFree(cgf&: *this, e, builtinID);
2814 case Builtin::BI__atomic_test_and_set:
2815 case Builtin::BI__atomic_clear:
2816 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2817 case Builtin::BI__atomic_thread_fence:
2818 case Builtin::BI__atomic_signal_fence:
2819 case Builtin::BI__c11_atomic_thread_fence:
2820 case Builtin::BI__c11_atomic_signal_fence: {
2821 cir::SyncScopeKind syncScope =
2822 builtinID == Builtin::BI__atomic_signal_fence ||
2823 builtinID == Builtin::BI__c11_atomic_signal_fence
2824 ? cir::SyncScopeKind::SingleThread
2825 : cir::SyncScopeKind::System;
2826 emitAtomicFenceOp(cgf&: *this, expr: e, syncScope);
2827 return RValue::get(v: nullptr);
2828 }
2829 case Builtin::BI__scoped_atomic_thread_fence:
2830 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2831 case Builtin::BI__builtin_signbit:
2832 case Builtin::BI__builtin_signbitf:
2833 case Builtin::BI__builtin_signbitl: {
2834 CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
2835 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
2836 mlir::Value value = emitScalarExpr(e: e->getArg(Arg: 0));
2837 mlir::Operation *signBitOp = cir::SignBitOp::create(builder, location: loc, input: value);
2838 mlir::Value result = builder.createBoolToInt(src: signBitOp->getResult(idx: 0),
2839 newTy: convertType(t: e->getType()));
2840 return RValue::get(v: result);
2841 }
2842 case Builtin::BI__warn_memset_zero_len:
2843 case Builtin::BI__annotation:
2844 return errorBuiltinNYI(cgf&: *this, e, builtinID);
2845
2846 case Builtin::BI__builtin_addcb:
2847 case Builtin::BI__builtin_addcs:
2848 case Builtin::BI__builtin_addc:
2849 case Builtin::BI__builtin_addcl:
2850 case Builtin::BI__builtin_addcll:
2851 case Builtin::BI__builtin_subcb:
2852 case Builtin::BI__builtin_subcs:
2853 case Builtin::BI__builtin_subc:
2854 case Builtin::BI__builtin_subcl:
2855 case Builtin::BI__builtin_subcll: {
2856 // Multiprecision add/sub-with-carry. Lower as two chained checked
2857 // add/sub overflow ops, matching classic CodeGen:
2858 // sum1, carry1 = x +/- y
2859 // result, carry2 = sum1 +/- carryin
2860 // *carryout = carry1 | carry2
2861 // All operands and the result share the builtin's integer type, so no
2862 // encompassing-type widening is needed.
2863 mlir::Value x = emitScalarExpr(e: e->getArg(Arg: 0));
2864 mlir::Value y = emitScalarExpr(e: e->getArg(Arg: 1));
2865 mlir::Value carryin = emitScalarExpr(e: e->getArg(Arg: 2));
2866 Address carryOutPtr = emitPointerWithAlignment(expr: e->getArg(Arg: 3));
2867
2868 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2869 mlir::Type resultTy = convertType(t: e->getType());
2870
2871 static constexpr unsigned addcBuiltins[] = {
2872 Builtin::BI__builtin_addcb, Builtin::BI__builtin_addcs,
2873 Builtin::BI__builtin_addc, Builtin::BI__builtin_addcl,
2874 Builtin::BI__builtin_addcll};
2875 bool isAdd = llvm::is_contained(Range: addcBuiltins, Element: builtinID);
2876
2877 mlir::Value sum1, carry1, sum2, carry2;
2878 if (isAdd) {
2879 std::tie(args&: sum1, args&: carry1) =
2880 emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultTy, lhs: x, rhs: y);
2881 std::tie(args&: sum2, args&: carry2) = emitOverflowOp<cir::AddOverflowOp>(
2882 builder, loc, resultTy, lhs: sum1, rhs: carryin);
2883 } else {
2884 std::tie(args&: sum1, args&: carry1) =
2885 emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultTy, lhs: x, rhs: y);
2886 std::tie(args&: sum2, args&: carry2) = emitOverflowOp<cir::SubOverflowOp>(
2887 builder, loc, resultTy, lhs: sum1, rhs: carryin);
2888 }
2889
2890 // Combine the two carry bits, then widen to the result integer type.
2891 mlir::Value carryOut = builder.createBoolToInt(
2892 src: builder.createOr(loc, lhs: carry1, rhs: carry2), newTy: resultTy);
2893 builder.createStore(loc, val: carryOut, dst: carryOutPtr);
2894 return RValue::get(v: sum2);
2895 }
2896
2897 case Builtin::BI__builtin_add_overflow:
2898 case Builtin::BI__builtin_sub_overflow:
2899 case Builtin::BI__builtin_mul_overflow: {
2900 const clang::Expr *leftArg = e->getArg(Arg: 0);
2901 const clang::Expr *rightArg = e->getArg(Arg: 1);
2902 const clang::Expr *resultArg = e->getArg(Arg: 2);
2903
2904 clang::QualType resultQTy =
2905 resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
2906
2907 WidthAndSignedness leftInfo =
2908 getIntegerWidthAndSignedness(astContext: cgm.getASTContext(), type: leftArg->getType());
2909 WidthAndSignedness rightInfo =
2910 getIntegerWidthAndSignedness(astContext: cgm.getASTContext(), type: rightArg->getType());
2911 WidthAndSignedness resultInfo =
2912 getIntegerWidthAndSignedness(astContext: cgm.getASTContext(), type: resultQTy);
2913
2914 // Note we compute the encompassing type with the consideration to the
2915 // result type, so later in LLVM lowering we don't get redundant integral
2916 // extension casts.
2917 WidthAndSignedness encompassingInfo =
2918 EncompassingIntegerType(types: {leftInfo, rightInfo, resultInfo});
2919
2920 auto encompassingCIRTy = cir::IntType::get(
2921 context: &getMLIRContext(), width: encompassingInfo.width, is_signed: encompassingInfo.isSigned);
2922 mlir::Type resultCIRTy = cgm.convertType(type: resultQTy);
2923
2924 mlir::Value x = emitScalarExpr(e: leftArg);
2925 mlir::Value y = emitScalarExpr(e: rightArg);
2926 Address resultPtr = emitPointerWithAlignment(expr: resultArg);
2927
2928 // Extend each operand to the encompassing type, if necessary.
2929 if (x.getType() != encompassingCIRTy) {
2930 x = builder.createCast(kind: mlir::isa<cir::BoolType>(Val: x.getType())
2931 ? cir::CastKind::bool_to_int
2932 : cir::CastKind::integral,
2933 src: x, newTy: encompassingCIRTy);
2934 }
2935
2936 if (y.getType() != encompassingCIRTy) {
2937 y = builder.createCast(kind: mlir::isa<cir::BoolType>(Val: y.getType())
2938 ? cir::CastKind::bool_to_int
2939 : cir::CastKind::integral,
2940 src: y, newTy: encompassingCIRTy);
2941 }
2942
2943 // Perform the operation on the extended values.
2944 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2945 mlir::Value result, overflow;
2946 switch (builtinID) {
2947 default:
2948 llvm_unreachable("Unknown overflow builtin id.");
2949 case Builtin::BI__builtin_add_overflow:
2950 std::tie(args&: result, args&: overflow) =
2951 emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
2952 break;
2953 case Builtin::BI__builtin_sub_overflow:
2954 std::tie(args&: result, args&: overflow) =
2955 emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
2956 break;
2957 case Builtin::BI__builtin_mul_overflow:
2958 std::tie(args&: result, args&: overflow) =
2959 emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
2960 break;
2961 }
2962
2963 // Here is a slight difference from the original clang CodeGen:
2964 // - In the original clang CodeGen, the checked arithmetic result is
2965 // first computed as a value of the encompassing type, and then it is
2966 // truncated to the actual result type with a second overflow checking.
2967 // - In CIRGen, the checked arithmetic operation directly produce the
2968 // checked arithmetic result in its expected type, which may be a
2969 // `cir.bool`.
2970 //
2971 // So we don't need a truncation and a second overflow checking here.
2972
2973 // Finally, store the result using the pointer.
2974 bool isVolatile =
2975 resultArg->getType()->getPointeeType().isVolatileQualified();
2976 builder.createStore(loc, val: result, dst: resultPtr, isVolatile);
2977
2978 return RValue::get(v: overflow);
2979 }
2980
2981 case Builtin::BI__builtin_uadd_overflow:
2982 case Builtin::BI__builtin_uaddl_overflow:
2983 case Builtin::BI__builtin_uaddll_overflow:
2984 case Builtin::BI__builtin_usub_overflow:
2985 case Builtin::BI__builtin_usubl_overflow:
2986 case Builtin::BI__builtin_usubll_overflow:
2987 case Builtin::BI__builtin_umul_overflow:
2988 case Builtin::BI__builtin_umull_overflow:
2989 case Builtin::BI__builtin_umulll_overflow:
2990 case Builtin::BI__builtin_sadd_overflow:
2991 case Builtin::BI__builtin_saddl_overflow:
2992 case Builtin::BI__builtin_saddll_overflow:
2993 case Builtin::BI__builtin_ssub_overflow:
2994 case Builtin::BI__builtin_ssubl_overflow:
2995 case Builtin::BI__builtin_ssubll_overflow:
2996 case Builtin::BI__builtin_smul_overflow:
2997 case Builtin::BI__builtin_smull_overflow:
2998 case Builtin::BI__builtin_smulll_overflow: {
2999 // Scalarize our inputs.
3000 mlir::Value x = emitScalarExpr(e: e->getArg(Arg: 0));
3001 mlir::Value y = emitScalarExpr(e: e->getArg(Arg: 1));
3002
3003 const clang::Expr *resultArg = e->getArg(Arg: 2);
3004 Address resultPtr = emitPointerWithAlignment(expr: resultArg);
3005
3006 clang::QualType resultQTy =
3007 resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
3008 auto resultCIRTy = mlir::cast<cir::IntType>(Val: cgm.convertType(type: resultQTy));
3009
3010 // Create the appropriate overflow-checked arithmetic operation.
3011 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
3012 mlir::Value result, overflow;
3013 switch (builtinID) {
3014 default:
3015 llvm_unreachable("Unknown overflow builtin id.");
3016 case Builtin::BI__builtin_uadd_overflow:
3017 case Builtin::BI__builtin_uaddl_overflow:
3018 case Builtin::BI__builtin_uaddll_overflow:
3019 case Builtin::BI__builtin_sadd_overflow:
3020 case Builtin::BI__builtin_saddl_overflow:
3021 case Builtin::BI__builtin_saddll_overflow:
3022 std::tie(args&: result, args&: overflow) =
3023 emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
3024 break;
3025 case Builtin::BI__builtin_usub_overflow:
3026 case Builtin::BI__builtin_usubl_overflow:
3027 case Builtin::BI__builtin_usubll_overflow:
3028 case Builtin::BI__builtin_ssub_overflow:
3029 case Builtin::BI__builtin_ssubl_overflow:
3030 case Builtin::BI__builtin_ssubll_overflow:
3031 std::tie(args&: result, args&: overflow) =
3032 emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
3033 break;
3034 case Builtin::BI__builtin_umul_overflow:
3035 case Builtin::BI__builtin_umull_overflow:
3036 case Builtin::BI__builtin_umulll_overflow:
3037 case Builtin::BI__builtin_smul_overflow:
3038 case Builtin::BI__builtin_smull_overflow:
3039 case Builtin::BI__builtin_smulll_overflow:
3040 std::tie(args&: result, args&: overflow) =
3041 emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultTy: resultCIRTy, lhs: x, rhs: y);
3042 break;
3043 }
3044
3045 bool isVolatile =
3046 resultArg->getType()->getPointeeType().isVolatileQualified();
3047 builder.createStore(loc, val: emitToMemory(value: result, ty: resultQTy), dst: resultPtr,
3048 isVolatile);
3049
3050 return RValue::get(v: overflow);
3051 }
3052
3053 case Builtin::BIaddressof:
3054 case Builtin::BI__addressof:
3055 case Builtin::BI__builtin_addressof:
3056 return RValue::get(v: emitLValue(e: e->getArg(Arg: 0)).getPointer());
3057 case Builtin::BI__builtin_function_start:
3058 return errorBuiltinNYI(cgf&: *this, e, builtinID);
3059 case Builtin::BI__builtin_operator_new:
3060 return emitNewOrDeleteBuiltinCall(
3061 type: e->getCallee()->getType()->castAs<FunctionProtoType>(), callExpr: e, op: OO_New);
3062 case Builtin::BI__builtin_operator_delete:
3063 emitNewOrDeleteBuiltinCall(
3064 type: e->getCallee()->getType()->castAs<FunctionProtoType>(), callExpr: e, op: OO_Delete);
3065 return RValue::get(v: nullptr);
3066 case Builtin::BI__builtin_is_aligned:
3067 case Builtin::BI__builtin_align_up:
3068 case Builtin::BI__builtin_align_down:
3069 case Builtin::BI__noop:
3070 case Builtin::BI__builtin_call_with_static_chain:
3071 case Builtin::BI_InterlockedExchange8:
3072 case Builtin::BI_InterlockedExchange16:
3073 case Builtin::BI_InterlockedExchange:
3074 case Builtin::BI_InterlockedExchangePointer:
3075 case Builtin::BI_InterlockedCompareExchangePointer:
3076 case Builtin::BI_InterlockedCompareExchangePointer_nf:
3077 case Builtin::BI_InterlockedCompareExchange8:
3078 case Builtin::BI_InterlockedCompareExchange16:
3079 case Builtin::BI_InterlockedCompareExchange:
3080 case Builtin::BI_InterlockedCompareExchange64:
3081 case Builtin::BI_InterlockedIncrement16:
3082 case Builtin::BI_InterlockedIncrement:
3083 case Builtin::BI_InterlockedDecrement16:
3084 case Builtin::BI_InterlockedDecrement:
3085 case Builtin::BI_InterlockedAnd8:
3086 case Builtin::BI_InterlockedAnd16:
3087 case Builtin::BI_InterlockedAnd:
3088 case Builtin::BI_InterlockedExchangeAdd8:
3089 case Builtin::BI_InterlockedExchangeAdd16:
3090 case Builtin::BI_InterlockedExchangeAdd:
3091 case Builtin::BI_InterlockedExchangeSub8:
3092 case Builtin::BI_InterlockedExchangeSub16:
3093 case Builtin::BI_InterlockedExchangeSub:
3094 case Builtin::BI_InterlockedOr8:
3095 case Builtin::BI_InterlockedOr16:
3096 case Builtin::BI_InterlockedOr:
3097 case Builtin::BI_InterlockedXor8:
3098 case Builtin::BI_InterlockedXor16:
3099 case Builtin::BI_InterlockedXor:
3100 case Builtin::BI_bittest64:
3101 case Builtin::BI_bittest:
3102 case Builtin::BI_bittestandcomplement64:
3103 case Builtin::BI_bittestandcomplement:
3104 case Builtin::BI_bittestandreset64:
3105 case Builtin::BI_bittestandreset:
3106 case Builtin::BI_bittestandset64:
3107 case Builtin::BI_bittestandset:
3108 case Builtin::BI_interlockedbittestandreset:
3109 case Builtin::BI_interlockedbittestandreset64:
3110 case Builtin::BI_interlockedbittestandreset64_acq:
3111 case Builtin::BI_interlockedbittestandreset64_rel:
3112 case Builtin::BI_interlockedbittestandreset64_nf:
3113 case Builtin::BI_interlockedbittestandset64:
3114 case Builtin::BI_interlockedbittestandset64_acq:
3115 case Builtin::BI_interlockedbittestandset64_rel:
3116 case Builtin::BI_interlockedbittestandset64_nf:
3117 case Builtin::BI_interlockedbittestandset:
3118 case Builtin::BI_interlockedbittestandset_acq:
3119 case Builtin::BI_interlockedbittestandset_rel:
3120 case Builtin::BI_interlockedbittestandset_nf:
3121 case Builtin::BI_interlockedbittestandreset_acq:
3122 case Builtin::BI_interlockedbittestandreset_rel:
3123 case Builtin::BI_interlockedbittestandreset_nf:
3124 case Builtin::BI__iso_volatile_load8:
3125 case Builtin::BI__iso_volatile_load16:
3126 case Builtin::BI__iso_volatile_load32:
3127 case Builtin::BI__iso_volatile_load64:
3128 case Builtin::BI__iso_volatile_store8:
3129 case Builtin::BI__iso_volatile_store16:
3130 case Builtin::BI__iso_volatile_store32:
3131 case Builtin::BI__iso_volatile_store64:
3132 case Builtin::BI__builtin_ptrauth_sign_constant:
3133 case Builtin::BI__builtin_ptrauth_auth:
3134 case Builtin::BI__builtin_ptrauth_auth_and_resign:
3135 case Builtin::BI__builtin_ptrauth_blend_discriminator:
3136 case Builtin::BI__builtin_ptrauth_sign_generic_data:
3137 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
3138 case Builtin::BI__builtin_ptrauth_strip:
3139 case Builtin::BI__builtin_get_vtable_pointer:
3140 case Builtin::BI__exception_code:
3141 case Builtin::BI_exception_code:
3142 case Builtin::BI__exception_info:
3143 case Builtin::BI_exception_info:
3144 case Builtin::BI__abnormal_termination:
3145 case Builtin::BI_abnormal_termination:
3146 return errorBuiltinNYI(cgf&: *this, e, builtinID);
3147 case Builtin::BI_setjmpex:
3148 case Builtin::BI_setjmp:
3149 if (getTarget().getTriple().isOSMSVCRT()) {
3150 cgm.errorNYI(e->getSourceRange(), "setjmp/setjmpex on MSVCRT");
3151 return getUndefRValue(ty: e->getType());
3152 }
3153 // Else break and this will be handled as a library call.
3154 break;
3155 case Builtin::BImove:
3156 case Builtin::BImove_if_noexcept:
3157 case Builtin::BIforward:
3158 case Builtin::BIforward_like:
3159 case Builtin::BIas_const:
3160 return RValue::get(v: emitLValue(e: e->getArg(Arg: 0)).getPointer());
3161 case Builtin::BI__GetExceptionInfo:
3162 case Builtin::BI__fastfail:
3163 return errorBuiltinNYI(cgf&: *this, e, builtinID);
3164 case Builtin::BI__builtin_coro_id:
3165 return RValue::get(v: emitCoroIDBuiltinCall(e).getResult());
3166 case Builtin::BI__builtin_coro_promise:
3167 return RValue::get(v: emitCoroPromiseBuiltinCall(e).getResult());
3168 case Builtin::BI__builtin_coro_resume: {
3169 emitCoroResumeBuiltinCall(e);
3170 return RValue::get(v: nullptr);
3171 }
3172 case Builtin::BI__builtin_coro_frame: {
3173 return emitCoroutineFrame();
3174 }
3175 case Builtin::BI__builtin_coro_noop:
3176 return RValue::get(v: emitCoroNoopBuiltinCall(e).getResult());
3177 case Builtin::BI__builtin_coro_free: {
3178 cir::CoroFreeOp coroFree = emitCoroFreeBuiltin(e);
3179 return coroFree ? RValue::get(v: coroFree.getResult())
3180 : getUndefRValue(ty: e->getType());
3181 }
3182 case Builtin::BI__builtin_coro_destroy: {
3183 emitCoroDestroyBuiltinCall(e);
3184 return RValue::get(v: nullptr);
3185 }
3186 case Builtin::BI__builtin_coro_done:
3187 return RValue::get(v: emitCoroDoneBuiltinCall(e).getResult());
3188 case Builtin::BI__builtin_coro_alloc: {
3189 cir::CoroAllocOp coroAlloc = emitCoroAllocBuiltinCall(e);
3190 return coroAlloc ? RValue::get(v: coroAlloc.getResult())
3191 : getUndefRValue(ty: e->getType());
3192 }
3193 case Builtin::BI__builtin_coro_begin: {
3194 cir::CoroBeginOp coroBeg = emitCoroBeginBuiltinCall(e);
3195 return coroBeg ? RValue::get(v: coroBeg.getResult())
3196 : getUndefRValue(ty: e->getType());
3197 }
3198 case Builtin::BI__builtin_coro_end:
3199 return RValue::get(v: emitCoroEndBuiltinCall(e).getResult());
3200 case Builtin::BI__builtin_coro_suspend: {
3201 mlir::Value result = emitCoroSuspendBuiltinCall(e).getResult();
3202 return RValue::get(
3203 v: builder.createIntCast(src: result, newTy: convertType(t: e->getType())));
3204 }
3205 case Builtin::BI__builtin_coro_size: {
3206 return RValue::get(v: emitCoroSizeBuiltinCall(e).getResult());
3207 }
3208 case Builtin::BI__builtin_coro_align:
3209 return RValue::get(v: emitCoroAlignBuiltinCall(e).getResult());
3210 case Builtin::BIread_pipe:
3211 case Builtin::BIwrite_pipe:
3212 case Builtin::BIreserve_read_pipe:
3213 case Builtin::BIreserve_write_pipe:
3214 case Builtin::BIwork_group_reserve_read_pipe:
3215 case Builtin::BIwork_group_reserve_write_pipe:
3216 case Builtin::BIsub_group_reserve_read_pipe:
3217 case Builtin::BIsub_group_reserve_write_pipe:
3218 case Builtin::BIcommit_read_pipe:
3219 case Builtin::BIcommit_write_pipe:
3220 case Builtin::BIwork_group_commit_read_pipe:
3221 case Builtin::BIwork_group_commit_write_pipe:
3222 case Builtin::BIsub_group_commit_read_pipe:
3223 case Builtin::BIsub_group_commit_write_pipe:
3224 case Builtin::BIget_pipe_num_packets:
3225 case Builtin::BIget_pipe_max_packets:
3226 case Builtin::BIto_global:
3227 case Builtin::BIto_local:
3228 case Builtin::BIto_private:
3229 case Builtin::BIenqueue_kernel:
3230 case Builtin::BIget_kernel_work_group_size:
3231 case Builtin::BIget_kernel_preferred_work_group_size_multiple:
3232 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
3233 case Builtin::BIget_kernel_sub_group_count_for_ndrange:
3234 case Builtin::BI__builtin_store_half:
3235 case Builtin::BI__builtin_store_halff:
3236 case Builtin::BI__builtin_load_half:
3237 case Builtin::BI__builtin_load_halff:
3238 return errorBuiltinNYI(cgf&: *this, e, builtinID);
3239 case Builtin::BI__builtin_printf:
3240 case Builtin::BIprintf:
3241 if (getTarget().getTriple().isNVPTX() ||
3242 getTarget().getTriple().isAMDGCN() ||
3243 (getTarget().getTriple().isSPIRV() &&
3244 getTarget().getTriple().getVendor() == llvm::Triple::AMD))
3245 return RValue::get(v: emitDevicePrintfCallExpr(expr: e));
3246 break;
3247 case Builtin::BI__builtin_canonicalize:
3248 case Builtin::BI__builtin_canonicalizef:
3249 case Builtin::BI__builtin_canonicalizef16:
3250 case Builtin::BI__builtin_canonicalizel:
3251 case Builtin::BI__builtin_thread_pointer:
3252 case Builtin::BI__builtin_os_log_format:
3253 case Builtin::BI__xray_customevent:
3254 case Builtin::BI__xray_typedevent:
3255 case Builtin::BI__builtin_ms_va_start:
3256 case Builtin::BI__builtin_ms_va_end:
3257 case Builtin::BI__builtin_ms_va_copy:
3258 case Builtin::BI__builtin_get_device_side_mangled_name:
3259 return errorBuiltinNYI(cgf&: *this, e, builtinID);
3260 }
3261
3262 // If this is an alias for a lib function (e.g. __builtin_sin), emit
3263 // the call using the normal call path, but using the unmangled
3264 // version of the function name.
3265 if (!shouldEmitBuiltinAsIR(builtinID, bi: getContext().BuiltinInfo, cgf: *this) &&
3266 getContext().BuiltinInfo.isLibFunction(ID: builtinID))
3267 return emitLibraryCall(cgf&: *this, fd, e,
3268 calleeValue: cgm.getBuiltinLibFunction(fd, builtinID));
3269
3270 // If this is a predefined lib function (e.g. malloc), emit the call
3271 // using exactly the normal call path.
3272 if (getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID))
3273 return emitLibraryCall(cgf&: *this, fd, e,
3274 calleeValue: emitScalarExpr(e: e->getCallee()).getDefiningOp());
3275
3276 // Check that a call to a target specific builtin has the correct target
3277 // features.
3278 // This is down here to avoid non-target specific builtins, however, if
3279 // generic builtins start to require generic target features then we
3280 // can move this up to the beginning of the function.
3281 checkTargetFeatures(e, targetDecl: fd);
3282
3283 // See if we have a target specific intrinsic.
3284 std::string name = getContext().BuiltinInfo.getName(ID: builtinID);
3285 Intrinsic::ID intrinsicID = Intrinsic::not_intrinsic;
3286 StringRef prefix =
3287 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
3288 if (!prefix.empty()) {
3289 intrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: prefix, BuiltinName: name);
3290 if (intrinsicID == Intrinsic::not_intrinsic && prefix == "spv" &&
3291 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
3292 intrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: name);
3293 // NOTE we don't need to perform a compatibility flag check here since the
3294 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
3295 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
3296 if (intrinsicID == Intrinsic::not_intrinsic)
3297 intrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: prefix, BuiltinName: name);
3298 }
3299
3300 if (intrinsicID != Intrinsic::not_intrinsic) {
3301 unsigned iceArguments = 0;
3302 ASTContext::GetBuiltinTypeError error;
3303 getContext().GetBuiltinType(ID: builtinID, Error&: error, IntegerConstantArgs: &iceArguments);
3304 assert(error == ASTContext::GE_None && "Should not codegen an error");
3305
3306 StringRef name = Intrinsic::getName(id: intrinsicID);
3307 // cir::LLVMIntrinsicCallOp expects intrinsic name to not have prefix
3308 // "llvm." For example, `llvm.nvvm.barrier0` should be passed as
3309 // `nvvm.barrier0`.
3310 assert(name.starts_with("llvm.") && "expected llvm. prefix");
3311 name = name.drop_front(/*strlen("llvm.")=*/N: 5);
3312
3313 cir::FuncType intrinsicType = getIntrinsicType(id: intrinsicID);
3314
3315 SmallVector<mlir::Value> args;
3316 const FunctionDecl *fd = e->getDirectCallee();
3317 for (unsigned i = 0; i < e->getNumArgs(); i++) {
3318 mlir::Value argValue =
3319 emitScalarOrConstFoldImmArg(iceArguments, idx: i, argExpr: e->getArg(Arg: i));
3320 // If the intrinsic arg type is different from the builtin arg type
3321 // we need to do a bit cast.
3322 mlir::Type argType = argValue.getType();
3323 mlir::Type expectedTy = intrinsicType.getInput(i);
3324
3325 // Correct integer signedness based on AST parameter type
3326 mlir::Type correctedExpectedTy = expectedTy;
3327 if (fd && i < fd->getNumParams()) {
3328 correctedExpectedTy = correctIntegerSignedness(
3329 iitType: expectedTy, astType: fd->getParamDecl(i)->getType(), context: &getMLIRContext());
3330 }
3331
3332 if (mlir::isa<cir::PointerType>(Val: expectedTy)) {
3333 bool argIsPointer = mlir::isa<cir::PointerType>(Val: argType);
3334 bool argIsVectorOfPointer = false;
3335 if (auto vecTy = dyn_cast<mlir::VectorType>(Val&: argType))
3336 argIsVectorOfPointer =
3337 mlir::isa<cir::PointerType>(Val: vecTy.getElementType());
3338
3339 if (!argIsPointer && !argIsVectorOfPointer) {
3340 cgm.errorNYI(
3341 e->getSourceRange(),
3342 "intrinsic expects a pointer type (NYI for non-pointer)");
3343 return getUndefRValue(ty: e->getType());
3344 }
3345
3346 // Pointer handling (address-space cast / bitcast fallback).
3347 if (argType != expectedTy)
3348 argValue = getCorrectedPtr(argValue, expectedTy, builder);
3349 } else {
3350 // Non-pointer expected type: if needed, bitcast to the corrected
3351 // expected type to match signedness/representation.
3352 if (argType != correctedExpectedTy)
3353 argValue = builder.createBitcast(src: argValue, newTy: correctedExpectedTy);
3354 }
3355
3356 args.push_back(Elt: argValue);
3357 }
3358
3359 // Correct the builtin type based on the AST function declaration's return
3360 // type, if available.
3361 mlir::Type correctedReturnType =
3362 correctReturnType(iitType: intrinsicType.getReturnType(), funcDecl: fd, context: &getMLIRContext());
3363
3364 cir::LLVMIntrinsicCallOp intrinsicCall = cir::LLVMIntrinsicCallOp::create(
3365 builder, location: getLoc(srcLoc: e->getExprLoc()), intrinsic_name: builder.getStringAttr(bytes: name),
3366 resType: correctedReturnType, operands: args);
3367
3368 mlir::Value intrinsicRes = intrinsicCall.getResult();
3369
3370 if (isa<cir::VoidType>(Val: correctedReturnType))
3371 return RValue::get(v: nullptr);
3372
3373 return RValue::get(v: intrinsicRes);
3374 }
3375
3376 // Some target-specific builtins can have aggregate return values, e.g.
3377 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
3378 // returnValue to be non-null, so that the target-specific emission code can
3379 // always just emit into it.
3380 cir::TypeEvaluationKind evalKind = getEvaluationKind(type: e->getType());
3381 if (evalKind == cir::TEK_Aggregate && returnValue.isNull()) {
3382 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
3383 return getUndefRValue(ty: e->getType());
3384 }
3385
3386 // Now see if we can emit a target-specific builtin.
3387 // FIXME: This is a temporary mechanism (double-optional semantics) that will
3388 // go away once everything is implemented:
3389 // 1. return `mlir::Value{}` for cases where we have issued the diagnostic.
3390 // 2. return `std::nullopt` in cases where we didn't issue a diagnostic
3391 // but also didn't handle the builtin.
3392 if (std::optional<mlir::Value> rst =
3393 emitTargetBuiltinExpr(builtinID, e, returnValue)) {
3394 mlir::Value v = rst.value();
3395 // A null value means either a void builtin or an already diagnosed NYI.
3396 // In the case of a void builtin, getUndefRValue returns
3397 // `RValue::get(nullptr)`.
3398 if (!v)
3399 return getUndefRValue(ty: e->getType());
3400
3401 switch (evalKind) {
3402 case cir::TEK_Scalar:
3403 if (mlir::isa<cir::VoidType>(Val: v.getType()))
3404 return RValue::get(v: nullptr);
3405 return RValue::get(v);
3406 case cir::TEK_Aggregate:
3407 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
3408 return getUndefRValue(ty: e->getType());
3409 case cir::TEK_Complex:
3410 llvm_unreachable("No current target builtin returns complex");
3411 }
3412 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
3413 }
3414
3415 cgm.errorNYI(e->getSourceRange(),
3416 std::string("unimplemented builtin call: ") +
3417 getContext().BuiltinInfo.getName(ID: builtinID));
3418 return getUndefRValue(ty: e->getType());
3419}
3420
3421static std::optional<mlir::Value>
3422emitTargetArchBuiltinExpr(CIRGenFunction *cgf, unsigned builtinID,
3423 const CallExpr *e, ReturnValueSlot &returnValue,
3424 llvm::Triple::ArchType arch) {
3425 // When compiling in HipStdPar mode we have to be conservative in rejecting
3426 // target specific features in the FE, and defer the possible error to the
3427 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
3428 // referenced by an accelerator executable function, we emit an error.
3429 // Returning nullptr here leads to the builtin being handled in
3430 // EmitStdParUnsupportedBuiltin.
3431 if (cgf->getLangOpts().HIPStdPar && cgf->getLangOpts().CUDAIsDevice &&
3432 arch != cgf->getTarget().getTriple().getArch())
3433 return std::nullopt;
3434
3435 switch (arch) {
3436 case llvm::Triple::arm:
3437 case llvm::Triple::armeb:
3438 case llvm::Triple::thumb:
3439 case llvm::Triple::thumbeb:
3440 // These are actually NYI, but that will be reported by emitBuiltinExpr.
3441 // At this point, we don't even know that the builtin is target-specific.
3442 return std::nullopt;
3443 case llvm::Triple::aarch64:
3444 case llvm::Triple::aarch64_32:
3445 case llvm::Triple::aarch64_be:
3446 return cgf->emitAArch64BuiltinExpr(builtinID, expr: e, returnValue, arch);
3447 case llvm::Triple::bpfeb:
3448 case llvm::Triple::bpfel:
3449 // These are actually NYI, but that will be reported by emitBuiltinExpr.
3450 // At this point, we don't even know that the builtin is target-specific.
3451 return std::nullopt;
3452
3453 case llvm::Triple::x86:
3454 case llvm::Triple::x86_64:
3455 return cgf->emitX86BuiltinExpr(builtinID, expr: e);
3456
3457 case llvm::Triple::ppc:
3458 case llvm::Triple::ppcle:
3459 case llvm::Triple::ppc64:
3460 case llvm::Triple::ppc64le:
3461 case llvm::Triple::r600:
3462 // These are actually NYI, but that will be reported by emitBuiltinExpr.
3463 // At this point, we don't even know that the builtin is target-specific.
3464 return std::nullopt;
3465 case llvm::Triple::amdgpu:
3466 return cgf->emitAMDGPUBuiltinExpr(builtinID, expr: e);
3467 case llvm::Triple::systemz:
3468 return std::nullopt;
3469 case llvm::Triple::nvptx:
3470 case llvm::Triple::nvptx64:
3471 return cgf->emitNVPTXBuiltinExpr(builtinID, expr: e);
3472 case llvm::Triple::wasm32:
3473 case llvm::Triple::wasm64:
3474 case llvm::Triple::hexagon:
3475 // These are actually NYI, but that will be reported by emitBuiltinExpr.
3476 // At this point, we don't even know that the builtin is target-specific.
3477 return std::nullopt;
3478 case llvm::Triple::riscv32:
3479 case llvm::Triple::riscv64:
3480 return cgf->emitRISCVBuiltinExpr(builtinID, expr: e);
3481 case llvm::Triple::spirv32:
3482 case llvm::Triple::spirv64:
3483 if (cgf->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
3484 return cgf->emitAMDGPUBuiltinExpr(builtinID, expr: e);
3485 [[fallthrough]];
3486 case llvm::Triple::spirv:
3487 return cgf->emitSPIRVBuiltinExpr(builtinID, expr: e);
3488 default:
3489 return std::nullopt;
3490 }
3491}
3492
3493std::optional<mlir::Value>
3494CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,
3495 ReturnValueSlot &returnValue) {
3496 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: builtinID)) {
3497 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
3498 return emitTargetArchBuiltinExpr(
3499 cgf: this, builtinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: builtinID), e,
3500 returnValue, arch: getContext().getAuxTargetInfo()->getTriple().getArch());
3501 }
3502
3503 return emitTargetArchBuiltinExpr(cgf: this, builtinID, e, returnValue,
3504 arch: getTarget().getTriple().getArch());
3505}
3506
3507mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(
3508 const unsigned iceArguments, const unsigned idx, const Expr *argExpr) {
3509 mlir::Value arg = {};
3510 if ((iceArguments & (1 << idx)) == 0) {
3511 arg = emitScalarExpr(e: argExpr);
3512 } else {
3513 // If this is required to be a constant, constant fold it so that we
3514 // know that the generated intrinsic gets a ConstantInt.
3515 const std::optional<llvm::APSInt> result =
3516 argExpr->getIntegerConstantExpr(Ctx: getContext());
3517 assert(result && "Expected argument to be a constant");
3518 arg = builder.getConstInt(loc: getLoc(srcLoc: argExpr->getSourceRange()), intVal: *result);
3519 }
3520 return arg;
3521}
3522
3523/// Given a builtin id for a function like "__builtin_fabsf", return a Function*
3524/// for "fabsf".
3525cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
3526 unsigned builtinID) {
3527 assert(astContext.BuiltinInfo.isLibFunction(builtinID));
3528
3529 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
3530 // to build this up so provide a small stack buffer to handle the vast
3531 // majority of names.
3532 llvm::SmallString<64> name;
3533 GlobalDecl d(fd);
3534
3535 // If the builtin has been declared explicitly with an assembler label,
3536 // use the mangled name. This differs from the plain label on platforms
3537 // that prefix labels.
3538 if (fd->hasAttr<AsmLabelAttr>())
3539 name = getMangledName(gd: d);
3540 else if (llvm::StringRef targetName =
3541 CodeGenUtils::getTargetBuiltinLibFunctionName(BuiltinID: builtinID,
3542 Target: getTarget());
3543 !targetName.empty())
3544 name = targetName;
3545 else
3546 name = astContext.BuiltinInfo.getName(ID: builtinID).substr(pos: 10);
3547
3548 mlir::Type type = convertType(type: fd->getType());
3549 return getOrCreateCIRFunction(mangledName: name, funcType: type, gd: d, /*forVTable=*/false);
3550}
3551
3552mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {
3553 mlir::Value argValue = evaluateExprAsBool(e);
3554 if (!sanOpts.has(K: SanitizerKind::Builtin))
3555 return argValue;
3556
3557 assert(!cir::MissingFeatures::sanitizers());
3558 cgm.errorNYI(e->getSourceRange(),
3559 "emitCheckedArgForAssume: sanitizers are NYI");
3560 return {};
3561}
3562
3563void CIRGenFunction::emitVAStart(mlir::Value vaList) {
3564 // LLVM codegen casts to *i8, no real gain on doing this for CIRGen this
3565 // early, defer to LLVM lowering.
3566 cir::VAStartOp::create(builder, location: vaList.getLoc(), arg_list: vaList);
3567}
3568
3569void CIRGenFunction::emitVAEnd(mlir::Value vaList) {
3570 cir::VAEndOp::create(builder, location: vaList.getLoc(), arg_list: vaList);
3571}
3572
3573// FIXME(cir): This completely abstracts away the ABI with a generic CIR Op. By
3574// default this lowers to llvm.va_arg which is incomplete and not ABI-compliant
3575// on most targets so cir.va_arg will need some ABI handling in LoweringPrepare
3576mlir::Value CIRGenFunction::emitVAArg(VAArgExpr *ve) {
3577 assert(!cir::MissingFeatures::msabi());
3578 assert(!cir::MissingFeatures::vlas());
3579 mlir::Location loc = cgm.getLoc(cLoc: ve->getExprLoc());
3580 mlir::Type type = convertType(t: ve->getType());
3581 mlir::Value vaList = emitVAListRef(e: ve->getSubExpr()).getPointer();
3582 return cir::VAArgOp::create(builder, location: loc, result: type, arg_list: vaList);
3583}
3584
3585mlir::Value CIRGenFunction::emitBuiltinObjectSize(const Expr *e, unsigned type,
3586 cir::IntType resType,
3587 mlir::Value emittedE,
3588 bool isDynamic) {
3589 // If this is a pass_object_size parameter, load the implicit size arg.
3590 //
3591 // BOS type compatibility: a pass_object_size annotation with one type can
3592 // satisfy a __builtin_object_size query with a different type when the
3593 // annotated type is a safe approximation. Type 0 (max, whole object) is
3594 // an overestimate for type 1 (max, closest surrounding subobject), and
3595 // type 3 (min, closest surrounding subobject) is an underestimate for
3596 // type 2 (min, whole object).
3597 enum BOSType {
3598 MaxWholeObject = 0,
3599 MaxSubobject = 1,
3600 MinWholeObject = 2,
3601 MinSubobject = 3,
3602 };
3603 if (auto *dre = dyn_cast<DeclRefExpr>(Val: e->IgnoreParenImpCasts())) {
3604 auto *param = dyn_cast<ParmVarDecl>(Val: dre->getDecl());
3605 auto *objSizeAttr = dre->getDecl()->getAttr<PassObjectSizeAttr>();
3606 if (param && objSizeAttr) {
3607 auto from = objSizeAttr->getType();
3608 bool compatible = from == static_cast<int>(type) ||
3609 (from == MaxWholeObject && type == MaxSubobject) ||
3610 (from == MinSubobject && type == MinWholeObject);
3611 if (compatible) {
3612 const ImplicitParamDecl *sizeDecl = sizeArguments.lookup(Val: param);
3613 assert(sizeDecl && "expected pass_object_size implicit param");
3614
3615 DeclMapTy::iterator declIter = localDeclMap.find(Val: sizeDecl);
3616 assert(declIter != localDeclMap.end());
3617 Address addr = declIter->second;
3618
3619 return emitLoadOfScalar(addr, /*volatile=*/isVolatile: false,
3620 ty: getContext().getSizeType(), loc: e->getBeginLoc(),
3621 baseInfo: LValueBaseInfo(AlignmentSource::Decl));
3622 }
3623 }
3624 }
3625
3626 // LLVM can't handle type=3 appropriately, and __builtin_object_size shouldn't
3627 // evaluate e for side-effects. In either case, just like original LLVM
3628 // lowering, we shouldn't lower to `cir.objsize` but to a constant instead.
3629 if (type == 3 || (!emittedE && e->HasSideEffects(Ctx: getContext())))
3630 return builder.getConstInt(loc: getLoc(srcLoc: e->getSourceRange()), t: resType,
3631 c: (type & 2) ? 0 : -1);
3632
3633 mlir::Value ptr = emittedE ? emittedE : emitScalarExpr(e);
3634 assert(mlir::isa<cir::PointerType>(ptr.getType()) &&
3635 "Non-pointer passed to __builtin_object_size?");
3636
3637 assert(!cir::MissingFeatures::countedBySize());
3638
3639 // Extract the min/max mode from type. CIR only supports type 0
3640 // (max, whole object) and type 2 (min, whole object), not type 1 or 3
3641 // (closest subobject variants).
3642 const bool min = ((type & 2) != 0);
3643 // For GCC compatibility, __builtin_object_size treats NULL as unknown size.
3644 auto op =
3645 cir::ObjSizeOp::create(builder, location: getLoc(srcLoc: e->getSourceRange()), result: resType, ptr,
3646 min, /*nullUnknown=*/nullunknown: true, dynamic: isDynamic);
3647 return op.getResult();
3648}
3649
3650mlir::Value CIRGenFunction::evaluateOrEmitBuiltinObjectSize(
3651 const Expr *e, unsigned type, cir::IntType resType, mlir::Value emittedE,
3652 bool isDynamic) {
3653 if (std::optional<uint64_t> objectSize =
3654 e->tryEvaluateObjectSize(Ctx: getContext(), Type: type))
3655 return builder.getConstInt(loc: getLoc(srcLoc: e->getSourceRange()), t: resType,
3656 c: *objectSize);
3657 return emitBuiltinObjectSize(e, type, resType, emittedE, isDynamic);
3658}
3659
3660// Emit a device printf as a cir.offload.printf. Unlike classic codegen, which
3661// emits each target's printf runtime interface here, CIR leaves that to the
3662// target-specific lowering of the op.
3663mlir::Value CIRGenFunction::emitDevicePrintfCallExpr(const CallExpr *expr) {
3664 assert(cgm.getTriple().isNVPTX() || cgm.getTriple().isAMDGCN() ||
3665 (cgm.getTriple().isSPIRV() &&
3666 cgm.getTriple().getVendor() == llvm::Triple::AMD));
3667 assert(expr->getBuiltinCallee() == Builtin::BIprintf ||
3668 expr->getBuiltinCallee() == Builtin::BI__builtin_printf);
3669 assert(expr->getNumArgs() >= 1); // printf always has at least one arg.
3670 CallArgList args;
3671 emitCallArgs(args,
3672 prototype: expr->getDirectCallee()->getType()->getAs<FunctionProtoType>(),
3673 argRange: expr->arguments(), callee: expr->getDirectCallee());
3674
3675 mlir::Location loc = getLoc(srcLoc: expr->getBeginLoc());
3676
3677 // We don't know how to emit non-scalar varargs, nor scalars that
3678 // cir.offload.printf does not accept.
3679 bool hasNonScalar =
3680 llvm::any_of(Range: llvm::drop_begin(RangeOrContainer&: args), P: [&](const CallArg &a) {
3681 if (a.hasLValue() || !a.getKnownRValue().isScalar())
3682 return true;
3683 mlir::Type ty = a.getKnownRValue().getValue().getType();
3684 return !mlir::isa<cir::IntType, cir::PointerType, cir::DataMemberType,
3685 cir::VectorType>(Val: ty) &&
3686 !cir::isAnyFloatingPointType(type: ty);
3687 });
3688 if (hasNonScalar) {
3689 cgm.errorUnsupported(s: expr, type: "non-scalar args to printf");
3690 return builder.getConstInt(loc, t: builder.getSInt32Ty(), c: 0);
3691 }
3692
3693 llvm::SmallVector<mlir::Value, 8> callArgs;
3694 for (const CallArg &a : args)
3695 callArgs.push_back(Elt: a.getKnownRValue().getValue());
3696
3697 return cir::OffloadPrintfOp::create(builder, location: loc, result: builder.getSInt32Ty(),
3698 format: callArgs.front(),
3699 args: llvm::drop_begin(RangeOrContainer&: callArgs));
3700}
3701