1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Emit Expr nodes with scalar CIR types as CIR code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenConstantEmitter.h"
14#include "CIRGenFixedPointBuilder.h"
15#include "CIRGenFunction.h"
16#include "CIRGenValue.h"
17
18#include "clang/AST/Expr.h"
19#include "clang/AST/StmtVisitor.h"
20#include "clang/CIR/Dialect/IR/CIRTypes.h"
21#include "clang/CIR/MissingFeatures.h"
22#include "clang/CodeGenUtils/ExprUtils.h"
23
24#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
25#include "mlir/IR/Location.h"
26#include "mlir/IR/Value.h"
27
28#include <cassert>
29#include <numeric>
30#include <utility>
31
32using namespace clang;
33using namespace clang::CIRGen;
34
35namespace {
36
37struct BinOpInfo {
38 mlir::Value lhs;
39 mlir::Value rhs;
40 SourceRange loc;
41 QualType fullType; // Type of operands and result
42 QualType compType; // Type used for computations. Element type
43 // for vectors, otherwise same as FullType.
44 BinaryOperator::Opcode opcode; // Opcode of BinOp to perform
45 FPOptions fpFeatures;
46 const Expr *e; // Entire expr, for error unsupported. May not be binop.
47
48 /// Check if the binop computes a division or a remainder.
49 bool isDivRemOp() const {
50 return opcode == BO_Div || opcode == BO_Rem || opcode == BO_DivAssign ||
51 opcode == BO_RemAssign;
52 }
53
54 /// Check if the binop can result in integer overflow.
55 bool mayHaveIntegerOverflow() const {
56 // Without constant input, we can't rule out overflow.
57 auto lhsci = lhs.getDefiningOp<cir::ConstantOp>();
58 auto rhsci = rhs.getDefiningOp<cir::ConstantOp>();
59 if (!lhsci || !rhsci)
60 return true;
61
62 assert(!cir::MissingFeatures::mayHaveIntegerOverflow());
63 // TODO(cir): For now we just assume that we might overflow
64 return true;
65 }
66
67 /// Check if at least one operand is a fixed point type. In such cases,
68 /// this operation did not follow usual arithmetic conversion and both
69 /// operands might not be of the same type.
70 bool isFixedPointOp() const {
71 // We cannot simply check the result type since comparison operations
72 // return an int.
73 if (const auto *binOp = llvm::dyn_cast<BinaryOperator>(Val: e)) {
74 QualType lhstype = binOp->getLHS()->getType();
75 QualType rhstype = binOp->getRHS()->getType();
76 return lhstype->isFixedPointType() || rhstype->isFixedPointType();
77 }
78 if (const auto *unop = llvm::dyn_cast<UnaryOperator>(Val: e))
79 return unop->getSubExpr()->getType()->isFixedPointType();
80 return false;
81 }
82};
83
84class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
85 CIRGenFunction &cgf;
86 CIRGenBuilderTy &builder;
87 // Unlike classic codegen we set this to false or use std::exchange to read
88 // the value instead of calling TestAndClearIgnoreResultAssign to make it
89 // explicit when the value is used
90 bool ignoreResultAssign;
91
92public:
93 ScalarExprEmitter(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
94 bool ignoreResultAssign = false)
95 : cgf(cgf), builder(builder), ignoreResultAssign(ignoreResultAssign) {}
96
97 //===--------------------------------------------------------------------===//
98 // Utilities
99 //===--------------------------------------------------------------------===//
100 mlir::Type convertType(QualType ty) { return cgf.convertType(t: ty); }
101
102 mlir::Value emitComplexToScalarConversion(mlir::Location loc,
103 mlir::Value value, CastKind kind,
104 QualType destTy);
105
106 mlir::Value emitNullValue(QualType ty, mlir::Location loc) {
107 return cgf.cgm.emitNullConstant(t: ty, loc);
108 }
109
110 mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType) {
111 return builder.createFloatingCast(v: result, destType: cgf.convertType(t: promotionType));
112 }
113
114 mlir::Value emitUnPromotedValue(mlir::Value result, QualType exprType) {
115 return builder.createFloatingCast(v: result, destType: cgf.convertType(t: exprType));
116 }
117
118 mlir::Value emitPromoted(const Expr *e, QualType promotionType);
119
120 mlir::Value maybePromoteBoolResult(mlir::Value value,
121 mlir::Type dstTy) const {
122 if (mlir::isa<cir::IntType>(Val: dstTy))
123 return builder.createBoolToInt(src: value, newTy: dstTy);
124 if (mlir::isa<cir::BoolType>(Val: dstTy))
125 return value;
126 llvm_unreachable("Can only promote integer or boolean types");
127 }
128
129 //===--------------------------------------------------------------------===//
130 // Visitor Methods
131 //===--------------------------------------------------------------------===//
132
133 mlir::Value Visit(Expr *e) {
134 return StmtVisitor<ScalarExprEmitter, mlir::Value>::Visit(S: e);
135 }
136
137 mlir::Value VisitStmt(Stmt *s) {
138 llvm_unreachable("Statement passed to ScalarExprEmitter");
139 }
140
141 mlir::Value VisitExpr(Expr *e) {
142 cgf.getCIRGenModule().errorNYI(
143 loc: e->getSourceRange(), feature: "scalar expression kind: ", name: e->getStmtClassName());
144 return {};
145 }
146
147 mlir::Value VisitConstantExpr(ConstantExpr *e) {
148 // A constant expression of type 'void' generates no code and produces no
149 // value.
150 if (e->getType()->isVoidType())
151 return {};
152
153 if (mlir::Attribute result = ConstantEmitter(cgf).tryEmitConstantExpr(ce: e)) {
154 if (e->isGLValue()) {
155 cgf.cgm.errorNYI(e->getSourceRange(),
156 "ScalarExprEmitter: constant expr GL Value");
157 return {};
158 }
159
160 return builder.getConstant(loc: cgf.getLoc(srcLoc: e->getSourceRange()),
161 attr: mlir::cast<mlir::TypedAttr>(Val&: result));
162 }
163
164 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: constant expr");
165 return {};
166 }
167
168 mlir::Value VisitPackIndexingExpr(PackIndexingExpr *e) {
169 return Visit(e: e->getSelectedExpr());
170 }
171
172 mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(e: pe->getSubExpr()); }
173
174 mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
175 return Visit(e: ge->getResultExpr());
176 }
177
178 /// Emits the address of the l-value, then loads and returns the result.
179 mlir::Value emitLoadOfLValue(const Expr *e) {
180 LValue lv = cgf.emitLValue(e);
181 // FIXME: add some akin to EmitLValueAlignmentAssumption(E, V);
182 return cgf.emitLoadOfLValue(lv, loc: e->getExprLoc()).getValue();
183 }
184
185 mlir::Value VisitCoawaitExpr(CoawaitExpr *s) {
186 return cgf.emitCoawaitExpr(e: *s).getValue();
187 }
188
189 mlir::Value VisitCoyieldExpr(CoyieldExpr *e) {
190 return cgf.emitCoyieldExpr(e: *e).getValue();
191 }
192
193 mlir::Value VisitUnaryCoawait(const UnaryOperator *e) {
194 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: unary coawait");
195 return {};
196 }
197
198 mlir::Value emitLoadOfLValue(LValue lv, SourceLocation loc) {
199 return cgf.emitLoadOfLValue(lv, loc).getValue();
200 }
201
202 // l-values
203 mlir::Value VisitDeclRefExpr(DeclRefExpr *e) {
204 if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(refExpr: e))
205 return cgf.emitScalarConstant(constant, e);
206
207 return emitLoadOfLValue(e);
208 }
209
210 mlir::Value VisitAddrLabelExpr(const AddrLabelExpr *e) {
211 auto func = cast<cir::FuncOp>(Val: cgf.curFn);
212 cir::BlockAddrInfoAttr blockInfoAttr = cir::BlockAddrInfoAttr::get(
213 context: &cgf.getMLIRContext(), func_name: func.getSymName(), label_name: e->getLabel()->getName());
214 // GotoSolver collects this cir.block_address op after FlattenCFG to keep
215 // the label and wire it as an indirect-branch successor.
216 return cir::BlockAddressOp::create(builder, location: cgf.getLoc(srcLoc: e->getSourceRange()),
217 addr: cgf.convertType(t: e->getType()),
218 block_addr_info: blockInfoAttr);
219 }
220
221 mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
222 mlir::Type type = cgf.convertType(t: e->getType());
223 return cir::ConstantOp::create(builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
224 value: cir::IntAttr::get(type, value: e->getValue()));
225 }
226
227 mlir::Value VisitFixedPointLiteral(const FixedPointLiteral *e) {
228 mlir::Type type = cgf.convertType(t: e->getType());
229 return cir::ConstantOp::create(builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
230 value: cir::IntAttr::get(type, value: e->getValue()));
231 }
232
233 mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) {
234 mlir::Type type = cgf.convertType(t: e->getType());
235 assert(mlir::isa<cir::FPTypeInterface>(type) &&
236 "expect floating-point type");
237 return cir::ConstantOp::create(builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
238 value: cir::FPAttr::get(type, value: e->getValue()));
239 }
240
241 mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
242 mlir::Type ty = cgf.convertType(t: e->getType());
243 // Character literals are always stored in an unsigned (even for signed
244 // char), so allow implicit truncation here.
245 auto intTy = mlir::cast<cir::IntTypeInterface>(Val&: ty);
246 llvm::APInt apValue(intTy.getWidth(), e->getValue(),
247 /*isSigned=*/false, /*implicitTrunc=*/true);
248 return cir::ConstantOp::create(builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
249 value: cir::IntAttr::get(type: ty, value: apValue));
250 }
251
252 mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) {
253 return builder.getBool(state: e->getValue(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
254 }
255
256 mlir::Value VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *e) {
257 if (e->getType()->isVoidType())
258 return {};
259
260 return emitNullValue(ty: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
261 }
262
263 mlir::Value VisitGNUNullExpr(const GNUNullExpr *e) {
264 return emitNullValue(ty: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
265 }
266
267 mlir::Value VisitOffsetOfExpr(OffsetOfExpr *e);
268
269 mlir::Value VisitSizeOfPackExpr(SizeOfPackExpr *e) {
270 return builder.getConstInt(loc: cgf.getLoc(srcLoc: e->getExprLoc()),
271 t: convertType(ty: e->getType()), c: e->getPackLength());
272 }
273 mlir::Value VisitPseudoObjectExpr(PseudoObjectExpr *e) {
274 return cgf.emitPseudoObjectRValue(e).getValue();
275 }
276 mlir::Value VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *e);
277 mlir::Value VisitEmbedExpr(EmbedExpr *e) {
278 assert(e->getDataElementCount() == 1);
279 auto it = e->begin();
280 llvm::APInt value = (*it)->getValue();
281 return builder.getConstInt(loc: cgf.getLoc(srcLoc: e->getExprLoc()), intVal: value,
282 isUnsigned: e->getType()->isUnsignedIntegerType());
283 }
284 mlir::Value VisitOpaqueValueExpr(OpaqueValueExpr *e) {
285 if (e->isGLValue())
286 return emitLoadOfLValue(lv: cgf.getOrCreateOpaqueLValueMapping(e),
287 loc: e->getExprLoc());
288
289 // Otherwise, assume the mapping is the scalar directly.
290 return cgf.getOrCreateOpaqueRValueMapping(e).getValue();
291 }
292
293 mlir::Value VisitObjCSelectorExpr(ObjCSelectorExpr *e) {
294 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc selector");
295 return {};
296 }
297 mlir::Value VisitObjCProtocolExpr(ObjCProtocolExpr *e) {
298 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc protocol");
299 return {};
300 }
301 mlir::Value VisitObjCIVarRefExpr(ObjCIvarRefExpr *e) {
302 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc ivar ref");
303 return {};
304 }
305 mlir::Value VisitObjCMessageExpr(ObjCMessageExpr *e) {
306 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc message");
307 return {};
308 }
309 mlir::Value VisitObjCIsaExpr(ObjCIsaExpr *e) {
310 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc isa");
311 return {};
312 }
313 mlir::Value VisitObjCAvailabilityCheckExpr(ObjCAvailabilityCheckExpr *e) {
314 cgf.cgm.errorNYI(e->getSourceRange(),
315 "ScalarExprEmitter: objc availability check");
316 return {};
317 }
318
319 mlir::Value VisitMatrixSubscriptExpr(MatrixSubscriptExpr *e) {
320 mlir::Value rowIdx = cgf.emitScalarExpr(e: e->getRowIdx());
321 mlir::Value columnIdx = cgf.emitScalarExpr(e: e->getColumnIdx());
322 mlir::Value matrix = Visit(e: e->getBase());
323 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
324 assert(!cir::MissingFeatures::emitMatrixIndexAssumption());
325 return builder.createMatrixExtract(loc, matrix, rowIdx, columnIdx);
326 }
327
328 mlir::Value VisitMatrixSingleSubscriptExpr(MatrixSingleSubscriptExpr *e) {
329 mlir::Value rowIdx = cgf.emitScalarExpr(e: e->getRowIdx());
330 assert(!cir::MissingFeatures::emitMatrixIndexAssumption());
331 mlir::Value matrix = Visit(e: e->getBase());
332 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
333 return builder.createMatrixExtractRow(loc, matrix, rowIdx);
334 }
335
336 mlir::Value VisitCastExpr(CastExpr *e);
337 mlir::Value VisitCallExpr(const CallExpr *e);
338
339 mlir::Value VisitStmtExpr(StmtExpr *e) {
340 CIRGenFunction::StmtExprEvaluation eval(cgf);
341 if (e->getType()->isVoidType()) {
342 (void)cgf.emitCompoundStmt(s: *e->getSubStmt());
343 return {};
344 }
345
346 Address retAlloca =
347 cgf.createMemTemp(t: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
348 (void)cgf.emitCompoundStmt(s: *e->getSubStmt(), lastValue: &retAlloca);
349
350 return cgf.emitLoadOfScalar(lvalue: cgf.makeAddrLValue(addr: retAlloca, ty: e->getType()),
351 loc: e->getExprLoc());
352 }
353
354 mlir::Value VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
355 ignoreResultAssign = false;
356
357 if (e->getBase()->getType()->isVectorType()) {
358 assert(!cir::MissingFeatures::scalableVectors());
359
360 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
361 const mlir::Value vecValue = Visit(e: e->getBase());
362 const mlir::Value indexValue = Visit(e: e->getIdx());
363 return cir::VecExtractOp::create(builder&: cgf.getBuilder(), location: loc, vec: vecValue,
364 index: indexValue);
365 }
366 // Just load the lvalue formed by the subscript expression.
367 return emitLoadOfLValue(e);
368 }
369
370 mlir::Value VisitShuffleVectorExpr(ShuffleVectorExpr *e) {
371 if (e->getNumSubExprs() == 2) {
372 // The undocumented form of __builtin_shufflevector.
373 mlir::Value inputVec = Visit(e: e->getExpr(Index: 0));
374 mlir::Value indexVec = Visit(e: e->getExpr(Index: 1));
375 return cir::VecShuffleDynamicOp::create(builder&: cgf.getBuilder(),
376 location: cgf.getLoc(srcLoc: e->getSourceRange()),
377 vec: inputVec, indices: indexVec);
378 }
379
380 mlir::Value vec1 = Visit(e: e->getExpr(Index: 0));
381 mlir::Value vec2 = Visit(e: e->getExpr(Index: 1));
382
383 // The documented form of __builtin_shufflevector, where the indices are
384 // a variable number of integer constants. The constants will be stored
385 // in an ArrayAttr.
386 SmallVector<mlir::Attribute, 8> indices;
387 for (unsigned i = 2; i < e->getNumSubExprs(); ++i) {
388 indices.push_back(
389 Elt: cir::IntAttr::get(type: cgf.getBuilder().getSInt64Ty(),
390 value: e->getExpr(Index: i)
391 ->EvaluateKnownConstInt(Ctx: cgf.getContext())
392 .getSExtValue()));
393 }
394
395 return cir::VecShuffleOp::create(builder&: cgf.getBuilder(),
396 location: cgf.getLoc(srcLoc: e->getSourceRange()),
397 result: cgf.convertType(t: e->getType()), vec1, vec2,
398 indices: cgf.getBuilder().getArrayAttr(value: indices));
399 }
400
401 mlir::Value VisitConvertVectorExpr(ConvertVectorExpr *e) {
402 // __builtin_convertvector is an element-wise cast, and is implemented as a
403 // regular cast. The back end handles casts of vectors correctly.
404 return emitScalarConversion(src: Visit(e: e->getSrcExpr()),
405 srcType: e->getSrcExpr()->getType(), dstType: e->getType(),
406 loc: e->getSourceRange().getBegin());
407 }
408
409 mlir::Value VisitExtVectorElementExpr(Expr *e) { return emitLoadOfLValue(e); }
410
411 mlir::Value VisitMatrixElementExpr(Expr *e) { return emitLoadOfLValue(e); }
412
413 mlir::Value VisitMemberExpr(MemberExpr *e);
414
415 mlir::Value VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
416 return emitLoadOfLValue(e);
417 }
418
419 mlir::Value VisitInitListExpr(InitListExpr *e);
420 mlir::Value emitVectorInitWithSubvectors(mlir::Location loc,
421 cir::VectorType vectorType,
422 ArrayRef<mlir::Value> inits);
423
424 mlir::Value VisitArrayInitIndexExpr(ArrayInitIndexExpr *e) {
425 assert(cgf.getArrayInitIndex() &&
426 "ArrayInitIndexExpr not inside an ArrayInitLoopExpr?");
427 return cgf.getArrayInitIndex();
428 }
429
430 mlir::Value VisitImplicitValueInitExpr(const ImplicitValueInitExpr *e) {
431 return emitNullValue(ty: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
432 }
433
434 mlir::Value VisitExplicitCastExpr(ExplicitCastExpr *e) {
435 return VisitCastExpr(e);
436 }
437
438 mlir::Value VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *e) {
439 return cgf.cgm.emitNullConstant(t: e->getType(),
440 loc: cgf.getLoc(srcLoc: e->getSourceRange()));
441 }
442
443 /// Perform a pointer to boolean conversion.
444 mlir::Value emitPointerToBoolConversion(mlir::Value v, QualType qt) {
445 // TODO(cir): comparing the ptr to null is done when lowering CIR to LLVM.
446 // We might want to have a separate pass for these types of conversions.
447 return cgf.getBuilder().createPtrToBoolCast(v);
448 }
449
450 mlir::Value emitFloatToBoolConversion(mlir::Value src, mlir::Location loc) {
451 cir::BoolType boolTy = builder.getBoolTy();
452 return cir::CastOp::create(builder, location: loc, result: boolTy,
453 kind: cir::CastKind::float_to_bool, src);
454 }
455
456 mlir::Value emitIntToBoolConversion(mlir::Value srcVal, mlir::Location loc) {
457 // The enumerator of an enum with an integral underlying type is lowered to
458 // that type, which can be bool. In that case the operand is already a
459 // !cir.bool, so return it -- int_to_bool requires a !cir.int source.
460 if (mlir::isa<cir::BoolType>(Val: srcVal.getType()))
461 return srcVal;
462
463 // Because of the type rules of C, we often end up computing a
464 // logical value, then zero extending it to int, then wanting it
465 // as a logical value again.
466 // TODO: optimize this common case here or leave it for later
467 // CIR passes?
468 cir::BoolType boolTy = builder.getBoolTy();
469 return cir::CastOp::create(builder, location: loc, result: boolTy, kind: cir::CastKind::int_to_bool,
470 src: srcVal);
471 }
472
473 /// Convert the specified expression value to a boolean (!cir.bool) truth
474 /// value. This is equivalent to "Val != 0".
475 mlir::Value emitConversionToBool(mlir::Value src, QualType srcType,
476 mlir::Location loc) {
477 assert(srcType.isCanonical() && "EmitScalarConversion strips typedefs");
478
479 if (srcType->isRealFloatingType())
480 return emitFloatToBoolConversion(src, loc);
481
482 if (llvm::isa<MemberPointerType>(Val: srcType)) {
483 cgf.getCIRGenModule().errorNYI(loc, feature: "member pointer to bool conversion");
484 return builder.getFalse(loc);
485 }
486
487 if (srcType->isIntegerType())
488 return emitIntToBoolConversion(srcVal: src, loc);
489
490 assert(::mlir::isa<cir::PointerType>(src.getType()));
491 return emitPointerToBoolConversion(v: src, qt: srcType);
492 }
493
494 // Emit a conversion from the specified type to the specified destination
495 // type, both of which are CIR scalar types.
496 struct ScalarConversionOpts {
497 bool treatBooleanAsSigned;
498 bool emitImplicitIntegerTruncationChecks;
499 bool emitImplicitIntegerSignChangeChecks;
500
501 ScalarConversionOpts()
502 : treatBooleanAsSigned(false),
503 emitImplicitIntegerTruncationChecks(false),
504 emitImplicitIntegerSignChangeChecks(false) {}
505
506 ScalarConversionOpts(clang::SanitizerSet sanOpts)
507 : treatBooleanAsSigned(false),
508 emitImplicitIntegerTruncationChecks(
509 sanOpts.hasOneOf(K: SanitizerKind::ImplicitIntegerTruncation)),
510 emitImplicitIntegerSignChangeChecks(
511 sanOpts.has(K: SanitizerKind::ImplicitIntegerSignChange)) {}
512 };
513
514 // Conversion from bool, integral, or floating-point to integral or
515 // floating-point. Conversions involving other types are handled elsewhere.
516 // Conversion to bool is handled elsewhere because that's a comparison against
517 // zero, not a simple cast. This handles both individual scalars and vectors.
518 mlir::Value emitScalarCast(mlir::Value src, QualType srcType,
519 QualType dstType, mlir::Type srcTy,
520 mlir::Type dstTy, ScalarConversionOpts opts) {
521 assert(!(mlir::isa<mlir::IntegerType>(srcTy) ||
522 mlir::isa<mlir::IntegerType>(dstTy)) &&
523 "Obsolete code. Don't use mlir::IntegerType with CIR.");
524
525 mlir::Type fullDstTy = dstTy;
526 if (mlir::isa<cir::MatrixType>(Val: srcTy) &&
527 mlir::isa<cir::MatrixType>(Val: dstTy)) {
528 // Use the element types of the matrices to figure out the CastKind.
529 srcTy = mlir::dyn_cast<cir::MatrixType>(Val&: srcTy).getElementType();
530 dstTy = mlir::dyn_cast<cir::MatrixType>(Val&: dstTy).getElementType();
531 } else if (mlir::isa<cir::VectorType>(Val: srcTy) &&
532 mlir::isa<cir::VectorType>(Val: dstTy)) {
533 // Use the element types of the vectors to figure out the CastKind.
534 srcTy = mlir::dyn_cast<cir::VectorType>(Val&: srcTy).getElementType();
535 dstTy = mlir::dyn_cast<cir::VectorType>(Val&: dstTy).getElementType();
536 }
537
538 std::optional<cir::CastKind> castKind;
539
540 // Start with a null fenv attr. If this is a floating point cast, we will
541 // get the attribute from the builder.
542 cir::FenvAttr fenvAttr;
543
544 if (mlir::isa<cir::BoolType>(Val: srcTy)) {
545 if (opts.treatBooleanAsSigned)
546 cgf.getCIRGenModule().errorNYI(feature: "signed bool");
547 if (cgf.getBuilder().isInt(i: dstTy)) {
548 castKind = cir::CastKind::bool_to_int;
549 } else if (mlir::isa<cir::FPTypeInterface>(Val: dstTy)) {
550 fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
551 castKind = cir::CastKind::bool_to_float;
552 } else {
553 llvm_unreachable("Internal error: Cast to unexpected type");
554 }
555 } else if (cgf.getBuilder().isInt(i: srcTy)) {
556 if (cgf.getBuilder().isInt(i: dstTy)) {
557 castKind = cir::CastKind::integral;
558 } else if (mlir::isa<cir::FPTypeInterface>(Val: dstTy)) {
559 fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
560 castKind = cir::CastKind::int_to_float;
561 } else if (mlir::isa<cir::BoolType>(Val: dstTy)) {
562 castKind = cir::CastKind::int_to_bool;
563 } else {
564 llvm_unreachable("Internal error: Cast to unexpected type");
565 }
566 } else if (mlir::isa<cir::FPTypeInterface>(Val: srcTy)) {
567 fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
568 if (cgf.getBuilder().isInt(i: dstTy)) {
569 // If we can't recognize overflow as undefined behavior, assume that
570 // overflow saturates. This protects against normal optimizations if we
571 // are compiling with non-standard FP semantics.
572 if (!cgf.cgm.getCodeGenOpts().StrictFloatCastOverflow)
573 cgf.getCIRGenModule().errorNYI(feature: "strict float cast overflow");
574 castKind = cir::CastKind::float_to_int;
575 } else if (mlir::isa<cir::FPTypeInterface>(Val: dstTy)) {
576 // TODO: split this to createFPExt/createFPTrunc
577 return builder.createFloatingCast(v: src, destType: fullDstTy);
578 } else if (mlir::isa<cir::BoolType>(Val: dstTy)) {
579 castKind = cir::CastKind::float_to_bool;
580 } else {
581 llvm_unreachable("Internal error: Cast to unexpected type");
582 }
583 } else {
584 llvm_unreachable("Internal error: Cast from unexpected type");
585 }
586
587 assert(castKind.has_value() && "Internal error: CastKind not set.");
588 return builder.createOrFold<cir::CastOp>(location: src.getLoc(), args&: fullDstTy, args&: *castKind,
589 args&: src, args&: fenvAttr);
590 }
591
592 mlir::Value
593 VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
594 return Visit(e: e->getReplacement());
595 }
596
597 mlir::Value VisitVAArgExpr(VAArgExpr *ve) {
598 QualType ty = ve->getType();
599
600 if (ty->isVariablyModifiedType()) {
601 cgf.cgm.errorNYI(ve->getSourceRange(),
602 "variably modified types in varargs");
603 }
604
605 return cgf.emitVAArg(ve);
606 }
607
608 mlir::Value VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *e) {
609 return Visit(e: e->getSemanticForm());
610 }
611
612 mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
613 mlir::Value
614 VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
615
616 // Unary Operators.
617 mlir::Value VisitUnaryPrePostIncDec(const UnaryOperator *e) {
618 LValue lv = cgf.emitLValue(e: e->getSubExpr());
619 return emitScalarPrePostIncDec(e, lv);
620 }
621 mlir::Value VisitUnaryPostDec(const UnaryOperator *e) {
622 return VisitUnaryPrePostIncDec(e);
623 }
624 mlir::Value VisitUnaryPostInc(const UnaryOperator *e) {
625 return VisitUnaryPrePostIncDec(e);
626 }
627 mlir::Value VisitUnaryPreDec(const UnaryOperator *e) {
628 return VisitUnaryPrePostIncDec(e);
629 }
630 mlir::Value VisitUnaryPreInc(const UnaryOperator *e) {
631 return VisitUnaryPrePostIncDec(e);
632 }
633
634 mlir::Value emitFixedPointIncDec(const UnaryOperator *e, mlir::Value value,
635 QualType type);
636 mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv) {
637 assert(!cir::MissingFeatures::openMPLastprivateConditional());
638
639 QualType type = e->getSubExpr()->getType();
640
641 mlir::Value value;
642 mlir::Value input;
643
644 if (const AtomicType *atomicTy = type->getAs<AtomicType>()) {
645 QualType valType = atomicTy->getValueType();
646 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
647 bool isInc = e->isIncrementOp();
648 bool isPre = e->isPrefix();
649
650 // Bools are always set-to-true, as decrement isn't legal on bools.
651 if (valType->isBooleanType()) {
652 assert(isInc);
653 // Atomic operations require an integer type; reinterpret the bool
654 // pointer as a pointer to its underlying storage integer type.
655 cir::IntType intTy = builder.getBoolMemoryTy();
656 mlir::Value one = builder.getConstInt(loc, t: intTy, c: 1);
657 Address intAddr = lv.getAddress().withElementType(builder, ElemTy: intTy);
658
659 if (isPre) {
660 // Pre-increment: atomically store true, return true.
661 cir::StoreOp store = builder.createStore(loc, val: one, dst: intAddr);
662 store.setMemOrder(cir::MemOrder::SequentiallyConsistent);
663 if (lv.isVolatileQualified())
664 store.setIsVolatile(true);
665 return builder.getTrue(loc);
666 }
667 // Post-increment: atomically exchange with true, return old value.
668 auto xchg = cir::AtomicXchgOp::create(
669 builder, location: loc, ptr: intAddr.getPointer(), val: one,
670 mem_order: cir::MemOrder::SequentiallyConsistent, sync_scope: cir::SyncScopeKind::System,
671 is_volatile: lv.isVolatileQualified());
672 return builder.createCast(kind: cir::CastKind::int_to_bool, src: xchg.getResult(),
673 newTy: builder.getBoolTy());
674 }
675
676 // Special case for atomic increment / decrement on integers, emit
677 // atomicrmw instructions. We skip this if we want to be doing overflow
678 // checking, and fall into the slow path with the atomic cmpxchg loop.
679 if (valType->isIntegerType() &&
680 !(valType->isUnsignedIntegerType() &&
681 cgf.sanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow)) &&
682 cgf.getLangOpts().getSignedOverflowBehavior() !=
683 LangOptions::SOB_Trapping) {
684 mlir::Type intTy = cgf.convertType(t: valType);
685 mlir::Value one = builder.getConstInt(loc, t: intTy, c: 1);
686 cir::AtomicFetchKind kind =
687 isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
688 auto rmw = cir::AtomicFetchOp::create(
689 builder, location: loc, ptr: lv.getPointer(), val: one, binop: kind,
690 mem_order: cir::MemOrder::SequentiallyConsistent, sync_scope: cir::SyncScopeKind::System,
691 is_volatile: lv.isVolatileQualified(), /*fetch_first=*/true);
692 mlir::Value oldVal = rmw->getResult(idx: 0);
693 // Prefix returns new value; postfix returns old value.
694 return isPre ? emitIntIncOrDec(e, input: oldVal) : oldVal;
695 }
696
697 // Special case for atomic increment/decrement on floats.
698 // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80).
699 if (valType->isFloatingType()) {
700 mlir::Type fpTy = cgf.convertType(t: valType);
701 auto fpType = mlir::cast<cir::FPTypeInterface>(Val&: fpTy);
702 // Bail on non-power-of-2 types (e.g., x86_fp80 is 80 bits).
703 if (llvm::has_single_bit(Value: fpType.getWidth())) {
704 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
705 mlir::Value amt = builder.getConstFP(
706 loc, t: fpTy, fpVal: llvm::APFloat(fpType.getFloatSemantics(), 1));
707 cir::AtomicFetchKind kind =
708 isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
709 auto rmw = cir::AtomicFetchOp::create(
710 builder, location: loc, ptr: lv.getPointer(), val: amt, binop: kind,
711 mem_order: cir::MemOrder::SequentiallyConsistent, sync_scope: cir::SyncScopeKind::System,
712 is_volatile: lv.isVolatileQualified(), /*fetch_first=*/true);
713 mlir::Value oldVal = rmw->getResult(idx: 0);
714 if (isPre)
715 return isInc ? builder.createFAdd(loc, lhs: oldVal, rhs: amt)
716 : builder.createFSub(loc, lhs: oldVal, rhs: amt);
717 return oldVal;
718 } else {
719 cgf.cgm.errorNYI(e->getSourceRange(),
720 "Atomic inc/dec of non-power-of-2 float");
721 }
722 }
723
724 // Otherwise we need a loop on compare-exchange.
725 cgf.cgm.errorNYI(e->getSourceRange(), "Atomic inc/dec");
726 value = cgf.emitLoadOfLValue(lv, loc: e->getExprLoc()).getValue();
727 input = value;
728 } else {
729 value = cgf.emitLoadOfLValue(lv, loc: e->getExprLoc()).getValue();
730 input = value;
731 }
732
733 // NOTE: When possible, more frequent cases are handled first.
734
735 // Special case of integer increment that we have to check first: bool++.
736 // Due to promotion rules, we get:
737 // bool++ -> bool = bool + 1
738 // -> bool = (int)bool + 1
739 // -> bool = ((int)bool + 1 != 0)
740 // An interesting aspect of this is that increment is always true.
741 // Decrement does not have this property.
742 if (e->isIncrementOp() && type->isBooleanType()) {
743 value = builder.getTrue(loc: cgf.getLoc(srcLoc: e->getExprLoc()));
744 } else if (type->isIntegerType()) {
745 QualType promotedType;
746 [[maybe_unused]] bool canPerformLossyDemotionCheck = false;
747 if (cgf.getContext().isPromotableIntegerType(T: type)) {
748 promotedType = cgf.getContext().getPromotedIntegerType(PromotableType: type);
749 assert(promotedType != type && "Shouldn't promote to the same type.");
750 canPerformLossyDemotionCheck = true;
751 canPerformLossyDemotionCheck &=
752 cgf.getContext().getCanonicalType(T: type) !=
753 cgf.getContext().getCanonicalType(T: promotedType);
754 canPerformLossyDemotionCheck &=
755 type->isIntegerType() && promotedType->isIntegerType();
756
757 // TODO(cir): Currently, we store bitwidths in CIR types only for
758 // integers. This might also be required for other types.
759
760 assert(
761 (!canPerformLossyDemotionCheck ||
762 type->isSignedIntegerOrEnumerationType() ||
763 promotedType->isSignedIntegerOrEnumerationType() ||
764 mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth() ==
765 mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth()) &&
766 "The following check expects that if we do promotion to different "
767 "underlying canonical type, at least one of the types (either "
768 "base or promoted) will be signed, or the bitwidths will match.");
769 }
770
771 assert(!cir::MissingFeatures::sanitizers());
772 if (e->canOverflow() && type->isSignedIntegerOrEnumerationType()) {
773 value = emitIncDecConsiderOverflowBehavior(e, inVal: value);
774 } else {
775 // NOTE(CIR): clang calls CreateAdd but folds this to a unary op
776 value = emitIntIncOrDec(e, input, /*nsw=*/false);
777 }
778 } else if (const PointerType *ptr = type->getAs<PointerType>()) {
779 QualType type = ptr->getPointeeType();
780 if (const VariableArrayType *vla =
781 cgf.getContext().getAsVariableArrayType(T: type)) {
782 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
783 mlir::Value numElts = cgf.getVLASize(type: vla).numElts;
784 if (!e->isIncrementOp())
785 numElts = cgf.getBuilder().createNeg(loc, value: numElts, /*nsw=*/true);
786 assert(!cir::MissingFeatures::sanitizers());
787 value = cgf.getBuilder().createPtrStride(loc, base: value, stride: numElts);
788 } else {
789 // For everything else, we can just do a simple increment.
790 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
791 int amount = e->isIncrementOp() ? 1 : -1;
792 mlir::Value amt = builder.getSInt32(c: amount, loc);
793 assert(!cir::MissingFeatures::sanitizers());
794 value = builder.createPtrStride(loc, base: value, stride: amt);
795 }
796 } else if (type->isVectorType()) {
797 if (type->hasIntegerRepresentation()) {
798 value = emitIntIncOrDec(e, input, /*nsw=*/false);
799 } else {
800 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
801 value = emitFloatIncOrDec(e, input);
802 }
803 } else if (type->isRealFloatingType()) {
804 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
805 value = emitFloatIncOrDec(e, input);
806 } else if (type->isFixedPointType()) {
807 value = emitFixedPointIncDec(e, value, type);
808 } else {
809 assert(type->castAs<ObjCObjectPointerType>());
810 cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec ObjectiveC pointer");
811 return {};
812 }
813
814 CIRGenFunction::SourceLocRAIIObject sourceloc{cgf, e->getSourceRange()};
815
816 // Store the updated result through the lvalue
817 if (lv.isBitField())
818 value = cgf.emitStoreThroughBitfieldLValue(src: RValue::get(v: value), dstresult: lv);
819 else
820 cgf.emitStoreThroughLValue(src: RValue::get(v: value), dst: lv);
821
822 // If this is a postinc, return the value read from memory, otherwise use
823 // the updated value.
824 return e->isPrefix() ? value : input;
825 }
826
827 mlir::Value emitIncDecConsiderOverflowBehavior(const UnaryOperator *e,
828 mlir::Value inVal) {
829 switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
830 case LangOptions::SOB_Defined:
831 return emitIntIncOrDec(e, input: inVal, /*nsw=*/false);
832 case LangOptions::SOB_Undefined:
833 assert(!cir::MissingFeatures::sanitizers());
834 return emitIntIncOrDec(e, input: inVal, /*nsw=*/true);
835 case LangOptions::SOB_Trapping:
836 if (!e->canOverflow())
837 return emitIntIncOrDec(e, input: inVal, /*nsw=*/true);
838 cgf.cgm.errorNYI(e->getSourceRange(), "inc/def overflow SOB_Trapping");
839 return {};
840 }
841 llvm_unreachable("Unexpected signed overflow behavior kind");
842 }
843
844 mlir::Value VisitUnaryAddrOf(const UnaryOperator *e) {
845 if (llvm::isa<MemberPointerType>(Val: e->getType()))
846 return cgf.cgm.emitMemberPointerConstant(e);
847
848 return cgf.emitLValue(e: e->getSubExpr()).getPointer();
849 }
850
851 mlir::Value VisitUnaryDeref(const UnaryOperator *e) {
852 if (e->getType()->isVoidType())
853 return Visit(e: e->getSubExpr()); // the actual value should be unused
854 return emitLoadOfLValue(e);
855 }
856
857 mlir::Value VisitUnaryPlus(const UnaryOperator *e) {
858 QualType promotionType = getPromotionType(ty: e->getSubExpr()->getType());
859 mlir::Value result = VisitUnaryPlus(e, promotionType);
860 if (result && !promotionType.isNull())
861 return emitUnPromotedValue(result, exprType: e->getType());
862 return result;
863 }
864
865 mlir::Value VisitUnaryPlus(const UnaryOperator *e, QualType promotionType) {
866 ignoreResultAssign = false;
867 if (!promotionType.isNull())
868 return cgf.emitPromotedScalarExpr(e: e->getSubExpr(), promotionType);
869 return Visit(e: e->getSubExpr());
870 }
871
872 mlir::Value VisitUnaryMinus(const UnaryOperator *e) {
873 QualType promotionType = getPromotionType(ty: e->getSubExpr()->getType());
874 mlir::Value result = VisitUnaryMinus(e, promotionType);
875 if (result && !promotionType.isNull())
876 return emitUnPromotedValue(result, exprType: e->getType());
877 return result;
878 }
879
880 mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
881 ignoreResultAssign = false;
882 mlir::Value operand;
883 if (!promotionType.isNull())
884 operand = cgf.emitPromotedScalarExpr(e: e->getSubExpr(), promotionType);
885 else
886 operand = Visit(e: e->getSubExpr());
887
888 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange().getBegin());
889
890 if (cir::isFPOrVectorOfFPType(type: operand.getType())) {
891 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
892 return builder.createOrFold<cir::FNegOp>(location: loc, args&: operand,
893 args: builder.getFastMathFlagsAttr());
894 }
895
896 // Saturated negation must saturate (-min for signed, any nonzero value for
897 // unsigned), as in classic codegen.
898 if (e->getType()->isSaturatedFixedPointType()) {
899 CIRGenFixedPointBuilder fpBuilder(builder, loc);
900 return fpBuilder.createNeg(
901 value: operand, sema: cgf.getContext().getFixedPointSemantics(Ty: e->getType()));
902 }
903
904 // TODO(cir): We might have to change this to support overflow trapping.
905 // Classic codegen routes unary minus through emitSub to ensure
906 // that the overflow behavior is handled correctly.
907 bool nsw = e->getType()->isSignedIntegerType() &&
908 cgf.getLangOpts().getSignedOverflowBehavior() !=
909 LangOptions::SOB_Defined;
910
911 return builder.createOrFold<cir::MinusOp>(location: loc, args&: operand, args&: nsw);
912 }
913
914 mlir::Value emitIntIncOrDec(const UnaryOperator *e, mlir::Value input,
915 bool nsw = false) {
916 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange().getBegin());
917 return e->isIncrementOp()
918 ? builder.createOrFold<cir::IncOp>(location: loc, args&: input, args&: nsw)
919 : builder.createOrFold<cir::DecOp>(location: loc, args&: input, args&: nsw);
920 }
921
922 mlir::Value emitFloatIncOrDec(const UnaryOperator *e, mlir::Value input) {
923 assert(cir::isFPOrVectorOfFPType(input.getType()) &&
924 "Expect floating-point operand");
925 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange().getBegin());
926
927 if (auto vecType = mlir::dyn_cast<cir::VectorType>(Val: input.getType())) {
928 mlir::Type fpScalarType = vecType.getElementType();
929 auto fpInterface = mlir::cast<cir::FPTypeInterface>(Val&: fpScalarType);
930 auto amount = llvm::APFloat::getOne(Sem: fpInterface.getFloatSemantics(),
931 /*Negative=*/e->isDecrementOp());
932 mlir::Value amtValue = builder.getConstFP(loc, t: fpScalarType, fpVal: amount);
933 amtValue = cir::VecSplatOp::create(builder, location: loc, result: vecType, value: amtValue);
934 return builder.createFAdd(loc, lhs: input, rhs: amtValue);
935 }
936
937 QualType type = e->getSubExpr()->getType();
938 // Another special case: half FP increment should be done via float.
939 if (type->isHalfType() && !cgf.getContext().getLangOpts().NativeHalfType)
940 input = builder.createFloatingCast(v: input, destType: builder.getSingleTy());
941
942 auto fpInterface = mlir::cast<cir::FPTypeInterface>(Val: input.getType());
943 auto amount = llvm::APFloat::getOne(Sem: fpInterface.getFloatSemantics(),
944 /*Negative=*/e->isDecrementOp());
945 mlir::Value amtValue = builder.getConstFP(loc, t: input.getType(), fpVal: amount);
946 mlir::Value output = builder.createFAdd(loc, lhs: input, rhs: amtValue);
947
948 if (type->isHalfType() && !cgf.getContext().getLangOpts().NativeHalfType)
949 output = builder.createFloatingCast(v: output, destType: builder.getFp16Ty());
950
951 return output;
952 }
953
954 mlir::Value VisitUnaryNot(const UnaryOperator *e) {
955 ignoreResultAssign = false;
956 mlir::Value op = Visit(e: e->getSubExpr());
957 return builder.createOrFold<cir::NotOp>(
958 location: cgf.getLoc(srcLoc: e->getSourceRange().getBegin()), args&: op);
959 }
960
961 mlir::Value VisitUnaryLNot(const UnaryOperator *e);
962
963 mlir::Value VisitUnaryReal(const UnaryOperator *e);
964 mlir::Value VisitUnaryImag(const UnaryOperator *e);
965 mlir::Value VisitRealImag(const UnaryOperator *e,
966 QualType promotionType = QualType());
967
968 mlir::Value VisitUnaryExtension(const UnaryOperator *e) {
969 return Visit(e: e->getSubExpr());
970 }
971
972 // C++
973 mlir::Value VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e) {
974 return emitLoadOfLValue(e);
975 }
976 mlir::Value VisitSourceLocExpr(SourceLocExpr *e) {
977 ASTContext &ctx = cgf.getContext();
978 APValue evaluated =
979 e->EvaluateInContext(Ctx: ctx, DefaultExpr: cgf.curSourceLocExprScope.getDefaultExpr());
980 mlir::Attribute attribute = ConstantEmitter(cgf).emitAbstract(
981 loc: e->getLocation(), value: evaluated, destType: e->getType());
982 mlir::TypedAttr typedAttr = mlir::cast<mlir::TypedAttr>(Val&: attribute);
983 return cir::ConstantOp::create(builder, location: cgf.getLoc(srcLoc: e->getExprLoc()),
984 value: typedAttr);
985 }
986 mlir::Value VisitCXXDefaultArgExpr(CXXDefaultArgExpr *dae) {
987 CIRGenFunction::CXXDefaultArgExprScope scope(cgf, dae);
988 return Visit(e: dae->getExpr());
989 }
990 mlir::Value VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die) {
991 CIRGenFunction::CXXDefaultInitExprScope scope(cgf, die);
992 return Visit(e: die->getExpr());
993 }
994
995 mlir::Value VisitCXXThisExpr(CXXThisExpr *te) { return cgf.loadCXXThis(); }
996
997 mlir::Value VisitExprWithCleanups(ExprWithCleanups *e);
998 mlir::Value VisitCXXNewExpr(const CXXNewExpr *e) {
999 return cgf.emitCXXNewExpr(e);
1000 }
1001 mlir::Value VisitCXXDeleteExpr(const CXXDeleteExpr *e) {
1002 cgf.emitCXXDeleteExpr(e);
1003 return {};
1004 }
1005 mlir::Value VisitTypeTraitExpr(const TypeTraitExpr *e) {
1006 // We diverge slightly from classic codegen here because CIR has stricter
1007 // typing. In LLVM IR, constant folding covers up some potential type
1008 // mismatches such as bool-to-int conversions that would fail the verifier
1009 // in CIR. To make things work, we need to be sure we only emit a bool value
1010 // if the expression type is bool.
1011 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1012 if (e->isStoredAsBoolean()) {
1013 if (e->getType()->isBooleanType())
1014 return builder.getBool(state: e->getBoolValue(), loc);
1015 assert(e->getType()->isIntegerType() &&
1016 "Expected int type for TypeTraitExpr");
1017 return builder.getConstInt(loc, t: cgf.convertType(t: e->getType()),
1018 c: (uint64_t)e->getBoolValue());
1019 }
1020 assert(e->getType()->isIntegerType() && "not a scalar type trait");
1021 return builder.getConstInt(loc, intVal: e->getAPValue().getInt());
1022 }
1023 mlir::Value
1024 VisitConceptSpecializationExpr(const ConceptSpecializationExpr *e) {
1025 return builder.getBool(state: e->isSatisfied(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
1026 }
1027 mlir::Value VisitRequiresExpr(const RequiresExpr *e) {
1028 return builder.getBool(state: e->isSatisfied(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
1029 }
1030 mlir::Value VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *e) {
1031 mlir::Type type = cgf.convertType(t: e->getType());
1032 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1033 return builder.getConstInt(loc, t: type, c: e->getValue());
1034 }
1035 mlir::Value VisitExpressionTraitExpr(const ExpressionTraitExpr *e) {
1036 return builder.getBool(state: e->getValue(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
1037 }
1038 mlir::Value VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *e) {
1039 cgf.cgm.errorNYI(e->getSourceRange(),
1040 "ScalarExprEmitter: cxx pseudo destructor");
1041 return {};
1042 }
1043 mlir::Value VisitCXXThrowExpr(const CXXThrowExpr *e) {
1044 cgf.emitCXXThrowExpr(e);
1045 return {};
1046 }
1047
1048 mlir::Value VisitCXXNoexceptExpr(CXXNoexceptExpr *e) {
1049 return builder.getBool(state: e->getValue(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
1050 }
1051
1052 mlir::Value emitFixedPointConversion(mlir::Value src, QualType srcTy,
1053 QualType dstTy, mlir::Location loc);
1054
1055 /// Emit a conversion from the specified type to the specified destination
1056 /// type, both of which are CIR scalar types.
1057 /// TODO: do we need ScalarConversionOpts here? Should be done in another
1058 /// pass.
1059 mlir::Value
1060 emitScalarConversion(mlir::Value src, QualType srcType, QualType dstType,
1061 SourceLocation loc,
1062 ScalarConversionOpts opts = ScalarConversionOpts()) {
1063 // All conversions involving fixed point types should be handled by the
1064 // emitFixedPoint family functions. This is done to prevent bloating up
1065 // this function more, and although fixed point numbers are represented by
1066 // integers, we do not want to follow any logic that assumes they should be
1067 // treated as integers.
1068 if (srcType->isFixedPointType()) {
1069 if (dstType->isBooleanType()) {
1070 cir::BoolType boolTy = builder.getBoolTy();
1071 return cir::CastOp::create(builder, location: cgf.getLoc(srcLoc: loc), result: boolTy,
1072 kind: cir::CastKind::int_to_bool, src);
1073 }
1074
1075 if (dstType->isFixedPointType() || dstType->isIntegerType() ||
1076 dstType->isRealFloatingType())
1077 return emitFixedPointConversion(src, srcTy: srcType, dstTy: dstType, loc: cgf.getLoc(srcLoc: loc));
1078
1079 llvm_unreachable("Unhandled scalar conversion from a fixed point type to "
1080 "another type.");
1081 } else if (dstType->isFixedPointType()) {
1082 if (srcType->isIntegerType() || srcType->isRealFloatingType()) {
1083 // This also includes converting booleans and enums to fixed point
1084 // types.
1085 return emitFixedPointConversion(src, srcTy: srcType, dstTy: dstType, loc: cgf.getLoc(srcLoc: loc));
1086 }
1087
1088 llvm_unreachable("Unhandled scalar conversion to a fixed point type from "
1089 "another type.");
1090 }
1091
1092 srcType = srcType.getCanonicalType();
1093 dstType = dstType.getCanonicalType();
1094 if (srcType == dstType) {
1095 if (opts.emitImplicitIntegerSignChangeChecks)
1096 cgf.getCIRGenModule().errorNYI(loc,
1097 "implicit integer sign change checks");
1098 return src;
1099 }
1100
1101 if (dstType->isVoidType())
1102 return {};
1103
1104 mlir::Type mlirSrcType = src.getType();
1105
1106 // Handle conversions to bool first, they are special: comparisons against
1107 // 0.
1108 if (dstType->isBooleanType())
1109 return emitConversionToBool(src, srcType, loc: cgf.getLoc(srcLoc: loc));
1110
1111 mlir::Type mlirDstType = cgf.convertType(t: dstType);
1112
1113 if (srcType->isHalfType() &&
1114 !cgf.getContext().getLangOpts().NativeHalfType) {
1115 // Cast to FP using the intrinsic if the half type itself isn't supported.
1116 if (!mlir::isa<cir::FPTypeInterface>(Val: mlirDstType)) {
1117 // Cast to other types through float, using FPExt, depending on whether
1118 // the half type itself is supported (as opposed to operations on half,
1119 // available with NativeHalfType).
1120 src = builder.createCast(loc: cgf.getLoc(srcLoc: loc), kind: cir::CastKind::floating, src,
1121 newTy: cgf.floatTy);
1122 srcType = cgf.getContext().FloatTy;
1123 mlirSrcType = cgf.floatTy;
1124 }
1125 }
1126
1127 // TODO(cir): LLVM codegen ignore conversions like int -> uint,
1128 // is there anything to be done for CIR here?
1129 if (mlirSrcType == mlirDstType) {
1130 if (opts.emitImplicitIntegerSignChangeChecks)
1131 cgf.getCIRGenModule().errorNYI(loc,
1132 "implicit integer sign change checks");
1133 return src;
1134 }
1135
1136 // Handle pointer conversions next: pointers can only be converted to/from
1137 // other pointers and integers. Check for pointer types in terms of LLVM, as
1138 // some native types (like Obj-C id) may map to a pointer type.
1139 if (auto dstPT = dyn_cast<cir::PointerType>(Val&: mlirDstType)) {
1140 cgf.getCIRGenModule().errorNYI(loc, "pointer casts");
1141 return builder.getNullPtr(ty: dstPT, loc: src.getLoc());
1142 }
1143
1144 if (isa<cir::PointerType>(Val: mlirSrcType)) {
1145 // Must be an ptr to int cast.
1146 assert(isa<cir::IntType>(mlirDstType) && "not ptr->int?");
1147 return builder.createPtrToInt(src, newTy: mlirDstType);
1148 }
1149
1150 // A scalar can be splatted to an extended vector of the same element type
1151 if (dstType->isExtVectorType() && !srcType->isVectorType()) {
1152 // Sema should add casts to make sure that the source expression's type
1153 // is the same as the vector's element type (sans qualifiers)
1154 assert(dstType->castAs<ExtVectorType>()->getElementType().getTypePtr() ==
1155 srcType.getTypePtr() &&
1156 "Splatted expr doesn't match with vector element type?");
1157
1158 cgf.getCIRGenModule().errorNYI(loc, "vector splatting");
1159 return {};
1160 }
1161
1162 if (srcType->isMatrixType() && dstType->isMatrixType()) {
1163 return emitScalarCast(src, srcType, dstType, srcTy: mlirSrcType, dstTy: mlirDstType,
1164 opts);
1165 }
1166 assert(!srcType->isMatrixType() && !dstType->isMatrixType() &&
1167 "Internal error: conversion between matrix type and scalar type");
1168
1169 // Finally, we have the arithmetic types or vectors of arithmetic types.
1170 mlir::Value res = nullptr;
1171 mlir::Type resTy = mlirDstType;
1172
1173 res = emitScalarCast(src, srcType, dstType, srcTy: mlirSrcType, dstTy: mlirDstType, opts);
1174
1175 if (mlirDstType != resTy) {
1176 res = builder.createCast(loc: cgf.getLoc(srcLoc: loc), kind: cir::CastKind::floating, src: res,
1177 newTy: resTy);
1178 }
1179
1180 if (opts.emitImplicitIntegerTruncationChecks)
1181 cgf.getCIRGenModule().errorNYI(loc, "implicit integer truncation checks");
1182
1183 if (opts.emitImplicitIntegerSignChangeChecks)
1184 cgf.getCIRGenModule().errorNYI(loc,
1185 "implicit integer sign change checks");
1186
1187 return res;
1188 }
1189
1190 BinOpInfo emitBinOps(const BinaryOperator *e,
1191 QualType promotionType = QualType()) {
1192 ignoreResultAssign = false;
1193 BinOpInfo result;
1194 result.lhs = cgf.emitPromotedScalarExpr(e: e->getLHS(), promotionType);
1195 result.rhs = cgf.emitPromotedScalarExpr(e: e->getRHS(), promotionType);
1196 if (!promotionType.isNull())
1197 result.fullType = promotionType;
1198 else
1199 result.fullType = e->getType();
1200 result.compType = result.fullType;
1201 if (const auto *vecType = result.fullType->getAs<VectorType>())
1202 result.compType = vecType->getElementType();
1203 result.opcode = e->getOpcode();
1204 result.loc = e->getSourceRange();
1205 result.fpFeatures = e->getFPFeaturesInEffect(LO: cgf.getLangOpts());
1206 result.e = e;
1207 return result;
1208 }
1209
1210 mlir::Value emitMul(const BinOpInfo &ops);
1211 mlir::Value emitDiv(const BinOpInfo &ops);
1212 mlir::Value emitRem(const BinOpInfo &ops);
1213 mlir::Value emitAdd(const BinOpInfo &ops);
1214 mlir::Value emitSub(const BinOpInfo &ops);
1215 mlir::Value emitShl(const BinOpInfo &ops);
1216 mlir::Value emitShr(const BinOpInfo &ops);
1217 mlir::Value emitAnd(const BinOpInfo &ops);
1218 mlir::Value emitXor(const BinOpInfo &ops);
1219 mlir::Value emitOr(const BinOpInfo &ops);
1220
1221 mlir::Value emitFixedPointBinOp(const BinOpInfo &ops);
1222
1223 LValue emitCompoundAssignLValue(
1224 const CompoundAssignOperator *e,
1225 mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &),
1226 mlir::Value &result);
1227 mlir::Value
1228 emitCompoundAssign(const CompoundAssignOperator *e,
1229 mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &));
1230
1231 // TODO(cir): Candidate to be in a common AST helper between CIR and LLVM
1232 // codegen.
1233 QualType getPromotionType(QualType ty) {
1234 const clang::ASTContext &ctx = cgf.getContext();
1235 if (auto *complexTy = ty->getAs<ComplexType>()) {
1236 QualType elementTy = complexTy->getElementType();
1237 if (elementTy.UseExcessPrecision(Ctx: ctx))
1238 return ctx.getComplexType(T: ctx.FloatTy);
1239 }
1240
1241 if (ty.UseExcessPrecision(Ctx: cgf.getContext())) {
1242 if (auto *vt = ty->getAs<VectorType>()) {
1243 unsigned numElements = vt->getNumElements();
1244 return ctx.getVectorType(VectorType: ctx.FloatTy, NumElts: numElements, VecKind: vt->getVectorKind());
1245 }
1246 return cgf.getContext().FloatTy;
1247 }
1248
1249 return QualType();
1250 }
1251
1252// Binary operators and binary compound assignment operators.
1253#define HANDLEBINOP(OP) \
1254 mlir::Value VisitBin##OP(const BinaryOperator *e) { \
1255 QualType promotionTy = getPromotionType(e->getType()); \
1256 auto result = emit##OP(emitBinOps(e, promotionTy)); \
1257 if (result && !promotionTy.isNull()) \
1258 result = emitUnPromotedValue(result, e->getType()); \
1259 return result; \
1260 } \
1261 mlir::Value VisitBin##OP##Assign(const CompoundAssignOperator *e) { \
1262 return emitCompoundAssign(e, &ScalarExprEmitter::emit##OP); \
1263 }
1264
1265 HANDLEBINOP(Mul)
1266 HANDLEBINOP(Div)
1267 HANDLEBINOP(Rem)
1268 HANDLEBINOP(Add)
1269 HANDLEBINOP(Sub)
1270 HANDLEBINOP(Shl)
1271 HANDLEBINOP(Shr)
1272 HANDLEBINOP(And)
1273 HANDLEBINOP(Xor)
1274 HANDLEBINOP(Or)
1275#undef HANDLEBINOP
1276
1277 cir::CmpOpKind clangCmpToCIRCmp(clang::BinaryOperatorKind clangCmp) {
1278 switch (clangCmp) {
1279 case BO_LT:
1280 return cir::CmpOpKind::lt;
1281 case BO_GT:
1282 return cir::CmpOpKind::gt;
1283 case BO_LE:
1284 return cir::CmpOpKind::le;
1285 case BO_GE:
1286 return cir::CmpOpKind::ge;
1287 case BO_EQ:
1288 return cir::CmpOpKind::eq;
1289 case BO_NE:
1290 return cir::CmpOpKind::ne;
1291 default:
1292 llvm_unreachable("unsupported comparison kind for cir.cmp");
1293 }
1294 }
1295 /// Returns <Signaling, Kind>.
1296 std::pair<bool, cir::FCmpOpKind>
1297 clangCmpToCIRFCmp(clang::BinaryOperatorKind clangCmp) {
1298 switch (clangCmp) {
1299 case BO_LT:
1300 return std::make_pair(/*Signaling=*/x: true, y: cir::FCmpOpKind::olt);
1301 case BO_GT:
1302 return std::make_pair(/*Signaling=*/x: true, y: cir::FCmpOpKind::ogt);
1303 case BO_LE:
1304 return std::make_pair(/*Signaling=*/x: true, y: cir::FCmpOpKind::ole);
1305 case BO_GE:
1306 return std::make_pair(/*Signaling=*/x: true, y: cir::FCmpOpKind::oge);
1307 case BO_EQ:
1308 return std::make_pair(/*Signaling=*/x: false, y: cir::FCmpOpKind::oeq);
1309 case BO_NE:
1310 return std::make_pair(/*Signaling=*/x: false, y: cir::FCmpOpKind::une);
1311 default:
1312 llvm_unreachable("unsupported comparison kind");
1313 }
1314 }
1315
1316 mlir::Value emitCmp(const BinaryOperator *e) {
1317 ignoreResultAssign = false;
1318 const mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1319 mlir::Value result;
1320 QualType lhsTy = e->getLHS()->getType();
1321 QualType rhsTy = e->getRHS()->getType();
1322
1323 cir::CmpOpKind kind = clangCmpToCIRCmp(clangCmp: e->getOpcode());
1324 if (lhsTy->getAs<MemberPointerType>()) {
1325 assert(!cir::MissingFeatures::dataMemberType());
1326 assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE);
1327 mlir::Value lhs = cgf.emitScalarExpr(e: e->getLHS());
1328 mlir::Value rhs = cgf.emitScalarExpr(e: e->getRHS());
1329 result = builder.createCompare(loc, kind, lhs, rhs);
1330 } else if (!lhsTy->isAnyComplexType() && !rhsTy->isAnyComplexType()) {
1331 BinOpInfo boInfo = emitBinOps(e);
1332 mlir::Value lhs = boInfo.lhs;
1333 mlir::Value rhs = boInfo.rhs;
1334
1335 if (lhsTy->isVectorType()) {
1336 if (!e->getType()->isVectorType()) {
1337 // If AltiVec, the comparison results in a numeric type, so we use
1338 // intrinsics comparing vectors and giving 0 or 1 as a result
1339 cgf.cgm.errorNYI(loc, feature: "AltiVec comparison");
1340 } else {
1341 // Other kinds of vectors. Element-wise comparison returning
1342 // a vector.
1343 auto resultTy =
1344 mlir::cast<cir::VectorType>(Val: cgf.convertType(t: boInfo.fullType));
1345 if (cir::isFPOrVectorOfFPType(type: lhs.getType())) {
1346 std::pair<bool, cir::FCmpOpKind> fpInfo =
1347 clangCmpToCIRFCmp(clangCmp: e->getOpcode());
1348 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
1349 result = builder.createVecFCompare(loc: cgf.getLoc(srcLoc: boInfo.loc), resultTy,
1350 kind: fpInfo.second, lhs, rhs,
1351 signaling: fpInfo.first);
1352 } else {
1353 result = builder.createVecCompare(loc: cgf.getLoc(srcLoc: boInfo.loc), resultTy,
1354 kind, lhs, rhs);
1355 }
1356 }
1357 } else if (boInfo.isFixedPointOp()) {
1358 result = emitFixedPointBinOp(ops: boInfo);
1359 } else {
1360 // integers and pointers
1361 if (cgf.cgm.getCodeGenOpts().StrictVTablePointers &&
1362 mlir::isa<cir::PointerType>(Val: lhs.getType()) &&
1363 mlir::isa<cir::PointerType>(Val: rhs.getType())) {
1364 cgf.cgm.errorNYI(loc, feature: "strict vtable pointer comparisons");
1365 }
1366 result = builder.createCompare(loc, kind, lhs, rhs);
1367 }
1368 } else {
1369 assert((e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE) &&
1370 "Complex Comparison: can only be an equality comparison");
1371
1372 mlir::Value lhs;
1373 if (lhsTy->isAnyComplexType()) {
1374 lhs = cgf.emitComplexExpr(e: e->getLHS());
1375 } else {
1376 mlir::Value lhsReal = Visit(e: e->getLHS());
1377 mlir::Value lhsImag = builder.getNullValue(ty: convertType(ty: lhsTy), loc);
1378 lhs = builder.createComplexCreate(loc, real: lhsReal, imag: lhsImag);
1379 }
1380
1381 mlir::Value rhs;
1382 if (rhsTy->isAnyComplexType()) {
1383 rhs = cgf.emitComplexExpr(e: e->getRHS());
1384 } else {
1385 mlir::Value rhsReal = Visit(e: e->getRHS());
1386 mlir::Value rhsImag = builder.getNullValue(ty: convertType(ty: rhsTy), loc);
1387 rhs = builder.createComplexCreate(loc, real: rhsReal, imag: rhsImag);
1388 }
1389
1390 result = builder.createCompare(loc, kind, lhs, rhs);
1391 }
1392
1393 return emitScalarConversion(src: result, srcType: cgf.getContext().BoolTy, dstType: e->getType(),
1394 loc: e->getExprLoc());
1395 }
1396
1397// Comparisons.
1398#define VISITCOMP(CODE) \
1399 mlir::Value VisitBin##CODE(const BinaryOperator *E) { return emitCmp(E); }
1400 VISITCOMP(LT)
1401 VISITCOMP(GT)
1402 VISITCOMP(LE)
1403 VISITCOMP(GE)
1404 VISITCOMP(EQ)
1405 VISITCOMP(NE)
1406#undef VISITCOMP
1407
1408 mlir::Value VisitBinAssign(const BinaryOperator *e) {
1409 const bool ignore = std::exchange(obj&: ignoreResultAssign, new_val: false);
1410
1411 mlir::Value rhs;
1412 LValue lhs;
1413
1414 switch (e->getLHS()->getType().getObjCLifetime()) {
1415 case Qualifiers::OCL_Strong:
1416 case Qualifiers::OCL_Autoreleasing:
1417 case Qualifiers::OCL_ExplicitNone:
1418 case Qualifiers::OCL_Weak:
1419 assert(!cir::MissingFeatures::objCLifetime());
1420 break;
1421 case Qualifiers::OCL_None:
1422 // __block variables need to have the rhs evaluated first, plus this
1423 // should improve codegen just a little.
1424 rhs = Visit(e: e->getRHS());
1425 assert(!cir::MissingFeatures::sanitizers());
1426 // TODO(cir): This needs to be emitCheckedLValue() once we support
1427 // sanitizers
1428 lhs = cgf.emitLValue(e: e->getLHS());
1429
1430 // Store the value into the LHS. Bit-fields are handled specially because
1431 // the result is altered by the store, i.e., [C99 6.5.16p1]
1432 // 'An assignment expression has the value of the left operand after the
1433 // assignment...'.
1434 if (lhs.isBitField()) {
1435 CIRGenFunction::SourceLocRAIIObject loc{cgf, e->getSourceRange()};
1436 rhs = cgf.emitStoreThroughBitfieldLValue(src: RValue::get(v: rhs), dstresult: lhs);
1437 } else {
1438 cgf.emitNullabilityCheck(lhs, rhs, loc: e->getExprLoc());
1439 CIRGenFunction::SourceLocRAIIObject loc{cgf, e->getSourceRange()};
1440 cgf.emitStoreThroughLValue(src: RValue::get(v: rhs), dst: lhs);
1441 }
1442 }
1443
1444 // If the result is clearly ignored, return now.
1445 if (ignore)
1446 return nullptr;
1447
1448 // The result of an assignment in C is the assigned r-value.
1449 if (!cgf.getLangOpts().CPlusPlus)
1450 return rhs;
1451
1452 // If the lvalue is non-volatile, return the computed value of the
1453 // assignment.
1454 if (!lhs.isVolatile())
1455 return rhs;
1456
1457 // Otherwise, reload the value.
1458 return emitLoadOfLValue(lv: lhs, loc: e->getExprLoc());
1459 }
1460
1461 mlir::Value VisitBinComma(const BinaryOperator *e) {
1462 cgf.emitIgnoredExpr(e: e->getLHS());
1463 // NOTE: We don't need to EnsureInsertPoint() like LLVM codegen.
1464 return Visit(e: e->getRHS());
1465 }
1466
1467 mlir::Value VisitBinLAnd(const clang::BinaryOperator *e) {
1468 if (e->getType()->isVectorType()) {
1469 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1470 mlir::Type lhsTy = cgf.convertType(t: e->getLHS()->getType());
1471 mlir::Value zeroVec = builder.getNullValue(ty: lhsTy, loc);
1472
1473 mlir::Value lhs = Visit(e: e->getLHS());
1474 mlir::Value rhs = Visit(e: e->getRHS());
1475
1476 auto resultTy =
1477 mlir::cast<cir::VectorType>(Val: cgf.convertType(t: e->getType()));
1478 if (cir::isFPOrVectorOfFPType(type: lhs.getType())) {
1479 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
1480 auto cmpOpKind = cir::FCmpOpKind::une;
1481 lhs = builder.createVecFCompare(loc, resultTy, kind: cmpOpKind, lhs, rhs: zeroVec);
1482 rhs = builder.createVecFCompare(loc, resultTy, kind: cmpOpKind, lhs: rhs, rhs: zeroVec);
1483 } else {
1484 auto cmpOpKind = cir::CmpOpKind::ne;
1485 lhs = builder.createVecCompare(loc, resultTy, kind: cmpOpKind, lhs, rhs: zeroVec);
1486 rhs = builder.createVecCompare(loc, resultTy, kind: cmpOpKind, lhs: rhs, rhs: zeroVec);
1487 }
1488 return builder.createAnd(loc, lhs, rhs);
1489 }
1490
1491 assert(!cir::MissingFeatures::instrumentation());
1492 mlir::Type resTy = cgf.convertType(t: e->getType());
1493 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1494
1495 mlir::Value lhsCondV = cgf.evaluateExprAsBool(e: e->getLHS());
1496
1497 CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
1498
1499 auto resOp = cir::TernaryOp::create(
1500 builder, location: loc, cond: lhsCondV, /*trueBuilder=*/
1501 [&](mlir::OpBuilder &b, mlir::Location loc) {
1502 CIRGenFunction::LexicalScope lexScope{cgf, loc,
1503 b.getInsertionBlock()};
1504 cgf.curLexScope->setAsTernary();
1505 eval.beginEvaluation();
1506 mlir::Value res = cgf.evaluateExprAsBool(e: e->getRHS());
1507 eval.endEvaluation();
1508 lexScope.forceCleanup(valuesToReload: {&res});
1509 cir::YieldOp::create(builder&: b, location: loc, args: res);
1510 },
1511 /*falseBuilder*/
1512 [&](mlir::OpBuilder &b, mlir::Location loc) {
1513 CIRGenFunction::LexicalScope lexScope{cgf, loc,
1514 b.getInsertionBlock()};
1515 cgf.curLexScope->setAsTernary();
1516 auto res = cir::ConstantOp::create(builder&: b, location: loc, value: builder.getFalseAttr());
1517 cir::YieldOp::create(builder&: b, location: loc, args: res.getRes());
1518 });
1519 return maybePromoteBoolResult(value: resOp.getResult(), dstTy: resTy);
1520 }
1521
1522 mlir::Value VisitBinLOr(const clang::BinaryOperator *e) {
1523 if (e->getType()->isVectorType()) {
1524 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1525 mlir::Type lhsTy = cgf.convertType(t: e->getLHS()->getType());
1526 mlir::Value zeroVec = builder.getNullValue(ty: lhsTy, loc);
1527
1528 mlir::Value lhs = Visit(e: e->getLHS());
1529 mlir::Value rhs = Visit(e: e->getRHS());
1530
1531 auto resultTy =
1532 mlir::cast<cir::VectorType>(Val: cgf.convertType(t: e->getType()));
1533 if (cir::isFPOrVectorOfFPType(type: lhs.getType())) {
1534 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
1535 auto cmpOpKind = cir::FCmpOpKind::une;
1536 lhs = builder.createVecFCompare(loc, resultTy, kind: cmpOpKind, lhs, rhs: zeroVec);
1537 rhs = builder.createVecFCompare(loc, resultTy, kind: cmpOpKind, lhs: rhs, rhs: zeroVec);
1538 } else {
1539 auto cmpOpKind = cir::CmpOpKind::ne;
1540 lhs = builder.createVecCompare(loc, resultTy, kind: cmpOpKind, lhs, rhs: zeroVec);
1541 rhs = builder.createVecCompare(loc, resultTy, kind: cmpOpKind, lhs: rhs, rhs: zeroVec);
1542 }
1543 return builder.createOr(loc, lhs, rhs);
1544 }
1545
1546 assert(!cir::MissingFeatures::instrumentation());
1547 mlir::Type resTy = cgf.convertType(t: e->getType());
1548 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1549
1550 mlir::Value lhsCondV = cgf.evaluateExprAsBool(e: e->getLHS());
1551
1552 CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
1553
1554 auto resOp = cir::TernaryOp::create(
1555 builder, location: loc, cond: lhsCondV, /*trueBuilder=*/
1556 [&](mlir::OpBuilder &b, mlir::Location loc) {
1557 CIRGenFunction::LexicalScope lexScope{cgf, loc,
1558 b.getInsertionBlock()};
1559 cgf.curLexScope->setAsTernary();
1560 auto res = cir::ConstantOp::create(builder&: b, location: loc, value: builder.getTrueAttr());
1561 cir::YieldOp::create(builder&: b, location: loc, args: res.getRes());
1562 },
1563 /*falseBuilder*/
1564 [&](mlir::OpBuilder &b, mlir::Location loc) {
1565 CIRGenFunction::LexicalScope lexScope{cgf, loc,
1566 b.getInsertionBlock()};
1567 cgf.curLexScope->setAsTernary();
1568 eval.beginEvaluation();
1569 mlir::Value res = cgf.evaluateExprAsBool(e: e->getRHS());
1570 eval.endEvaluation();
1571 lexScope.forceCleanup(valuesToReload: {&res});
1572 cir::YieldOp::create(builder&: b, location: loc, args: res);
1573 });
1574
1575 return maybePromoteBoolResult(value: resOp.getResult(), dstTy: resTy);
1576 }
1577
1578 mlir::Value VisitBinPtrMemD(const BinaryOperator *e) {
1579 return emitLoadOfLValue(e);
1580 }
1581
1582 mlir::Value VisitBinPtrMemI(const BinaryOperator *e) {
1583 return emitLoadOfLValue(e);
1584 }
1585
1586 // Other Operators.
1587 mlir::Value VisitBlockExpr(const BlockExpr *e) {
1588 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: block");
1589 return {};
1590 }
1591
1592 mlir::Value VisitChooseExpr(ChooseExpr *e) {
1593 return Visit(e: e->getChosenSubExpr());
1594 }
1595
1596 mlir::Value VisitObjCStringLiteral(const ObjCStringLiteral *e) {
1597 cgf.cgm.errorNYI(e->getSourceRange(),
1598 "ScalarExprEmitter: objc string literal");
1599 return {};
1600 }
1601 mlir::Value VisitObjCBoxedExpr(ObjCBoxedExpr *e) {
1602 cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc boxed");
1603 return {};
1604 }
1605 mlir::Value VisitObjCArrayLiteral(ObjCArrayLiteral *e) {
1606 cgf.cgm.errorNYI(e->getSourceRange(),
1607 "ScalarExprEmitter: objc array literal");
1608 return {};
1609 }
1610 mlir::Value VisitObjCDictionaryLiteral(ObjCDictionaryLiteral *e) {
1611 cgf.cgm.errorNYI(e->getSourceRange(),
1612 "ScalarExprEmitter: objc dictionary literal");
1613 return {};
1614 }
1615
1616 mlir::Value convertVec3AndVec4(CIRGenBuilderTy &builder, mlir::Location loc,
1617 mlir::Value src, unsigned numElementsDst) {
1618 static constexpr int64_t mask[] = {0, 1, 2, -1};
1619 return builder.createVecShuffle(
1620 loc, vec1: src, mask: llvm::ArrayRef<int64_t>(mask, numElementsDst));
1621 }
1622
1623 // Create cast instructions for converting MLIR value \p Src to MLIR type \p
1624 // DstTy. \p Src has the same size as \p DstTy. Both are single value types
1625 // but could be scalar or vectors of different lengths, and either can be
1626 // pointer.
1627 //
1628 // There are 4 cases:
1629 // 1. non-pointer -> non-pointer : needs 1 bitcast
1630 // 2. pointer -> pointer : needs 1 bitcast or addrspacecast
1631 // 3. pointer -> non-pointer
1632 // a) pointer -> intptr_t : needs 1 ptrtoint
1633 // b) pointer -> non-intptr_t : needs 1 ptrtoint then 1 bitcast
1634 // 4. non-pointer -> pointer
1635 // a) intptr_t -> pointer : needs 1 inttoptr
1636 // b) non-intptr_t -> pointer : needs 1 bitcast then 1 inttoptr
1637 //
1638 // Note: for cases 3b and 4b two casts are required since LLVM casts do not
1639 // allow casting directly between pointer types and non-integer non-pointer
1640 // types.
1641 mlir::Value createCastsForTypeOfSameSize(mlir::Value src, mlir::Type dstTy) {
1642 mlir::Type srcTy = src.getType();
1643
1644 // Case 1.
1645 if (!isa<cir::PointerType>(Val: srcTy) && !isa<cir::PointerType>(Val: dstTy))
1646 return builder.createBitcast(src, newTy: dstTy);
1647
1648 // Case 2.
1649 if (isa<cir::PointerType>(Val: srcTy) && isa<cir::PointerType>(Val: dstTy)) {
1650 cgf.cgm.errorNYI(
1651 feature: "ScalarExprEmitter: createCastsForTypeOfSameSize Case 2");
1652 return {};
1653 }
1654
1655 // Case 3.
1656 if (isa<cir::PointerType>(Val: srcTy) && !isa<cir::PointerType>(Val: dstTy)) {
1657 if (!isa<cir::IntType>(Val: dstTy)) {
1658 cgf.cgm.errorNYI(
1659 feature: "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a");
1660 }
1661
1662 cgf.cgm.errorNYI(
1663 feature: "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a and 3b");
1664 return {};
1665 }
1666
1667 // Case 4b.
1668 if (!isa<cir::IntType>(Val: srcTy)) {
1669 cgf.cgm.errorNYI(
1670 feature: "ScalarExprEmitter: createCastsForTypeOfSameSize Case 4a");
1671 return {};
1672 }
1673 // Cases 4a and 4b.
1674 return builder.createIntToPtr(src, newTy: dstTy);
1675 }
1676
1677 mlir::Value VisitAsTypeExpr(AsTypeExpr *e) {
1678 mlir::Value src = cgf.emitScalarExpr(e: e->getSrcExpr());
1679 mlir::Type srcTy = src.getType();
1680 mlir::Type dstTy = cgf.convertType(t: e->getType());
1681
1682 unsigned numElementsSrc = isa<cir::VectorType>(Val: srcTy)
1683 ? cast<cir::VectorType>(Val&: srcTy).getFixedSize()
1684 : 0;
1685 unsigned numElementsDst = isa<cir::VectorType>(Val: dstTy)
1686 ? cast<cir::VectorType>(Val&: dstTy).getFixedSize()
1687 : 0;
1688
1689 // Use bit vector expansion for ext_vector_type boolean vectors.
1690 if (e->getType()->isExtVectorBoolType()) {
1691 cgf.cgm.errorNYI(e->getSourceRange(),
1692 "ScalarExprEmitter: VisitAsTypeExpr ExtVectorBoolType");
1693 return {};
1694 }
1695
1696 // Going from vec3 to non-vec3 is a special case and requires a shuffle
1697 // vector to get a vec4, then a bitcast if the target type is different.
1698 if (numElementsSrc == 3 && numElementsDst != 3) {
1699 cgf.cgm.errorNYI(e->getSourceRange(),
1700 "ScalarExprEmitter: VisitAsTypeExpr numElemsSrc = 3, "
1701 "numElemsDst != 3");
1702 return {};
1703 }
1704
1705 // Going from non-vec3 to vec3 is a special case and requires a bitcast
1706 // to vec4 if the original type is not vec4, then a shuffle vector to
1707 // get a vec3.
1708 if (numElementsSrc != 3 && numElementsDst == 3) {
1709 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
1710 auto dstElemTy = cast<cir::VectorType>(Val&: dstTy).getElementType();
1711 auto dstVec4Ty = cir::VectorType::getFixed(elementType: dstElemTy, size: 4);
1712 src = createCastsForTypeOfSameSize(src, dstTy: dstVec4Ty);
1713 src = convertVec3AndVec4(builder, loc, src, numElementsDst: 3);
1714 return src;
1715 }
1716
1717 return createCastsForTypeOfSameSize(src, dstTy);
1718 }
1719
1720 mlir::Value VisitAtomicExpr(AtomicExpr *e) {
1721 return cgf.emitAtomicExpr(e).getValue();
1722 }
1723};
1724
1725LValue ScalarExprEmitter::emitCompoundAssignLValue(
1726 const CompoundAssignOperator *e,
1727 mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &),
1728 mlir::Value &result) {
1729 if (e->getComputationResultType()->isAnyComplexType())
1730 return cgf.emitScalarCompoundAssignWithComplex(e, result);
1731
1732 QualType lhsTy = e->getLHS()->getType();
1733 BinOpInfo opInfo;
1734
1735 // Emit the RHS first. __block variables need to have the rhs evaluated
1736 // first, plus this should improve codegen a little.
1737
1738 QualType promotionTypeCR = getPromotionType(ty: e->getComputationResultType());
1739 if (promotionTypeCR.isNull())
1740 promotionTypeCR = e->getComputationResultType();
1741
1742 QualType promotionTypeLHS = getPromotionType(ty: e->getComputationLHSType());
1743 QualType promotionTypeRHS = getPromotionType(ty: e->getRHS()->getType());
1744
1745 if (!promotionTypeRHS.isNull())
1746 opInfo.rhs = cgf.emitPromotedScalarExpr(e: e->getRHS(), promotionType: promotionTypeRHS);
1747 else
1748 opInfo.rhs = Visit(e: e->getRHS());
1749
1750 opInfo.fullType = promotionTypeCR;
1751 opInfo.compType = opInfo.fullType;
1752 if (const auto *vecType = opInfo.fullType->getAs<VectorType>())
1753 opInfo.compType = vecType->getElementType();
1754 opInfo.opcode = e->getOpcode();
1755 opInfo.fpFeatures = e->getFPFeaturesInEffect(LO: cgf.getLangOpts());
1756 opInfo.e = e;
1757 opInfo.loc = e->getSourceRange();
1758
1759 // Load/convert the LHS
1760 LValue lhsLV = cgf.emitLValue(e: e->getLHS());
1761
1762 if (lhsTy->getAs<AtomicType>()) {
1763 cgf.cgm.errorNYI(loc: result.getLoc(), feature: "atomic lvalue assign");
1764 return LValue();
1765 }
1766
1767 opInfo.lhs = emitLoadOfLValue(lv: lhsLV, loc: e->getExprLoc());
1768
1769 CIRGenFunction::SourceLocRAIIObject sourceloc{cgf, e->getSourceRange()};
1770 SourceLocation loc = e->getExprLoc();
1771 if (!promotionTypeLHS.isNull())
1772 opInfo.lhs = emitScalarConversion(src: opInfo.lhs, srcType: lhsTy, dstType: promotionTypeLHS, loc);
1773 else
1774 opInfo.lhs = emitScalarConversion(src: opInfo.lhs, srcType: lhsTy,
1775 dstType: e->getComputationLHSType(), loc);
1776
1777 // Expand the binary operator.
1778 result = (this->*func)(opInfo);
1779
1780 // Convert the result back to the LHS type,
1781 // potentially with Implicit Conversion sanitizer check.
1782 result = emitScalarConversion(src: result, srcType: promotionTypeCR, dstType: lhsTy, loc,
1783 opts: ScalarConversionOpts(cgf.sanOpts));
1784
1785 // Store the result value into the LHS lvalue. Bit-fields are handled
1786 // specially because the result is altered by the store, i.e., [C99 6.5.16p1]
1787 // 'An assignment expression has the value of the left operand after the
1788 // assignment...'.
1789 if (lhsLV.isBitField())
1790 cgf.emitStoreThroughBitfieldLValue(src: RValue::get(v: result), dstresult: lhsLV);
1791 else
1792 cgf.emitStoreThroughLValue(src: RValue::get(v: result), dst: lhsLV);
1793
1794 assert(!cir::MissingFeatures::openMPLastprivateConditional());
1795
1796 return lhsLV;
1797}
1798
1799mlir::Value ScalarExprEmitter::emitComplexToScalarConversion(mlir::Location lov,
1800 mlir::Value value,
1801 CastKind kind,
1802 QualType destTy) {
1803 cir::CastKind castOpKind;
1804 switch (kind) {
1805 case CK_FloatingComplexToReal:
1806 castOpKind = cir::CastKind::float_complex_to_real;
1807 break;
1808 case CK_IntegralComplexToReal:
1809 castOpKind = cir::CastKind::int_complex_to_real;
1810 break;
1811 case CK_FloatingComplexToBoolean:
1812 castOpKind = cir::CastKind::float_complex_to_bool;
1813 break;
1814 case CK_IntegralComplexToBoolean:
1815 castOpKind = cir::CastKind::int_complex_to_bool;
1816 break;
1817 default:
1818 llvm_unreachable("invalid complex-to-scalar cast kind");
1819 }
1820
1821 return builder.createCast(loc: lov, kind: castOpKind, src: value, newTy: cgf.convertType(t: destTy));
1822}
1823
1824mlir::Value ScalarExprEmitter::emitPromoted(const Expr *e,
1825 QualType promotionType) {
1826 e = e->IgnoreParens();
1827 if (const auto *bo = dyn_cast<BinaryOperator>(Val: e)) {
1828 switch (bo->getOpcode()) {
1829#define HANDLE_BINOP(OP) \
1830 case BO_##OP: \
1831 return emit##OP(emitBinOps(bo, promotionType));
1832 HANDLE_BINOP(Add)
1833 HANDLE_BINOP(Sub)
1834 HANDLE_BINOP(Mul)
1835 HANDLE_BINOP(Div)
1836#undef HANDLE_BINOP
1837 default:
1838 break;
1839 }
1840 } else if (const auto *uo = dyn_cast<UnaryOperator>(Val: e)) {
1841 switch (uo->getOpcode()) {
1842 case UO_Imag:
1843 case UO_Real:
1844 return VisitRealImag(e: uo, promotionType);
1845 case UO_Minus:
1846 return VisitUnaryMinus(e: uo, promotionType);
1847 case UO_Plus:
1848 return VisitUnaryPlus(e: uo, promotionType);
1849 default:
1850 break;
1851 }
1852 }
1853 mlir::Value result = Visit(e: const_cast<Expr *>(e));
1854 if (result) {
1855 if (!promotionType.isNull())
1856 return emitPromotedValue(result, promotionType);
1857 return emitUnPromotedValue(result, exprType: e->getType());
1858 }
1859 return result;
1860}
1861
1862mlir::Value ScalarExprEmitter::emitCompoundAssign(
1863 const CompoundAssignOperator *e,
1864 mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &)) {
1865
1866 bool ignore = std::exchange(obj&: ignoreResultAssign, new_val: false);
1867 mlir::Value rhs;
1868 LValue lhs = emitCompoundAssignLValue(e, func, result&: rhs);
1869
1870 // If the result is clearly ignored, return now.
1871 if (ignore)
1872 return {};
1873
1874 // The result of an assignment in C is the assigned r-value.
1875 if (!cgf.getLangOpts().CPlusPlus)
1876 return rhs;
1877
1878 // If the lvalue is non-volatile, return the computed value of the assignment.
1879 if (!lhs.isVolatile())
1880 return rhs;
1881
1882 // Otherwise, reload the value.
1883 return emitLoadOfLValue(lv: lhs, loc: e->getExprLoc());
1884}
1885
1886mlir::Value
1887ScalarExprEmitter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *e) {
1888 ASTContext &context = cgf.getContext();
1889 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
1890 cir::GlobalOp globalConstStr =
1891 cgf.cgm.createGlobalString(loc, str: e->ComputeName(Context&: context), name: "__usn_str");
1892
1893 // Unlike LLVM IR, CIR pointers are typed, so the bitcast and the address
1894 // space cast are emitted as separate operations when both are needed.
1895 mlir::Value addr = builder.createGetGlobal(loc, global: globalConstStr);
1896 auto exprTy = mlir::cast<cir::PointerType>(Val: convertType(ty: e->getType()));
1897 auto addrTy = mlir::cast<cir::PointerType>(Val: addr.getType());
1898 if (addrTy.getAddrSpace() != exprTy.getAddrSpace())
1899 addr = cgf.performAddrSpaceCast(
1900 v: addr, destTy: builder.getPointerTo(ty: addrTy.getPointee(), as: exprTy.getAddrSpace()));
1901 return builder.createBitcast(loc, src: addr, newTy: exprTy);
1902}
1903
1904mlir::Value ScalarExprEmitter::VisitExprWithCleanups(ExprWithCleanups *e) {
1905 CIRGenFunction::FullExprCleanupScope scope(cgf, e->getSubExpr());
1906 mlir::Value v = Visit(e: e->getSubExpr());
1907 // Defend against dominance problems caused by jumps out of expression
1908 // evaluation through the shared cleanup block.
1909 scope.exit(valuesToReload: {&v});
1910 return v;
1911}
1912
1913} // namespace
1914
1915LValue
1916CIRGenFunction::emitCompoundAssignmentLValue(const CompoundAssignOperator *e) {
1917 ScalarExprEmitter emitter(*this, builder);
1918 mlir::Value result;
1919 switch (e->getOpcode()) {
1920#define COMPOUND_OP(Op) \
1921 case BO_##Op##Assign: \
1922 return emitter.emitCompoundAssignLValue(e, &ScalarExprEmitter::emit##Op, \
1923 result)
1924 COMPOUND_OP(Mul);
1925 COMPOUND_OP(Div);
1926 COMPOUND_OP(Rem);
1927 COMPOUND_OP(Add);
1928 COMPOUND_OP(Sub);
1929 COMPOUND_OP(Shl);
1930 COMPOUND_OP(Shr);
1931 COMPOUND_OP(And);
1932 COMPOUND_OP(Xor);
1933 COMPOUND_OP(Or);
1934#undef COMPOUND_OP
1935
1936 case BO_PtrMemD:
1937 case BO_PtrMemI:
1938 case BO_Mul:
1939 case BO_Div:
1940 case BO_Rem:
1941 case BO_Add:
1942 case BO_Sub:
1943 case BO_Shl:
1944 case BO_Shr:
1945 case BO_LT:
1946 case BO_GT:
1947 case BO_LE:
1948 case BO_GE:
1949 case BO_EQ:
1950 case BO_NE:
1951 case BO_Cmp:
1952 case BO_And:
1953 case BO_Xor:
1954 case BO_Or:
1955 case BO_LAnd:
1956 case BO_LOr:
1957 case BO_Assign:
1958 case BO_Comma:
1959 llvm_unreachable("Not valid compound assignment operators");
1960 }
1961 llvm_unreachable("Unhandled compound assignment operator");
1962}
1963
1964/// Emit the computation of the specified expression of scalar type.
1965mlir::Value CIRGenFunction::emitScalarExpr(const Expr *e,
1966 bool ignoreResultAssign) {
1967 assert(e && hasScalarEvaluationKind(e->getType()) &&
1968 "Invalid scalar expression to emit");
1969
1970 return ScalarExprEmitter(*this, builder, ignoreResultAssign)
1971 .Visit(e: const_cast<Expr *>(e));
1972}
1973
1974mlir::Value CIRGenFunction::emitPromotedScalarExpr(const Expr *e,
1975 QualType promotionType) {
1976 if (!promotionType.isNull())
1977 return ScalarExprEmitter(*this, builder).emitPromoted(e, promotionType);
1978 return ScalarExprEmitter(*this, builder).Visit(e: const_cast<Expr *>(e));
1979}
1980
1981[[maybe_unused]] static bool mustVisitNullValue(const Expr *e) {
1982 // If a null pointer expression's type is the C++0x nullptr_t and
1983 // the expression is not a simple literal, it must be evaluated
1984 // for its potential side effects.
1985 if (isa<IntegerLiteral>(Val: e) || isa<CXXNullPtrLiteralExpr>(Val: e))
1986 return false;
1987 return e->getType()->isNullPtrType();
1988}
1989
1990/// If \p e is a widened promoted integer, get its base (unpromoted) type.
1991static std::optional<QualType>
1992getUnwidenedIntegerType(const ASTContext &astContext, const Expr *e) {
1993 const Expr *base = e->IgnoreImpCasts();
1994 if (e == base)
1995 return std::nullopt;
1996
1997 QualType baseTy = base->getType();
1998 if (!astContext.isPromotableIntegerType(T: baseTy) ||
1999 astContext.getTypeSize(T: baseTy) >= astContext.getTypeSize(T: e->getType()))
2000 return std::nullopt;
2001
2002 return baseTy;
2003}
2004
2005/// Check if \p e is a widened promoted integer.
2006[[maybe_unused]] static bool isWidenedIntegerOp(const ASTContext &astContext,
2007 const Expr *e) {
2008 return getUnwidenedIntegerType(astContext, e).has_value();
2009}
2010
2011/// Check if we can skip the overflow check for \p Op.
2012[[maybe_unused]] static bool canElideOverflowCheck(const ASTContext &astContext,
2013 const BinOpInfo &op) {
2014 assert((isa<UnaryOperator>(op.e) || isa<BinaryOperator>(op.e)) &&
2015 "Expected a unary or binary operator");
2016
2017 // If the binop has constant inputs and we can prove there is no overflow,
2018 // we can elide the overflow check.
2019 if (!op.mayHaveIntegerOverflow())
2020 return true;
2021
2022 // If a unary op has a widened operand, the op cannot overflow.
2023 if (const auto *uo = dyn_cast<UnaryOperator>(Val: op.e))
2024 return !uo->canOverflow();
2025
2026 // We usually don't need overflow checks for binops with widened operands.
2027 // Multiplication with promoted unsigned operands is a special case.
2028 const auto *bo = cast<BinaryOperator>(Val: op.e);
2029 std::optional<QualType> optionalLHSTy =
2030 getUnwidenedIntegerType(astContext, e: bo->getLHS());
2031 if (!optionalLHSTy)
2032 return false;
2033
2034 std::optional<QualType> optionalRHSTy =
2035 getUnwidenedIntegerType(astContext, e: bo->getRHS());
2036 if (!optionalRHSTy)
2037 return false;
2038
2039 QualType lhsTy = *optionalLHSTy;
2040 QualType rhsTy = *optionalRHSTy;
2041
2042 // This is the simple case: binops without unsigned multiplication, and with
2043 // widened operands. No overflow check is needed here.
2044 if ((op.opcode != BO_Mul && op.opcode != BO_MulAssign) ||
2045 !lhsTy->isUnsignedIntegerType() || !rhsTy->isUnsignedIntegerType())
2046 return true;
2047
2048 // For unsigned multiplication the overflow check can be elided if either one
2049 // of the unpromoted types are less than half the size of the promoted type.
2050 unsigned promotedSize = astContext.getTypeSize(T: op.e->getType());
2051 return (2 * astContext.getTypeSize(T: lhsTy)) < promotedSize ||
2052 (2 * astContext.getTypeSize(T: rhsTy)) < promotedSize;
2053}
2054
2055/// Emit pointer + index arithmetic.
2056static mlir::Value emitPointerArithmetic(CIRGenFunction &cgf,
2057 const BinOpInfo &op,
2058 bool isSubtraction) {
2059 // Must have binary (not unary) expr here. Unary pointer
2060 // increment/decrement doesn't use this path.
2061 const BinaryOperator *expr = cast<BinaryOperator>(Val: op.e);
2062
2063 mlir::Value pointer = op.lhs;
2064 Expr *pointerOperand = expr->getLHS();
2065 mlir::Value index = op.rhs;
2066 Expr *indexOperand = expr->getRHS();
2067
2068 // In the case of subtraction, the FE has ensured that the LHS is always the
2069 // pointer. However, addition can have the pointer on either side. We will
2070 // always have a pointer operand and an integer operand, so if the LHS wasn't
2071 // a pointer, we need to swap our values.
2072 if (!isSubtraction && !mlir::isa<cir::PointerType>(Val: pointer.getType())) {
2073 std::swap(a&: pointer, b&: index);
2074 std::swap(a&: pointerOperand, b&: indexOperand);
2075 }
2076 assert(mlir::isa<cir::PointerType>(pointer.getType()) &&
2077 "Need a pointer operand");
2078 assert(mlir::isa<cir::IntType>(index.getType()) && "Need an integer operand");
2079
2080 // Some versions of glibc and gcc use idioms (particularly in their malloc
2081 // routines) that add a pointer-sized integer (known to be a pointer value)
2082 // to a null pointer in order to cast the value back to an integer or as
2083 // part of a pointer alignment algorithm. This is undefined behavior, but
2084 // we'd like to be able to compile programs that use it.
2085 //
2086 // Normally, we'd generate a GEP with a null-pointer base here in response
2087 // to that code, but it's also UB to dereference a pointer created that
2088 // way. Instead (as an acknowledged hack to tolerate the idiom) we will
2089 // generate a direct cast of the integer value to a pointer.
2090 //
2091 // The idiom (p = nullptr + N) is not met if any of the following are true:
2092 //
2093 // The operation is subtraction.
2094 // The index is not pointer-sized.
2095 // The pointer type is not byte-sized.
2096 //
2097 if (BinaryOperator::isNullPointerArithmeticExtension(
2098 Ctx&: cgf.getContext(), Opc: op.opcode, LHS: expr->getLHS(), RHS: expr->getRHS()))
2099 return cgf.getBuilder().createIntToPtr(src: index, newTy: pointer.getType());
2100
2101 // Differently from LLVM codegen, ABI bits for index sizes is handled during
2102 // LLVM lowering.
2103
2104 // If this is subtraction, negate the index.
2105 if (isSubtraction)
2106 index = cgf.getBuilder().createNeg(loc: cgf.getLoc(srcLoc: op.e->getExprLoc()), value: index);
2107
2108 assert(!cir::MissingFeatures::sanitizers());
2109
2110 const PointerType *pointerType =
2111 pointerOperand->getType()->getAs<PointerType>();
2112 if (!pointerType) {
2113 cgf.cgm.errorNYI(feature: "Objective-C:pointer arithmetic with non-pointer type");
2114 return nullptr;
2115 }
2116
2117 QualType elementType = pointerType->getPointeeType();
2118 if (const VariableArrayType *vla =
2119 cgf.getContext().getAsVariableArrayType(T: elementType)) {
2120 mlir::Value numElements = cgf.getVLASize(type: vla).numElts;
2121 mlir::Location loc = cgf.getLoc(srcLoc: op.e->getExprLoc());
2122 index = cgf.getBuilder().createCast(kind: cir::CastKind::integral, src: index,
2123 newTy: numElements.getType());
2124 // GEP indexes are signed, and scaling an index isn't permitted to
2125 // signed-overflow, so we use the same semantics for our explicit
2126 // multiply. We suppress this if overflow is not undefined behavior.
2127 cir::OverflowFlags overflowFlags = cgf.getLangOpts().PointerOverflowDefined
2128 ? cir::OverflowFlags::None
2129 : cir::OverflowFlags::NoSignedWrap;
2130 index = cgf.getBuilder().createMul(loc, lhs: index, rhs: numElements, flags: overflowFlags);
2131 assert(!cir::MissingFeatures::sanitizers());
2132 return cir::PtrStrideOp::create(builder&: cgf.getBuilder(), location: loc, result: pointer.getType(),
2133 base: pointer, stride: index);
2134 }
2135
2136 assert(!cir::MissingFeatures::sanitizers());
2137 return cir::PtrStrideOp::create(builder&: cgf.getBuilder(),
2138 location: cgf.getLoc(srcLoc: op.e->getExprLoc()),
2139 result: pointer.getType(), base: pointer, stride: index);
2140}
2141
2142static bool isIntegerVectorBinOp(mlir::Type ty) {
2143 auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: ty);
2144 return vecTy && mlir::isa<cir::IntType>(Val: vecTy.getElementType());
2145}
2146
2147// Construct a cir.fmuladd op to represent a fused mul-add of `mulOp` and
2148// `addend`. Use negMul and negAdd to negate the first operand of the mul or
2149// the addend respectively. This allows fmuladd to represent a*b-c, or c-a*b.
2150// Patterns in LLVM should catch the negated forms and translate them to
2151// efficient operations.
2152static mlir::Value buildFMulAdd(mlir::Location addLoc, cir::FMulOp mulOp,
2153 mlir::Value addend, CIRGenBuilderTy &builder,
2154 bool negMul, bool negAdd) {
2155 mlir::Location loc = builder.getFusedLoc(locs: {mulOp.getLoc(), addLoc});
2156 mlir::Value mulOp0 = mulOp.getLhs();
2157 mlir::Value mulOp1 = mulOp.getRhs();
2158 if (negMul)
2159 mulOp0 = builder.createFNeg(loc, operand: mulOp0);
2160 if (negAdd)
2161 addend = builder.createFNeg(loc, operand: addend);
2162
2163 // Carry the mul's fenv attribute so a constrained fmul yields a constrained
2164 // fmuladd; the builder is under the add's FP options, not the mul's.
2165 mlir::Value fmuladd = cir::FMulAddOp::create(
2166 builder, location: loc, result: addend.getType(), a: mulOp0, b: mulOp1, c: addend,
2167 fenv: mulOp.getFenvAttr(), fastmath_flags: builder.getFastMathFlagsAttr());
2168 mulOp.erase();
2169 return fmuladd;
2170}
2171
2172// Check whether it would be legal to emit a cir.fmuladd op to represent op
2173// and if so, build it.
2174//
2175// Checks that (a) the operation is fusable, and (b) -ffp-contract=on.
2176// Does NOT check the type of the operation - it's assumed that this function
2177// will be called from contexts where it's known that the type is contractable.
2178static mlir::Value tryEmitFMulAdd(mlir::Location loc, const BinOpInfo &op,
2179 CIRGenBuilderTy &builder,
2180 bool isSub = false) {
2181 assert((op.opcode == BO_Add || op.opcode == BO_AddAssign ||
2182 op.opcode == BO_Sub || op.opcode == BO_SubAssign) &&
2183 "Only fadd/fsub can be the root of an fmuladd.");
2184
2185 // Check whether this op is fusable, i.e. -ffp-contract=on. -ffp-contract=fast
2186 // is represented by the `contract` flag on the fmul/fadd instead.
2187 if (!op.fpFeatures.allowFPContractWithinStatement())
2188 return nullptr;
2189
2190 mlir::Value lhs = op.lhs;
2191 mlir::Value rhs = op.rhs;
2192
2193 // Peek through fneg to look for fmul. Make sure the fneg has no other users,
2194 // and that it is the only use of its operand.
2195 bool negLHS = false;
2196 if (auto lhsNeg = lhs.getDefiningOp<cir::FNegOp>()) {
2197 if (lhsNeg.getResult().use_empty() && lhsNeg.getInput().hasOneUse()) {
2198 lhs = lhsNeg.getInput();
2199 negLHS = true;
2200 }
2201 }
2202
2203 bool negRHS = false;
2204 if (auto rhsNeg = rhs.getDefiningOp<cir::FNegOp>()) {
2205 if (rhsNeg.getResult().use_empty() && rhsNeg.getInput().hasOneUse()) {
2206 rhs = rhsNeg.getInput();
2207 negRHS = true;
2208 }
2209 }
2210
2211 // We have a potentially fusable op. Look for a mul on one of the operands.
2212 // Also make sure that the mul result isn't used directly. In that case,
2213 // there's no point creating a muladd operation.
2214 if (auto lhsMul = lhs.getDefiningOp<cir::FMulOp>()) {
2215 if (lhsMul.getResult().use_empty() || negLHS) {
2216 // If we looked through fneg, erase it.
2217 if (negLHS)
2218 op.lhs.getDefiningOp<cir::FNegOp>().erase();
2219 return buildFMulAdd(addLoc: loc, mulOp: lhsMul, addend: op.rhs, builder, negMul: negLHS, negAdd: isSub);
2220 }
2221 }
2222 if (auto rhsMul = rhs.getDefiningOp<cir::FMulOp>()) {
2223 if (rhsMul.getResult().use_empty() || negRHS) {
2224 // If we looked through fneg, erase it.
2225 if (negRHS)
2226 op.rhs.getDefiningOp<cir::FNegOp>().erase();
2227 return buildFMulAdd(addLoc: loc, mulOp: rhsMul, addend: op.lhs, builder, negMul: isSub ^ negRHS, negAdd: false);
2228 }
2229 }
2230
2231 return nullptr;
2232}
2233
2234mlir::Value ScalarExprEmitter::emitFixedPointIncDec(const UnaryOperator *e,
2235 mlir::Value value,
2236 QualType type) {
2237 // Fixed-point types are tricky. In some cases, it isn't possible to
2238 // represent a 1 or a -1 in the type at all. Piggyback off of
2239 // EmitFixedPointBinOp to avoid having to reimplement saturation.
2240 BinOpInfo info;
2241 info.loc = e->getSourceRange();
2242 info.fpFeatures = e->getFPFeaturesInEffect(LO: cgf.getLangOpts());
2243 info.e = e;
2244 info.compType = info.fullType = e->getType();
2245 info.opcode = e->isIncrementOp() ? BO_Add : BO_Sub;
2246 info.lhs = value;
2247 info.rhs = builder.getConstInt(loc: cgf.getLoc(srcLoc: info.loc), t: value.getType(), c: 1);
2248 // If the type is signed, it's better to represent this as +(-1) or -(-1),
2249 // since -1 is guaranteed to be representable.
2250 // FIXME(cir): This is a carry-over from Classic codegen, we should probably
2251 // figure out if ++ -> -(-1) and -- -> +(-1) is REALLY what we want? For now
2252 // we are just doing what classic does here.
2253 if (type->isSignedFixedPointType()) {
2254 info.opcode = e->isIncrementOp() ? BO_Sub : BO_Add;
2255 info.rhs = builder.createNeg(loc: cgf.getLoc(srcLoc: info.loc), value: info.rhs);
2256 }
2257 // Now, convert from our invented integer literal to the type of the unary
2258 // op. This will upscale and saturate if necessary. This value can become
2259 // undef in some cases.
2260 CIRGenFixedPointBuilder fpbuilder(builder, cgf.getLoc(srcLoc: info.loc));
2261 auto dstSema = cgf.getContext().getFixedPointSemantics(Ty: info.fullType);
2262 info.rhs =
2263 fpbuilder.createIntegerToFixed(src: info.rhs, /*srcIsSigned=*/true, dstSema);
2264 return emitFixedPointBinOp(ops: info);
2265}
2266
2267mlir::Value ScalarExprEmitter::emitFixedPointConversion(mlir::Value src,
2268 QualType srcTy,
2269 QualType dstTy,
2270 mlir::Location loc) {
2271 assert(srcTy->isFixedPointType() || dstTy->isFixedPointType());
2272
2273 CIRGenFixedPointBuilder fpBuilder(builder, loc);
2274 mlir::Value result;
2275 if (srcTy->isRealFloatingType()) {
2276 assert(dstTy->isFixedPointType());
2277 result = fpBuilder.createFloatingToFixed(
2278 src, dstSema: cgf.getContext().getFixedPointSemantics(Ty: dstTy));
2279 } else if (dstTy->isRealFloatingType()) {
2280 assert(srcTy->isFixedPointType());
2281 result = fpBuilder.createFixedToFloating(
2282 src, srcSema: cgf.getContext().getFixedPointSemantics(Ty: srcTy),
2283 dstTy: cgf.convertType(t: dstTy));
2284 } else {
2285 llvm::FixedPointSemantics srcFPSema =
2286 cgf.getContext().getFixedPointSemantics(Ty: srcTy);
2287 llvm::FixedPointSemantics dstFPSema =
2288 cgf.getContext().getFixedPointSemantics(Ty: dstTy);
2289
2290 if (dstTy->isIntegerType())
2291 result = fpBuilder.createFixedToInteger(
2292 src, srcSema: srcFPSema, dstWidth: dstFPSema.getWidth(), dstIsSigned: dstFPSema.isSigned());
2293 else if (srcTy->isIntegerType())
2294 result =
2295 fpBuilder.createIntegerToFixed(src, srcIsSigned: srcFPSema.isSigned(), dstSema: dstFPSema);
2296 else {
2297 assert(srcTy->isFixedPointType() && dstTy->isFixedPointType());
2298 result = fpBuilder.createFixedToFixed(src, srcSema: srcFPSema, dstSema: dstFPSema);
2299 }
2300 }
2301 return result;
2302}
2303
2304mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) {
2305 const mlir::Location loc = cgf.getLoc(srcLoc: ops.loc);
2306 if (!isIntegerVectorBinOp(ty: ops.lhs.getType()) &&
2307 ops.compType->isSignedIntegerOrEnumerationType()) {
2308 switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
2309 case LangOptions::SOB_Defined:
2310 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2311 return builder.createMul(loc, lhs: ops.lhs, rhs: ops.rhs);
2312 [[fallthrough]];
2313 case LangOptions::SOB_Undefined:
2314 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2315 return builder.createNSWMul(loc, lhs: ops.lhs, rhs: ops.rhs);
2316 [[fallthrough]];
2317 case LangOptions::SOB_Trapping:
2318 if (canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2319 return builder.createNSWMul(loc, lhs: ops.lhs, rhs: ops.rhs);
2320 cgf.cgm.errorNYI(feature: "sanitizers");
2321 }
2322 }
2323
2324 if (ops.fullType->isConstantMatrixType()) {
2325 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
2326
2327 if (isa<cir::MatrixType>(Val: ops.lhs.getType()) &&
2328 isa<cir::MatrixType>(Val: ops.rhs.getType()))
2329 return builder.createMatrixMultiply(loc, lhs: ops.lhs, rhs: ops.rhs);
2330
2331 auto [lhs, rhs] =
2332 builder.splatMatrixOpOperandsIfNecessary(loc, lhs: ops.lhs, rhs: ops.rhs);
2333 if (cir::isFPOrVectorOrMatrixOfFPType(type: lhs.getType()))
2334 return builder.createFMul(loc, lhs, rhs);
2335 return builder.createMul(loc, lhs, rhs);
2336 }
2337
2338 if (ops.compType->isUnsignedIntegerType() &&
2339 cgf.sanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow) &&
2340 !canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2341 cgf.cgm.errorNYI(feature: "unsigned int overflow sanitizer");
2342
2343 if (cir::isFPOrVectorOfFPType(type: ops.lhs.getType())) {
2344 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
2345 return builder.createFMul(loc, lhs: ops.lhs, rhs: ops.rhs);
2346 }
2347
2348 if (ops.isFixedPointOp())
2349 return emitFixedPointBinOp(ops);
2350
2351 return cir::MulOp::create(builder, location: cgf.getLoc(srcLoc: ops.loc),
2352 result: cgf.convertType(t: ops.fullType), lhs: ops.lhs, rhs: ops.rhs);
2353}
2354mlir::Value ScalarExprEmitter::emitDiv(const BinOpInfo &ops) {
2355 const mlir::Location loc = cgf.getLoc(srcLoc: ops.loc);
2356 if (cir::isFPOrVectorOfFPType(type: ops.lhs.getType())) {
2357 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
2358 return builder.createFDiv(loc, lhs: ops.lhs, rhs: ops.rhs);
2359 }
2360
2361 if (ops.fullType->isConstantMatrixType()) {
2362 assert(mlir::isa<cir::MatrixType>(ops.lhs.getType()) &&
2363 "first operand must be a matrix");
2364 assert(cir::isAnyIntegerOrFloatingPointType(ops.rhs.getType()) &&
2365 "second operand must be an arithmetic type");
2366
2367 auto [lhs, rhs] =
2368 builder.splatMatrixOpOperandsIfNecessary(loc, lhs: ops.lhs, rhs: ops.rhs);
2369
2370 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
2371 if (cir::isFPOrVectorOrMatrixOfFPType(type: lhs.getType()))
2372 return builder.createFDiv(loc, lhs, rhs);
2373 return builder.createDiv(loc, lhs, rhs);
2374 }
2375
2376 if (ops.isFixedPointOp())
2377 return emitFixedPointBinOp(ops);
2378
2379 return cir::DivOp::create(builder, location: loc, result: cgf.convertType(t: ops.fullType),
2380 lhs: ops.lhs, rhs: ops.rhs);
2381}
2382mlir::Value ScalarExprEmitter::emitRem(const BinOpInfo &ops) {
2383 const mlir::Location loc = cgf.getLoc(srcLoc: ops.loc);
2384 if (cir::isFPOrVectorOfFPType(type: ops.lhs.getType())) {
2385 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
2386 return builder.createFRem(loc, lhs: ops.lhs, rhs: ops.rhs);
2387 }
2388 return cir::RemOp::create(builder, location: loc, result: cgf.convertType(t: ops.fullType),
2389 lhs: ops.lhs, rhs: ops.rhs);
2390}
2391
2392mlir::Value ScalarExprEmitter::emitFixedPointBinOp(const BinOpInfo &ops) {
2393 // This is either a binary operation where at least one of the operands is
2394 // a fixed-point type, or a unary operation where the operand is a fixed-point
2395 // type. The result type of a binary operation is determined by
2396 // Sema::handleFixedPointConversions().
2397 QualType resultTy = ops.compType;
2398 QualType lhsTy, rhsTy;
2399 if (const auto *binOp = dyn_cast<BinaryOperator>(Val: ops.e)) {
2400 rhsTy = binOp->getRHS()->getType();
2401 if (const auto *cao = dyn_cast<CompoundAssignOperator>(Val: binOp)) {
2402 // For compound assignment, the effective type of the LHS at this point
2403 // is the computation LHS type, not the actual LHS type, and the final
2404 // result type is not the type of the expression but rather the
2405 // computation result type.
2406 lhsTy = cao->getComputationLHSType();
2407 resultTy = cao->getComputationResultType();
2408 } else {
2409 lhsTy = binOp->getLHS()->getType();
2410 }
2411 } else if (const auto *unOp = dyn_cast<UnaryOperator>(Val: ops.e)) {
2412 lhsTy = unOp->getSubExpr()->getType();
2413 rhsTy = unOp->getSubExpr()->getType();
2414 }
2415 ASTContext &ctx = cgf.getContext();
2416 mlir::Value lhs = ops.lhs;
2417 mlir::Value rhs = ops.rhs;
2418
2419 auto lhsFixedSema = ctx.getFixedPointSemantics(Ty: lhsTy);
2420 auto rhsFixedSema = ctx.getFixedPointSemantics(Ty: rhsTy);
2421 auto resultFixedSema = ctx.getFixedPointSemantics(Ty: resultTy);
2422 auto commonFixedSema = lhsFixedSema.getCommonSemantics(Other: rhsFixedSema);
2423
2424 // Perform the actual operation.
2425 mlir::Value result;
2426 CIRGenFixedPointBuilder fpbuilder(builder, cgf.getLoc(srcLoc: ops.loc));
2427 switch (ops.opcode) {
2428 case BO_AddAssign:
2429 case BO_Add:
2430 result = fpbuilder.createAdd(lhs, lhsSema: lhsFixedSema, rhs, rhsSema: rhsFixedSema);
2431 break;
2432 case BO_SubAssign:
2433 case BO_Sub:
2434 result = fpbuilder.createSub(lhs, lhsSema: lhsFixedSema, rhs, rhsSema: rhsFixedSema);
2435 break;
2436 case BO_MulAssign:
2437 case BO_Mul:
2438 result = fpbuilder.createMul(lhs, lhsSema: lhsFixedSema, rhs, rhsSema: rhsFixedSema);
2439 break;
2440 case BO_DivAssign:
2441 case BO_Div:
2442 result = fpbuilder.createDiv(lhs, lhsSema: lhsFixedSema, rhs, rhsSema: rhsFixedSema);
2443 break;
2444 case BO_ShlAssign:
2445 case BO_Shl:
2446 result = fpbuilder.createShl(lhs, lhsSema: lhsFixedSema, rhs);
2447 break;
2448 case BO_ShrAssign:
2449 case BO_Shr:
2450 result = fpbuilder.createShr(lhs, rhs);
2451 break;
2452 case BO_LT:
2453 case BO_GT:
2454 case BO_LE:
2455 case BO_GE:
2456 case BO_EQ:
2457 case BO_NE:
2458 return fpbuilder.createCmp(lhs, lhsSema: lhsFixedSema, rhs, rhsSema: rhsFixedSema,
2459 kind: clangCmpToCIRCmp(clangCmp: ops.opcode));
2460 case BO_Cmp:
2461 case BO_LAnd:
2462 case BO_LOr:
2463 llvm_unreachable("Found unimplemented fixed point binary operation");
2464 case BO_PtrMemD:
2465 case BO_PtrMemI:
2466 case BO_Rem:
2467 case BO_Xor:
2468 case BO_And:
2469 case BO_Or:
2470 case BO_Assign:
2471 case BO_RemAssign:
2472 case BO_AndAssign:
2473 case BO_XorAssign:
2474 case BO_OrAssign:
2475 case BO_Comma:
2476 llvm_unreachable(
2477 "Found unsupported binary operation for fixed point types.");
2478 }
2479
2480 bool isShift = BinaryOperator::isShiftOp(Opc: ops.opcode) ||
2481 BinaryOperator::isShiftAssignOp(Opc: ops.opcode);
2482 // Convert to the result type.
2483 return fpbuilder.createFixedToFixed(
2484 src: result, srcSema: isShift ? lhsFixedSema : commonFixedSema, dstSema: resultFixedSema);
2485}
2486
2487mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) {
2488 if (mlir::isa<cir::PointerType>(Val: ops.lhs.getType()) ||
2489 mlir::isa<cir::PointerType>(Val: ops.rhs.getType()))
2490 return emitPointerArithmetic(cgf, op: ops, /*isSubtraction=*/
2491 false);
2492
2493 const mlir::Location loc = cgf.getLoc(srcLoc: ops.loc);
2494 if (!isIntegerVectorBinOp(ty: ops.lhs.getType()) &&
2495 ops.compType->isSignedIntegerOrEnumerationType()) {
2496 switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
2497 case LangOptions::SOB_Defined:
2498 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2499 return builder.createAdd(loc, lhs: ops.lhs, rhs: ops.rhs);
2500 [[fallthrough]];
2501 case LangOptions::SOB_Undefined:
2502 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2503 return builder.createNSWAdd(loc, lhs: ops.lhs, rhs: ops.rhs);
2504 [[fallthrough]];
2505 case LangOptions::SOB_Trapping:
2506 if (canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2507 return builder.createNSWAdd(loc, lhs: ops.lhs, rhs: ops.rhs);
2508 cgf.cgm.errorNYI(feature: "sanitizers");
2509 }
2510 }
2511 if (ops.fullType->isConstantMatrixType()) {
2512 // Like llvm::MatrixBuilder::CreateAdd, splat a scalar operand to the matrix
2513 // type before adding.
2514 auto [lhs, rhs] =
2515 builder.splatMatrixOpOperandsIfNecessary(loc, lhs: ops.lhs, rhs: ops.rhs);
2516
2517 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
2518 if (cir::isFPOrVectorOrMatrixOfFPType(type: lhs.getType()))
2519 return builder.createFAdd(loc, lhs, rhs);
2520 return builder.createAdd(loc, lhs, rhs);
2521 }
2522
2523 if (ops.compType->isUnsignedIntegerType() &&
2524 cgf.sanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow) &&
2525 !canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2526 cgf.cgm.errorNYI(feature: "unsigned int overflow sanitizer");
2527
2528 if (cir::isFPOrVectorOfFPType(type: ops.lhs.getType())) {
2529 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
2530 // Try to form an fmuladd.
2531 if (mlir::Value fmuladd = tryEmitFMulAdd(loc, op: ops, builder))
2532 return fmuladd;
2533 return builder.createFAdd(loc, lhs: ops.lhs, rhs: ops.rhs);
2534 }
2535
2536 if (ops.isFixedPointOp())
2537 return emitFixedPointBinOp(ops);
2538
2539 return builder.createAdd(loc, lhs: ops.lhs, rhs: ops.rhs);
2540}
2541
2542mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) {
2543 const mlir::Location loc = cgf.getLoc(srcLoc: ops.loc);
2544 // The LHS is always a pointer if either side is.
2545 if (!mlir::isa<cir::PointerType>(Val: ops.lhs.getType())) {
2546 if (!isIntegerVectorBinOp(ty: ops.lhs.getType()) &&
2547 ops.compType->isSignedIntegerOrEnumerationType()) {
2548 switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
2549 case LangOptions::SOB_Defined: {
2550 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2551 return builder.createSub(loc, lhs: ops.lhs, rhs: ops.rhs);
2552 [[fallthrough]];
2553 }
2554 case LangOptions::SOB_Undefined:
2555 if (!cgf.sanOpts.has(K: SanitizerKind::SignedIntegerOverflow))
2556 return builder.createNSWSub(loc, lhs: ops.lhs, rhs: ops.rhs);
2557 [[fallthrough]];
2558 case LangOptions::SOB_Trapping:
2559 if (canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2560 return builder.createNSWSub(loc, lhs: ops.lhs, rhs: ops.rhs);
2561 cgf.cgm.errorNYI(feature: "sanitizers");
2562 }
2563 }
2564
2565 if (ops.fullType->isConstantMatrixType()) {
2566 // Like llvm::MatrixBuilder::CreateSub, splat a scalar operand to the
2567 // matrix type before adding.
2568 auto [lhs, rhs] =
2569 builder.splatMatrixOpOperandsIfNecessary(loc, lhs: ops.lhs, rhs: ops.rhs);
2570
2571 CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
2572 if (cir::isFPOrVectorOrMatrixOfFPType(type: lhs.getType()))
2573 return builder.createFSub(loc, lhs, rhs);
2574 return builder.createSub(loc, lhs, rhs);
2575 }
2576
2577 if (ops.compType->isUnsignedIntegerType() &&
2578 cgf.sanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow) &&
2579 !canElideOverflowCheck(astContext: cgf.getContext(), op: ops))
2580 cgf.cgm.errorNYI(feature: "unsigned int overflow sanitizer");
2581
2582 if (cir::isFPOrVectorOfFPType(type: ops.lhs.getType())) {
2583 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
2584 // Try to form an fmuladd.
2585 if (mlir::Value fmuladd =
2586 tryEmitFMulAdd(loc, op: ops, builder, /*isSub=*/true))
2587 return fmuladd;
2588 return builder.createFSub(loc, lhs: ops.lhs, rhs: ops.rhs);
2589 }
2590
2591 if (ops.isFixedPointOp())
2592 return emitFixedPointBinOp(ops);
2593
2594 return builder.createSub(loc, lhs: ops.lhs, rhs: ops.rhs);
2595 }
2596
2597 // If the RHS is not a pointer, then we have normal pointer
2598 // arithmetic.
2599 if (!mlir::isa<cir::PointerType>(Val: ops.rhs.getType()))
2600 return emitPointerArithmetic(cgf, op: ops, /*isSubtraction=*/true);
2601
2602 // Otherwise, this is a pointer subtraction
2603
2604 // Do the raw subtraction part.
2605 //
2606 // TODO(cir): note for LLVM lowering out of this; when expanding this into
2607 // LLVM we shall take VLA's, division by element size, etc.
2608 //
2609 // See more in `EmitSub` in CGExprScalar.cpp.
2610 assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
2611 return cir::PtrDiffOp::create(builder, location: cgf.getLoc(srcLoc: ops.loc), result: cgf.ptrDiffTy,
2612 lhs: ops.lhs, rhs: ops.rhs);
2613}
2614
2615mlir::Value ScalarExprEmitter::emitShl(const BinOpInfo &ops) {
2616 // TODO: This misses out on the sanitizer check below.
2617 if (ops.isFixedPointOp())
2618 return emitFixedPointBinOp(ops);
2619
2620 // CIR accepts shift between different types, meaning nothing special
2621 // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
2622 // promote or truncate the RHS to the same size as the LHS.
2623
2624 bool sanitizeSignedBase = cgf.sanOpts.has(K: SanitizerKind::ShiftBase) &&
2625 ops.compType->hasSignedIntegerRepresentation() &&
2626 !cgf.getLangOpts().isSignedOverflowDefined() &&
2627 !cgf.getLangOpts().CPlusPlus20;
2628 bool sanitizeUnsignedBase =
2629 cgf.sanOpts.has(K: SanitizerKind::UnsignedShiftBase) &&
2630 ops.compType->hasUnsignedIntegerRepresentation();
2631 bool sanitizeBase = sanitizeSignedBase || sanitizeUnsignedBase;
2632 bool sanitizeExponent = cgf.sanOpts.has(K: SanitizerKind::ShiftExponent);
2633
2634 // OpenCL 6.3j: shift values are effectively % word size of LHS.
2635 if (cgf.getLangOpts().OpenCL)
2636 cgf.cgm.errorNYI(feature: "opencl");
2637 else if ((sanitizeBase || sanitizeExponent) &&
2638 mlir::isa<cir::IntType>(Val: ops.lhs.getType()))
2639 cgf.cgm.errorNYI(feature: "sanitizers");
2640
2641 return builder.createShiftLeft(loc: cgf.getLoc(srcLoc: ops.loc), lhs: ops.lhs, rhs: ops.rhs);
2642}
2643
2644mlir::Value ScalarExprEmitter::emitShr(const BinOpInfo &ops) {
2645 // TODO: This misses out on the sanitizer check below.
2646 if (ops.isFixedPointOp())
2647 return emitFixedPointBinOp(ops);
2648
2649 // CIR accepts shift between different types, meaning nothing special
2650 // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
2651 // promote or truncate the RHS to the same size as the LHS.
2652
2653 // OpenCL 6.3j: shift values are effectively % word size of LHS.
2654 if (cgf.getLangOpts().OpenCL)
2655 cgf.cgm.errorNYI(feature: "opencl");
2656 else if (cgf.sanOpts.has(K: SanitizerKind::ShiftExponent) &&
2657 mlir::isa<cir::IntType>(Val: ops.lhs.getType()))
2658 cgf.cgm.errorNYI(feature: "sanitizers");
2659
2660 // Note that we don't need to distinguish unsigned treatment at this
2661 // point since it will be handled later by LLVM lowering.
2662 return builder.createShiftRight(loc: cgf.getLoc(srcLoc: ops.loc), lhs: ops.lhs, rhs: ops.rhs);
2663}
2664
2665mlir::Value ScalarExprEmitter::emitAnd(const BinOpInfo &ops) {
2666 return cir::AndOp::create(builder, location: cgf.getLoc(srcLoc: ops.loc), lhs: ops.lhs, rhs: ops.rhs);
2667}
2668mlir::Value ScalarExprEmitter::emitXor(const BinOpInfo &ops) {
2669 return cir::XorOp::create(builder, location: cgf.getLoc(srcLoc: ops.loc), lhs: ops.lhs, rhs: ops.rhs);
2670}
2671mlir::Value ScalarExprEmitter::emitOr(const BinOpInfo &ops) {
2672 return cir::OrOp::create(builder, location: cgf.getLoc(srcLoc: ops.loc), lhs: ops.lhs, rhs: ops.rhs);
2673}
2674
2675// Emit code for an explicit or implicit cast. Implicit
2676// casts have to handle a more broad range of conversions than explicit
2677// casts, as they handle things like function to ptr-to-function decay
2678// etc.
2679mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
2680 Expr *subExpr = ce->getSubExpr();
2681 QualType destTy = ce->getType();
2682 CastKind kind = ce->getCastKind();
2683 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ce);
2684
2685 // These cases are generally not written to ignore the result of evaluating
2686 // their sub-expressions, so we clear this now.
2687 ignoreResultAssign = false;
2688
2689 switch (kind) {
2690 case clang::CK_Dependent:
2691 llvm_unreachable("dependent cast kind in CIR gen!");
2692 case clang::CK_BuiltinFnToFnPtr:
2693 llvm_unreachable("builtin functions are handled elsewhere");
2694 case CK_LValueBitCast:
2695 case CK_LValueToRValueBitCast: {
2696 LValue sourceLVal = cgf.emitLValue(e: subExpr);
2697 Address sourceAddr = sourceLVal.getAddress();
2698
2699 mlir::Type destElemTy = cgf.convertTypeForMem(t: destTy);
2700 Address destAddr = sourceAddr.withElementType(builder&: cgf.getBuilder(), ElemTy: destElemTy);
2701 LValue destLVal = cgf.makeAddrLValue(addr: destAddr, ty: destTy);
2702 assert(!cir::MissingFeatures::opTBAA());
2703 return emitLoadOfLValue(lv: destLVal, loc: ce->getExprLoc());
2704 }
2705
2706 case CK_CPointerToObjCPointerCast:
2707 case CK_BlockPointerToObjCPointerCast:
2708 case CK_AnyPointerToBlockPointerCast:
2709 case CK_BitCast: {
2710 mlir::Value src = Visit(e: const_cast<Expr *>(subExpr));
2711 mlir::Type dstTy = cgf.convertType(t: destTy);
2712
2713 // Function and object pointers may differ in address space.
2714 auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(Val: src.getType());
2715 auto dstPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: dstTy);
2716 if (srcPtrTy && dstPtrTy &&
2717 srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace())
2718 src = cgf.performAddrSpaceCast(
2719 v: src,
2720 destTy: builder.getPointerTo(ty: srcPtrTy.getPointee(), as: dstPtrTy.getAddrSpace()));
2721
2722 if (cgf.sanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
2723 cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
2724 "sanitizer support");
2725
2726 if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
2727 cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
2728 "strict vtable pointers");
2729
2730 // Update heapallocsite metadata when there is an explicit pointer cast.
2731 assert(!cir::MissingFeatures::addHeapAllocSiteMetadata());
2732
2733 // If Src is a fixed vector and Dst is a scalable vector, and both have the
2734 // same element type, use the llvm.vector.insert intrinsic to perform the
2735 // bitcast.
2736 assert(!cir::MissingFeatures::scalableVectors());
2737
2738 // If Src is a scalable vector and Dst is a fixed vector, and both have the
2739 // same element type, use the llvm.vector.extract intrinsic to perform the
2740 // bitcast.
2741 assert(!cir::MissingFeatures::scalableVectors());
2742
2743 // Perform VLAT <-> VLST bitcast through memory.
2744 // TODO: since the llvm.experimental.vector.{insert,extract} intrinsics
2745 // require the element types of the vectors to be the same, we
2746 // need to keep this around for bitcasts between VLAT <-> VLST where
2747 // the element types of the vectors are not the same, until we figure
2748 // out a better way of doing these casts.
2749 assert(!cir::MissingFeatures::scalableVectors());
2750
2751 return cgf.getBuilder().createBitcast(loc: cgf.getLoc(srcLoc: subExpr->getSourceRange()),
2752 src, newTy: dstTy);
2753 }
2754 case CK_AddressSpaceConversion: {
2755 Expr::EvalResult result;
2756 if (subExpr->EvaluateAsRValue(Result&: result, Ctx: cgf.getContext()) &&
2757 result.Val.isNullPointer()) {
2758 // If e has side effect, it is emitted even if its final result is a
2759 // null pointer. In that case, a DCE pass should be able to
2760 // eliminate the useless instructions emitted during translating E.
2761 if (result.HasSideEffects)
2762 Visit(e: subExpr);
2763 return cgf.cgm.getNullPointer(
2764 ptrTy: mlir::cast<cir::PointerType>(Val: convertType(ty: destTy)), qt: destTy,
2765 loc: cgf.getLoc(srcLoc: subExpr->getExprLoc()));
2766 }
2767 return cgf.performAddrSpaceCast(v: Visit(e: subExpr), destTy: convertType(ty: destTy));
2768 }
2769
2770 case CK_AtomicToNonAtomic:
2771 case CK_NonAtomicToAtomic:
2772 case CK_UserDefinedConversion:
2773 return Visit(e: const_cast<Expr *>(subExpr));
2774 case CK_NoOp:
2775 return ce->changesVolatileQualification() ? emitLoadOfLValue(e: ce)
2776 : Visit(e: subExpr);
2777 case CK_IntegralToPointer: {
2778 mlir::Type destCIRTy = cgf.convertType(t: destTy);
2779 mlir::Value src = Visit(e: const_cast<Expr *>(subExpr));
2780
2781 // Properly resize by casting to an int of the same size as the pointer.
2782 // Clang's IntegralToPointer includes 'bool' as the source, but in CIR
2783 // 'bool' is not an integral type. So check the source type to get the
2784 // correct CIR conversion.
2785 mlir::Type middleTy = cgf.cgm.getDataLayout().getIntPtrType(ty: destCIRTy);
2786 mlir::Value middleVal = builder.createCast(
2787 kind: subExpr->getType()->isBooleanType() ? cir::CastKind::bool_to_int
2788 : cir::CastKind::integral,
2789 src, newTy: middleTy);
2790
2791 if (cgf.cgm.getCodeGenOpts().StrictVTablePointers) {
2792 cgf.cgm.errorNYI(subExpr->getSourceRange(),
2793 "IntegralToPointer: strict vtable pointers");
2794 return {};
2795 }
2796
2797 return builder.createIntToPtr(src: middleVal, newTy: destCIRTy);
2798 }
2799
2800 case CK_BaseToDerived: {
2801 const CXXRecordDecl *derivedClassDecl = destTy->getPointeeCXXRecordDecl();
2802 assert(derivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
2803 Address base = cgf.emitPointerWithAlignment(expr: subExpr);
2804 Address derived = cgf.getAddressOfDerivedClass(
2805 loc: cgf.getLoc(srcLoc: ce->getSourceRange()), baseAddr: base, derived: derivedClassDecl, path: ce->path(),
2806 nullCheckValue: cgf.shouldNullCheckClassCastValue(ce));
2807
2808 // C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is
2809 // performed and the object is not of the derived type.
2810 assert(!cir::MissingFeatures::sanitizers());
2811
2812 return cgf.getAsNaturalPointerTo(addr: derived, pointeeType: ce->getType()->getPointeeType());
2813 }
2814 case CK_UncheckedDerivedToBase:
2815 case CK_DerivedToBase: {
2816 // The EmitPointerWithAlignment path does this fine; just discard
2817 // the alignment.
2818 return cgf.getAsNaturalPointerTo(addr: cgf.emitPointerWithAlignment(expr: ce),
2819 pointeeType: ce->getType()->getPointeeType());
2820 }
2821 case CK_Dynamic: {
2822 Address v = cgf.emitPointerWithAlignment(expr: subExpr);
2823 const auto *dce = cast<CXXDynamicCastExpr>(Val: ce);
2824 return cgf.emitDynamicCast(thisAddr: v, dce);
2825 }
2826 case CK_ArrayToPointerDecay:
2827 return cgf.emitArrayToPointerDecay(e: subExpr).getPointer();
2828
2829 case CK_NullToPointer: {
2830 if (mustVisitNullValue(e: subExpr))
2831 cgf.emitIgnoredExpr(e: subExpr);
2832
2833 // Note that DestTy is used as the MLIR type instead of a custom
2834 // nullptr type.
2835 auto ty = mlir::cast<cir::PointerType>(Val: cgf.convertType(t: destTy));
2836 return cgf.cgm.getNullPointer(ptrTy: ty, qt: destTy,
2837 loc: cgf.getLoc(srcLoc: subExpr->getExprLoc()));
2838 }
2839
2840 case CK_NullToMemberPointer: {
2841 if (mustVisitNullValue(e: subExpr))
2842 cgf.emitIgnoredExpr(e: subExpr);
2843
2844 assert(!cir::MissingFeatures::cxxABI());
2845
2846 const MemberPointerType *mpt = ce->getType()->getAs<MemberPointerType>();
2847 mlir::Location loc = cgf.getLoc(srcLoc: subExpr->getExprLoc());
2848 return cgf.getBuilder().getConstant(
2849 loc, attr: cgf.cgm.emitNullMemberAttr(t: destTy, mpt));
2850 }
2851
2852 case CK_ReinterpretMemberPointer: {
2853 mlir::Value src = Visit(e: subExpr);
2854 return builder.createBitcast(loc: cgf.getLoc(srcLoc: subExpr->getExprLoc()), src,
2855 newTy: cgf.convertType(t: destTy));
2856 }
2857 case CK_BaseToDerivedMemberPointer:
2858 case CK_DerivedToBaseMemberPointer: {
2859 mlir::Value src = Visit(e: subExpr);
2860
2861 assert(!cir::MissingFeatures::memberFuncPtrAuthInfo());
2862
2863 QualType derivedTy =
2864 kind == CK_DerivedToBaseMemberPointer ? subExpr->getType() : destTy;
2865 const auto *mpType = derivedTy->castAs<MemberPointerType>();
2866 NestedNameSpecifier qualifier = mpType->getQualifier();
2867 assert(qualifier && "member pointer without class qualifier");
2868 const Type *qualifierType = qualifier.getAsType();
2869 assert(qualifierType && "member pointer qualifier is not a type");
2870 const CXXRecordDecl *derivedClass = qualifierType->getAsCXXRecordDecl();
2871 CharUnits offset =
2872 cgf.cgm.computeNonVirtualBaseClassOffset(derivedClass, path: ce->path());
2873
2874 mlir::Location loc = cgf.getLoc(srcLoc: subExpr->getExprLoc());
2875 mlir::Type resultTy = cgf.convertType(t: destTy);
2876 mlir::IntegerAttr offsetAttr = builder.getIndexAttr(value: offset.getQuantity());
2877
2878 if (subExpr->getType()->isMemberFunctionPointerType()) {
2879 if (kind == CK_BaseToDerivedMemberPointer)
2880 return cir::DerivedMethodOp::create(builder, location: loc, result: resultTy, src,
2881 offset: offsetAttr);
2882 return cir::BaseMethodOp::create(builder, location: loc, result: resultTy, src, offset: offsetAttr);
2883 }
2884
2885 if (kind == CK_BaseToDerivedMemberPointer)
2886 return cir::DerivedDataMemberOp::create(builder, location: loc, result: resultTy, src,
2887 offset: offsetAttr);
2888 return cir::BaseDataMemberOp::create(builder, location: loc, result: resultTy, src,
2889 offset: offsetAttr);
2890 }
2891
2892 case CK_LValueToRValue:
2893 assert(cgf.getContext().hasSameUnqualifiedType(subExpr->getType(), destTy));
2894 assert(subExpr->isGLValue() && "lvalue-to-rvalue applied to r-value!");
2895 return Visit(e: const_cast<Expr *>(subExpr));
2896
2897 case CK_IntegralCast: {
2898 ScalarConversionOpts opts;
2899 if (auto *ice = dyn_cast<ImplicitCastExpr>(Val: ce)) {
2900 if (!ice->isPartOfExplicitCast())
2901 opts = ScalarConversionOpts(cgf.sanOpts);
2902 }
2903 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2904 loc: ce->getExprLoc(), opts);
2905 }
2906
2907 case CK_FloatingComplexToReal:
2908 case CK_IntegralComplexToReal:
2909 case CK_FloatingComplexToBoolean:
2910 case CK_IntegralComplexToBoolean: {
2911 mlir::Value value = cgf.emitComplexExpr(e: subExpr);
2912 return emitComplexToScalarConversion(lov: cgf.getLoc(srcLoc: ce->getExprLoc()), value,
2913 kind, destTy);
2914 }
2915
2916 case CK_FloatingRealToComplex:
2917 case CK_FloatingComplexCast:
2918 case CK_IntegralRealToComplex:
2919 case CK_IntegralComplexCast:
2920 case CK_IntegralComplexToFloatingComplex:
2921 case CK_FloatingComplexToIntegralComplex:
2922 llvm_unreachable("scalar cast to non-scalar value");
2923
2924 case CK_PointerToIntegral: {
2925 assert(!destTy->isBooleanType() && "bool should use PointerToBool");
2926 if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
2927 cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
2928 "strict vtable pointers");
2929 return builder.createPtrToInt(src: Visit(e: subExpr), newTy: cgf.convertType(t: destTy));
2930 }
2931 case CK_ToVoid:
2932 cgf.emitIgnoredExpr(e: subExpr);
2933 return {};
2934
2935 case CK_FixedPointCast:
2936 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2937 loc: ce->getExprLoc());
2938
2939 case CK_FixedPointToBoolean:
2940 assert(subExpr->getType()->isFixedPointType() &&
2941 "Expected src type to be fixed point type");
2942 assert(destTy->isBooleanType() && "Expected dest type to be boolean type");
2943 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2944 loc: ce->getExprLoc());
2945
2946 case CK_FixedPointToIntegral:
2947 assert(subExpr->getType()->isFixedPointType() &&
2948 "Expected src type to be fixed point type");
2949 assert(destTy->isIntegerType() && "Expected dest type to be an integer");
2950 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2951 loc: ce->getExprLoc());
2952
2953 case CK_IntegralToFixedPoint:
2954 assert(subExpr->getType()->isIntegerType() &&
2955 "Expected src type to be an integer");
2956 assert(destTy->isFixedPointType() &&
2957 "Expected dest type to be fixed point type");
2958 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2959 loc: ce->getExprLoc());
2960
2961 case CK_IntegralToFloating:
2962 case CK_FloatingToIntegral:
2963 case CK_FloatingCast:
2964 case CK_FixedPointToFloating:
2965 case CK_FloatingToFixedPoint: {
2966 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ce);
2967 return emitScalarConversion(src: Visit(e: subExpr), srcType: subExpr->getType(), dstType: destTy,
2968 loc: ce->getExprLoc());
2969 }
2970
2971 case CK_IntegralToBoolean:
2972 return emitIntToBoolConversion(srcVal: Visit(e: subExpr),
2973 loc: cgf.getLoc(srcLoc: ce->getSourceRange()));
2974
2975 case CK_PointerToBoolean:
2976 return emitPointerToBoolConversion(v: Visit(e: subExpr), qt: subExpr->getType());
2977 case CK_FloatingToBoolean:
2978 return emitFloatToBoolConversion(src: Visit(e: subExpr),
2979 loc: cgf.getLoc(srcLoc: subExpr->getExprLoc()));
2980 case CK_MemberPointerToBoolean: {
2981 mlir::Value memPtr = Visit(e: subExpr);
2982 return builder.createCast(loc: cgf.getLoc(srcLoc: ce->getSourceRange()),
2983 kind: cir::CastKind::member_ptr_to_bool, src: memPtr,
2984 newTy: cgf.convertType(t: destTy));
2985 }
2986
2987 case CK_HLSLAggregateSplatCast:
2988 case CK_VectorSplat: {
2989 // Splat the element across to all elements
2990 mlir::Type destCIRTy = cgf.convertType(t: destTy);
2991 return cir::VecSplatOp::create(builder,
2992 location: cgf.getLoc(srcLoc: subExpr->getSourceRange()),
2993 result: destCIRTy, value: Visit(e: subExpr));
2994 }
2995 case CK_FunctionToPointerDecay:
2996 return cgf.emitLValue(e: subExpr).getPointer();
2997
2998 case CK_HLSLVectorTruncation: {
2999 assert((destTy->isVectorType() || destTy->isBuiltinType()) &&
3000 "Destination type must be a vector or builtin type.");
3001 mlir::Value vec = Visit(e: subExpr);
3002 mlir::Location loc = cgf.getLoc(srcLoc: subExpr->getSourceRange());
3003 if (auto *vecTy = destTy->getAs<VectorType>()) {
3004 unsigned numElts = vecTy->getNumElements();
3005 llvm::SmallVector<int64_t, 4> mask(numElts);
3006 std::iota(first: mask.begin(), last: mask.end(), value: 0);
3007 return builder.createVecShuffle(loc, vec1: vec, mask);
3008 }
3009 return builder.createExtractElement(loc, vec, /*idx=*/0);
3010 }
3011 case CK_HLSLMatrixTruncation: {
3012 assert((destTy->isMatrixType() || destTy->isBuiltinType()) &&
3013 "Destination type must be a matrix or builtin type.");
3014 mlir::Value matrix = Visit(e: subExpr);
3015 mlir::Location loc = cgf.getLoc(srcLoc: subExpr->getSourceRange());
3016 if (const auto *matTy = destTy->getAs<ConstantMatrixType>()) {
3017 llvm::SmallVector<int64_t> mask(matTy->getNumElementsFlattened());
3018 unsigned numCols = matTy->getNumColumns();
3019 unsigned numRows = matTy->getNumRows();
3020
3021 auto *srcMatTy = subExpr->getType()->getAs<ConstantMatrixType>();
3022 assert(srcMatTy && "Source type must be a matrix type.");
3023 assert(numRows <= srcMatTy->getNumRows());
3024 assert(numCols <= srcMatTy->getNumColumns());
3025
3026 for (unsigned row = 0; row < numRows; row++)
3027 for (unsigned column = 0; column < numCols; column++)
3028 mask[matTy->getColumnMajorFlattenedIndex(Row: row, Column: column)] =
3029 srcMatTy->getColumnMajorFlattenedIndex(Row: row, Column: column);
3030
3031 auto elementTy =
3032 mlir::cast<cir::MatrixType>(Val: matrix.getType()).getElementType();
3033 auto resultTy = cir::MatrixType::get(context: builder.getContext(), element_type: elementTy,
3034 num_rows: numRows, num_columns: numCols);
3035 return builder.createVecShuffle(loc, vec1: matrix, resultTy, mask);
3036 }
3037
3038 cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
3039 "ScalarExprEmitter::VisitCastExpr: HLSL "
3040 "Matrix truncation with non-matrix dest");
3041 return {};
3042 }
3043 default:
3044 cgf.getCIRGenModule().errorNYI(loc: subExpr->getSourceRange(),
3045 feature: "CastExpr: ", name: ce->getCastKindName());
3046 }
3047 return {};
3048}
3049
3050mlir::Value ScalarExprEmitter::VisitCallExpr(const CallExpr *e) {
3051 if (e->getCallReturnType(Ctx: cgf.getContext())->isReferenceType())
3052 return emitLoadOfLValue(e);
3053
3054 auto v = cgf.emitCallExpr(e).getValue();
3055 assert(!cir::MissingFeatures::emitLValueAlignmentAssumption());
3056 return v;
3057}
3058
3059mlir::Value ScalarExprEmitter::VisitMemberExpr(MemberExpr *e) {
3060 if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(me: e)) {
3061 cgf.emitIgnoredExpr(e: e->getBase());
3062 return cgf.emitScalarConstant(constant, e);
3063 } else {
3064 Expr::EvalResult result;
3065 if (e->EvaluateAsInt(Result&: result, Ctx: cgf.getContext(), AllowSideEffects: Expr::SE_AllowSideEffects)) {
3066 llvm::APSInt value = result.Val.getInt();
3067 cgf.emitIgnoredExpr(e: e->getBase());
3068 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
3069 // The constant is folded from an APSInt with the source-type's bit width
3070 // (1 for bool), but the AST's expression type is what later consumers of
3071 // this value see. For a bool member we have to emit a !cir.bool constant
3072 // -- otherwise downstream ops (cir.call into a bool parameter, cir.if /
3073 // cir.ternary on the value, ...) would all reject the !cir.int<u, 1> the
3074 // raw APSInt would produce.
3075 if (e->getType()->isBooleanType())
3076 return builder.getBool(state: value.getBoolValue(), loc);
3077 return builder.getConstInt(loc, intVal: value);
3078 }
3079 }
3080
3081 return emitLoadOfLValue(e);
3082}
3083
3084mlir::Value
3085ScalarExprEmitter::emitVectorInitWithSubvectors(mlir::Location loc,
3086 cir::VectorType vectorType,
3087 ArrayRef<mlir::Value> inits) {
3088 const unsigned resElts = vectorType.getFixedSize();
3089 mlir::Value result =
3090 builder.getConstant(loc, attr: cir::PoisonAttr::get(type: vectorType));
3091 unsigned curIdx = 0;
3092 for (mlir::Value init : inits) {
3093 auto initType = mlir::dyn_cast<cir::VectorType>(Val: init.getType());
3094 if (!initType) {
3095 result = builder.createInsertElement(loc, vec: result, newElt: init, idx: curIdx++);
3096 continue;
3097 }
3098
3099 const unsigned initElts = initType.getFixedSize();
3100 assert(curIdx + initElts <= resElts &&
3101 "Sema should reject excess elements");
3102 SmallVector<int64_t, 16> mask(resElts, -1);
3103 std::iota(first: mask.begin(), last: mask.begin() + initElts, value: 0);
3104 mlir::Value ext = builder.createVecShuffle(loc, vec1: init, mask);
3105
3106 // Poison stays on the RHS of the first shuffle, matching classic codegen.
3107 const unsigned offset = curIdx == 0 ? 0 : resElts;
3108 std::iota(first: mask.begin(), last: mask.begin() + curIdx, value: 0);
3109 std::iota(first: mask.begin() + curIdx, last: mask.begin() + curIdx + initElts, value: offset);
3110 if (curIdx == 0)
3111 std::swap(a&: result, b&: ext);
3112 result = builder.createVecShuffle(loc, vec1: result, vec2: ext, mask);
3113 curIdx += initElts;
3114 }
3115
3116 if (curIdx < resElts) {
3117 const mlir::Value zeroValue =
3118 builder.getNullValue(ty: vectorType.getElementType(), loc);
3119 for (; curIdx < resElts; ++curIdx)
3120 result = builder.createInsertElement(loc, vec: result, newElt: zeroValue, idx: curIdx);
3121 }
3122 return result;
3123}
3124
3125mlir::Value ScalarExprEmitter::VisitInitListExpr(InitListExpr *e) {
3126 const unsigned numInitElements = e->getNumInits();
3127
3128 [[maybe_unused]] const bool ignore = std::exchange(obj&: ignoreResultAssign, new_val: false);
3129 assert((ignore == false ||
3130 (numInitElements == 0 && e->getType()->isVoidType())) &&
3131 "init list ignored");
3132
3133 if (e->hadArrayRangeDesignator()) {
3134 cgf.cgm.errorNYI(e->getSourceRange(), "ArrayRangeDesignator");
3135 return {};
3136 }
3137
3138 if (e->getType()->isVectorType()) {
3139 const auto vectorType =
3140 mlir::cast<cir::VectorType>(Val: cgf.convertType(t: e->getType()));
3141
3142 SmallVector<mlir::Value, 16> elements;
3143 bool hasVectorInit = false;
3144 for (Expr *init : e->inits()) {
3145 elements.push_back(Elt: Visit(e: init));
3146 hasVectorInit |= mlir::isa<cir::VectorType>(Val: elements.back().getType());
3147 }
3148
3149 if (hasVectorInit)
3150 return emitVectorInitWithSubvectors(loc: cgf.getLoc(srcLoc: e->getSourceRange()),
3151 vectorType, inits: elements);
3152
3153 // Zero-initialize any remaining values.
3154 const uint64_t vectorSize = vectorType.getFixedSize();
3155 if (numInitElements < vectorSize) {
3156 const mlir::Value zeroValue = cgf.getBuilder().getNullValue(
3157 ty: vectorType.getElementType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
3158 std::fill_n(first: std::back_inserter(x&: elements), n: vectorSize - numInitElements,
3159 value: zeroValue);
3160 }
3161
3162 return cir::VecCreateOp::create(builder&: cgf.getBuilder(),
3163 location: cgf.getLoc(srcLoc: e->getSourceRange()), result: vectorType,
3164 elements);
3165 }
3166
3167 assert(!cir::MissingFeatures::scalableVectors());
3168
3169 if (e->getType()->isVoidType())
3170 return {};
3171
3172 // C++11 value-initialization for the scalar.
3173 if (numInitElements == 0)
3174 return emitNullValue(ty: e->getType(), loc: cgf.getLoc(srcLoc: e->getExprLoc()));
3175
3176 return Visit(e: e->getInit(Init: 0));
3177}
3178
3179mlir::Value CIRGenFunction::emitScalarConversion(mlir::Value src,
3180 QualType srcTy, QualType dstTy,
3181 SourceLocation loc) {
3182 assert(CIRGenFunction::hasScalarEvaluationKind(srcTy) &&
3183 CIRGenFunction::hasScalarEvaluationKind(dstTy) &&
3184 "Invalid scalar expression to emit");
3185 return ScalarExprEmitter(*this, builder)
3186 .emitScalarConversion(src, srcType: srcTy, dstType: dstTy, loc);
3187}
3188
3189mlir::Value CIRGenFunction::emitComplexToScalarConversion(mlir::Value src,
3190 QualType srcTy,
3191 QualType dstTy,
3192 SourceLocation loc) {
3193 assert(srcTy->isAnyComplexType() && hasScalarEvaluationKind(dstTy) &&
3194 "Invalid complex -> scalar conversion");
3195
3196 QualType complexElemTy = srcTy->castAs<ComplexType>()->getElementType();
3197 if (dstTy->isBooleanType()) {
3198 auto kind = complexElemTy->isFloatingType()
3199 ? cir::CastKind::float_complex_to_bool
3200 : cir::CastKind::int_complex_to_bool;
3201 return builder.createCast(loc: getLoc(srcLoc: loc), kind, src, newTy: convertType(t: dstTy));
3202 }
3203
3204 auto kind = complexElemTy->isFloatingType()
3205 ? cir::CastKind::float_complex_to_real
3206 : cir::CastKind::int_complex_to_real;
3207 mlir::Value real =
3208 builder.createCast(loc: getLoc(srcLoc: loc), kind, src, newTy: convertType(t: complexElemTy));
3209 return emitScalarConversion(src: real, srcTy: complexElemTy, dstTy, loc);
3210}
3211
3212mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
3213 // Perform vector logical not on comparison with zero vector.
3214 if (e->getType()->isVectorType() &&
3215 e->getType()->castAs<VectorType>()->getVectorKind() ==
3216 VectorKind::Generic) {
3217 mlir::Value oper = Visit(e: e->getSubExpr());
3218 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
3219 auto operVecTy = mlir::cast<cir::VectorType>(Val: oper.getType());
3220 mlir::Value zeroVec = builder.getNullValue(ty: operVecTy, loc);
3221 auto resultTy = mlir::cast<cir::VectorType>(Val: cgf.convertType(t: e->getType()));
3222 if (cir::isFPOrVectorOfFPType(type: operVecTy.getElementType())) {
3223 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
3224 return builder.createVecFCompare(loc, resultTy, kind: cir::FCmpOpKind::oeq,
3225 lhs: oper, rhs: zeroVec);
3226 }
3227
3228 return builder.createVecCompare(loc, resultTy, kind: cir::CmpOpKind::eq, lhs: oper,
3229 rhs: zeroVec);
3230 }
3231
3232 // Compare operand to zero.
3233 mlir::Value boolVal = cgf.evaluateExprAsBool(e: e->getSubExpr());
3234
3235 // Invert value.
3236 boolVal = builder.createNot(value: boolVal);
3237
3238 // ZExt result to the expr type.
3239 return maybePromoteBoolResult(value: boolVal, dstTy: cgf.convertType(t: e->getType()));
3240}
3241
3242mlir::Value ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *e) {
3243 // Try folding the offsetof to a constant.
3244 Expr::EvalResult evalResult;
3245 if (e->EvaluateAsInt(Result&: evalResult, Ctx: cgf.getContext())) {
3246 mlir::Type type = cgf.convertType(t: e->getType());
3247 llvm::APSInt value = evalResult.Val.getInt();
3248 return builder.getConstAPInt(loc: cgf.getLoc(srcLoc: e->getExprLoc()), typ: type, val: value);
3249 }
3250
3251 cgf.getCIRGenModule().errorNYI(
3252 e->getSourceRange(),
3253 "ScalarExprEmitter::VisitOffsetOfExpr Can't eval expr as int");
3254 return {};
3255}
3256
3257mlir::Value ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *e) {
3258 QualType promotionTy = getPromotionType(ty: e->getSubExpr()->getType());
3259 mlir::Value result = VisitRealImag(e, promotionType: promotionTy);
3260 if (result && !promotionTy.isNull())
3261 result = emitUnPromotedValue(result, exprType: e->getType());
3262 return result;
3263}
3264
3265mlir::Value ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *e) {
3266 QualType promotionTy = getPromotionType(ty: e->getSubExpr()->getType());
3267 mlir::Value result = VisitRealImag(e, promotionType: promotionTy);
3268 if (result && !promotionTy.isNull())
3269 result = emitUnPromotedValue(result, exprType: e->getType());
3270 return result;
3271}
3272
3273mlir::Value ScalarExprEmitter::VisitRealImag(const UnaryOperator *e,
3274 QualType promotionTy) {
3275 assert(
3276 (e->getOpcode() == clang::UO_Real || e->getOpcode() == clang::UO_Imag) &&
3277 "Invalid UnaryOp kind for ComplexType Real or Imag");
3278
3279 Expr *op = e->getSubExpr();
3280 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
3281 if (op->getType()->isAnyComplexType()) {
3282 // If it's an l-value, load through the appropriate subobject l-value.
3283 // Note that we have to ask `e` because `op` might be an l-value that
3284 // this won't work for, e.g. an Obj-C property
3285 mlir::Value complex = cgf.emitComplexExpr(e: op);
3286 if (e->isGLValue() && !promotionTy.isNull()) {
3287 promotionTy = promotionTy->isAnyComplexType()
3288 ? promotionTy
3289 : cgf.getContext().getComplexType(T: promotionTy);
3290 complex = cgf.emitPromotedValue(result: complex, promotionType: promotionTy);
3291 }
3292
3293 return e->getOpcode() == clang::UO_Real
3294 ? builder.createComplexReal(loc, operand: complex)
3295 : builder.createComplexImag(loc, operand: complex);
3296 }
3297
3298 if (e->getOpcode() == UO_Real) {
3299 mlir::Value operand = promotionTy.isNull()
3300 ? Visit(e: op)
3301 : cgf.emitPromotedScalarExpr(e: op, promotionType: promotionTy);
3302 return builder.createComplexReal(loc, operand);
3303 }
3304
3305 // __imag on a scalar returns zero. Emit the subexpr to ensure side
3306 // effects are evaluated, but not the actual value.
3307 mlir::Value operand;
3308 if (op->isGLValue()) {
3309 operand = cgf.emitLValue(e: op).getPointer();
3310 operand = cir::LoadOp::create(builder, location: loc, addr: operand);
3311 } else if (!promotionTy.isNull()) {
3312 operand = cgf.emitPromotedScalarExpr(e: op, promotionType: promotionTy);
3313 } else {
3314 operand = cgf.emitScalarExpr(e: op);
3315 }
3316 return builder.createComplexImag(loc, operand);
3317}
3318
3319/// Return the size or alignment of the type of argument of the sizeof
3320/// expression as an integer.
3321mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
3322 const UnaryExprOrTypeTraitExpr *e) {
3323 const QualType typeToSize = e->getTypeOfArgument();
3324 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
3325 if (auto kind = e->getKind();
3326 kind == UETT_SizeOf || kind == UETT_DataSizeOf || kind == UETT_CountOf) {
3327 if (const VariableArrayType *vat =
3328 cgf.getContext().getAsVariableArrayType(T: typeToSize)) {
3329 // For _Countof, we only want to evaluate if the extent is actually
3330 // variable as opposed to a multi-dimensional array whose extent is
3331 // constant but whose element type is variable.
3332 bool evaluateExtent = true;
3333 if (kind == UETT_CountOf && vat->getElementType()->isArrayType()) {
3334 evaluateExtent =
3335 !vat->getSizeExpr()->isIntegerConstantExpr(Ctx: cgf.getContext());
3336 }
3337
3338 if (evaluateExtent) {
3339 if (e->isArgumentType()) {
3340 // sizeof(type) - make sure to emit the VLA size.
3341 cgf.emitVariablyModifiedType(ty: typeToSize);
3342 } else {
3343 // C99 6.5.3.4p2: If the argument is an expression of type
3344 // VLA, it is evaluated.
3345 cgf.emitIgnoredExpr(e: e->getArgumentExpr());
3346 }
3347
3348 // For _Countof, we just want to return the size of a single dimension.
3349 if (kind == UETT_CountOf)
3350 return cgf.getVLAElements1D(vla: vat).numElts;
3351
3352 // For sizeof and __datasizeof, we need to scale the number of elements
3353 // by the size of the array element type.
3354 CIRGenFunction::VlaSizePair vlaSize = cgf.getVLASize(type: vat);
3355 mlir::Value numElts = vlaSize.numElts;
3356
3357 // Scale the number of non-VLA elements by the non-VLA element size.
3358 CharUnits eltSize = cgf.getContext().getTypeSizeInChars(T: vlaSize.type);
3359 if (!eltSize.isOne()) {
3360 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
3361 mlir::Value eltSizeValue =
3362 builder.getConstAPInt(loc: numElts.getLoc(), typ: numElts.getType(),
3363 val: cgf.cgm.getSize(size: eltSize).getValue());
3364 return builder.createMul(loc, lhs: eltSizeValue, rhs: numElts,
3365 flags: cir::OverflowFlags::NoUnsignedWrap);
3366 }
3367
3368 return numElts;
3369 }
3370 }
3371 } else if (e->getKind() == UETT_OpenMPRequiredSimdAlign) {
3372 clang::CharUnits::QuantityType alignment =
3373 cgf.getContext()
3374 .toCharUnitsFromBits(BitSize: cgf.getContext().getOpenMPDefaultSimdAlign(
3375 T: e->getTypeOfArgument()->getPointeeType()))
3376 .getQuantity();
3377 return builder.getConstantInt(loc, ty: cgf.cgm.sizeTy, value: alignment);
3378 } else if (e->getKind() == UETT_VectorElements) {
3379 auto vecTy = cast<cir::VectorType>(Val: convertType(ty: e->getTypeOfArgument()));
3380 if (vecTy.isScalable()) {
3381 cgf.getCIRGenModule().errorNYI(
3382 e->getSourceRange(),
3383 "VisitUnaryExprOrTypeTraitExpr: sizeOf scalable vector");
3384 return builder.getConstant(
3385 loc, attr: cir::IntAttr::get(type: cgf.cgm.sizeTy,
3386 value: e->EvaluateKnownConstInt(Ctx: cgf.getContext())));
3387 }
3388
3389 return builder.getConstant(
3390 loc, attr: cir::IntAttr::get(type: cgf.cgm.sizeTy, value: vecTy.getFixedSize()));
3391 }
3392
3393 // The result type is size_t (target-dependent width); use it so the IntAttr
3394 // width matches the APInt from EvaluateKnownConstInt.
3395 return builder.getConstant(
3396 loc, attr: cir::IntAttr::get(type: cgf.cgm.sizeTy,
3397 value: e->EvaluateKnownConstInt(Ctx: cgf.getContext())));
3398}
3399
3400mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator(
3401 const AbstractConditionalOperator *e) {
3402 CIRGenBuilderTy &builder = cgf.getBuilder();
3403 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
3404 ignoreResultAssign = false;
3405
3406 // Bind the common expression if necessary.
3407 CIRGenFunction::OpaqueValueMapping binding(cgf, e);
3408
3409 Expr *condExpr = e->getCond();
3410 Expr *lhsExpr = e->getTrueExpr();
3411 Expr *rhsExpr = e->getFalseExpr();
3412
3413 // If the condition constant folds and can be elided, try to avoid emitting
3414 // the condition and the dead arm.
3415 bool condExprBool;
3416 if (cgf.constantFoldsToBool(cond: condExpr, resultBool&: condExprBool)) {
3417 Expr *live = lhsExpr, *dead = rhsExpr;
3418 if (!condExprBool)
3419 std::swap(a&: live, b&: dead);
3420
3421 // If the dead side doesn't have labels we need, just emit the Live part.
3422 if (!cgf.containsLabel(s: dead)) {
3423 if (condExprBool)
3424 assert(!cir::MissingFeatures::incrementProfileCounter());
3425 mlir::Value result = Visit(e: live);
3426
3427 // If the live part is a throw expression, it acts like it has a void
3428 // type, so evaluating it returns a null Value. However, a conditional
3429 // with non-void type must return a non-null Value.
3430 if (!result && !e->getType()->isVoidType()) {
3431 result = builder.getConstant(
3432 loc, attr: cir::PoisonAttr::get(context: builder.getContext(),
3433 type: cgf.convertType(t: e->getType())));
3434 }
3435
3436 return result;
3437 }
3438 }
3439
3440 QualType condType = condExpr->getType();
3441
3442 // OpenCL: If the condition is a vector, we can treat this condition like
3443 // the select function.
3444 if (cgf.getLangOpts().OpenCL &&
3445 (condType->isVectorType() || condType->isExtVectorType())) {
3446 assert(!cir::MissingFeatures::incrementProfileCounter());
3447
3448 mlir::Value condValue = cgf.emitScalarExpr(e: condExpr);
3449 mlir::Value lhsValue = Visit(e: lhsExpr);
3450 mlir::Value rhsValue = Visit(e: rhsExpr);
3451
3452 // Test the MSB with a signed compare, as classic codegen does.
3453 mlir::Type vecTy =
3454 convertType(ty: cgf.getContext().getCorrespondingSignedType(T: condType));
3455 condValue = builder.createBitcast(src: condValue, newTy: vecTy);
3456 mlir::Value zeroVec = builder.getNullValue(ty: vecTy, loc);
3457 auto testMSB = cir::VecCmpOp::create(
3458 builder, location: loc, result: vecTy, kind: cir::CmpOpKind::lt, lhs: condValue, rhs: zeroVec);
3459 mlir::Value tmp = builder.createIntCast(src: testMSB, newTy: vecTy);
3460 mlir::Value tmp2 = builder.createNot(value: tmp);
3461
3462 // Bitcast float or differently signed operands to the condition type.
3463 mlir::Value rhsTmp = rhsValue;
3464 mlir::Value lhsTmp = lhsValue;
3465 bool wasCast = false;
3466 if (rhsValue.getType() != vecTy) {
3467 rhsTmp = builder.createBitcast(src: rhsValue, newTy: tmp2.getType());
3468 lhsTmp = builder.createBitcast(src: lhsValue, newTy: tmp.getType());
3469 wasCast = true;
3470 }
3471
3472 mlir::Value tmp3 = builder.createAnd(loc, lhs: rhsTmp, rhs: tmp2);
3473 mlir::Value tmp4 = builder.createAnd(loc, lhs: lhsTmp, rhs: tmp);
3474 mlir::Value tmp5 = builder.createOr(loc, lhs: tmp3, rhs: tmp4);
3475 if (wasCast)
3476 tmp5 = builder.createBitcast(src: tmp5, newTy: rhsValue.getType());
3477 return tmp5;
3478 }
3479
3480 if (condType->isVectorType() || condType->isSveVLSBuiltinType()) {
3481 if (!condType->isVectorType()) {
3482 assert(!cir::MissingFeatures::vecTernaryOp());
3483 cgf.cgm.errorNYI(loc, feature: "TernaryOp for SVE vector");
3484 return {};
3485 }
3486
3487 mlir::Value condValue = Visit(e: condExpr);
3488 mlir::Value lhsValue = Visit(e: lhsExpr);
3489 mlir::Value rhsValue = Visit(e: rhsExpr);
3490 return cir::VecTernaryOp::create(builder, location: loc, cond: condValue, lhs: lhsValue,
3491 rhs: rhsValue);
3492 }
3493
3494 // If this is a really simple expression (like x ? 4 : 5), emit this as a
3495 // select instead of as control flow. We can only do this if it is cheap
3496 // and safe to evaluate the LHS and RHS unconditionally.
3497 if (CodeGenUtils::isCheapEnoughToEvaluateUnconditionally(E: lhsExpr,
3498 Ctx: cgf.getContext()) &&
3499 CodeGenUtils::isCheapEnoughToEvaluateUnconditionally(E: rhsExpr,
3500 Ctx: cgf.getContext())) {
3501 mlir::Value condV = cgf.evaluateExprAsBool(e: condExpr);
3502 assert(!cir::MissingFeatures::incrementProfileCounter());
3503
3504 mlir::Value lhs = Visit(e: lhsExpr);
3505 mlir::Value rhs = Visit(e: rhsExpr);
3506 if (!lhs) {
3507 // If the conditional has void type, make sure we return a null Value.
3508 assert(!rhs && "lhs and rhs types must match");
3509 return {};
3510 }
3511
3512 return builder.createSelect(loc, condition: condV, trueValue: lhs, falseValue: rhs);
3513 }
3514
3515 mlir::Value condV = cgf.emitOpOnBoolExpr(loc, cond: condExpr);
3516 CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
3517
3518 auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc, Expr *expr) {
3519 CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()};
3520 cgf.curLexScope->setAsTernary();
3521
3522 mlir::Value branch;
3523 {
3524 // Emit any cleanups that were needed on this branch so we can spill
3525 // and reload the return value.
3526 CIRGenFunction::RunCleanupsScope branchCleanups(cgf);
3527 assert(!cir::MissingFeatures::incrementProfileCounter());
3528 eval.beginEvaluation();
3529 branch = Visit(e: expr);
3530 eval.endEvaluation();
3531 branchCleanups.forceCleanup(valuesToReload: {&branch});
3532 }
3533
3534 if (branch)
3535 cir::YieldOp::create(builder&: b, location: loc, args: branch);
3536 };
3537
3538 cir::TernaryOp ternary = cir::TernaryOp::create(
3539 builder, location: loc, cond: condV,
3540 /*trueBuilder=*/
3541 [&](mlir::OpBuilder &b, mlir::Location loc) {
3542 emitBranch(b, loc, lhsExpr);
3543 },
3544 /*falseBuilder=*/
3545 [&](mlir::OpBuilder &b, mlir::Location loc) {
3546 emitBranch(b, loc, rhsExpr);
3547 });
3548
3549 // Only a void arm can be left unterminated (a noreturn arm already ends
3550 // in cir.unreachable); close it with an empty cir.yield.
3551 for (mlir::Region *region :
3552 {&ternary.getTrueRegion(), &ternary.getFalseRegion()}) {
3553 mlir::Block &lastBlock = region->back();
3554 if (lastBlock.empty() ||
3555 !lastBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) {
3556 mlir::OpBuilder::InsertionGuard guard(builder);
3557 builder.setInsertionPointToEnd(&lastBlock);
3558 cir::YieldOp::create(builder, location: loc);
3559 }
3560 }
3561
3562 return ternary.getResult();
3563}
3564
3565mlir::Value CIRGenFunction::emitScalarPrePostIncDec(const UnaryOperator *e,
3566 LValue lv) {
3567 return ScalarExprEmitter(*this, builder).emitScalarPrePostIncDec(e, lv);
3568}
3569