1//===- CIRGenExprAggregrate.cpp - Emit CIR Code from Aggregate Expressions ===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as CIR code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14#include "CIRGenConstantEmitter.h"
15#include "CIRGenFunction.h"
16#include "CIRGenValue.h"
17#include "mlir/IR/Builders.h"
18#include "clang/CIR/Dialect/IR/CIRAttrs.h"
19
20#include "clang/AST/Expr.h"
21#include "clang/AST/ExprCXX.h"
22#include "clang/AST/RecordLayout.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/CodeGenUtils/ExprUtils.h"
25#include "llvm/IR/Value.h"
26#include <cstdint>
27
28using namespace clang;
29using namespace clang::CIRGen;
30
31namespace {
32class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
33
34 CIRGenFunction &cgf;
35 AggValueSlot dest;
36
37 // Calls `fn` with a valid return value slot, potentially creating a temporary
38 // to do so. If a temporary is created, an appropriate copy into `Dest` will
39 // be emitted, as will lifetime markers.
40 //
41 // The given function should take a ReturnValueSlot, and return an RValue that
42 // points to said slot.
43 void withReturnValueSlot(const Expr *e,
44 llvm::function_ref<RValue(ReturnValueSlot)> fn);
45
46 AggValueSlot ensureSlot(mlir::Location loc, QualType t) {
47 if (!dest.isIgnored())
48 return dest;
49 return cgf.createAggTemp(ty: t, loc, name: "agg.tmp.ensured");
50 }
51
52 void ensureDest(mlir::Location loc, QualType ty) {
53 if (!dest.isIgnored())
54 return;
55 dest = cgf.createAggTemp(ty, loc, name: "agg.tmp.ensured");
56 }
57
58public:
59 AggExprEmitter(CIRGenFunction &cgf, AggValueSlot dest)
60 : cgf(cgf), dest(dest) {}
61
62 /// Given an expression with aggregate type that represents a value lvalue,
63 /// this method emits the address of the lvalue, then loads the result into
64 /// DestPtr.
65 void emitAggLoadOfLValue(const Expr *e);
66
67 void emitArrayInit(Address destPtr, cir::ArrayType arrayTy, QualType arrayQTy,
68 Expr *exprToVisit, ArrayRef<Expr *> args,
69 Expr *arrayFiller);
70
71 void emitFinalDestCopy(QualType type, RValue src);
72
73 /// Perform the final copy to DestPtr, if desired.
74 void emitFinalDestCopy(QualType type, const LValue &src,
75 CIRGenFunction::ExprValueKind srcValueKind =
76 CIRGenFunction::EVK_NonRValue);
77
78 void emitCopy(QualType type, const AggValueSlot &dest,
79 const AggValueSlot &src);
80
81 void emitInitializationToLValue(Expr *e, LValue lv);
82
83 void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
84
85 void emitComparisonResult(const Expr *e, mlir::Location loc,
86 const ComparisonCategoryInfo &cmpInfo,
87 mlir::Value resultValue);
88
89 void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(S: e); }
90
91 void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
92 emitAggLoadOfLValue(e);
93 }
94
95 void VisitCallExpr(const CallExpr *e);
96 void VisitStmtExpr(const StmtExpr *e) {
97 CIRGenFunction::StmtExprEvaluation eval(cgf);
98 Address retAlloca =
99 cgf.createMemTemp(t: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()));
100 (void)cgf.emitCompoundStmt(s: *e->getSubStmt(), lastValue: &retAlloca, slot: dest);
101 }
102
103 void VisitBinAssign(const BinaryOperator *e) {
104 // For an assignment to work, the value on the right has
105 // to be compatible with the value on the left.
106 assert(cgf.getContext().hasSameUnqualifiedType(e->getLHS()->getType(),
107 e->getRHS()->getType()) &&
108 "Invalid assignment");
109
110 if (CodeGenUtils::isBlockVarRef(E: e->getLHS()) &&
111 e->getRHS()->HasSideEffects(Ctx: cgf.getContext())) {
112 cgf.cgm.errorNYI(e->getSourceRange(),
113 "block var reference with side effects");
114 return;
115 }
116
117 LValue lhs = cgf.emitLValue(e: e->getLHS());
118
119 // If we have an atomic type, evaluate into the destination and then
120 // do an atomic copy.
121 if (lhs.getType()->isAtomicType() ||
122 cgf.isLValueSuitableForInlineAtomic(lv: lhs)) {
123 ensureDest(loc: cgf.getLoc(srcLoc: e->getExprLoc()), ty: e->getRHS()->getType());
124 Visit(e: e->getRHS());
125 cgf.emitAtomicStore(rvalue: dest.asRValue(), dest: lhs, /*isInit=*/false);
126 return;
127 }
128
129 // Codegen the RHS so that it stores directly into the LHS.
130 assert(!cir::MissingFeatures::aggValueSlotGC());
131 AggValueSlot lhsSlot = AggValueSlot::forLValue(
132 LV: lhs, isDestructed: AggValueSlot::IsDestructed, isAliased: AggValueSlot::IsAliased,
133 mayOverlap: AggValueSlot::MayOverlap);
134
135 // A non-volatile aggregate destination might have volatile member.
136 if (!lhsSlot.isVolatile() && cgf.hasVolatileMember(t: e->getLHS()->getType()))
137 lhsSlot.setVolatile(true);
138
139 cgf.emitAggExpr(e: e->getRHS(), slot: lhsSlot);
140
141 // Copy into the destination if the assignment isn't ignored.
142 emitFinalDestCopy(type: e->getType(), src: lhs);
143
144 if (!dest.isIgnored() && !dest.isExternallyDestructed() &&
145 e->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
146 cgf.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: dest.getAddress(),
147 type: e->getType());
148 }
149
150 void VisitDeclRefExpr(DeclRefExpr *e) { emitAggLoadOfLValue(e); }
151
152 void VisitInitListExpr(InitListExpr *e);
153 void VisitCXXConstructExpr(const CXXConstructExpr *e);
154
155 void visitCXXParenListOrInitListExpr(Expr *e, ArrayRef<Expr *> args,
156 FieldDecl *initializedFieldInUnion,
157 Expr *arrayFiller);
158 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die) {
159 CIRGenFunction::CXXDefaultInitExprScope Scope(cgf, die);
160 Visit(e: die->getExpr());
161 }
162 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *e) {
163 // Ensure that we have a slot, but if we already do, remember
164 // whether it was externally destructed.
165 bool wasExternallyDestructed = dest.isExternallyDestructed();
166 ensureDest(loc: cgf.getLoc(srcLoc: e->getSourceRange()), ty: e->getType());
167
168 // We're going to push a destructor if there isn't already one.
169 dest.setExternallyDestructed();
170
171 Visit(e: e->getSubExpr());
172
173 // Push that destructor we promised.
174 if (!wasExternallyDestructed)
175 cgf.emitCXXTemporary(temporary: e->getTemporary(), tempType: e->getType(), ptr: dest.getAddress());
176 }
177 void VisitLambdaExpr(LambdaExpr *e);
178 void VisitExprWithCleanups(ExprWithCleanups *e);
179
180 /// Attempt to look through various unimportant expressions to find a
181 /// cast of the given kind.
182 static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
183 op = op->IgnoreParenNoopCasts(Ctx: ctx);
184 if (auto *castE = dyn_cast<CastExpr>(Val: op)) {
185 if (castE->getCastKind() == kind)
186 return castE->getSubExpr();
187 }
188 return nullptr;
189 }
190
191 // Stubs -- These should be moved up when they are implemented.
192 void VisitCastExpr(CastExpr *e) {
193 switch (e->getCastKind()) {
194 case CK_LValueToRValueBitCast: {
195 if (dest.isIgnored()) {
196 cgf.emitAnyExpr(e: e->getSubExpr(), aggSlot: AggValueSlot::ignored(),
197 /*ignoreResult=*/true);
198 break;
199 }
200
201 LValue sourceLV = cgf.emitLValue(e: e->getSubExpr());
202 Address sourceAddress =
203 sourceLV.getAddress().withElementType(builder&: cgf.getBuilder(), ElemTy: cgf.voidTy);
204 Address destAddress =
205 dest.getAddress().withElementType(builder&: cgf.getBuilder(), ElemTy: cgf.voidTy);
206
207 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
208
209 mlir::Value sizeVal = cgf.getBuilder().getConstInt(
210 loc, t: cgf.sizeTy,
211 c: cgf.getContext().getTypeSizeInChars(T: e->getType()).getQuantity());
212 cgf.getBuilder().createMemCpy(loc, dst: destAddress, src: sourceAddress, len: sizeVal);
213
214 break;
215 }
216
217 case CK_NonAtomicToAtomic:
218 case CK_AtomicToNonAtomic: {
219 bool isToAtomic = (e->getCastKind() == CK_NonAtomicToAtomic);
220
221 // Determine the atomic and value types.
222 QualType atomicType = e->getSubExpr()->getType();
223 QualType valueType = e->getType();
224 if (isToAtomic)
225 std::swap(a&: atomicType, b&: valueType);
226
227 assert(atomicType->isAtomicType());
228 assert(cgf.getContext().hasSameUnqualifiedType(
229 valueType, atomicType->castAs<AtomicType>()->getValueType()));
230
231 // Just recurse normally if we're ignoring the result or the
232 // atomic type doesn't change representation.
233 if (dest.isIgnored() || !cgf.cgm.isPaddedAtomicType(type: atomicType))
234 return Visit(e: e->getSubExpr());
235
236 // These two cases are reverses of each other; try to peephole them.
237 CastKind peepholeTarget =
238 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
239
240 // These two cases are reverses of each other; try to peephole them.
241 if (Expr *op =
242 findPeephole(op: e->getSubExpr(), kind: peepholeTarget, ctx: cgf.getContext())) {
243 assert(cgf.getContext().hasSameUnqualifiedType(op->getType(),
244 e->getType()) &&
245 "peephole significantly changed types?");
246 return Visit(e: op);
247 }
248
249 // If we're converting an r-value of non-atomic type to an r-value
250 // of atomic type, just emit directly into the relevant sub-object.
251 if (isToAtomic) {
252 AggValueSlot valueDest = dest;
253 if (!valueDest.isIgnored() && cgf.cgm.isPaddedAtomicType(type: atomicType)) {
254 // Zero-initialize. (Strictly speaking, we only need to initialize
255 // the padding at the end, but this is simpler.)
256 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
257 if (!dest.isZeroed())
258 cgf.emitNullInitialization(loc, destPtr: dest.getAddress(), ty: atomicType);
259
260 Address valueAddr = cgf.getBuilder().createGetMember(
261 loc, base: valueDest.getAddress(), name: "value_addr", index: 0,
262 layout: cgf.cgm.getDataLayout());
263
264 assert(!cir::MissingFeatures::aggValueSlotGC());
265 valueDest = AggValueSlot::forAddr(
266 addr: valueAddr, quals: valueDest.getQualifiers(),
267 isDestructed: valueDest.isExternallyDestructed(),
268 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
269 isZeroed: AggValueSlot::IsZeroed);
270 }
271
272 cgf.emitAggExpr(e: e->getSubExpr(), slot: valueDest);
273 return;
274 }
275
276 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
277 AggValueSlot atomicSlot = cgf.createAggTemp(ty: atomicType, loc);
278 cgf.emitAggExpr(e: e->getSubExpr(), slot: atomicSlot);
279
280 Address valueAddr = cgf.getBuilder().createGetMember(
281 loc, base: atomicSlot.getAddress(), name: "value_addr", index: 0,
282 layout: cgf.cgm.getDataLayout());
283 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
284 return emitFinalDestCopy(type: valueType, src: rvalue);
285 }
286 case CK_LValueToRValue:
287 // If we're loading from a volatile type, force the destination
288 // into existence.
289 if (e->getSubExpr()->getType().isVolatileQualified())
290 cgf.cgm.errorNYI(e->getSourceRange(),
291 "AggExprEmitter: volatile lvalue-to-rvalue cast");
292 [[fallthrough]];
293 case CK_NoOp:
294 case CK_UserDefinedConversion:
295 case CK_ConstructorConversion:
296 assert(cgf.getContext().hasSameUnqualifiedType(e->getSubExpr()->getType(),
297 e->getType()) &&
298 "Implicit cast types must be compatible");
299 Visit(e: e->getSubExpr());
300 break;
301 case CK_ToUnion: {
302 if (dest.isIgnored()) {
303 cgf.emitAnyExpr(e: e->getSubExpr(), aggSlot: AggValueSlot::ignored(),
304 /*ignoreResult=*/true);
305 break;
306 }
307 QualType ty = e->getSubExpr()->getType();
308 Address castPtr = dest.getAddress().withElementType(builder&: cgf.getBuilder(),
309 ElemTy: cgf.convertType(t: ty));
310 emitInitializationToLValue(e: e->getSubExpr(),
311 lv: cgf.makeAddrLValue(addr: castPtr, ty));
312 break;
313 }
314 default:
315 cgf.cgm.errorNYI(e->getSourceRange(),
316 std::string("AggExprEmitter: VisitCastExpr: ") +
317 e->getCastKindName());
318 break;
319 }
320 }
321 void VisitStmt(Stmt *s) {
322 cgf.cgm.errorNYI(s->getSourceRange(),
323 std::string("AggExprEmitter::VisitStmt: ") +
324 s->getStmtClassName());
325 }
326 void VisitParenExpr(ParenExpr *pe) { Visit(e: pe->getSubExpr()); }
327 void VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
328 Visit(e: ge->getResultExpr());
329 }
330 void VisitCoawaitExpr(CoawaitExpr *e) {
331 cgf.emitCoawaitExpr(e: *e, aggSlot: dest, ignoreResult: dest.isIgnored());
332 }
333 void VisitCoyieldExpr(CoyieldExpr *e) {
334 cgf.emitCoyieldExpr(e: *e, aggSlot: dest, ignoreResult: dest.isIgnored());
335 }
336 void VisitUnaryCoawait(UnaryOperator *e) { Visit(e: e->getSubExpr()); }
337 void VisitUnaryExtension(UnaryOperator *e) { Visit(e: e->getSubExpr()); }
338 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
339 Visit(e: e->getReplacement());
340 }
341 void VisitPackIndexingExpr(PackIndexingExpr *e) {
342 Visit(e: e->getSelectedExpr());
343 }
344 void VisitConstantExpr(ConstantExpr *e) {
345 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
346 ensureDest(loc, ty: e->getType());
347
348 if (mlir::Attribute result = ConstantEmitter(cgf).tryEmitConstantExpr(ce: e)) {
349 QualType ty = e->getType();
350
351 // If the destination's tail padding may overlap another object's
352 // storage (e.g. a [[no_unique_address]] member or base), only store
353 // the type's data size rather than its full size, so we don't
354 // clobber bytes that belong to that other object. In particular, a
355 // genuinely empty class has a data size of zero, so nothing should
356 // be stored at all.
357 if (dest.mayOverlap()) {
358 CharUnits dataSize =
359 cgf.getContext().getTypeInfoDataSizeInChars(T: ty).Width;
360 if (dataSize.isZero())
361 return;
362
363 if (dataSize != cgf.getContext().getTypeSizeInChars(T: ty)) {
364 Address temp = cgf.createMemTemp(t: ty, loc);
365 mlir::Value resultVal = cgf.getBuilder().getConstant(
366 loc, attr: mlir::cast<mlir::TypedAttr>(Val&: result));
367 cgf.getBuilder().createStore(loc, val: resultVal, dst: temp);
368 cgf.getBuilder().createCopy(dst: dest.getAddress(), src: temp,
369 /*isVolatile=*/false,
370 /*skipTailPadding=*/true);
371 return;
372 }
373 }
374
375 mlir::Value resultVal = cgf.getBuilder().getConstant(
376 loc, attr: mlir::cast<mlir::TypedAttr>(Val&: result));
377 LValue destLVal = cgf.makeAddrLValue(addr: dest.getAddress(), ty);
378 cgf.emitStoreThroughLValue(src: RValue::get(v: resultVal), dst: destLVal);
379 return;
380 }
381
382 // It isn't clear that it is possible to get to here, but this branch is
383 // present in classic codegen, so we leave it here too.
384 return Visit(e: e->getSubExpr());
385 }
386 void VisitMemberExpr(MemberExpr *e) { emitAggLoadOfLValue(e); }
387 void VisitUnaryDeref(UnaryOperator *e) { emitAggLoadOfLValue(e); }
388 void VisitStringLiteral(StringLiteral *e) { emitAggLoadOfLValue(e); }
389 void VisitCompoundLiteralExpr(CompoundLiteralExpr *e);
390
391 void VisitPredefinedExpr(const PredefinedExpr *e) { emitAggLoadOfLValue(e); }
392 void VisitBinaryOperator(const BinaryOperator *e) {
393 if (e->getOpcode() == BO_PtrMemD || e->getOpcode() == BO_PtrMemI)
394 VisitPointerToDataMemberBinaryOperator(e);
395 else
396 cgf.cgm.errorUnsupported(s: e, type: "aggregate binary expression");
397 }
398 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *e) {
399 LValue lv = cgf.emitPointerToDataMemberBinaryExpr(e);
400 emitFinalDestCopy(type: e->getType(), src: lv);
401 }
402 void VisitBinComma(const BinaryOperator *e) {
403 cgf.emitIgnoredExpr(e: e->getLHS());
404 Visit(e: e->getRHS());
405 }
406 void VisitBinCmp(const BinaryOperator *e) {
407 assert(cgf.getContext().hasSameType(e->getLHS()->getType(),
408 e->getRHS()->getType()));
409 const ComparisonCategoryInfo &cmpInfo =
410 cgf.getContext().CompCategories.getInfoForType(Ty: e->getType());
411 assert(cmpInfo.Record->isTriviallyCopyable() &&
412 "cannot copy non-trivially copyable aggregate");
413
414 QualType argTy = e->getLHS()->getType();
415
416 if (!argTy->isIntegralOrEnumerationType() && !argTy->isRealFloatingType() &&
417 !argTy->isNullPtrType() && !argTy->isPointerType() &&
418 !argTy->isMemberPointerType() && !argTy->isAnyComplexType())
419 cgf.cgm.errorNYI(e->getBeginLoc(), "aggregate three-way comparison");
420
421 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
422 CIRGenBuilderTy &builder = cgf.getBuilder();
423
424 if (e->getType()->isAnyComplexType())
425 cgf.cgm.errorNYI(e->getBeginLoc(), "VisitBinCmp: complex type");
426
427 if (e->getType()->isAggregateType())
428 cgf.cgm.errorNYI(e->getBeginLoc(), "VisitBinCmp: aggregate type");
429
430 mlir::Value lhs = cgf.emitAnyExpr(e: e->getLHS()).getValue();
431 mlir::Value rhs = cgf.emitAnyExpr(e: e->getRHS()).getValue();
432
433 mlir::Value resultScalar;
434 if (argTy->isNullPtrType()) {
435 resultScalar =
436 builder.getConstInt(loc, intVal: cmpInfo.getEqualOrEquiv()->getIntValue());
437 } else {
438 llvm::APSInt ltRes = cmpInfo.getLess()->getIntValue();
439 llvm::APSInt eqRes = cmpInfo.getEqualOrEquiv()->getIntValue();
440 llvm::APSInt gtRes = cmpInfo.getGreater()->getIntValue();
441 if (!cmpInfo.isPartial()) {
442 cir::CmpOrdering ordering = cmpInfo.isStrong()
443 ? cir::CmpOrdering::Strong
444 : cir::CmpOrdering::Weak;
445 resultScalar = builder.createThreeWayCmpTotalOrdering(
446 loc, lhs, rhs, ltRes, eqRes, gtRes, ordering);
447 } else {
448 // Partial ordering.
449 llvm::APSInt unorderedRes = cmpInfo.getUnordered()->getIntValue();
450 resultScalar = builder.createThreeWayCmpPartialOrdering(
451 loc, lhs, rhs, ltRes, eqRes, gtRes, unorderedRes);
452 }
453 }
454
455 emitComparisonResult(e, loc, cmpInfo, resultValue: resultScalar);
456 }
457
458 void VisitTypeTraitExpr(const TypeTraitExpr *e) {
459 assert(e->isStoredAsComparisonResult() &&
460 "expected a strong_ordering type trait with a stored value");
461
462 const ComparisonCategoryInfo &cmpInfo =
463 cgf.getContext().CompCategories.getInfoForType(Ty: e->getType());
464 const auto result =
465 ComparisonCategoryResult(e->getAPValue().getInt().getZExtValue());
466 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
467 mlir::Value resultValue = cgf.getBuilder().getConstInt(
468 loc, intVal: cmpInfo.getValueInfo(ValueKind: result)->getIntValue());
469
470 emitComparisonResult(e, loc, cmpInfo, resultValue);
471 }
472
473 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *e) {
474 Visit(e: e->getSemanticForm());
475 }
476 void VisitObjCMessageExpr(ObjCMessageExpr *e) {
477 cgf.cgm.errorNYI(e->getSourceRange(),
478 "AggExprEmitter: VisitObjCMessageExpr");
479 }
480 void VisitObjCIVarRefExpr(ObjCIvarRefExpr *e) {
481 cgf.cgm.errorNYI(e->getSourceRange(),
482 "AggExprEmitter: VisitObjCIVarRefExpr");
483 }
484
485 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *e) {
486 AggValueSlot dest = ensureSlot(loc: cgf.getLoc(srcLoc: e->getExprLoc()), t: e->getType());
487 LValue destLV = cgf.makeAddrLValue(addr: dest.getAddress(), ty: e->getType());
488 emitInitializationToLValue(e: e->getBase(), lv: destLV);
489 VisitInitListExpr(e: e->getUpdater());
490 }
491 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *e) {
492 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
493
494 CIRGenFunction::OpaqueValueMapping binding(cgf, e);
495
496 // Emit the condition before opening the conditional evaluation, so that
497 // the cleanup scope of any temporary the condition creates encloses the
498 // one the evaluation opens.
499 mlir::Value condV = cgf.emitOpOnBoolExpr(loc, cond: e->getCond());
500 CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
501
502 // Save whether the destination's lifetime is externally managed.
503 bool isExternallyDestructed = dest.isExternallyDestructed();
504 bool destructNonTrivialCStruct =
505 !isExternallyDestructed &&
506 e->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
507 isExternallyDestructed |= destructNonTrivialCStruct;
508
509 // emitIfOnBoolValue terminates each region; an unconditional yield here
510 // would keep alive the dead block a noreturn arm leaves behind.
511 cgf.emitIfOnBoolValue(
512 condV, loc,
513 /*thenBuilder=*/
514 [&](mlir::OpBuilder &b, mlir::Location loc) {
515 eval.beginEvaluation();
516 {
517 CIRGenFunction::LexicalScope lexScope{cgf, loc,
518 b.getInsertionBlock()};
519 cgf.curLexScope->setAsTernary();
520 dest.setExternallyDestructed(isExternallyDestructed);
521 assert(!cir::MissingFeatures::incrementProfileCounter());
522 Visit(e: e->getTrueExpr());
523 }
524 eval.endEvaluation();
525 },
526 thenLoc: loc,
527 /*elseBuilder=*/
528 [&](mlir::OpBuilder &b, mlir::Location loc) {
529 eval.beginEvaluation();
530 {
531 CIRGenFunction::LexicalScope lexScope{cgf, loc,
532 b.getInsertionBlock()};
533 cgf.curLexScope->setAsTernary();
534
535 // If the result of an agg expression is unused, then the emission
536 // of the LHS might need to create a destination slot. That's fine
537 // with us, and we can safely emit the RHS into the same slot, but
538 // we shouldn't claim that it's already being destructed.
539 dest.setExternallyDestructed(isExternallyDestructed);
540 assert(!cir::MissingFeatures::incrementProfileCounter());
541 Visit(e: e->getFalseExpr());
542 }
543 eval.endEvaluation();
544 },
545 elseLoc: loc);
546
547 if (destructNonTrivialCStruct)
548 cgf.cgm.errorNYI(
549 e->getSourceRange(),
550 "Abstract conditional aggregate: destructNonTrivialCStruct");
551 }
552 void VisitChooseExpr(const ChooseExpr *e) { Visit(e: e->getChosenSubExpr()); }
553 void VisitCXXParenListInitExpr(CXXParenListInitExpr *e) {
554 visitCXXParenListOrInitListExpr(e, args: e->getInitExprs(),
555 initializedFieldInUnion: e->getInitializedFieldInUnion(),
556 arrayFiller: e->getArrayFiller());
557 }
558
559 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *e) {
560 CIRGenFunction::OpaqueValueMapping binding(cgf, e->getCommonExpr());
561 uint64_t numElements = e->getArraySize().getZExtValue();
562
563 if (!numElements)
564 return;
565
566 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
567
568 if (!e->getType()->isConstantArrayType())
569 cgf.cgm.errorNYI(e->getSourceRange(),
570 "VisitArrayInitLoopExpr: Non-constant array");
571
572 Address dest = ensureSlot(loc, t: e->getType()).getAddress();
573 cir::ArrayType arrayTy = cast<cir::ArrayType>(Val: dest.getElementType());
574
575 emitArrayInit(destPtr: dest, arrayTy, arrayQTy: e->getType(),
576 exprToVisit: const_cast<ArrayInitLoopExpr *>(e), args: {}, arrayFiller: e->getSubExpr());
577 }
578
579 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *e) {
580 QualType ty = e->getType();
581 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
582 AggValueSlot slot = ensureSlot(loc, t: ty);
583 emitNullInitializationToLValue(loc,
584 lv: cgf.makeAddrLValue(addr: slot.getAddress(), ty));
585 }
586 void VisitNoInitExpr(NoInitExpr *e) {
587 cgf.cgm.errorNYI(e->getSourceRange(), "AggExprEmitter: VisitNoInitExpr");
588 }
589 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *dae) {
590 CIRGenFunction::CXXDefaultArgExprScope scope(cgf, dae);
591 Visit(e: dae->getExpr());
592 }
593 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *e) {
594 AggValueSlot slot =
595 ensureSlot(loc: cgf.getLoc(srcLoc: e->getSourceRange()), t: e->getType());
596 cgf.emitInheritedCXXConstructorCall(d: e->getConstructor(),
597 forVirtualBase: e->constructsVBase(), thisAddr: slot.getAddress(),
598 inheritedFromVBase: e->inheritedFromVBase(), e);
599 }
600
601 /// Emit the initializer for a std::initializer_list initialized with a
602 /// real initializer list.
603 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *e) {
604 ASTContext &ctx = cgf.getContext();
605 CIRGenBuilderTy &builder = cgf.getBuilder();
606 mlir::Location loc = cgf.getLoc(srcLoc: e->getExprLoc());
607
608 LValue array = cgf.emitLValue(e: e->getSubExpr());
609 assert(array.isSimple() && "initializer_list array not a simple lvalue");
610 Address arrayPtr = array.getAddress();
611
612 const ConstantArrayType *arrayType =
613 ctx.getAsConstantArrayType(T: e->getSubExpr()->getType());
614 assert(arrayType && "std::initializer_list constructed from non-array");
615
616 auto *record = e->getType()->castAsRecordDecl();
617 assert(record->getNumFields() == 2 &&
618 "Expected std::initializer_list to only have two fields");
619
620 RecordDecl::field_iterator field = record->field_begin();
621 assert(field != record->field_end() &&
622 ctx.hasSameType(field->getType()->getPointeeType(),
623 arrayType->getElementType()) &&
624 "Expected std::initializer_list first field to be const E *");
625
626 // Start pointer.
627 AggValueSlot dest = ensureSlot(loc, t: e->getType());
628 LValue destLV = cgf.makeAddrLValue(addr: dest.getAddress(), ty: e->getType());
629 LValue start =
630 cgf.emitLValueForFieldInitialization(base: destLV, field: *field, fieldName: field->getName());
631
632 mlir::Value arrayStart = arrayPtr.emitRawPointer();
633 cgf.emitStoreThroughLValue(src: RValue::get(v: arrayStart), dst: start);
634 ++field;
635 assert(field != record->field_end() &&
636 "Expected std::initializer_list to have two fields");
637
638 cir::ConstantOp size = builder.getConstInt(loc, intVal: arrayType->getSize());
639 LValue endOrLength =
640 cgf.emitLValueForFieldInitialization(base: destLV, field: *field, fieldName: field->getName());
641 if (ctx.hasSameType(T1: field->getType(), T2: ctx.getSizeType())) {
642 // Length.
643 cgf.emitStoreThroughLValue(src: RValue::get(v: size), dst: endOrLength);
644 } else {
645 // End pointer.
646 assert(field->getType()->isPointerType() &&
647 ctx.hasSameType(field->getType()->getPointeeType(),
648 arrayType->getElementType()) &&
649 "Expected std::initializer_list second field to be const E *");
650 mlir::Value arrayEnd = builder.createPtrStride(loc, base: arrayStart, stride: size);
651 cgf.emitStoreThroughLValue(src: RValue::get(v: arrayEnd), dst: endOrLength);
652 }
653 }
654
655 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *e) {
656 cgf.cgm.errorNYI(e->getSourceRange(),
657 "AggExprEmitter: VisitCXXScalarValueInitExpr");
658 }
659 void VisitCXXTypeidExpr(CXXTypeidExpr *e) { emitAggLoadOfLValue(e); }
660 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e) {
661 Visit(e: e->getSubExpr());
662 }
663 void VisitOpaqueValueExpr(OpaqueValueExpr *e) {
664 cgf.cgm.errorNYI(e->getSourceRange(),
665 "AggExprEmitter: VisitOpaqueValueExpr");
666 }
667
668 void VisitPseudoObjectExpr(PseudoObjectExpr *e) {
669 cgf.cgm.errorNYI(e->getSourceRange(),
670 "AggExprEmitter: VisitPseudoObjectExpr");
671 }
672
673 void VisitVAArgExpr(VAArgExpr *e) {
674 // emitVAArg returns an aggregate value (not a pointer) at the CIR level.
675 // ABI-specific pointer handling will be done later in LoweringPrepare.
676 mlir::Value vaArgValue = cgf.emitVAArg(ve: e);
677
678 // Create a temporary alloca to hold the aggregate value.
679 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
680 Address tmpAddr = cgf.createMemTemp(t: e->getType(), loc, name: "vaarg.tmp");
681
682 // Store the va_arg result into the temporary.
683 cgf.emitAggregateStore(value: vaArgValue, dest: tmpAddr);
684
685 // Create an LValue from the temporary address.
686 LValue tmpLValue = cgf.makeAddrLValue(addr: tmpAddr, ty: e->getType());
687
688 // Copy the aggregate value from temporary to destination.
689 emitFinalDestCopy(type: e->getType(), src: tmpLValue);
690 }
691
692 void VisitCXXThrowExpr(const CXXThrowExpr *e) { cgf.emitCXXThrowExpr(e); }
693 void VisitAtomicExpr(AtomicExpr *e) {
694 RValue result = cgf.emitAtomicExpr(e);
695 emitFinalDestCopy(type: e->getType(), src: result);
696 }
697};
698
699} // namespace
700
701/// Given an expression with aggregate type that represents a value lvalue, this
702/// method emits the address of the lvalue, then loads the result into DestPtr.
703void AggExprEmitter::emitAggLoadOfLValue(const Expr *e) {
704 LValue lv = cgf.emitLValue(e);
705
706 // If the type of the l-value is atomic, then do an atomic load.
707 if (lv.getType()->isAtomicType() || cgf.isLValueSuitableForInlineAtomic(lv)) {
708 cgf.emitAtomicLoad(lvalue: lv, loc: e->getExprLoc(), slot: dest);
709 return;
710 }
711
712 emitFinalDestCopy(type: e->getType(), src: lv);
713}
714
715void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
716 if (dest.isPotentiallyAliased() && e->getType().isPODType(Context: cgf.getContext())) {
717 // For a POD type, just emit a load of the lvalue + a copy, because our
718 // compound literal might alias the destination.
719 emitAggLoadOfLValue(e);
720 return;
721 }
722
723 AggValueSlot slot = ensureSlot(loc: cgf.getLoc(srcLoc: e->getSourceRange()), t: e->getType());
724
725 // Block-scope compound literals are destroyed at the end of the enclosing
726 // scope in C.
727 bool destruct =
728 !cgf.getLangOpts().CPlusPlus && !slot.isExternallyDestructed();
729 if (destruct)
730 slot.setExternallyDestructed();
731
732 cgf.emitAggExpr(e: e->getInitializer(), slot);
733
734 if (destruct)
735 if ([[maybe_unused]] QualType::DestructionKind dtorKind =
736 e->getType().isDestructedType())
737 cgf.cgm.errorNYI(e->getSourceRange(), "compound literal with destructor");
738}
739
740void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
741 QualType arrayQTy, Expr *e,
742 ArrayRef<Expr *> args, Expr *arrayFiller) {
743 CIRGenBuilderTy &builder = cgf.getBuilder();
744 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
745
746 const uint64_t numInitElements = args.size();
747
748 bool setArrayInitLoopExprScope = isa<ArrayInitLoopExpr>(Val: e);
749
750 const QualType elementType =
751 cgf.getContext().getAsArrayType(T: arrayQTy)->getElementType();
752
753 const QualType elementPtrType = cgf.getContext().getPointerType(T: elementType);
754
755 const mlir::Type cirElementType = cgf.convertType(t: elementType);
756 const cir::PointerType cirElementPtrType =
757 builder.getPointerTo(ty: cirElementType, as: destPtr.getAddressSpace());
758
759 auto begin = cir::CastOp::create(builder, location: loc, result: cirElementPtrType,
760 kind: cir::CastKind::array_to_ptrdecay,
761 src: destPtr.getPointer());
762
763 const CharUnits elementSize =
764 cgf.getContext().getTypeSizeInChars(T: elementType);
765 const CharUnits elementAlign =
766 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
767
768 // Destroy already-constructed elements if a later initializer throws.
769 // The cleanup is deactivated when this initialization finishes.
770 QualType::DestructionKind dtorKind = elementType.isDestructedType();
771 Address endOfInit = Address::invalid();
772 CIRGenFunction::CleanupDeactivationScope deactivateCleanups(cgf);
773
774 if (dtorKind && cgf.getLangOpts().Exceptions) {
775 endOfInit = cgf.createTempAlloca(ty: cirElementPtrType, align: cgf.getPointerAlign(),
776 loc, name: "arrayinit.endOfInit");
777 builder.createStore(loc, val: begin, dst: endOfInit);
778
779 cgf.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
780 elementAlign,
781 destroyer: cgf.getDestroyer(kind: dtorKind));
782 }
783
784 // The 'current element to initialize'. The invariants on this
785 // variable are complicated. Essentially, after each iteration of
786 // the loop, it points to the last initialized element, except
787 // that it points to the beginning of the array before any
788 // elements have been initialized.
789 mlir::Value element = begin;
790
791 // Don't build the 'one' before the cycle to avoid
792 // emmiting the redundant `cir.const 1` instrs.
793 mlir::Value one;
794
795 // Emit the explicit initializers.
796 for (uint64_t i = 0; i != numInitElements; ++i) {
797 // Advance to the next element.
798 if (i > 0) {
799 one = builder.getConstantInt(loc, ty: cgf.ptrDiffTy, value: i);
800 element = builder.createPtrStride(loc, base: begin, stride: one);
801
802 // Tell the cleanup that it needs to destroy up to this element.
803 if (endOfInit.isValid())
804 builder.createStore(loc, val: element, dst: endOfInit);
805 }
806
807 const Address address = Address(element, cirElementType, elementAlign);
808 const LValue elementLV = cgf.makeAddrLValue(addr: address, ty: elementType);
809 emitInitializationToLValue(e: args[i], lv: elementLV);
810 }
811
812 const uint64_t numArrayElements = arrayTy.getSize();
813
814 // Check whether there's a non-trivial array-fill expression.
815 const bool hasTrivialFiller = CodeGenUtils::isTrivialFiller(E: arrayFiller);
816
817 // Any remaining elements need to be zero-initialized, possibly
818 // using the filler expression. We can skip this if the we're
819 // emitting to zeroed memory.
820 if (numInitElements != numArrayElements &&
821 !(dest.isZeroed() && hasTrivialFiller &&
822 cgf.getTypes().isZeroInitializable(ty: elementType))) {
823 // Advance to the start of the rest of the array.
824 if (numInitElements) {
825 one = builder.getConstantInt(loc, ty: cgf.ptrDiffTy, value: 1);
826 element = cir::PtrStrideOp::create(builder, location: loc, result: cirElementPtrType,
827 base: element, stride: one);
828
829 if (endOfInit.isValid())
830 builder.createStore(loc, val: element, dst: endOfInit);
831 }
832
833 // Allocate the temporary variable
834 // to store the pointer to first unitialized element
835 const Address tmpAddr = cgf.createTempAlloca(
836 ty: cirElementPtrType, align: cgf.getPointerAlign(), loc, name: "arrayinit.temp");
837 LValue tmpLV = cgf.makeAddrLValue(addr: tmpAddr, ty: elementPtrType);
838 cgf.emitStoreThroughLValue(src: RValue::get(v: element), dst: tmpLV);
839
840 // Compute the end of array
841 cir::ConstantOp numArrayElementsConst = builder.getConstInt(
842 loc, t: mlir::cast<cir::IntType>(Val&: cgf.ptrDiffTy), c: numArrayElements);
843 mlir::Value end = cir::PtrStrideOp::create(builder, location: loc, result: cirElementPtrType,
844 base: begin, stride: numArrayElementsConst);
845
846 builder.createDoWhile(
847 loc,
848 /*condBuilder=*/
849 [&](mlir::OpBuilder &b, mlir::Location loc) {
850 cir::LoadOp currentElement = builder.createLoad(loc, addr: tmpAddr);
851 cir::CmpOp cmp = cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::ne,
852 lhs: currentElement, rhs: end);
853 builder.createCondition(condition: cmp);
854 },
855 /*bodyBuilder=*/
856 [&](mlir::OpBuilder &b, mlir::Location loc) {
857 cir::LoadOp currentElement = builder.createLoad(loc, addr: tmpAddr);
858
859 // Emit the actual filler expression.
860 LValue elementLV = cgf.makeAddrLValue(
861 addr: Address(currentElement, cirElementType, elementAlign),
862 ty: elementType);
863
864 mlir::Value idx;
865 if (setArrayInitLoopExprScope)
866 idx = cir::PtrDiffOp::create(builder&: b, location: loc, result: cgf.ptrDiffTy, lhs: currentElement,
867 rhs: begin);
868
869 CIRGenFunction::ArrayInitLoopExprScope loopExprScope(
870 cgf, setArrayInitLoopExprScope, idx);
871
872 // C++17 [class.temporary]p5:
873 // "when a default constructor is called to initialize an element of
874 // an array with no corresponding initializer [...] the destruction
875 // of every temporary created in a default argument is sequenced
876 // before the construction of the next array element, if any."
877 if (arrayFiller) {
878 CIRGenFunction::FullExprCleanupScope scope(cgf, arrayFiller);
879 scope.deferLifetimeExtendedCleanups();
880 emitInitializationToLValue(e: arrayFiller, lv: elementLV);
881 } else {
882 emitNullInitializationToLValue(loc, lv: elementLV);
883 }
884
885 // Advance pointer and store them to temporary variable
886 cir::ConstantOp one = builder.getConstInt(
887 loc, t: mlir::cast<cir::IntType>(Val&: cgf.ptrDiffTy), c: 1);
888 auto nextElement = cir::PtrStrideOp::create(
889 builder, location: loc, result: cirElementPtrType, base: currentElement, stride: one);
890
891 // Tell the EH cleanup that we finished with the last element.
892 if (endOfInit.isValid())
893 builder.createStore(loc, val: nextElement, dst: endOfInit);
894
895 cgf.emitStoreThroughLValue(src: RValue::get(v: nextElement), dst: tmpLV);
896
897 builder.createYield(loc);
898 });
899 }
900}
901
902/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
903void AggExprEmitter::emitFinalDestCopy(QualType type, RValue src) {
904 assert(src.isAggregate() && "value must be aggregate value!");
905 LValue srcLV = cgf.makeAddrLValue(addr: src.getAggregateAddress(), ty: type);
906 emitFinalDestCopy(type, src: srcLV, srcValueKind: CIRGenFunction::EVK_RValue);
907}
908
909/// Perform the final copy to destPtr, if desired.
910void AggExprEmitter::emitFinalDestCopy(
911 QualType type, const LValue &src,
912 CIRGenFunction::ExprValueKind srcValueKind) {
913 // If dest is ignored, then we're evaluating an aggregate expression
914 // in a context that doesn't care about the result. Note that loads
915 // from volatile l-values force the existence of a non-ignored
916 // destination.
917 if (dest.isIgnored())
918 return;
919
920 if (srcValueKind == CIRGenFunction::EVK_RValue) {
921 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
922 cgf.cgm.errorNYI(feature: "emitFinalDestCopy: EVK_RValue & PCK_Struct");
923 }
924 } else {
925 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
926 cgf.cgm.errorNYI(feature: "emitFinalDestCopy: !EVK_RValue & PCK_Struct");
927 }
928 }
929
930 assert(!cir::MissingFeatures::aggValueSlotVolatile());
931 assert(!cir::MissingFeatures::aggEmitFinalDestCopyRValue());
932 assert(!cir::MissingFeatures::aggValueSlotGC());
933
934 AggValueSlot srcAgg = AggValueSlot::forLValue(LV: src, isDestructed: AggValueSlot::IsDestructed,
935 isAliased: AggValueSlot::IsAliased,
936 mayOverlap: AggValueSlot::MayOverlap);
937 emitCopy(type, dest, src: srcAgg);
938}
939
940/// Perform a copy from the source into the destination.
941///
942/// \param type - the type of the aggregate being copied; qualifiers are
943/// ignored
944void AggExprEmitter::emitCopy(QualType type, const AggValueSlot &dest,
945 const AggValueSlot &src) {
946 assert(!cir::MissingFeatures::aggValueSlotGC());
947
948 // If the result of the assignment is used, copy the LHS there also.
949 // It's volatile if either side is. Use the minimum alignment of
950 // the two sides.
951 LValue destLV = cgf.makeAddrLValue(addr: dest.getAddress(), ty: type);
952 LValue srcLV = cgf.makeAddrLValue(addr: src.getAddress(), ty: type);
953 assert(!cir::MissingFeatures::aggValueSlotVolatile());
954 cgf.emitAggregateCopy(dest: destLV, src: srcLV, eltTy: type, mayOverlap: dest.mayOverlap(),
955 isVolatile: dest.isVolatile() || src.isVolatile());
956}
957
958void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
959 const QualType type = lv.getType();
960
961 if (isa<ImplicitValueInitExpr, CXXScalarValueInitExpr>(Val: e)) {
962 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
963 return emitNullInitializationToLValue(loc, lv);
964 }
965
966 if (isa<NoInitExpr>(Val: e))
967 return;
968
969 if (type->isReferenceType()) {
970 RValue rv = cgf.emitReferenceBindingToExpr(e);
971 return cgf.emitStoreThroughLValue(src: rv, dst: lv);
972 }
973
974 switch (cgf.getEvaluationKind(type)) {
975 case cir::TEK_Complex:
976 cgf.emitComplexExprIntoLValue(e, dest: lv, /*isInit*/ true);
977 break;
978 case cir::TEK_Aggregate:
979 cgf.emitAggExpr(e, slot: AggValueSlot::forLValue(LV: lv, isDestructed: AggValueSlot::IsDestructed,
980 isAliased: AggValueSlot::IsNotAliased,
981 mayOverlap: AggValueSlot::MayOverlap,
982 isZeroed: dest.isZeroed()));
983
984 return;
985 case cir::TEK_Scalar:
986 if (lv.isSimple())
987 cgf.emitScalarInit(init: e, lvalue: lv);
988 else
989 cgf.emitStoreThroughLValue(src: RValue::get(v: cgf.emitScalarExpr(e)), dst: lv);
990 return;
991 }
992}
993
994void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *e) {
995 AggValueSlot slot = ensureSlot(loc: cgf.getLoc(srcLoc: e->getSourceRange()), t: e->getType());
996 cgf.emitCXXConstructExpr(e, dest: slot);
997}
998
999void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
1000 LValue lv) {
1001 const QualType type = lv.getType();
1002
1003 // If the destination slot is already zeroed out before the aggregate is
1004 // copied into it, we don't have to emit any zeros here.
1005 if (dest.isZeroed() && cgf.getTypes().isZeroInitializable(ty: type))
1006 return;
1007
1008 if (cgf.hasScalarEvaluationKind(type)) {
1009 // For non-aggregates, we can store the appropriate null constant.
1010 mlir::Value null = cgf.cgm.emitNullConstant(t: type, loc);
1011 if (lv.isSimple()) {
1012 cgf.emitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1013 return;
1014 }
1015
1016 cgf.emitStoreThroughBitfieldLValue(src: RValue::get(v: null), dstresult: lv);
1017 return;
1018 }
1019
1020 // There's a potential optimization opportunity in combining
1021 // memsets; that would be easy for arrays, but relatively
1022 // difficult for structures with the current code.
1023 cgf.emitNullInitialization(loc, destPtr: lv.getAddress(), ty: lv.getType());
1024}
1025
1026void AggExprEmitter::emitComparisonResult(const Expr *e, mlir::Location loc,
1027 const ComparisonCategoryInfo &cmpInfo,
1028 mlir::Value resultValue) {
1029 // Create the return value in the destination slot.
1030 ensureDest(loc, ty: e->getType());
1031 LValue destLVal = cgf.makeAddrLValue(addr: dest.getAddress(), ty: e->getType());
1032
1033 // Emit the address of the first (and only) field in the comparison category
1034 // type, and initialize it from the constant integer value produced above.
1035 const FieldDecl *resultField = *cmpInfo.Record->field_begin();
1036 LValue fieldLVal = cgf.emitLValueForFieldInitialization(
1037 base: destLVal, field: resultField, fieldName: resultField->getName());
1038 cgf.emitStoreThroughLValue(src: RValue::get(v: resultValue), dst: fieldLVal);
1039}
1040
1041void AggExprEmitter::VisitLambdaExpr(LambdaExpr *e) {
1042 CIRGenFunction::SourceLocRAIIObject loc{cgf, e->getSourceRange()};
1043 AggValueSlot slot = ensureSlot(loc: cgf.getLoc(srcLoc: e->getSourceRange()), t: e->getType());
1044 LValue slotLV = cgf.makeAddrLValue(addr: slot.getAddress(), ty: e->getType());
1045
1046 // We'll need to enter cleanup scopes in case any of the element
1047 // initializers throws an exception or contains branch out of the expressions.
1048 CIRGenFunction::CleanupDeactivationScope deactivationScope(cgf);
1049
1050 for (auto [curField, capture, captureInit] : llvm::zip(
1051 t: e->getLambdaClass()->fields(), u: e->captures(), args: e->capture_inits())) {
1052 // Pick a name for the field.
1053 llvm::StringRef fieldName = curField->getName();
1054 if (capture.capturesVariable()) {
1055 assert(!curField->isBitField() && "lambdas don't have bitfield members!");
1056 ValueDecl *v = capture.getCapturedVar();
1057 fieldName = v->getName();
1058 cgf.cgm.lambdaFieldToName[curField] = fieldName;
1059 } else if (capture.capturesThis()) {
1060 cgf.cgm.lambdaFieldToName[curField] = "this";
1061 } else if (capture.capturesVLAType()) {
1062 cgf.cgm.lambdaFieldToName[curField] = "vla-size";
1063 } else {
1064 cgf.cgm.errorNYI(e->getSourceRange(), "Unhandled capture kind");
1065 cgf.cgm.lambdaFieldToName[curField] = "unhandled-capture-kind";
1066 }
1067
1068 // Emit initialization
1069 LValue lv =
1070 cgf.emitLValueForFieldInitialization(base: slotLV, field: curField, fieldName);
1071 if (curField->hasCapturedVLAType()) {
1072 cgf.emitLambdaVLACapture(vat: curField->getCapturedVLAType(), lv);
1073 continue;
1074 }
1075
1076 emitInitializationToLValue(e: captureInit, lv);
1077
1078 // Push a destructor if necessary.
1079 if (QualType::DestructionKind dtorKind =
1080 curField->getType().isDestructedType()) {
1081 assert(lv.isSimple());
1082 cgf.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: lv.getAddress(),
1083 type: curField->getType(),
1084 destroyer: cgf.getDestroyer(kind: dtorKind), useEHCleanupForArray: false);
1085 }
1086 }
1087}
1088
1089void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *e) {
1090 CIRGenFunction::FullExprCleanupScope fullExprScope(cgf, e->getSubExpr());
1091 Visit(e: e->getSubExpr());
1092}
1093
1094void AggExprEmitter::VisitCallExpr(const CallExpr *e) {
1095 if (e->getCallReturnType(Ctx: cgf.getContext())->isReferenceType()) {
1096 cgf.cgm.errorNYI(e->getSourceRange(), "reference return type");
1097 return;
1098 }
1099
1100 withReturnValueSlot(
1101 e, fn: [&](ReturnValueSlot slot) { return cgf.emitCallExpr(e, returnValue: slot); });
1102}
1103
1104void AggExprEmitter::withReturnValueSlot(
1105 const Expr *e, llvm::function_ref<RValue(ReturnValueSlot)> fn) {
1106 QualType retTy = e->getType();
1107
1108 assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
1109 bool requiresDestruction =
1110 retTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
1111 if (requiresDestruction)
1112 cgf.cgm.errorNYI(
1113 e->getSourceRange(),
1114 "withReturnValueSlot: return value requiring destruction is NYI");
1115
1116 // If it makes no observable difference, save a memcpy + temporary.
1117 //
1118 // We need to always provide our own temporary if destruction is required.
1119 // Otherwise, fn will emit its own, notice that it's "unused", and end its
1120 // lifetime before we have the chance to emit a proper destructor call.
1121 assert(!cir::MissingFeatures::aggValueSlotAlias());
1122 assert(!cir::MissingFeatures::aggValueSlotGC());
1123
1124 Address retAddr = dest.getAddress();
1125 assert(!cir::MissingFeatures::emitLifetimeMarkers());
1126
1127 assert(!cir::MissingFeatures::aggValueSlotVolatile());
1128 assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
1129 fn(ReturnValueSlot(retAddr));
1130}
1131
1132void AggExprEmitter::VisitInitListExpr(InitListExpr *e) {
1133 if (e->hadArrayRangeDesignator())
1134 llvm_unreachable("GNU array range designator extension");
1135
1136 if (e->isTransparent())
1137 return Visit(e: e->getInit(Init: 0));
1138
1139 visitCXXParenListOrInitListExpr(
1140 e, args: e->inits(), initializedFieldInUnion: e->getInitializedFieldInUnion(), arrayFiller: e->getArrayFiller());
1141}
1142
1143void AggExprEmitter::visitCXXParenListOrInitListExpr(
1144 Expr *e, ArrayRef<Expr *> args, FieldDecl *initializedFieldInUnion,
1145 Expr *arrayFiller) {
1146
1147 const mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
1148 const AggValueSlot dest = ensureSlot(loc, t: e->getType());
1149
1150 if (e->getType()->isConstantArrayType()) {
1151 cir::ArrayType arrayTy =
1152 cast<cir::ArrayType>(Val: dest.getAddress().getElementType());
1153 emitArrayInit(destPtr: dest.getAddress(), arrayTy, arrayQTy: e->getType(), e, args,
1154 arrayFiller);
1155 return;
1156 } else if (e->getType()->isVariableArrayType()) {
1157 cgf.cgm.errorNYI(e->getSourceRange(),
1158 "visitCXXParenListOrInitListExpr variable array type");
1159 return;
1160 }
1161
1162 if (e->getType()->isArrayType()) {
1163 cgf.cgm.errorNYI(e->getSourceRange(),
1164 "visitCXXParenListOrInitListExpr array type");
1165 return;
1166 }
1167
1168 assert(e->getType()->isRecordType() && "Only support structs/unions here!");
1169
1170 // Do struct initialization; this code just sets each individual member
1171 // to the approprate value. This makes bitfield support automatic;
1172 // the disadvantage is that the generated code is more difficult for
1173 // the optimizer, especially with bitfields.
1174 unsigned numInitElements = args.size();
1175 auto *record = e->getType()->castAsRecordDecl();
1176
1177 // We'll need to enter cleanup scopes in case any of the element
1178 // initializers throws an exception.
1179 CIRGenFunction::CleanupDeactivationScope deactivateCleanups(cgf);
1180
1181 unsigned curInitIndex = 0;
1182
1183 // Emit initialization of base classes.
1184 if (auto *cxxrd = dyn_cast<CXXRecordDecl>(Val: record)) {
1185 assert(numInitElements >= cxxrd->getNumBases() &&
1186 "missing initializer for base class");
1187 for (auto &base : cxxrd->bases()) {
1188 assert(!base.isVirtual() && "should not see vbases here");
1189 CXXRecordDecl *baseRD = base.getType()->getAsCXXRecordDecl();
1190 Address address = cgf.getAddressOfDirectBaseInCompleteClass(
1191 loc, value: dest.getAddress(), derived: cxxrd, base: baseRD,
1192 /*baseIsVirtual=*/false);
1193 assert(!cir::MissingFeatures::aggValueSlotGC());
1194 AggValueSlot aggSlot = AggValueSlot::forAddr(
1195 addr: address, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1196 isAliased: AggValueSlot::IsNotAliased,
1197 mayOverlap: cgf.getOverlapForBaseInit(rd: cxxrd, baseRD, isVirtual: false));
1198 cgf.emitAggExpr(e: args[curInitIndex++], slot: aggSlot);
1199
1200 if (QualType::DestructionKind dtorKind =
1201 base.getType().isDestructedType())
1202 cgf.pushDestroyAndDeferDeactivation(dtorKind, addr: address, type: base.getType());
1203 }
1204 }
1205
1206 // Prepare a 'this' for CXXDefaultInitExprs.
1207 CIRGenFunction::FieldConstructionScope fcScope(cgf, dest.getAddress());
1208
1209 LValue destLV = cgf.makeAddrLValue(addr: dest.getAddress(), ty: e->getType());
1210
1211 if (record->isUnion()) {
1212 // Only initialize one field of a union. The field itself is
1213 // specified by the initializer list.
1214 if (!initializedFieldInUnion) {
1215 // Empty union; we have nothing to do.
1216
1217 // Make sure that it's really an empty and not a failure of
1218 // semantic analysis.
1219 assert(llvm::all_of(record->fields(),
1220 [](const FieldDecl *f) {
1221 return f->isUnnamedBitField() ||
1222 f->isAnonymousStructOrUnion();
1223 }) &&
1224 "Only unnamed bitfields or anonymous class allowed");
1225 return;
1226 }
1227
1228 // FIXME: volatility
1229 FieldDecl *initedField = initializedFieldInUnion;
1230
1231 LValue fieldLV = cgf.emitLValueForFieldInitialization(
1232 base: destLV, field: initedField, fieldName: initedField->getName());
1233
1234 if (numInitElements) {
1235 // Store the initializer into the field
1236 emitInitializationToLValue(e: args[0], lv: fieldLV);
1237 } else {
1238 // Default-initialize to null.
1239 emitNullInitializationToLValue(loc, lv: fieldLV);
1240 }
1241 return;
1242 }
1243
1244 // Here we iterate over the fields; this makes it simpler to both
1245 // default-initialize fields and skip over unnamed fields.
1246 for (const FieldDecl *field : record->fields()) {
1247 // We're done once we hit the flexible array member.
1248 if (field->getType()->isIncompleteArrayType())
1249 break;
1250
1251 // Always skip anonymous bitfields.
1252 if (field->isUnnamedBitField())
1253 continue;
1254
1255 // We're done if we reach the end of the explicit initializers, we
1256 // have a zeroed object, and the rest of the fields are
1257 // zero-initializable.
1258 if (curInitIndex == numInitElements && dest.isZeroed() &&
1259 cgf.getTypes().isZeroInitializable(ty: e->getType()))
1260 break;
1261 LValue lv =
1262 cgf.emitLValueForFieldInitialization(base: destLV, field, fieldName: field->getName());
1263 // We never generate write-barriers for initialized fields.
1264 assert(!cir::MissingFeatures::setNonGC());
1265
1266 if (curInitIndex < numInitElements) {
1267 // Store the initializer into the field.
1268 CIRGenFunction::SourceLocRAIIObject loc{cgf, record->getSourceRange()};
1269 emitInitializationToLValue(e: args[curInitIndex++], lv);
1270 } else {
1271 // We're out of initializers; default-initialize to null
1272 emitNullInitializationToLValue(loc: cgf.getLoc(srcLoc: e->getSourceRange()), lv);
1273 }
1274
1275 // Push a destructor if necessary.
1276 // FIXME: if we have an array of structures, all explicitly
1277 // initialized, we can end up pushing a linear number of cleanups.
1278 if (QualType::DestructionKind dtorKind =
1279 field->getType().isDestructedType()) {
1280 assert(lv.isSimple());
1281 cgf.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: lv.getAddress(),
1282 type: field->getType(),
1283 destroyer: cgf.getDestroyer(kind: dtorKind), useEHCleanupForArray: false);
1284 }
1285
1286 // From classic codegen, maybe not useful for CIR:
1287 // If the GEP didn't get used because of a dead zero init or something
1288 // else, clean it up for -O0 builds and general tidiness.
1289 }
1290}
1291
1292// TODO(cir): This could be shared with classic codegen.
1293AggValueSlot::Overlap_t CIRGenFunction::getOverlapForBaseInit(
1294 const CXXRecordDecl *rd, const CXXRecordDecl *baseRD, bool isVirtual) {
1295 // If the most-derived object is a field declared with [[no_unique_address]],
1296 // the tail padding of any virtual base could be reused for other subobjects
1297 // of that field's class.
1298 if (isVirtual)
1299 return AggValueSlot::MayOverlap;
1300
1301 // Empty bases can overlap earlier bases.
1302 if (baseRD->isEmpty())
1303 return AggValueSlot::MayOverlap;
1304
1305 // If the base class is laid out entirely within the nvsize of the derived
1306 // class, its tail padding cannot yet be initialized, so we can issue
1307 // stores at the full width of the base class.
1308 const ASTRecordLayout &layout = getContext().getASTRecordLayout(D: rd);
1309 if (layout.getBaseClassOffset(Base: baseRD) +
1310 getContext().getASTRecordLayout(D: baseRD).getSize() <=
1311 layout.getNonVirtualSize())
1312 return AggValueSlot::DoesNotOverlap;
1313
1314 // The tail padding may contain values we need to preserve.
1315 return AggValueSlot::MayOverlap;
1316}
1317
1318void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
1319 AggExprEmitter(*this, slot).Visit(e: const_cast<Expr *>(e));
1320}
1321
1322void CIRGenFunction::emitAggregateCopy(LValue dest, LValue src, QualType ty,
1323 AggValueSlot::Overlap_t mayOverlap,
1324 bool isVolatile) {
1325 // TODO(cir): this function needs improvements, commented code for now since
1326 // this will be touched again soon.
1327 assert(!ty->isAnyComplexType() && "Unexpected copy of complex");
1328
1329 Address destPtr = dest.getAddress();
1330 Address srcPtr = src.getAddress();
1331
1332 if (getLangOpts().CPlusPlus) {
1333 if (auto *record = ty->getAsCXXRecordDecl()) {
1334 assert((record->hasTrivialCopyConstructor() ||
1335 record->hasTrivialCopyAssignment() ||
1336 record->hasTrivialMoveConstructor() ||
1337 record->hasTrivialMoveAssignment() ||
1338 record->hasAttr<TrivialABIAttr>() || record->isUnion()) &&
1339 "Trying to aggregate-copy a type without a trivial copy/move "
1340 "constructor or assignment operator");
1341 // Ignore empty classes in C++.
1342 if (record->isEmpty())
1343 return;
1344 }
1345 }
1346
1347 assert(!cir::MissingFeatures::cudaSupport());
1348
1349 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
1350 // C99 6.5.16.1p3, which states "If the value being stored in an object is
1351 // read from another object that overlaps in anyway the storage of the first
1352 // object, then the overlap shall be exact and the two objects shall have
1353 // qualified or unqualified versions of a compatible type."
1354 //
1355 // memcpy is not defined if the source and destination pointers are exactly
1356 // equal, but other compilers do this optimization, and almost every memcpy
1357 // implementation handles this case safely. If there is a libc that does not
1358 // safely handle this, we can add a target hook.
1359
1360 // Get data size info for this aggregate. Don't copy the tail padding if this
1361 // might be a potentially-overlapping subobject, since the tail padding might
1362 // be occupied by a different object. Otherwise, copying it is fine.
1363 TypeInfoChars typeInfo;
1364 if (mayOverlap)
1365 typeInfo = getContext().getTypeInfoDataSizeInChars(T: ty);
1366 else
1367 typeInfo = getContext().getTypeInfoInChars(T: ty);
1368
1369 assert(!cir::MissingFeatures::aggValueSlotVolatile());
1370
1371 // Don't do any of the memmove_collectable tests if GC isn't set.
1372 if (cgm.getLangOpts().getGC() != LangOptions::NonGC)
1373 cgm.errorNYI(feature: "emitAggregateCopy: GC");
1374
1375 // If the data size (excluding tail padding) differs from the full type size,
1376 // use skip_tail_padding to avoid clobbering tail padding that may be occupied
1377 // by other objects (e.g. fields marked with [[no_unique_address]]).
1378 CharUnits dataSize = typeInfo.Width;
1379 bool skipTailPadding =
1380 mayOverlap && dataSize != getContext().getTypeSizeInChars(T: ty);
1381 // NOTE(cir): original codegen would normally convert destPtr and srcPtr to
1382 // i8* since memcpy operates on bytes. We don't need that in CIR because
1383 // cir.copy will operate on any CIR pointer that points to a sized type.
1384 builder.createCopy(dst: destPtr, src: srcPtr, isVolatile, skipTailPadding);
1385
1386 assert(!cir::MissingFeatures::opTBAA());
1387}
1388
1389// TODO(cir): This could be shared with classic codegen.
1390AggValueSlot::Overlap_t
1391CIRGenFunction::getOverlapForFieldInit(const FieldDecl *fd) {
1392 if (!fd->hasAttr<NoUniqueAddressAttr>() || !fd->getType()->isRecordType())
1393 return AggValueSlot::DoesNotOverlap;
1394
1395 // Empty fields can overlap earlier fields.
1396 if (fd->getType()->getAsCXXRecordDecl()->isEmpty())
1397 return AggValueSlot::MayOverlap;
1398
1399 // If the field lies entirely within the enclosing class's nvsize, its tail
1400 // padding cannot overlap any already-initialized object. (The only subobjects
1401 // with greater addresses that might already be initialized are vbases.)
1402 const RecordDecl *classRD = fd->getParent();
1403 const ASTRecordLayout &layout = getContext().getASTRecordLayout(D: classRD);
1404 if (layout.getFieldOffset(FieldNo: fd->getFieldIndex()) +
1405 getContext().getTypeSize(T: fd->getType()) <=
1406 (uint64_t)getContext().toBits(CharSize: layout.getNonVirtualSize()))
1407 return AggValueSlot::DoesNotOverlap;
1408
1409 // The tail padding may contain values we need to preserve.
1410 return AggValueSlot::MayOverlap;
1411}
1412
1413LValue CIRGenFunction::emitAggExprToLValue(const Expr *e) {
1414 assert(hasAggregateEvaluationKind(e->getType()) && "Invalid argument!");
1415 Address temp = createMemTemp(t: e->getType(), loc: getLoc(srcLoc: e->getSourceRange()));
1416 LValue lv = makeAddrLValue(addr: temp, ty: e->getType());
1417 emitAggExpr(e, slot: AggValueSlot::forLValue(LV: lv, isDestructed: AggValueSlot::IsNotDestructed,
1418 isAliased: AggValueSlot::IsNotAliased,
1419 mayOverlap: AggValueSlot::DoesNotOverlap));
1420 return lv;
1421}
1422