1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
23#include "clang/AST/ASTContext.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
26#include "clang/AST/DeclTemplate.h"
27#include "clang/AST/StmtVisitor.h"
28#include "clang/CodeGenUtils/ExprUtils.h"
29#include "llvm/IR/Constants.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/GlobalVariable.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35using namespace clang;
36using namespace CodeGen;
37
38//===----------------------------------------------------------------------===//
39// Aggregate Expression Emitter
40//===----------------------------------------------------------------------===//
41
42namespace {
43class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
44 CodeGenFunction &CGF;
45 CGBuilderTy &Builder;
46 AggValueSlot Dest;
47 bool IsResultUnused;
48
49 AggValueSlot EnsureSlot(QualType T) {
50 if (!Dest.isIgnored())
51 return Dest;
52 return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
53 }
54 void EnsureDest(QualType T) {
55 if (!Dest.isIgnored())
56 return;
57 Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
58 }
59
60 // Calls `Fn` with a valid return value slot, potentially creating a temporary
61 // to do so. If a temporary is created, an appropriate copy into `Dest` will
62 // be emitted, as will lifetime markers.
63 //
64 // The given function should take a ReturnValueSlot, and return an RValue that
65 // points to said slot.
66 void withReturnValueSlot(const Expr *E,
67 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
68
69 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
70 const FieldDecl *NextField);
71
72public:
73 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
74 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
75 IsResultUnused(IsResultUnused) {}
76
77 //===--------------------------------------------------------------------===//
78 // Utilities
79 //===--------------------------------------------------------------------===//
80
81 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
82 /// represents a value lvalue, this method emits the address of the lvalue,
83 /// then loads the result into DestPtr.
84 void EmitAggLoadOfLValue(const Expr *E);
85
86 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
87 /// SrcIsRValue is true if source comes from an RValue.
88 void EmitFinalDestCopy(QualType type, const LValue &src,
89 CodeGenFunction::ExprValueKind SrcValueKind =
90 CodeGenFunction::EVK_NonRValue);
91 void EmitFinalDestCopy(QualType type, RValue src);
92 void EmitCopy(QualType type, const AggValueSlot &dest,
93 const AggValueSlot &src);
94
95 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
96 Expr *ExprToVisit, ArrayRef<Expr *> Args,
97 Expr *ArrayFiller);
98
99 void EmitComparisonResult(const Expr *E,
100 const ComparisonCategoryInfo &CmpInfo,
101 llvm::Value *ResultValue);
102
103 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
104 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
105 return AggValueSlot::NeedsGCBarriers;
106 return AggValueSlot::DoesNotNeedGCBarriers;
107 }
108
109 bool TypeRequiresGCollection(QualType T);
110
111 //===--------------------------------------------------------------------===//
112 // Visitor Methods
113 //===--------------------------------------------------------------------===//
114
115 void Visit(Expr *E) {
116 ApplyDebugLocation DL(CGF, E);
117 StmtVisitor<AggExprEmitter>::Visit(S: E);
118 }
119
120 void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, Type: "aggregate expression"); }
121 void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
122 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
123 Visit(E: GE->getResultExpr());
124 }
125 void VisitCoawaitExpr(CoawaitExpr *E) {
126 CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
127 }
128 void VisitCoyieldExpr(CoyieldExpr *E) {
129 CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
130 }
131 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
132 void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
133 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
134 return Visit(E: E->getReplacement());
135 }
136
137 void VisitConstantExpr(ConstantExpr *E) {
138 EnsureDest(T: E->getType());
139
140 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
141 CGF.CreateCoercedStore(
142 Src: Result, SrcFETy: E->getType(), Dst: Dest.getAddress(),
143 DstSize: llvm::TypeSize::getFixed(
144 ExactSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType())
145 .getQuantity()),
146 DstIsVolatile: E->getType().isVolatileQualified());
147 return;
148 }
149 return Visit(E: E->getSubExpr());
150 }
151
152 // l-values.
153 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
154 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(E: ME); }
155 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
156 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
157 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
158 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
159 EmitAggLoadOfLValue(E);
160 }
161 void VisitPredefinedExpr(const PredefinedExpr *E) { EmitAggLoadOfLValue(E); }
162
163 // Operators.
164 void VisitCastExpr(CastExpr *E);
165 void VisitCallExpr(const CallExpr *E);
166 void VisitStmtExpr(const StmtExpr *E);
167 void VisitBinaryOperator(const BinaryOperator *BO);
168 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
169 void VisitBinAssign(const BinaryOperator *E);
170 void VisitBinComma(const BinaryOperator *E);
171 void VisitBinCmp(const BinaryOperator *E);
172 void VisitTypeTraitExpr(const TypeTraitExpr *E);
173 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
174 Visit(E: E->getSemanticForm());
175 }
176
177 void VisitObjCMessageExpr(ObjCMessageExpr *E);
178 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { EmitAggLoadOfLValue(E); }
179
180 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
181 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
182 void VisitChooseExpr(const ChooseExpr *CE);
183 void VisitInitListExpr(InitListExpr *E);
184 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
185 FieldDecl *InitializedFieldInUnion,
186 Expr *ArrayFiller);
187 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
188 llvm::Value *outerBegin = nullptr);
189 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
190 void VisitNoInitExpr(NoInitExpr *E) {} // Do nothing.
191 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
192 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
193 Visit(E: DAE->getExpr());
194 }
195 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
196 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
197 Visit(E: DIE->getExpr());
198 }
199 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
200 void VisitCXXConstructExpr(const CXXConstructExpr *E);
201 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
202 void VisitLambdaExpr(LambdaExpr *E);
203 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
204 void VisitExprWithCleanups(ExprWithCleanups *E);
205 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
206 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
207 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
208 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
209
210 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
211 if (E->isGLValue()) {
212 LValue LV = CGF.EmitPseudoObjectLValue(e: E);
213 return EmitFinalDestCopy(type: E->getType(), src: LV);
214 }
215
216 AggValueSlot Slot = EnsureSlot(T: E->getType());
217 bool NeedsDestruction =
218 !Slot.isExternallyDestructed() &&
219 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
220 if (NeedsDestruction)
221 Slot.setExternallyDestructed();
222 CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
223 if (NeedsDestruction)
224 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Slot.getAddress(),
225 type: E->getType());
226 }
227
228 void VisitVAArgExpr(VAArgExpr *E);
229 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
230 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
231 Expr *ArrayFiller);
232
233 void EmitInitializationToLValue(Expr *E, LValue Address);
234 void EmitNullInitializationToLValue(LValue Address);
235 // case Expr::ChooseExprClass:
236 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
237 void VisitAtomicExpr(AtomicExpr *E) {
238 RValue Res = CGF.EmitAtomicExpr(E);
239 EmitFinalDestCopy(type: E->getType(), src: Res);
240 }
241 void VisitPackIndexingExpr(PackIndexingExpr *E) {
242 Visit(E: E->getSelectedExpr());
243 }
244};
245} // end anonymous namespace.
246
247//===----------------------------------------------------------------------===//
248// Utilities
249//===----------------------------------------------------------------------===//
250
251/// EmitAggLoadOfLValue - Given an expression with aggregate type that
252/// represents a value lvalue, this method emits the address of the lvalue,
253/// then loads the result into DestPtr.
254void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
255 LValue LV = CGF.EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load);
256
257 // If the type of the l-value is atomic, then do an atomic load.
258 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
259 CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
260 return;
261 }
262
263 if (E->getType().getAddressSpace() == LangAS::hlsl_constant)
264 if (CGF.CGM.getHLSLRuntime().emitBufferCopy(CGF, E, SrcLV: LV, DestSlot&: Dest))
265 return;
266
267 EmitFinalDestCopy(type: E->getType(), src: LV);
268}
269
270/// True if the given aggregate type requires special GC API calls.
271bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
272 // Only record types have members that might require garbage collection.
273 const auto *Record = T->getAsRecordDecl();
274 if (!Record)
275 return false;
276
277 // Don't mess with non-trivial C++ types.
278 if (isa<CXXRecordDecl>(Val: Record) &&
279 (cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
280 !cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
281 return false;
282
283 // Check whether the type has an object member.
284 return Record->hasObjectMember();
285}
286
287void AggExprEmitter::withReturnValueSlot(
288 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
289 QualType RetTy = E->getType();
290 bool RequiresDestruction =
291 !Dest.isExternallyDestructed() &&
292 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
293
294 // If it makes no observable difference, save a memcpy + temporary.
295 //
296 // We need to always provide our own temporary if destruction is required.
297 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
298 // its lifetime before we have the chance to emit a proper destructor call.
299 //
300 // We also need a temporary if the destination is in a different address space
301 // from the sret AS. Use the target hook to get the actual sret AS for this
302 // return type.
303 const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl();
304 LangAS SRetLangAS = CGF.CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
305 unsigned SRetAS = CGF.getContext().getTargetAddressSpace(AS: SRetLangAS);
306 bool CanAggregateCopy =
307 RD ? (RD->hasTrivialCopyConstructor() ||
308 RD->hasTrivialMoveConstructor() || RD->hasTrivialCopyAssignment() ||
309 RD->hasTrivialMoveAssignment() || RD->hasAttr<TrivialABIAttr>() ||
310 RD->isUnion())
311 : RetTy.isTriviallyCopyableType(Context: CGF.getContext());
312 bool DestASMismatch = !Dest.isIgnored() && CanAggregateCopy &&
313 Dest.getAddress()
314 .getBasePointer()
315 ->stripPointerCasts()
316 ->getType()
317 ->getPointerAddressSpace() != SRetAS;
318 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
319 (RequiresDestruction && Dest.isIgnored()) || DestASMismatch;
320
321 Address RetAddr = Address::invalid();
322
323 EHScopeStack::stable_iterator LifetimeEndBlock;
324 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
325 if (!UseTemp) {
326 RetAddr = Dest.getAddress();
327 if (RetAddr.isValid() && RetAddr.getAddressSpace() != SRetAS) {
328 llvm::Type *SRetPtrTy =
329 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: SRetAS);
330 RetAddr = RetAddr.withPointer(
331 NewPointer: CGF.performAddrSpaceCast(Src: RetAddr.getBasePointer(), DestTy: SRetPtrTy),
332 IsKnownNonNull: RetAddr.isKnownNonNull());
333 }
334 } else {
335 RetAddr = CGF.CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
336 if (CGF.EmitLifetimeStart(Addr: RetAddr.getBasePointer())) {
337 LifetimeStartInst =
338 cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
339 assert(LifetimeStartInst->getIntrinsicID() ==
340 llvm::Intrinsic::lifetime_start &&
341 "Last insertion wasn't a lifetime.start?");
342
343 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
344 kind: NormalEHLifetimeMarker, A: RetAddr);
345 LifetimeEndBlock = CGF.EHStack.stable_begin();
346 }
347 }
348
349 RValue Src =
350 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
351 Dest.isExternallyDestructed()));
352
353 if (!UseTemp)
354 return;
355
356 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
357 Src.getAggregatePointer(E->getType(), CGF));
358 EmitFinalDestCopy(type: E->getType(), src: Src);
359
360 if (!RequiresDestruction && LifetimeStartInst) {
361 // If there's no dtor to run, the copy was the last use of our temporary.
362 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
363 // eagerly.
364 CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
365 CGF.EmitLifetimeEnd(Addr: RetAddr.getBasePointer());
366 }
367}
368
369/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
370void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
371 assert(src.isAggregate() && "value must be aggregate value!");
372 LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
373 EmitFinalDestCopy(type, src: srcLV, SrcValueKind: CodeGenFunction::EVK_RValue);
374}
375
376/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
377void AggExprEmitter::EmitFinalDestCopy(
378 QualType type, const LValue &src,
379 CodeGenFunction::ExprValueKind SrcValueKind) {
380 // If Dest is ignored, then we're evaluating an aggregate expression
381 // in a context that doesn't care about the result. Note that loads
382 // from volatile l-values force the existence of a non-ignored
383 // destination.
384 if (Dest.isIgnored())
385 return;
386
387 // Copy non-trivial C structs here.
388 LValue DstLV = CGF.MakeAddrLValue(
389 Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
390
391 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
392 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
393 if (Dest.isPotentiallyAliased())
394 CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
395 else
396 CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
397 return;
398 }
399 } else {
400 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
401 if (Dest.isPotentiallyAliased())
402 CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
403 else
404 CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
405 return;
406 }
407 }
408
409 AggValueSlot srcAgg = AggValueSlot::forLValue(
410 LV: src, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type), isAliased: AggValueSlot::IsAliased,
411 mayOverlap: AggValueSlot::MayOverlap);
412 EmitCopy(type, dest: Dest, src: srcAgg);
413}
414
415/// Perform a copy from the source into the destination.
416///
417/// \param type - the type of the aggregate being copied; qualifiers are
418/// ignored
419void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
420 const AggValueSlot &src) {
421 if (dest.requiresGCollection()) {
422 CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
423 llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
424 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr: dest.getAddress(),
425 SrcPtr: src.getAddress(), Size: size);
426 return;
427 }
428
429 // If the result of the assignment is used, copy the LHS there also.
430 // It's volatile if either side is. Use the minimum alignment of
431 // the two sides.
432 LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
433 LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
434 CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
435 isVolatile: dest.isVolatile() || src.isVolatile());
436}
437
438/// Emit the initializer for a std::initializer_list initialized with a
439/// real initializer list.
440void AggExprEmitter::VisitCXXStdInitializerListExpr(
441 CXXStdInitializerListExpr *E) {
442 // Emit an array containing the elements. The array is externally destructed
443 // if the std::initializer_list object is.
444 ASTContext &Ctx = CGF.getContext();
445 LValue Array = CGF.EmitLValue(E: E->getSubExpr());
446 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
447 Address ArrayPtr = Array.getAddress();
448
449 const ConstantArrayType *ArrayType =
450 Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
451 assert(ArrayType && "std::initializer_list constructed from non-array");
452
453 auto *Record = E->getType()->castAsRecordDecl();
454 RecordDecl::field_iterator Field = Record->field_begin();
455 assert(Field != Record->field_end() &&
456 Ctx.hasSameType(Field->getType()->getPointeeType(),
457 ArrayType->getElementType()) &&
458 "Expected std::initializer_list first field to be const E *");
459
460 // Start pointer.
461 AggValueSlot Dest = EnsureSlot(T: E->getType());
462 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
463 LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
464 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
465 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
466 ++Field;
467 assert(Field != Record->field_end() &&
468 "Expected std::initializer_list to have two fields");
469
470 llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
471 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
472 if (Ctx.hasSameType(T1: Field->getType(), T2: Ctx.getSizeType())) {
473 // Length.
474 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
475
476 } else {
477 // End pointer.
478 assert(Field->getType()->isPointerType() &&
479 Ctx.hasSameType(Field->getType()->getPointeeType(),
480 ArrayType->getElementType()) &&
481 "Expected std::initializer_list second field to be const E *");
482 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
483 llvm::Value *IdxEnd[] = {Zero, Size};
484 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
485 Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
486 Name: "arrayend");
487 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
488 }
489
490 assert(++Field == Record->field_end() &&
491 "Expected std::initializer_list to only have two fields");
492}
493
494// emit an elementwise cast where the RHS is a scalar or vector
495// or emit an aggregate splat cast
496static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,
497 LValue DestVal,
498 llvm::Value *SrcVal,
499 QualType SrcTy,
500 SourceLocation Loc) {
501 // Flatten our destination
502 SmallVector<LValue, 16> StoreList;
503 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
504
505 bool isVector = false;
506 if (auto *VT = SrcTy->getAs<VectorType>()) {
507 isVector = true;
508 SrcTy = VT->getElementType();
509 assert(StoreList.size() <= VT->getNumElements() &&
510 "Cannot perform HLSL flat cast when vector source \
511 object has less elements than flattened destination \
512 object.");
513 }
514
515 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
516 LValue DestLVal = StoreList[I];
517 llvm::Value *Load =
518 isVector ? CGF.Builder.CreateExtractElement(Vec: SrcVal, Idx: I, Name: "vec.load")
519 : SrcVal;
520 llvm::Value *Cast =
521 CGF.EmitScalarConversion(Src: Load, SrcTy, DstTy: DestLVal.getType(), Loc);
522 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
523 }
524}
525
526// emit a flat cast where the RHS is an aggregate
527static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
528 LValue SrcVal, SourceLocation Loc) {
529 // Flatten our destination
530 SmallVector<LValue, 16> StoreList;
531 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
532 // Flatten our src
533 SmallVector<LValue, 16> LoadList;
534 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
535
536 assert(StoreList.size() <= LoadList.size() &&
537 "Cannot perform HLSL elementwise cast when flattened source object \
538 has less elements than flattened destination object.");
539 // apply casts to what we load from LoadList
540 // and store result in Dest
541 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
542 LValue DestLVal = StoreList[I];
543 LValue SrcLVal = LoadList[I];
544 RValue RVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc);
545 assert(RVal.isScalar() && "All flattened source values should be scalars");
546 llvm::Value *Val = RVal.getScalarVal();
547 llvm::Value *Cast = CGF.EmitScalarConversion(Src: Val, SrcTy: SrcLVal.getType(),
548 DstTy: DestLVal.getType(), Loc);
549 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
550 }
551}
552
553/// Emit initialization of an array from an initializer list. ExprToVisit must
554/// be either an InitListEpxr a CXXParenInitListExpr.
555void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
556 QualType ArrayQTy, Expr *ExprToVisit,
557 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
558 uint64_t NumInitElements = Args.size();
559
560 uint64_t NumArrayElements = AType->getNumElements();
561 for (const auto *Init : Args) {
562 if (const auto *Embed = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
563 NumInitElements += Embed->getDataElementCount() - 1;
564 if (NumInitElements > NumArrayElements) {
565 NumInitElements = NumArrayElements;
566 break;
567 }
568 }
569 }
570
571 assert(NumInitElements <= NumArrayElements);
572
573 QualType elementType =
574 CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
575 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
576 CharUnits elementAlign =
577 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
578 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
579
580 // Consider initializing the array by copying from a global. For this to be
581 // more efficient than per-element initialization, the size of the elements
582 // with explicit initializers should be large enough.
583 if (NumInitElements * elementSize.getQuantity() > 16 &&
584 elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
585 CodeGen::CodeGenModule &CGM = CGF.CGM;
586 ConstantEmitter Emitter(CGF);
587 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
588 T: CGM.getContext().removeAddrSpaceQualType(T: ArrayQTy),
589 AddressSpace: CGM.GetGlobalConstantAddressSpace());
590 LangAS AS = GVArrayQTy.getAddressSpace();
591 if (llvm::Constant *C =
592 Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: GVArrayQTy)) {
593 auto GV = new llvm::GlobalVariable(
594 CGM.getModule(), C->getType(),
595 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
596 "constinit",
597 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
598 CGM.getContext().getTargetAddressSpace(AS));
599 Emitter.finalize(global: GV);
600 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: GVArrayQTy);
601 GV->setAlignment(Align.getAsAlign());
602 Address GVAddr(GV, GV->getValueType(), Align);
603 EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: GVArrayQTy));
604 return;
605 }
606 }
607
608 // Exception safety requires us to destroy all the
609 // already-constructed members if an initializer throws.
610 // For that, we'll need an EH cleanup.
611 QualType::DestructionKind dtorKind = elementType.isDestructedType();
612 Address endOfInit = Address::invalid();
613 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
614
615 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
616 if (dtorKind) {
617 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
618 // In principle we could tell the cleanup where we are more
619 // directly, but the control flow can get so varied here that it
620 // would actually be quite complex. Therefore we go through an
621 // alloca.
622 llvm::Instruction *dominatingIP =
623 Builder.CreateFlagLoad(Addr: llvm::ConstantInt::getNullValue(Ty: CGF.Int8PtrTy));
624 endOfInit = CGF.CreateTempAlloca(Ty: begin->getType(), align: CGF.getPointerAlign(),
625 Name: "arrayinit.endOfInit");
626 Builder.CreateStore(Val: begin, Addr: endOfInit);
627 CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
628 elementAlignment: elementAlign,
629 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
630 cast<EHCleanupScope>(Val&: *CGF.EHStack.find(sp: CGF.EHStack.stable_begin()))
631 .AddAuxAllocas(Allocas: allocaTracker.Take());
632
633 CGF.DeferredDeactivationCleanupStack.push_back(
634 Elt: {.Cleanup: CGF.EHStack.stable_begin(), .DominatingIP: dominatingIP});
635 }
636
637 llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
638
639 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
640 llvm::Value *element = begin;
641 if (ArrayIndex > 0) {
642 if (CGF.getLangOpts().EmitLogicalPointer)
643 element = Builder.CreateStructuredGEP(
644 BaseType: AType, PtrBase: begin, Indices: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
645 Name: "arrayinit.element");
646 else
647 element = Builder.CreateInBoundsGEP(
648 Ty: llvmElementType, Ptr: begin,
649 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
650 Name: "arrayinit.element");
651
652 // Tell the cleanup that it needs to destroy up to this
653 // element. TODO: some of these stores can be trivially
654 // observed to be unnecessary.
655 if (endOfInit.isValid())
656 Builder.CreateStore(Val: element, Addr: endOfInit);
657 }
658
659 LValue elementLV = CGF.MakeAddrLValue(
660 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
661 EmitInitializationToLValue(E: Init, Address: elementLV);
662 return true;
663 };
664
665 unsigned ArrayIndex = 0;
666 // Emit the explicit initializers.
667 for (uint64_t i = 0; i != NumInitElements; ++i) {
668 if (ArrayIndex >= NumInitElements)
669 break;
670 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Args[i]->IgnoreParenImpCasts())) {
671 EmbedS->doForEachDataElement(C&: Emit, StartingIndexInArray&: ArrayIndex);
672 } else {
673 Emit(Args[i], ArrayIndex);
674 ArrayIndex++;
675 }
676 }
677
678 // Check whether there's a non-trivial array-fill expression.
679 bool hasTrivialFiller = CodeGenUtils::isTrivialFiller(E: ArrayFiller);
680
681 // Any remaining elements need to be zero-initialized, possibly
682 // using the filler expression. We can skip this if the we're
683 // emitting to zeroed memory.
684 if (NumInitElements != NumArrayElements &&
685 !(Dest.isZeroed() && hasTrivialFiller &&
686 CGF.getTypes().isZeroInitializable(T: elementType))) {
687
688 // Use an actual loop. This is basically
689 // do { *array++ = filler; } while (array != end);
690
691 // Advance to the start of the rest of the array.
692 llvm::Value *element = begin;
693 if (NumInitElements) {
694 element = Builder.CreateInBoundsGEP(
695 Ty: llvmElementType, Ptr: element,
696 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumInitElements),
697 Name: "arrayinit.start");
698 if (endOfInit.isValid())
699 Builder.CreateStore(Val: element, Addr: endOfInit);
700 }
701
702 // Compute the end of the array.
703 llvm::Value *end = Builder.CreateInBoundsGEP(
704 Ty: llvmElementType, Ptr: begin,
705 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
706
707 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
708 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
709
710 // Jump into the body.
711 CGF.EmitBlock(BB: bodyBB);
712 llvm::PHINode *currentElement =
713 Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
714 currentElement->addIncoming(V: element, BB: entryBB);
715
716 if (CGF.CGM.shouldEmitConvergenceTokens())
717 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
718
719 // Emit the actual filler expression.
720 {
721 // C++1z [class.temporary]p5:
722 // when a default constructor is called to initialize an element of
723 // an array with no corresponding initializer [...] the destruction of
724 // every temporary created in a default argument is sequenced before
725 // the construction of the next array element, if any
726 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
727 LValue elementLV = CGF.MakeAddrLValue(
728 Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
729 if (ArrayFiller)
730 EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
731 else
732 EmitNullInitializationToLValue(Address: elementLV);
733 }
734
735 // Move on to the next element.
736 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
737 Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
738
739 // Tell the EH cleanup that we finished with the last element.
740 if (endOfInit.isValid())
741 Builder.CreateStore(Val: nextElement, Addr: endOfInit);
742
743 // Leave the loop if we're done.
744 llvm::Value *done =
745 Builder.CreateICmpEQ(LHS: nextElement, RHS: end, Name: "arrayinit.done");
746 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
747 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
748 currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
749
750 if (CGF.CGM.shouldEmitConvergenceTokens())
751 CGF.ConvergenceTokenStack.pop_back();
752
753 CGF.EmitBlock(BB: endBB);
754 }
755}
756
757//===----------------------------------------------------------------------===//
758// Visitor Methods
759//===----------------------------------------------------------------------===//
760
761void AggExprEmitter::VisitMaterializeTemporaryExpr(
762 MaterializeTemporaryExpr *E) {
763 Visit(E: E->getSubExpr());
764}
765
766void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
767 // If this is a unique OVE, just visit its source expression.
768 if (e->isUnique())
769 Visit(E: e->getSourceExpr());
770 else
771 EmitFinalDestCopy(type: e->getType(), src: CGF.getOrCreateOpaqueLValueMapping(e));
772}
773
774void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
775 if (Dest.isPotentiallyAliased()) {
776 // Just emit a load of the lvalue + a copy, because our compound literal
777 // might alias the destination.
778 EmitAggLoadOfLValue(E);
779 return;
780 }
781
782 AggValueSlot Slot = EnsureSlot(T: E->getType());
783
784 // Block-scope compound literals are destroyed at the end of the enclosing
785 // scope in C.
786 bool Destruct =
787 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
788 if (Destruct)
789 Slot.setExternallyDestructed();
790
791 CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
792
793 if (Destruct)
794 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
795 CGF.pushLifetimeExtendedDestroy(dtorKind: DtorKind, addr: Slot.getAddress(),
796 type: E->getType());
797}
798
799/// Attempt to look through various unimportant expressions to find a
800/// cast of the given kind.
801static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
802 op = op->IgnoreParenNoopCasts(Ctx: ctx);
803 if (auto castE = dyn_cast<CastExpr>(Val: op)) {
804 if (castE->getCastKind() == kind)
805 return castE->getSubExpr();
806 }
807 return nullptr;
808}
809
810void AggExprEmitter::VisitCastExpr(CastExpr *E) {
811 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
812 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
813 switch (E->getCastKind()) {
814 case CK_Dynamic: {
815 // FIXME: Can this actually happen? We have no test coverage for it.
816 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
817 LValue LV =
818 CGF.EmitCheckedLValue(E: E->getSubExpr(), TCK: CodeGenFunction::TCK_Load);
819 // FIXME: Do we also need to handle property references here?
820 if (LV.isSimple())
821 CGF.EmitDynamicCast(V: LV.getAddress(), DCE: cast<CXXDynamicCastExpr>(Val: E));
822 else
823 CGF.CGM.ErrorUnsupported(S: E, Type: "non-simple lvalue dynamic_cast");
824
825 if (!Dest.isIgnored())
826 CGF.CGM.ErrorUnsupported(S: E, Type: "lvalue dynamic_cast with a destination");
827 break;
828 }
829
830 case CK_ToUnion: {
831 // Evaluate even if the destination is ignored.
832 if (Dest.isIgnored()) {
833 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
834 /*ignoreResult=*/true);
835 break;
836 }
837
838 // GCC union extension
839 QualType Ty = E->getSubExpr()->getType();
840 Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
841 EmitInitializationToLValue(E: E->getSubExpr(),
842 Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
843 break;
844 }
845
846 case CK_LValueToRValueBitCast: {
847 if (Dest.isIgnored()) {
848 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
849 /*ignoreResult=*/true);
850 break;
851 }
852
853 LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
854 Address SourceAddress = SourceLV.getAddress().withElementType(ElemTy: CGF.Int8Ty);
855 Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
856 llvm::Value *SizeVal = llvm::ConstantInt::get(
857 Ty: CGF.SizeTy,
858 V: CGF.getContext().getTypeSizeInChars(T: E->getType()).getQuantity());
859 Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
860 break;
861 }
862
863 case CK_DerivedToBase: {
864 assert(CGF.getLangOpts().HLSL &&
865 "Derived/Base casts in EmitAggExpr are only supported in HLSL");
866
867 // Create a temporary for the derived record, switch it out with the current
868 // Dest slot, and emit the derived value.
869 QualType DerivedTy = E->getSubExpr()->getType();
870 RawAddress DerivedAddr = CGF.CreateMemTempWithoutCast(T: DerivedTy);
871 AggValueSlot DerivedTmpSlot = AggValueSlot::forAddr(
872 addr: DerivedAddr, quals: DerivedTy.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
873 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
874 mayOverlap: AggValueSlot::DoesNotOverlap);
875
876 AggValueSlot DestBaseSlot = Dest;
877 Dest = DerivedTmpSlot;
878
879 Visit(E: E->getSubExpr());
880
881 // Perform derived-to-base address conversion to get the address
882 // of the base record within the derived record. In HLSL this should
883 // always be same as the derived because of single inheritance, but let's
884 // do it properly.
885 Address BaseAddrInDerived = CGF.GetAddressOfBaseClass(
886 Value: DerivedTmpSlot.getAddress(), Derived: DerivedTy->castAsCXXRecordDecl(),
887 PathBegin: E->path_begin(), PathEnd: E->path_end(),
888 /*NullCheckValue=*/false, Loc: E->getExprLoc());
889
890 AggValueSlot SrcBaseSlot = AggValueSlot::forAddr(
891 addr: BaseAddrInDerived, quals: E->getType().getQualifiers(),
892 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
893 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
894
895 // Copy the base class to the original destination slot and restore it.
896 EmitCopy(type: E->getType(), dest: DestBaseSlot, src: SrcBaseSlot);
897 Dest = DestBaseSlot;
898 break;
899 }
900
901 case CK_BaseToDerived:
902 case CK_UncheckedDerivedToBase: {
903 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
904 "should have been unpacked before we got here");
905 }
906
907 case CK_NonAtomicToAtomic:
908 case CK_AtomicToNonAtomic: {
909 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
910
911 // Determine the atomic and value types.
912 QualType atomicType = E->getSubExpr()->getType();
913 QualType valueType = E->getType();
914 if (isToAtomic)
915 std::swap(a&: atomicType, b&: valueType);
916
917 assert(atomicType->isAtomicType());
918 assert(CGF.getContext().hasSameUnqualifiedType(
919 valueType, atomicType->castAs<AtomicType>()->getValueType()));
920
921 // Just recurse normally if we're ignoring the result or the
922 // atomic type doesn't change representation.
923 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
924 return Visit(E: E->getSubExpr());
925 }
926
927 CastKind peepholeTarget =
928 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
929
930 // These two cases are reverses of each other; try to peephole them.
931 if (Expr *op =
932 findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
933 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
934 E->getType()) &&
935 "peephole significantly changed types?");
936 return Visit(E: op);
937 }
938
939 // If we're converting an r-value of non-atomic type to an r-value
940 // of atomic type, just emit directly into the relevant sub-object.
941 if (isToAtomic) {
942 AggValueSlot valueDest = Dest;
943 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
944 // Zero-initialize. (Strictly speaking, we only need to initialize
945 // the padding at the end, but this is simpler.)
946 if (!Dest.isZeroed())
947 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
948
949 // Build a GEP to refer to the subobject.
950 Address valueAddr =
951 CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
952 valueDest = AggValueSlot::forAddr(
953 addr: valueAddr, quals: valueDest.getQualifiers(),
954 isDestructed: valueDest.isExternallyDestructed(), needsGC: valueDest.requiresGCollection(),
955 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
956 isZeroed: AggValueSlot::IsZeroed);
957 }
958
959 CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
960 return;
961 }
962
963 // Otherwise, we're converting an atomic type to a non-atomic type.
964 // Make an atomic temporary, emit into that, and then copy the value out.
965 AggValueSlot atomicSlot =
966 CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
967 CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
968
969 Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
970 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
971 return EmitFinalDestCopy(type: valueType, src: rvalue);
972 }
973 case CK_AddressSpaceConversion:
974 return Visit(E: E->getSubExpr());
975
976 case CK_LValueToRValue:
977 // If we're loading from a volatile type, force the destination
978 // into existence.
979 if (E->getSubExpr()->getType().isVolatileQualified()) {
980 bool Destruct =
981 !Dest.isExternallyDestructed() &&
982 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
983 if (Destruct)
984 Dest.setExternallyDestructed();
985 EnsureDest(T: E->getType());
986 Visit(E: E->getSubExpr());
987
988 if (Destruct)
989 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
990 type: E->getType());
991
992 return;
993 }
994
995 [[fallthrough]];
996
997 case CK_HLSLArrayRValue:
998 if (CGF.getLangOpts().HLSL &&
999 E->getSubExpr()->getType()->isHLSLResourceRecordArray())
1000 if (CGF.CGM.getHLSLRuntime().emitGlobalResourceArray(CGF, E, DestSlot&: Dest))
1001 break;
1002 Visit(E: E->getSubExpr());
1003 break;
1004 case CK_HLSLAggregateSplatCast: {
1005 Expr *Src = E->getSubExpr();
1006 QualType SrcTy = Src->getType();
1007 RValue RV = CGF.EmitAnyExpr(E: Src);
1008 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1009 SourceLocation Loc = E->getExprLoc();
1010
1011 assert(RV.isScalar() && SrcTy->isScalarType() &&
1012 "RHS of HLSL splat cast must be a scalar.");
1013 llvm::Value *SrcVal = RV.getScalarVal();
1014 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1015 break;
1016 }
1017 case CK_HLSLElementwiseCast: {
1018 Expr *Src = E->getSubExpr();
1019 QualType SrcTy = Src->getType();
1020 RValue RV = CGF.EmitAnyExpr(E: Src);
1021 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1022 SourceLocation Loc = E->getExprLoc();
1023
1024 if (RV.isScalar()) {
1025 llvm::Value *SrcVal = RV.getScalarVal();
1026 assert(SrcTy->isVectorType() &&
1027 "HLSL Elementwise cast doesn't handle splatting.");
1028 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1029 } else {
1030 assert(RV.isAggregate() &&
1031 "Can't perform HLSL Aggregate cast on a complex type.");
1032 Address SrcVal = RV.getAggregateAddress();
1033 EmitHLSLElementwiseCast(CGF, DestVal: DestLVal, SrcVal: CGF.MakeAddrLValue(Addr: SrcVal, T: SrcTy),
1034 Loc);
1035 }
1036 break;
1037 }
1038 case CK_NoOp:
1039 case CK_UserDefinedConversion:
1040 case CK_ConstructorConversion:
1041 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
1042 E->getType()) &&
1043 "Implicit cast types must be compatible");
1044 Visit(E: E->getSubExpr());
1045 break;
1046
1047 case CK_LValueBitCast:
1048 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
1049
1050 case CK_Dependent:
1051 case CK_BitCast:
1052 case CK_ArrayToPointerDecay:
1053 case CK_FunctionToPointerDecay:
1054 case CK_NullToPointer:
1055 case CK_NullToMemberPointer:
1056 case CK_BaseToDerivedMemberPointer:
1057 case CK_DerivedToBaseMemberPointer:
1058 case CK_MemberPointerToBoolean:
1059 case CK_ReinterpretMemberPointer:
1060 case CK_IntegralToPointer:
1061 case CK_PointerToIntegral:
1062 case CK_PointerToBoolean:
1063 case CK_ToVoid:
1064 case CK_VectorSplat:
1065 case CK_IntegralCast:
1066 case CK_BooleanToSignedIntegral:
1067 case CK_IntegralToBoolean:
1068 case CK_IntegralToFloating:
1069 case CK_FloatingToIntegral:
1070 case CK_FloatingToBoolean:
1071 case CK_FloatingCast:
1072 case CK_CPointerToObjCPointerCast:
1073 case CK_BlockPointerToObjCPointerCast:
1074 case CK_AnyPointerToBlockPointerCast:
1075 case CK_ObjCObjectLValueCast:
1076 case CK_FloatingRealToComplex:
1077 case CK_FloatingComplexToReal:
1078 case CK_FloatingComplexToBoolean:
1079 case CK_FloatingComplexCast:
1080 case CK_FloatingComplexToIntegralComplex:
1081 case CK_IntegralRealToComplex:
1082 case CK_IntegralComplexToReal:
1083 case CK_IntegralComplexToBoolean:
1084 case CK_IntegralComplexCast:
1085 case CK_IntegralComplexToFloatingComplex:
1086 case CK_ARCProduceObject:
1087 case CK_ARCConsumeObject:
1088 case CK_ARCReclaimReturnedObject:
1089 case CK_ARCExtendBlockObject:
1090 case CK_CopyAndAutoreleaseBlockObject:
1091 case CK_BuiltinFnToFnPtr:
1092 case CK_ZeroToOCLOpaqueType:
1093 case CK_MatrixCast:
1094 case CK_HLSLVectorTruncation:
1095 case CK_HLSLMatrixTruncation:
1096 case CK_IntToOCLSampler:
1097 case CK_FloatingToFixedPoint:
1098 case CK_FixedPointToFloating:
1099 case CK_FixedPointCast:
1100 case CK_FixedPointToBoolean:
1101 case CK_FixedPointToIntegral:
1102 case CK_IntegralToFixedPoint:
1103 llvm_unreachable("cast kind invalid for aggregate types");
1104 }
1105}
1106
1107void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1108 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
1109 EmitAggLoadOfLValue(E);
1110 return;
1111 }
1112
1113 withReturnValueSlot(
1114 E, EmitCall: [&](ReturnValueSlot Slot) { return CGF.EmitCallExpr(E, ReturnValue: Slot); });
1115}
1116
1117void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1118 withReturnValueSlot(E, EmitCall: [&](ReturnValueSlot Slot) {
1119 return CGF.EmitObjCMessageExpr(E, Return: Slot);
1120 });
1121}
1122
1123void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1124 CGF.EmitIgnoredExpr(E: E->getLHS());
1125 Visit(E: E->getRHS());
1126}
1127
1128void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1129 CodeGenFunction::StmtExprEvaluation eval(CGF);
1130 CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
1131}
1132
1133enum CompareKind {
1134 CK_Less,
1135 CK_Greater,
1136 CK_Equal,
1137};
1138
1139static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1140 const BinaryOperator *E, llvm::Value *LHS,
1141 llvm::Value *RHS, CompareKind Kind,
1142 const char *NameSuffix = "") {
1143 QualType ArgTy = E->getLHS()->getType();
1144 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1145 ArgTy = CT->getElementType();
1146
1147 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1148 assert(Kind == CK_Equal &&
1149 "member pointers may only be compared for equality");
1150 return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1151 CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1152 }
1153
1154 // Compute the comparison instructions for the specified comparison kind.
1155 struct CmpInstInfo {
1156 const char *Name;
1157 llvm::CmpInst::Predicate FCmp;
1158 llvm::CmpInst::Predicate SCmp;
1159 llvm::CmpInst::Predicate UCmp;
1160 };
1161 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1162 using FI = llvm::FCmpInst;
1163 using II = llvm::ICmpInst;
1164 switch (Kind) {
1165 case CK_Less:
1166 return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1167 case CK_Greater:
1168 return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1169 case CK_Equal:
1170 return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1171 }
1172 llvm_unreachable("Unrecognised CompareKind enum");
1173 }();
1174
1175 if (ArgTy->hasFloatingRepresentation())
1176 return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1177 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1178 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1179 auto Inst =
1180 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1181 return Builder.CreateICmp(P: Inst, LHS, RHS,
1182 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1183 }
1184
1185 llvm_unreachable("unsupported aggregate binary expression should have "
1186 "already been handled");
1187}
1188
1189void AggExprEmitter::EmitComparisonResult(const Expr *E,
1190 const ComparisonCategoryInfo &CmpInfo,
1191 llvm::Value *ResultValue) {
1192 // Create the return value in the destination slot.
1193 EnsureDest(T: E->getType());
1194 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1195
1196 // Emit the address of the first (and only) field in the comparison category
1197 // type, and initialize it from the constant integer value selected above.
1198 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1199 Base: DestLV, Field: *CmpInfo.Record->field_begin());
1200 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ResultValue), Dst: FieldLV,
1201 /*IsInit=*/isInit: true);
1202}
1203
1204void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1205 using llvm::BasicBlock;
1206 using llvm::PHINode;
1207 using llvm::Value;
1208 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1209 E->getRHS()->getType()));
1210 const ComparisonCategoryInfo &CmpInfo =
1211 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1212 assert(CmpInfo.Record->isTriviallyCopyable() &&
1213 "cannot copy non-trivially copyable aggregate");
1214
1215 QualType ArgTy = E->getLHS()->getType();
1216
1217 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1218 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1219 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1220 return CGF.ErrorUnsupported(S: E, Type: "aggregate three-way comparison");
1221 }
1222 bool IsComplex = ArgTy->isAnyComplexType();
1223
1224 // Evaluate the operands to the expression and extract their values.
1225 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1226 RValue RV = CGF.EmitAnyExpr(E);
1227 if (RV.isScalar())
1228 return {RV.getScalarVal(), nullptr};
1229 if (RV.isAggregate())
1230 return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1231 assert(RV.isComplex());
1232 return RV.getComplexVal();
1233 };
1234 auto LHSValues = EmitOperand(E->getLHS()),
1235 RHSValues = EmitOperand(E->getRHS());
1236
1237 auto EmitCmp = [&](CompareKind K) {
1238 Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1239 Kind: K, NameSuffix: IsComplex ? ".r" : "");
1240 if (!IsComplex)
1241 return Cmp;
1242 assert(K == CompareKind::CK_Equal);
1243 Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1244 RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1245 return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1246 };
1247 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1248 return Builder.getInt(AI: VInfo->getIntValue());
1249 };
1250
1251 Value *Select;
1252 if (ArgTy->isNullPtrType()) {
1253 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1254 } else if (!CmpInfo.isPartial()) {
1255 Value *SelectOne =
1256 Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1257 False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1258 Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1259 True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1260 False: SelectOne, Name: "sel.eq");
1261 } else {
1262 Value *SelectEq = Builder.CreateSelect(
1263 C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1264 False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1265 Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1266 True: EmitCmpRes(CmpInfo.getGreater()),
1267 False: SelectEq, Name: "sel.gt");
1268 Select = Builder.CreateSelect(
1269 C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1270 }
1271
1272 EmitComparisonResult(E, CmpInfo, ResultValue: Select);
1273}
1274
1275void AggExprEmitter::VisitTypeTraitExpr(const TypeTraitExpr *E) {
1276 assert(E->isStoredAsComparisonResult() &&
1277 "expected a strong_ordering type trait with a stored value");
1278
1279 const ComparisonCategoryInfo &CmpInfo =
1280 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1281 const auto Result =
1282 ComparisonCategoryResult(E->getAPValue().getInt().getZExtValue());
1283 llvm::Value *ResultValue =
1284 Builder.getInt(AI: CmpInfo.getValueInfo(ValueKind: Result)->getIntValue());
1285
1286 EmitComparisonResult(E, CmpInfo, ResultValue);
1287}
1288
1289void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1290 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1291 VisitPointerToDataMemberBinaryOperator(BO: E);
1292 else
1293 CGF.ErrorUnsupported(S: E, Type: "aggregate binary expression");
1294}
1295
1296void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1297 const BinaryOperator *E) {
1298 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1299 EmitFinalDestCopy(type: E->getType(), src: LV);
1300}
1301
1302void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1303 ApplyAtomGroup Grp(CGF.getDebugInfo());
1304 // For an assignment to work, the value on the right has
1305 // to be compatible with the value on the left.
1306 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1307 E->getRHS()->getType()) &&
1308 "Invalid assignment");
1309
1310 // If the LHS might be a __block variable, and the RHS can
1311 // potentially cause a block copy, we need to evaluate the RHS first
1312 // so that the assignment goes the right place.
1313 // This is pretty semantically fragile.
1314 if (CodeGenUtils::isBlockVarRef(E: E->getLHS()) &&
1315 E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1316 // Ensure that we have a destination, and evaluate the RHS into that.
1317 EnsureDest(T: E->getRHS()->getType());
1318 Visit(E: E->getRHS());
1319
1320 // Now emit the LHS and copy into it.
1321 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1322
1323 // That copy is an atomic copy if the LHS is atomic.
1324 if (LHS.getType()->isAtomicType() ||
1325 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1326 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1327 return;
1328 }
1329
1330 EmitCopy(type: E->getLHS()->getType(),
1331 dest: AggValueSlot::forLValue(LV: LHS, isDestructed: AggValueSlot::IsDestructed,
1332 needsGC: needsGC(T: E->getLHS()->getType()),
1333 isAliased: AggValueSlot::IsAliased,
1334 mayOverlap: AggValueSlot::MayOverlap),
1335 src: Dest);
1336 return;
1337 }
1338
1339 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1340
1341 // If we have an atomic type, evaluate into the destination and then
1342 // do an atomic copy.
1343 if (LHS.getType()->isAtomicType() ||
1344 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1345 EnsureDest(T: E->getRHS()->getType());
1346 Visit(E: E->getRHS());
1347 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1348 return;
1349 }
1350
1351 // Codegen the RHS so that it stores directly into the LHS.
1352 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1353 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1354 isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1355 // A non-volatile aggregate destination might have volatile member.
1356 if (!LHSSlot.isVolatile() && CGF.hasVolatileMember(T: E->getLHS()->getType()))
1357 LHSSlot.setVolatile(true);
1358
1359 CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1360
1361 // Copy into the destination if the assignment isn't ignored.
1362 EmitFinalDestCopy(type: E->getType(), src: LHS);
1363
1364 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1365 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1366 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1367 type: E->getType());
1368}
1369
1370void AggExprEmitter::VisitAbstractConditionalOperator(
1371 const AbstractConditionalOperator *E) {
1372 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1373 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1374 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1375
1376 // Bind the common expression if necessary.
1377 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1378
1379 CodeGenFunction::ConditionalEvaluation eval(CGF);
1380 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1381 TrueCount: CGF.getProfileCount(S: E));
1382
1383 // Save whether the destination's lifetime is externally managed.
1384 bool isExternallyDestructed = Dest.isExternallyDestructed();
1385 bool destructNonTrivialCStruct =
1386 !isExternallyDestructed &&
1387 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1388 isExternallyDestructed |= destructNonTrivialCStruct;
1389 Dest.setExternallyDestructed(isExternallyDestructed);
1390
1391 eval.begin(CGF);
1392 CGF.EmitBlock(BB: LHSBlock);
1393 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
1394 Visit(E: E->getTrueExpr());
1395 eval.end(CGF);
1396
1397 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1398 CGF.Builder.CreateBr(Dest: ContBlock);
1399
1400 // If the result of an agg expression is unused, then the emission
1401 // of the LHS might need to create a destination slot. That's fine
1402 // with us, and we can safely emit the RHS into the same slot, but
1403 // we shouldn't claim that it's already being destructed.
1404 Dest.setExternallyDestructed(isExternallyDestructed);
1405
1406 eval.begin(CGF);
1407 CGF.EmitBlock(BB: RHSBlock);
1408 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
1409 Visit(E: E->getFalseExpr());
1410 eval.end(CGF);
1411
1412 if (destructNonTrivialCStruct)
1413 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1414 type: E->getType());
1415
1416 CGF.EmitBlock(BB: ContBlock);
1417}
1418
1419void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1420 Visit(E: CE->getChosenSubExpr());
1421}
1422
1423void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1424 Address ArgValue = Address::invalid();
1425 CGF.EmitVAArg(VE, VAListAddr&: ArgValue, Slot: Dest);
1426
1427 // If EmitVAArg fails, emit an error.
1428 if (!ArgValue.isValid()) {
1429 CGF.ErrorUnsupported(S: VE, Type: "aggregate va_arg expression");
1430 return;
1431 }
1432}
1433
1434void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1435 // Ensure that we have a slot, but if we already do, remember
1436 // whether it was externally destructed.
1437 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1438 EnsureDest(T: E->getType());
1439
1440 // We're going to push a destructor if there isn't already one.
1441 Dest.setExternallyDestructed();
1442
1443 Visit(E: E->getSubExpr());
1444
1445 // Push that destructor we promised.
1446 if (!wasExternallyDestructed)
1447 CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1448}
1449
1450void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1451 AggValueSlot Slot = EnsureSlot(T: E->getType());
1452 CGF.EmitCXXConstructExpr(E, Dest: Slot);
1453}
1454
1455void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1456 const CXXInheritedCtorInitExpr *E) {
1457 AggValueSlot Slot = EnsureSlot(T: E->getType());
1458 CGF.EmitInheritedCXXConstructorCall(D: E->getConstructor(), ForVirtualBase: E->constructsVBase(),
1459 This: Slot.getAddress(),
1460 InheritedFromVBase: E->inheritedFromVBase(), E);
1461}
1462
1463void AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1464 AggValueSlot Slot = EnsureSlot(T: E->getType());
1465 LValue SlotLV = CGF.MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType());
1466
1467 // We'll need to enter cleanup scopes in case any of the element
1468 // initializers throws an exception or contains branch out of the expressions.
1469 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1470
1471 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1472 for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1473 e = E->capture_init_end();
1474 i != e; ++i, ++CurField) {
1475 // Emit initialization
1476 LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1477 if (CurField->hasCapturedVLAType()) {
1478 CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1479 continue;
1480 }
1481
1482 EmitInitializationToLValue(E: *i, Address: LV);
1483
1484 // Push a destructor if necessary.
1485 if (QualType::DestructionKind DtorKind =
1486 CurField->getType().isDestructedType()) {
1487 assert(LV.isSimple());
1488 if (DtorKind)
1489 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1490 type: CurField->getType(),
1491 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: false);
1492 }
1493 }
1494}
1495
1496void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1497 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1498 Visit(E: E->getSubExpr());
1499}
1500
1501void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1502 QualType T = E->getType();
1503 AggValueSlot Slot = EnsureSlot(T);
1504 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1505}
1506
1507void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1508 QualType T = E->getType();
1509 AggValueSlot Slot = EnsureSlot(T);
1510 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1511}
1512
1513/// Determine whether the given cast kind is known to always convert values
1514/// with all zero bits in their value representation to values with all zero
1515/// bits in their value representation.
1516static bool castPreservesZero(const CastExpr *CE) {
1517 switch (CE->getCastKind()) {
1518 // No-ops.
1519 case CK_NoOp:
1520 case CK_UserDefinedConversion:
1521 case CK_ConstructorConversion:
1522 case CK_BitCast:
1523 case CK_ToUnion:
1524 case CK_ToVoid:
1525 // Conversions between (possibly-complex) integral, (possibly-complex)
1526 // floating-point, and bool.
1527 case CK_BooleanToSignedIntegral:
1528 case CK_FloatingCast:
1529 case CK_FloatingComplexCast:
1530 case CK_FloatingComplexToBoolean:
1531 case CK_FloatingComplexToIntegralComplex:
1532 case CK_FloatingComplexToReal:
1533 case CK_FloatingRealToComplex:
1534 case CK_FloatingToBoolean:
1535 case CK_FloatingToIntegral:
1536 case CK_IntegralCast:
1537 case CK_IntegralComplexCast:
1538 case CK_IntegralComplexToBoolean:
1539 case CK_IntegralComplexToFloatingComplex:
1540 case CK_IntegralComplexToReal:
1541 case CK_IntegralRealToComplex:
1542 case CK_IntegralToBoolean:
1543 case CK_IntegralToFloating:
1544 // Reinterpreting integers as pointers and vice versa.
1545 case CK_IntegralToPointer:
1546 case CK_PointerToIntegral:
1547 // Language extensions.
1548 case CK_VectorSplat:
1549 case CK_MatrixCast:
1550 case CK_NonAtomicToAtomic:
1551 case CK_AtomicToNonAtomic:
1552 case CK_HLSLVectorTruncation:
1553 case CK_HLSLMatrixTruncation:
1554 case CK_HLSLElementwiseCast:
1555 case CK_HLSLAggregateSplatCast:
1556 return true;
1557
1558 case CK_BaseToDerivedMemberPointer:
1559 case CK_DerivedToBaseMemberPointer:
1560 case CK_MemberPointerToBoolean:
1561 case CK_NullToMemberPointer:
1562 case CK_ReinterpretMemberPointer:
1563 // FIXME: ABI-dependent.
1564 return false;
1565
1566 case CK_AnyPointerToBlockPointerCast:
1567 case CK_BlockPointerToObjCPointerCast:
1568 case CK_CPointerToObjCPointerCast:
1569 case CK_ObjCObjectLValueCast:
1570 case CK_IntToOCLSampler:
1571 case CK_ZeroToOCLOpaqueType:
1572 // FIXME: Check these.
1573 return false;
1574
1575 case CK_FixedPointCast:
1576 case CK_FixedPointToBoolean:
1577 case CK_FixedPointToFloating:
1578 case CK_FixedPointToIntegral:
1579 case CK_FloatingToFixedPoint:
1580 case CK_IntegralToFixedPoint:
1581 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1582 return false;
1583
1584 case CK_AddressSpaceConversion:
1585 case CK_BaseToDerived:
1586 case CK_DerivedToBase:
1587 case CK_Dynamic:
1588 case CK_NullToPointer:
1589 case CK_PointerToBoolean:
1590 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1591 // same representation in all involved address spaces.
1592 return false;
1593
1594 case CK_ARCConsumeObject:
1595 case CK_ARCExtendBlockObject:
1596 case CK_ARCProduceObject:
1597 case CK_ARCReclaimReturnedObject:
1598 case CK_CopyAndAutoreleaseBlockObject:
1599 case CK_ArrayToPointerDecay:
1600 case CK_FunctionToPointerDecay:
1601 case CK_BuiltinFnToFnPtr:
1602 case CK_Dependent:
1603 case CK_LValueBitCast:
1604 case CK_LValueToRValue:
1605 case CK_LValueToRValueBitCast:
1606 case CK_UncheckedDerivedToBase:
1607 case CK_HLSLArrayRValue:
1608 return false;
1609 }
1610 llvm_unreachable("Unhandled clang::CastKind enum");
1611}
1612
1613/// isSimpleZero - If emitting this value will obviously just cause a store of
1614/// zero to memory, return true. This can return false if uncertain, so it just
1615/// handles simple cases.
1616static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1617 E = E->IgnoreParens();
1618 while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1619 if (!castPreservesZero(CE))
1620 break;
1621 E = CE->getSubExpr()->IgnoreParens();
1622 }
1623
1624 // 0
1625 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1626 return IL->getValue() == 0;
1627 // +0.0
1628 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1629 return FL->getValue().isPosZero();
1630 // int()
1631 if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1632 CGF.getTypes().isZeroInitializable(T: E->getType()))
1633 return true;
1634 // (int*)0 - Null pointer expressions.
1635 if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1636 return ICE->getCastKind() == CK_NullToPointer &&
1637 CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1638 !E->HasSideEffects(Ctx: CGF.getContext());
1639 // '\0'
1640 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1641 return CL->getValue() == 0;
1642
1643 // Otherwise, hard case: conservatively return false.
1644 return false;
1645}
1646
1647void AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1648 QualType type = LV.getType();
1649 // FIXME: Ignore result?
1650 // FIXME: Are initializers affected by volatile?
1651 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1652 // Storing "i32 0" to a zero'd memory location is a noop.
1653 return;
1654 } else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1655 return EmitNullInitializationToLValue(Address: LV);
1656 } else if (isa<NoInitExpr>(Val: E)) {
1657 // Do nothing.
1658 return;
1659 } else if (type->isReferenceType()) {
1660 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1661 return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1662 }
1663
1664 CGF.EmitInitializationToLValue(E, LV, IsZeroed: Dest.isZeroed());
1665}
1666
1667void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1668 QualType type = lv.getType();
1669
1670 // If the destination slot is already zeroed out before the aggregate is
1671 // copied into it, we don't have to emit any zeros here.
1672 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1673 return;
1674
1675 if (CGF.hasScalarEvaluationKind(T: type)) {
1676 // For non-aggregates, we can store the appropriate null constant.
1677 llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1678 // Note that the following is not equivalent to
1679 // EmitStoreThroughBitfieldLValue for ARC types.
1680 if (lv.isBitField()) {
1681 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1682 } else {
1683 assert(lv.isSimple());
1684 CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1685 }
1686 } else {
1687 // There's a potential optimization opportunity in combining
1688 // memsets; that would be easy for arrays, but relatively
1689 // difficult for structures with the current code.
1690 CGF.EmitNullInitialization(DestPtr: lv.getAddress(), Ty: lv.getType());
1691 }
1692}
1693
1694void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1695 VisitCXXParenListOrInitListExpr(ExprToVisit: E, Args: E->getInitExprs(),
1696 InitializedFieldInUnion: E->getInitializedFieldInUnion(),
1697 ArrayFiller: E->getArrayFiller());
1698}
1699
1700void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1701 if (E->hadArrayRangeDesignator())
1702 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
1703
1704 if (E->isTransparent())
1705 return Visit(E: E->getInit(Init: 0));
1706
1707 VisitCXXParenListOrInitListExpr(
1708 ExprToVisit: E, Args: E->inits(), InitializedFieldInUnion: E->getInitializedFieldInUnion(), ArrayFiller: E->getArrayFiller());
1709}
1710
1711void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1712 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1713 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1714#if 0
1715 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1716 // (Length of globals? Chunks of zeroed-out space?).
1717 //
1718 // If we can, prefer a copy from a global; this is a lot less code for long
1719 // globals, and it's easier for the current optimizers to analyze.
1720 if (llvm::Constant *C =
1721 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1722 llvm::GlobalVariable* GV =
1723 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1724 llvm::GlobalValue::InternalLinkage, C, "");
1725 EmitFinalDestCopy(ExprToVisit->getType(),
1726 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1727 return;
1728 }
1729#endif
1730
1731 // HLSL initialization lists in the AST are an expansion which can contain
1732 // side-effecting expressions wrapped in opaque value expressions. To properly
1733 // emit these we need to emit the opaque values before we emit the argument
1734 // expressions themselves. This is a little hacky, but it prevents us needing
1735 // to do a bigger AST-level change for a language feature that we need
1736 // deprecate in the near future. See related HLSL language proposals:
1737 // * 0005-strict-initializer-lists.md
1738 // * https://github.com/microsoft/hlsl-specs/pull/325
1739 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(Val: ExprToVisit))
1740 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(
1741 CGF, E: cast<InitListExpr>(Val: ExprToVisit));
1742
1743 AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1744
1745 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1746
1747 // Handle initialization of an array.
1748 if (ExprToVisit->getType()->isConstantArrayType()) {
1749 auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1750 EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1751 Args: InitExprs, ArrayFiller);
1752 return;
1753 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1754 // A variable array type that has an initializer can only do empty
1755 // initialization. And because this feature is not exposed as an extension
1756 // in C++, we can safely memset the array memory to zero.
1757 assert(InitExprs.size() == 0 &&
1758 "you can only use an empty initializer with VLAs");
1759 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1760 return;
1761 }
1762
1763 assert(ExprToVisit->getType()->isRecordType() &&
1764 "Only support structs/unions here!");
1765
1766 // Do struct initialization; this code just sets each individual member
1767 // to the approprate value. This makes bitfield support automatic;
1768 // the disadvantage is that the generated code is more difficult for
1769 // the optimizer, especially with bitfields.
1770 unsigned NumInitElements = InitExprs.size();
1771 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1772
1773 // We'll need to enter cleanup scopes in case any of the element
1774 // initializers throws an exception.
1775 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1776
1777 unsigned curInitIndex = 0;
1778
1779 // Emit initialization of base classes.
1780 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1781 assert(NumInitElements >= CXXRD->getNumBases() &&
1782 "missing initializer for base class");
1783 for (auto &Base : CXXRD->bases()) {
1784 assert(!Base.isVirtual() && "should not see vbases here");
1785 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1786 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1787 Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1788 /*isBaseVirtual*/ BaseIsVirtual: false);
1789 AggValueSlot AggSlot = AggValueSlot::forAddr(
1790 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1791 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1792 mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1793 CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1794
1795 if (QualType::DestructionKind dtorKind =
1796 Base.getType().isDestructedType())
1797 CGF.pushDestroyAndDeferDeactivation(dtorKind, addr: V, type: Base.getType());
1798 }
1799 }
1800
1801 // Prepare a 'this' for CXXDefaultInitExprs.
1802 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1803
1804 const bool ZeroInitPadding =
1805 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1806
1807 if (record->isUnion()) {
1808 // Only initialize one field of a union. The field itself is
1809 // specified by the initializer list.
1810 if (!InitializedFieldInUnion) {
1811 // Empty union; we have nothing to do.
1812
1813#ifndef NDEBUG
1814 // Make sure that it's really an empty and not a failure of
1815 // semantic analysis.
1816 for (const auto *Field : record->fields())
1817 assert(
1818 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1819 "Only unnamed bitfields or anonymous class allowed");
1820#endif
1821 return;
1822 }
1823
1824 // FIXME: volatility
1825 FieldDecl *Field = InitializedFieldInUnion;
1826
1827 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1828 if (NumInitElements) {
1829 // Store the initializer into the field
1830 EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1831 if (ZeroInitPadding) {
1832 uint64_t TotalSize = CGF.getContext().toBits(
1833 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1834 uint64_t FieldSize = CGF.getContext().getTypeSize(T: FieldLoc.getType());
1835 DoZeroInitPadding(PaddingStart&: FieldSize, PaddingEnd: TotalSize, NextField: nullptr);
1836 }
1837 } else {
1838 // Default-initialize to null.
1839 if (ZeroInitPadding)
1840 EmitNullInitializationToLValue(lv: DestLV);
1841 else
1842 EmitNullInitializationToLValue(lv: FieldLoc);
1843 }
1844 return;
1845 }
1846
1847 // Here we iterate over the fields; this makes it simpler to both
1848 // default-initialize fields and skip over unnamed fields.
1849 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(D: record);
1850 uint64_t PaddingStart = 0;
1851
1852 for (const auto *field : record->fields()) {
1853 // We're done once we hit the flexible array member.
1854 if (field->getType()->isIncompleteArrayType())
1855 break;
1856
1857 // Always skip anonymous bitfields.
1858 if (field->isUnnamedBitField())
1859 continue;
1860
1861 // We're done if we reach the end of the explicit initializers, we
1862 // have a zeroed object, and the rest of the fields are
1863 // zero-initializable.
1864 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1865 CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1866 break;
1867
1868 if (ZeroInitPadding)
1869 DoZeroInitPadding(PaddingStart,
1870 PaddingEnd: Layout.getFieldOffset(FieldNo: field->getFieldIndex()), NextField: field);
1871
1872 LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1873 // We never generate write-barries for initialized fields.
1874 LV.setNonGC(true);
1875
1876 if (curInitIndex < NumInitElements) {
1877 // Store the initializer into the field.
1878 EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1879 } else {
1880 // We're out of initializers; default-initialize to null
1881 EmitNullInitializationToLValue(lv: LV);
1882 }
1883
1884 // Push a destructor if necessary.
1885 // FIXME: if we have an array of structures, all explicitly
1886 // initialized, we can end up pushing a linear number of cleanups.
1887 if (QualType::DestructionKind dtorKind =
1888 field->getType().isDestructedType()) {
1889 assert(LV.isSimple());
1890 if (dtorKind) {
1891 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1892 type: field->getType(),
1893 destroyer: CGF.getDestroyer(destructionKind: dtorKind), useEHCleanupForArray: false);
1894 }
1895 }
1896 }
1897 if (ZeroInitPadding) {
1898 uint64_t TotalSize = CGF.getContext().toBits(
1899 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1900 DoZeroInitPadding(PaddingStart, PaddingEnd: TotalSize, NextField: nullptr);
1901 }
1902}
1903
1904void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1905 uint64_t PaddingEnd,
1906 const FieldDecl *NextField) {
1907
1908 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1909 CharUnits Start = CGF.getContext().toCharUnitsFromBits(BitSize: StartBit);
1910 CharUnits End = CGF.getContext().toCharUnitsFromBits(BitSize: EndBit);
1911 Address Addr = Dest.getAddress().withElementType(ElemTy: CGF.CharTy);
1912 if (!Start.isZero())
1913 Addr = Builder.CreateConstGEP(Addr, Index: Start.getQuantity());
1914 llvm::Constant *SizeVal = Builder.getInt64(C: (End - Start).getQuantity());
1915 CGF.Builder.CreateMemSet(Dest: Addr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
1916 };
1917
1918 if (NextField != nullptr && NextField->isBitField()) {
1919 // For bitfield, zero init StorageSize before storing the bits. So we don't
1920 // need to handle big/little endian.
1921 const CGRecordLayout &RL =
1922 CGF.getTypes().getCGRecordLayout(NextField->getParent());
1923 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: NextField);
1924 uint64_t StorageStart = CGF.getContext().toBits(CharSize: Info.StorageOffset);
1925 if (StorageStart + Info.StorageSize > PaddingStart) {
1926 if (StorageStart > PaddingStart)
1927 InitBytes(PaddingStart, StorageStart);
1928 Address Addr = Dest.getAddress();
1929 if (!Info.StorageOffset.isZero())
1930 Addr = Builder.CreateConstGEP(Addr: Addr.withElementType(ElemTy: CGF.CharTy),
1931 Index: Info.StorageOffset.getQuantity());
1932 Addr = Addr.withElementType(
1933 ElemTy: llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: Info.StorageSize));
1934 Builder.CreateStore(Val: Builder.getIntN(N: Info.StorageSize, C: 0), Addr);
1935 PaddingStart = StorageStart + Info.StorageSize;
1936 }
1937 return;
1938 }
1939
1940 if (PaddingStart < PaddingEnd)
1941 InitBytes(PaddingStart, PaddingEnd);
1942 if (NextField != nullptr)
1943 PaddingStart =
1944 PaddingEnd + CGF.getContext().getTypeSize(T: NextField->getType());
1945}
1946
1947void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1948 llvm::Value *outerBegin) {
1949 // Emit the common subexpression.
1950 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1951
1952 Address destPtr = EnsureSlot(T: E->getType()).getAddress();
1953 uint64_t numElements = E->getArraySize().getZExtValue();
1954
1955 if (!numElements)
1956 return;
1957
1958 // destPtr is an array*. Construct an elementType* by drilling down a level.
1959 llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1960 llvm::Value *indices[] = {zero, zero};
1961 llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
1962 Ptr: destPtr.emitRawPointer(CGF),
1963 IdxList: indices, Name: "arrayinit.begin");
1964
1965 // Prepare to special-case multidimensional array initialization: we avoid
1966 // emitting multiple destructor loops in that case.
1967 if (!outerBegin)
1968 outerBegin = begin;
1969 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
1970
1971 QualType elementType =
1972 CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
1973 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
1974 CharUnits elementAlign =
1975 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1976 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
1977
1978 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1979 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
1980
1981 // Jump into the body.
1982 CGF.EmitBlock(BB: bodyBB);
1983 llvm::PHINode *index =
1984 Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
1985 index->addIncoming(V: zero, BB: entryBB);
1986 llvm::Value *element =
1987 Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
1988
1989 if (CGF.CGM.shouldEmitConvergenceTokens())
1990 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
1991
1992 // Prepare for a cleanup.
1993 QualType::DestructionKind dtorKind = elementType.isDestructedType();
1994 EHScopeStack::stable_iterator cleanup;
1995 if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
1996 if (outerBegin->getType() != element->getType())
1997 outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
1998 CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
1999 elementAlignment: elementAlign,
2000 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
2001 cleanup = CGF.EHStack.stable_begin();
2002 } else {
2003 dtorKind = QualType::DK_none;
2004 }
2005
2006 // Emit the actual filler expression.
2007 {
2008 // Temporaries created in an array initialization loop are destroyed
2009 // at the end of each iteration.
2010 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2011 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2012 LValue elementLV = CGF.MakeAddrLValue(
2013 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
2014
2015 if (InnerLoop) {
2016 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2017 auto elementSlot = AggValueSlot::forLValue(
2018 LV: elementLV, isDestructed: AggValueSlot::IsDestructed,
2019 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2020 mayOverlap: AggValueSlot::DoesNotOverlap);
2021 AggExprEmitter(CGF, elementSlot, false)
2022 .VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
2023 } else
2024 EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
2025 }
2026
2027 // Move on to the next element.
2028 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2029 LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
2030 index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
2031
2032 // Leave the loop if we're done.
2033 llvm::Value *done = Builder.CreateICmpEQ(
2034 LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
2035 Name: "arrayinit.done");
2036 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
2037 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
2038
2039 if (CGF.CGM.shouldEmitConvergenceTokens())
2040 CGF.ConvergenceTokenStack.pop_back();
2041
2042 CGF.EmitBlock(BB: endBB);
2043
2044 // Leave the partial-array cleanup if we entered one.
2045 if (dtorKind)
2046 CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
2047}
2048
2049void AggExprEmitter::VisitDesignatedInitUpdateExpr(
2050 DesignatedInitUpdateExpr *E) {
2051 AggValueSlot Dest = EnsureSlot(T: E->getType());
2052
2053 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
2054 EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
2055 VisitInitListExpr(E: E->getUpdater());
2056}
2057
2058//===----------------------------------------------------------------------===//
2059// Entry Points into this File
2060//===----------------------------------------------------------------------===//
2061
2062/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2063/// non-zero bytes that will be stored when outputting the initializer for the
2064/// specified initializer expression.
2065static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
2066 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
2067 E = MTE->getSubExpr();
2068 E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
2069
2070 // 0 and 0.0 won't require any non-zero stores!
2071 if (isSimpleZero(E, CGF))
2072 return CharUnits::Zero();
2073
2074 // If this is an initlist expr, sum up the size of sizes of the (present)
2075 // elements. If this is something weird, assume the whole thing is non-zero.
2076 const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
2077 while (ILE && ILE->isTransparent())
2078 ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
2079 if (!ILE || !CGF.getTypes().isZeroInitializable(T: ILE->getType()))
2080 return CGF.getContext().getTypeSizeInChars(T: E->getType());
2081
2082 // InitListExprs for structs have to be handled carefully. If there are
2083 // reference members, we need to consider the size of the reference, not the
2084 // referencee. InitListExprs for unions and arrays can't have references.
2085 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2086 if (!RT->isUnionType()) {
2087 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2088 CharUnits NumNonZeroBytes = CharUnits::Zero();
2089
2090 unsigned ILEElement = 0;
2091 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
2092 while (ILEElement != CXXRD->getNumBases())
2093 NumNonZeroBytes +=
2094 GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
2095 for (const auto *Field : SD->fields()) {
2096 // We're done once we hit the flexible array member or run out of
2097 // InitListExpr elements.
2098 if (Field->getType()->isIncompleteArrayType() ||
2099 ILEElement == ILE->getNumInits())
2100 break;
2101 if (Field->isUnnamedBitField())
2102 continue;
2103
2104 const Expr *E = ILE->getInit(Init: ILEElement++);
2105
2106 // Reference values are always non-null and have the width of a pointer.
2107 if (Field->getType()->isReferenceType())
2108 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2109 BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2110 else
2111 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2112 }
2113
2114 return NumNonZeroBytes;
2115 }
2116 }
2117
2118 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2119 CharUnits NumNonZeroBytes = CharUnits::Zero();
2120 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2121 NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2122 return NumNonZeroBytes;
2123}
2124
2125/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2126/// zeros in it, emit a memset and avoid storing the individual zeros.
2127///
2128static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2129 CodeGenFunction &CGF) {
2130 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2131 // volatile stores.
2132 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2133 return;
2134
2135 // C++ objects with a user-declared constructor don't need zero'ing.
2136 if (CGF.getLangOpts().CPlusPlus)
2137 if (const RecordType *RT = CGF.getContext()
2138 .getBaseElementType(QT: E->getType())
2139 ->getAsCanonical<RecordType>()) {
2140 const auto *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2141 if (RD->hasUserDeclaredConstructor())
2142 return;
2143 }
2144
2145 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2146 CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2147 if (Size <= CharUnits::fromQuantity(Quantity: 16))
2148 return;
2149
2150 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2151 // we prefer to emit memset + individual stores for the rest.
2152 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2153 if (NumNonZeroBytes * 4 > Size)
2154 return;
2155
2156 // Okay, it seems like a good idea to use an initial memset, emit the call.
2157 llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2158
2159 Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2160 CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2161
2162 // Tell the AggExprEmitter that the slot is known zero.
2163 Slot.setZeroed();
2164}
2165
2166/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2167/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2168/// the value of the aggregate expression is not needed. If VolatileDest is
2169/// true, DestPtr cannot be 0.
2170void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2171 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2172 "Invalid aggregate expression to emit");
2173 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2174 "slot has bits but no address");
2175
2176 // Optimize the slot if possible.
2177 CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2178
2179 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr *>(E));
2180}
2181
2182LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2183 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2184 Address Temp = CreateMemTempWithoutCast(T: E->getType());
2185 LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2186 EmitAggExpr(E, Slot: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsNotDestructed,
2187 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2188 isAliased: AggValueSlot::IsNotAliased,
2189 mayOverlap: AggValueSlot::DoesNotOverlap));
2190 return LV;
2191}
2192
2193void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,
2194 const LValue &Src,
2195 ExprValueKind SrcKind) {
2196 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2197 .EmitFinalDestCopy(type: Type, src: Src, SrcValueKind: SrcKind);
2198}
2199
2200AggValueSlot::Overlap_t
2201CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2202 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2203 return AggValueSlot::DoesNotOverlap;
2204
2205 // Empty fields can overlap earlier fields.
2206 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2207 return AggValueSlot::MayOverlap;
2208
2209 // If the field lies entirely within the enclosing class's nvsize, its tail
2210 // padding cannot overlap any already-initialized object. (The only subobjects
2211 // with greater addresses that might already be initialized are vbases.)
2212 const RecordDecl *ClassRD = FD->getParent();
2213 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2214 if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2215 getContext().getTypeSize(T: FD->getType()) <=
2216 (uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2217 return AggValueSlot::DoesNotOverlap;
2218
2219 // The tail padding may contain values we need to preserve.
2220 return AggValueSlot::MayOverlap;
2221}
2222
2223AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2224 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2225 // If the most-derived object is a field declared with [[no_unique_address]],
2226 // the tail padding of any virtual base could be reused for other subobjects
2227 // of that field's class.
2228 if (IsVirtual)
2229 return AggValueSlot::MayOverlap;
2230
2231 // Empty bases can overlap earlier bases.
2232 if (BaseRD->isEmpty())
2233 return AggValueSlot::MayOverlap;
2234
2235 // If the base class is laid out entirely within the nvsize of the derived
2236 // class, its tail padding cannot yet be initialized, so we can issue
2237 // stores at the full width of the base class.
2238 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2239 if (Layout.getBaseClassOffset(Base: BaseRD) +
2240 getContext().getASTRecordLayout(D: BaseRD).getSize() <=
2241 Layout.getNonVirtualSize())
2242 return AggValueSlot::DoesNotOverlap;
2243
2244 // The tail padding may contain values we need to preserve.
2245 return AggValueSlot::MayOverlap;
2246}
2247
2248void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2249 AggValueSlot::Overlap_t MayOverlap,
2250 bool isVolatile) {
2251 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2252
2253 Address DestPtr = Dest.getAddress();
2254 Address SrcPtr = Src.getAddress();
2255
2256 if (getLangOpts().CPlusPlus) {
2257 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2258 assert((Record->hasTrivialCopyConstructorForCall() ||
2259 Record->hasTrivialCopyAssignment() ||
2260 Record->hasTrivialMoveConstructorForCall() ||
2261 Record->hasTrivialMoveAssignment() || Record->isUnion() ||
2262 // HLSL uses aggregate-copy for user-defined record types.
2263 (getLangOpts().HLSL && !Record->isHLSLBuiltinRecord())) &&
2264 "Trying to aggregate-copy a type without a trivial copy/move "
2265 "constructor or assignment operator");
2266 // Ignore empty classes in C++.
2267 if (Record->isEmpty())
2268 return;
2269 }
2270 }
2271
2272 if (getLangOpts().CUDAIsDevice) {
2273 if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2274 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2275 Src))
2276 return;
2277 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2278 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2279 Src))
2280 return;
2281 }
2282 }
2283
2284 assert(Ty.getAddressSpace() != LangAS::hlsl_constant &&
2285 "copies of aggregates in hlsl_constant address space should be "
2286 "handled earlier by the HLSL runtime");
2287
2288 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2289 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2290 // read from another object that overlaps in anyway the storage of the first
2291 // object, then the overlap shall be exact and the two objects shall have
2292 // qualified or unqualified versions of a compatible type."
2293 //
2294 // memcpy is not defined if the source and destination pointers are exactly
2295 // equal, but other compilers do this optimization, and almost every memcpy
2296 // implementation handles this case safely. If there is a libc that does not
2297 // safely handle this, we can add a target hook.
2298
2299 // Get data size info for this aggregate. Don't copy the tail padding if this
2300 // might be a potentially-overlapping subobject, since the tail padding might
2301 // be occupied by a different object. Otherwise, copying it is fine.
2302 TypeInfoChars TypeInfo;
2303 if (MayOverlap)
2304 TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2305 else
2306 TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2307
2308 llvm::Value *SizeVal = nullptr;
2309 if (TypeInfo.Width.isZero()) {
2310 // But note that getTypeInfo returns 0 for a VLA.
2311 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2312 Val: getContext().getAsArrayType(T: Ty))) {
2313 QualType BaseEltTy;
2314 SizeVal = emitArrayLength(arrayType: VAT, baseType&: BaseEltTy, addr&: DestPtr);
2315 TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2316 assert(!TypeInfo.Width.isZero());
2317 SizeVal = Builder.CreateNUWMul(
2318 LHS: SizeVal,
2319 RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2320 }
2321 }
2322 if (!SizeVal) {
2323 SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2324 }
2325
2326 // FIXME: If we have a volatile struct, the optimizer can remove what might
2327 // appear to be `extra' memory ops:
2328 //
2329 // volatile struct { int i; } a, b;
2330 //
2331 // int main() {
2332 // a = b;
2333 // a = b;
2334 // }
2335 //
2336 // we need to use a different call here. We use isVolatile to indicate when
2337 // either the source or the destination is volatile.
2338
2339 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2340 SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2341
2342 // Don't do any of the memmove_collectable tests if GC isn't set.
2343 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2344 // fall through
2345 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2346 if (Record->hasObjectMember()) {
2347 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2348 Size: SizeVal);
2349 return;
2350 }
2351 } else if (Ty->isArrayType()) {
2352 QualType BaseType = getContext().getBaseElementType(QT: Ty);
2353 if (const auto *Record = BaseType->getAsRecordDecl()) {
2354 if (Record->hasObjectMember()) {
2355 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2356 Size: SizeVal);
2357 return;
2358 }
2359 }
2360 }
2361
2362 auto *Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2363 addInstToCurrentSourceAtom(KeyInstruction: Inst, Backup: nullptr);
2364 emitPFPPostCopyUpdates(DestPtr, SrcPtr, Ty);
2365
2366 // Determine the metadata to describe the position of any padding in this
2367 // memcpy, as well as the TBAA tags for the members of the struct, in case
2368 // the optimizer wishes to expand it in to scalar memory operations.
2369 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2370 Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2371
2372 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2373 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2374 DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2375 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2376 }
2377}
2378