1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
23#include "clang/AST/ASTContext.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
26#include "clang/AST/DeclTemplate.h"
27#include "clang/AST/StmtVisitor.h"
28#include "clang/CodeGenUtils/ExprUtils.h"
29#include "llvm/IR/Constants.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/GlobalVariable.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35using namespace clang;
36using namespace CodeGen;
37
38//===----------------------------------------------------------------------===//
39// Aggregate Expression Emitter
40//===----------------------------------------------------------------------===//
41
42namespace {
43class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
44 CodeGenFunction &CGF;
45 CGBuilderTy &Builder;
46 AggValueSlot Dest;
47 bool IsResultUnused;
48
49 AggValueSlot EnsureSlot(QualType T) {
50 if (!Dest.isIgnored())
51 return Dest;
52 return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
53 }
54 void EnsureDest(QualType T) {
55 if (!Dest.isIgnored())
56 return;
57 Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
58 }
59
60 // Calls `Fn` with a valid return value slot, potentially creating a temporary
61 // to do so. If a temporary is created, an appropriate copy into `Dest` will
62 // be emitted, as will lifetime markers.
63 //
64 // The given function should take a ReturnValueSlot, and return an RValue that
65 // points to said slot.
66 void withReturnValueSlot(const Expr *E,
67 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
68
69 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
70 const FieldDecl *NextField);
71
72public:
73 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
74 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
75 IsResultUnused(IsResultUnused) {}
76
77 //===--------------------------------------------------------------------===//
78 // Utilities
79 //===--------------------------------------------------------------------===//
80
81 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
82 /// represents a value lvalue, this method emits the address of the lvalue,
83 /// then loads the result into DestPtr.
84 void EmitAggLoadOfLValue(const Expr *E);
85
86 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
87 /// SrcIsRValue is true if source comes from an RValue.
88 void EmitFinalDestCopy(QualType type, const LValue &src,
89 CodeGenFunction::ExprValueKind SrcValueKind =
90 CodeGenFunction::EVK_NonRValue);
91 void EmitFinalDestCopy(QualType type, RValue src);
92 void EmitCopy(QualType type, const AggValueSlot &dest,
93 const AggValueSlot &src);
94
95 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
96 Expr *ExprToVisit, ArrayRef<Expr *> Args,
97 Expr *ArrayFiller);
98
99 void EmitComparisonResult(const Expr *E,
100 const ComparisonCategoryInfo &CmpInfo,
101 llvm::Value *ResultValue);
102
103 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
104 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
105 return AggValueSlot::NeedsGCBarriers;
106 return AggValueSlot::DoesNotNeedGCBarriers;
107 }
108
109 bool TypeRequiresGCollection(QualType T);
110
111 //===--------------------------------------------------------------------===//
112 // Visitor Methods
113 //===--------------------------------------------------------------------===//
114
115 void Visit(Expr *E) {
116 ApplyDebugLocation DL(CGF, E);
117 StmtVisitor<AggExprEmitter>::Visit(S: E);
118 }
119
120 void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, Type: "aggregate expression"); }
121 void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
122 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
123 Visit(E: GE->getResultExpr());
124 }
125 void VisitCoawaitExpr(CoawaitExpr *E) {
126 CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
127 }
128 void VisitCoyieldExpr(CoyieldExpr *E) {
129 CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
130 }
131 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
132 void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
133 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
134 return Visit(E: E->getReplacement());
135 }
136
137 void VisitConstantExpr(ConstantExpr *E) {
138 EnsureDest(T: E->getType());
139
140 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
141 CGF.CreateCoercedStore(
142 Src: Result, SrcFETy: E->getType(), Dst: Dest.getAddress(),
143 DstSize: llvm::TypeSize::getFixed(
144 ExactSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType())
145 .getQuantity()),
146 DstIsVolatile: E->getType().isVolatileQualified());
147 return;
148 }
149 return Visit(E: E->getSubExpr());
150 }
151
152 // l-values.
153 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
154 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(E: ME); }
155 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
156 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
157 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
158 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
159 EmitAggLoadOfLValue(E);
160 }
161 void VisitPredefinedExpr(const PredefinedExpr *E) { EmitAggLoadOfLValue(E); }
162
163 // Operators.
164 void VisitCastExpr(CastExpr *E);
165 void VisitCallExpr(const CallExpr *E);
166 void VisitStmtExpr(const StmtExpr *E);
167 void VisitBinaryOperator(const BinaryOperator *BO);
168 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
169 void VisitBinAssign(const BinaryOperator *E);
170 void VisitBinComma(const BinaryOperator *E);
171 void VisitBinCmp(const BinaryOperator *E);
172 void VisitTypeTraitExpr(const TypeTraitExpr *E);
173 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
174 Visit(E: E->getSemanticForm());
175 }
176
177 void VisitObjCMessageExpr(ObjCMessageExpr *E);
178 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { EmitAggLoadOfLValue(E); }
179
180 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
181 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
182 void VisitChooseExpr(const ChooseExpr *CE);
183 void VisitInitListExpr(InitListExpr *E);
184 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
185 FieldDecl *InitializedFieldInUnion,
186 Expr *ArrayFiller);
187 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
188 llvm::Value *outerBegin = nullptr);
189 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
190 void VisitNoInitExpr(NoInitExpr *E) {} // Do nothing.
191 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
192 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
193 Visit(E: DAE->getExpr());
194 }
195 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
196 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
197 Visit(E: DIE->getExpr());
198 }
199 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
200 void VisitCXXConstructExpr(const CXXConstructExpr *E);
201 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
202 void VisitLambdaExpr(LambdaExpr *E);
203 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
204 void VisitExprWithCleanups(ExprWithCleanups *E);
205 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
206 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
207 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
208 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
209
210 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
211 if (E->isGLValue()) {
212 LValue LV = CGF.EmitPseudoObjectLValue(e: E);
213 return EmitFinalDestCopy(type: E->getType(), src: LV);
214 }
215
216 AggValueSlot Slot = EnsureSlot(T: E->getType());
217 bool NeedsDestruction =
218 !Slot.isExternallyDestructed() &&
219 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
220 if (NeedsDestruction)
221 Slot.setExternallyDestructed();
222 CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
223 if (NeedsDestruction)
224 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Slot.getAddress(),
225 type: E->getType());
226 }
227
228 void VisitVAArgExpr(VAArgExpr *E);
229 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
230
231 void EmitInitializationToLValue(Expr *E, LValue Address);
232 void EmitNullInitializationToLValue(LValue Address);
233 // case Expr::ChooseExprClass:
234 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
235 void VisitAtomicExpr(AtomicExpr *E) {
236 RValue Res = CGF.EmitAtomicExpr(E);
237 EmitFinalDestCopy(type: E->getType(), src: Res);
238 }
239 void VisitPackIndexingExpr(PackIndexingExpr *E) {
240 Visit(E: E->getSelectedExpr());
241 }
242};
243} // end anonymous namespace.
244
245//===----------------------------------------------------------------------===//
246// Utilities
247//===----------------------------------------------------------------------===//
248
249/// EmitAggLoadOfLValue - Given an expression with aggregate type that
250/// represents a value lvalue, this method emits the address of the lvalue,
251/// then loads the result into DestPtr.
252void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
253 LValue LV = CGF.EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load);
254
255 // If the type of the l-value is atomic, then do an atomic load.
256 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
257 CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
258 return;
259 }
260
261 if (E->getType().getAddressSpace() == LangAS::hlsl_constant)
262 if (CGF.CGM.getHLSLRuntime().emitBufferCopy(CGF, E, SrcLV: LV, DestSlot&: Dest))
263 return;
264
265 EmitFinalDestCopy(type: E->getType(), src: LV);
266}
267
268/// True if the given aggregate type requires special GC API calls.
269bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
270 // Only record types have members that might require garbage collection.
271 const auto *Record = T->getAsRecordDecl();
272 if (!Record)
273 return false;
274
275 // Don't mess with non-trivial C++ types.
276 if (isa<CXXRecordDecl>(Val: Record) &&
277 (cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
278 !cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
279 return false;
280
281 // Check whether the type has an object member.
282 return Record->hasObjectMember();
283}
284
285void AggExprEmitter::withReturnValueSlot(
286 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
287 QualType RetTy = E->getType();
288 bool RequiresDestruction =
289 !Dest.isExternallyDestructed() &&
290 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
291
292 // If it makes no observable difference, save a memcpy + temporary.
293 //
294 // We need to always provide our own temporary if destruction is required.
295 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
296 // its lifetime before we have the chance to emit a proper destructor call.
297 //
298 // We also need a temporary if the destination is in a different address space
299 // from the sret AS. Use the target hook to get the actual sret AS for this
300 // return type.
301 const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl();
302 LangAS SRetLangAS = CGF.CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
303 unsigned SRetAS = CGF.getContext().getTargetAddressSpace(AS: SRetLangAS);
304 bool CanAggregateCopy =
305 RD ? (RD->hasTrivialCopyConstructor() ||
306 RD->hasTrivialMoveConstructor() || RD->hasTrivialCopyAssignment() ||
307 RD->hasTrivialMoveAssignment() || RD->hasAttr<TrivialABIAttr>() ||
308 RD->isUnion())
309 : RetTy.isTriviallyCopyableType(Context: CGF.getContext());
310 bool DestASMismatch = !Dest.isIgnored() && CanAggregateCopy &&
311 Dest.getAddress()
312 .getBasePointer()
313 ->stripPointerCasts()
314 ->getType()
315 ->getPointerAddressSpace() != SRetAS;
316 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
317 (RequiresDestruction && Dest.isIgnored()) || DestASMismatch;
318
319 Address RetAddr = Address::invalid();
320
321 EHScopeStack::stable_iterator LifetimeEndBlock;
322 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
323 if (!UseTemp) {
324 RetAddr = Dest.getAddress();
325 if (RetAddr.isValid() && RetAddr.getAddressSpace() != SRetAS) {
326 llvm::Type *SRetPtrTy =
327 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: SRetAS);
328 RetAddr = RetAddr.withPointer(
329 NewPointer: CGF.performAddrSpaceCast(Src: RetAddr.getBasePointer(), DestTy: SRetPtrTy),
330 IsKnownNonNull: RetAddr.isKnownNonNull());
331 }
332 } else {
333 RetAddr = CGF.CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
334 if (CGF.EmitLifetimeStart(Addr: RetAddr.getBasePointer())) {
335 LifetimeStartInst =
336 cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
337 assert(LifetimeStartInst->getIntrinsicID() ==
338 llvm::Intrinsic::lifetime_start &&
339 "Last insertion wasn't a lifetime.start?");
340
341 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
342 kind: NormalEHLifetimeMarker, A: RetAddr);
343 LifetimeEndBlock = CGF.EHStack.stable_begin();
344 }
345 }
346
347 RValue Src =
348 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
349 Dest.isExternallyDestructed()));
350
351 if (!UseTemp)
352 return;
353
354 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
355 Src.getAggregatePointer(E->getType(), CGF));
356 EmitFinalDestCopy(type: E->getType(), src: Src);
357
358 if (!RequiresDestruction && LifetimeStartInst) {
359 // If there's no dtor to run, the copy was the last use of our temporary.
360 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
361 // eagerly.
362 CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
363 CGF.EmitLifetimeEnd(Addr: RetAddr.getBasePointer());
364 }
365}
366
367/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
368void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
369 assert(src.isAggregate() && "value must be aggregate value!");
370 LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
371 EmitFinalDestCopy(type, src: srcLV, SrcValueKind: CodeGenFunction::EVK_RValue);
372}
373
374/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
375void AggExprEmitter::EmitFinalDestCopy(
376 QualType type, const LValue &src,
377 CodeGenFunction::ExprValueKind SrcValueKind) {
378 // If Dest is ignored, then we're evaluating an aggregate expression
379 // in a context that doesn't care about the result. Note that loads
380 // from volatile l-values force the existence of a non-ignored
381 // destination.
382 if (Dest.isIgnored())
383 return;
384
385 // Copy non-trivial C structs here.
386 LValue DstLV = CGF.MakeAddrLValue(
387 Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
388
389 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
390 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
391 if (Dest.isPotentiallyAliased())
392 CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
393 else
394 CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
395 return;
396 }
397 } else {
398 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
399 if (Dest.isPotentiallyAliased())
400 CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
401 else
402 CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
403 return;
404 }
405 }
406
407 AggValueSlot srcAgg = AggValueSlot::forLValue(
408 LV: src, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type), isAliased: AggValueSlot::IsAliased,
409 mayOverlap: AggValueSlot::MayOverlap);
410 EmitCopy(type, dest: Dest, src: srcAgg);
411}
412
413/// Perform a copy from the source into the destination.
414///
415/// \param type - the type of the aggregate being copied; qualifiers are
416/// ignored
417void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
418 const AggValueSlot &src) {
419 if (dest.requiresGCollection()) {
420 CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
421 llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
422 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr: dest.getAddress(),
423 SrcPtr: src.getAddress(), Size: size);
424 return;
425 }
426
427 // If the result of the assignment is used, copy the LHS there also.
428 // It's volatile if either side is. Use the minimum alignment of
429 // the two sides.
430 LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
431 LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
432 CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
433 isVolatile: dest.isVolatile() || src.isVolatile());
434}
435
436/// Emit the initializer for a std::initializer_list initialized with a
437/// real initializer list.
438void AggExprEmitter::VisitCXXStdInitializerListExpr(
439 CXXStdInitializerListExpr *E) {
440 // Emit an array containing the elements. The array is externally destructed
441 // if the std::initializer_list object is.
442 ASTContext &Ctx = CGF.getContext();
443 LValue Array = CGF.EmitLValue(E: E->getSubExpr());
444 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
445 Address ArrayPtr = Array.getAddress();
446
447 const ConstantArrayType *ArrayType =
448 Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
449 assert(ArrayType && "std::initializer_list constructed from non-array");
450
451 auto *Record = E->getType()->castAsRecordDecl();
452 RecordDecl::field_iterator Field = Record->field_begin();
453 assert(Field != Record->field_end() &&
454 Ctx.hasSameType(Field->getType()->getPointeeType(),
455 ArrayType->getElementType()) &&
456 "Expected std::initializer_list first field to be const E *");
457
458 // Start pointer.
459 AggValueSlot Dest = EnsureSlot(T: E->getType());
460 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
461 LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
462 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
463 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
464 ++Field;
465 assert(Field != Record->field_end() &&
466 "Expected std::initializer_list to have two fields");
467
468 llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
469 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
470 if (Ctx.hasSameType(T1: Field->getType(), T2: Ctx.getSizeType())) {
471 // Length.
472 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
473
474 } else {
475 // End pointer.
476 assert(Field->getType()->isPointerType() &&
477 Ctx.hasSameType(Field->getType()->getPointeeType(),
478 ArrayType->getElementType()) &&
479 "Expected std::initializer_list second field to be const E *");
480 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
481 llvm::Value *IdxEnd[] = {Zero, Size};
482 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
483 Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
484 Name: "arrayend");
485 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
486 }
487
488 assert(++Field == Record->field_end() &&
489 "Expected std::initializer_list to only have two fields");
490}
491
492// emit an elementwise cast where the RHS is a scalar or vector
493// or emit an aggregate splat cast
494static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,
495 LValue DestVal,
496 llvm::Value *SrcVal,
497 QualType SrcTy,
498 SourceLocation Loc) {
499 // Flatten our destination
500 SmallVector<LValue, 16> StoreList;
501 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
502
503 bool isVector = false;
504 if (auto *VT = SrcTy->getAs<VectorType>()) {
505 isVector = true;
506 SrcTy = VT->getElementType();
507 assert(StoreList.size() <= VT->getNumElements() &&
508 "Cannot perform HLSL flat cast when vector source \
509 object has less elements than flattened destination \
510 object.");
511 }
512
513 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
514 LValue DestLVal = StoreList[I];
515 llvm::Value *Load =
516 isVector ? CGF.Builder.CreateExtractElement(Vec: SrcVal, Idx: I, Name: "vec.load")
517 : SrcVal;
518 llvm::Value *Cast =
519 CGF.EmitScalarConversion(Src: Load, SrcTy, DstTy: DestLVal.getType(), Loc);
520 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
521 }
522}
523
524// emit a flat cast where the RHS is an aggregate
525static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
526 LValue SrcVal, SourceLocation Loc) {
527 // Flatten our destination
528 SmallVector<LValue, 16> StoreList;
529 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
530 // Flatten our src
531 SmallVector<LValue, 16> LoadList;
532 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
533
534 assert(StoreList.size() <= LoadList.size() &&
535 "Cannot perform HLSL elementwise cast when flattened source object \
536 has less elements than flattened destination object.");
537 // apply casts to what we load from LoadList
538 // and store result in Dest
539 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
540 LValue DestLVal = StoreList[I];
541 LValue SrcLVal = LoadList[I];
542 RValue RVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc);
543 assert(RVal.isScalar() && "All flattened source values should be scalars");
544 llvm::Value *Val = RVal.getScalarVal();
545 llvm::Value *Cast = CGF.EmitScalarConversion(Src: Val, SrcTy: SrcLVal.getType(),
546 DstTy: DestLVal.getType(), Loc);
547 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
548 }
549}
550
551/// Emit initialization of an array from an initializer list. ExprToVisit must
552/// be either an InitListEpxr a CXXParenInitListExpr.
553void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
554 QualType ArrayQTy, Expr *ExprToVisit,
555 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
556 uint64_t NumInitElements = Args.size();
557
558 uint64_t NumArrayElements = AType->getNumElements();
559 for (const auto *Init : Args) {
560 if (const auto *Embed = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
561 NumInitElements += Embed->getDataElementCount() - 1;
562 if (NumInitElements > NumArrayElements) {
563 NumInitElements = NumArrayElements;
564 break;
565 }
566 }
567 }
568
569 assert(NumInitElements <= NumArrayElements);
570
571 QualType elementType =
572 CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
573 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
574 CharUnits elementAlign =
575 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
576 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
577
578 // Consider initializing the array by copying from a global. For this to be
579 // more efficient than per-element initialization, the size of the elements
580 // with explicit initializers should be large enough.
581 if (NumInitElements * elementSize.getQuantity() > 16 &&
582 elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
583 CodeGen::CodeGenModule &CGM = CGF.CGM;
584 ConstantEmitter Emitter(CGF);
585 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
586 T: CGM.getContext().removeAddrSpaceQualType(T: ArrayQTy),
587 AddressSpace: CGM.GetGlobalConstantAddressSpace());
588 LangAS AS = GVArrayQTy.getAddressSpace();
589 if (llvm::Constant *C =
590 Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: GVArrayQTy)) {
591 auto GV = new llvm::GlobalVariable(
592 CGM.getModule(), C->getType(),
593 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
594 "constinit",
595 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
596 CGM.getContext().getTargetAddressSpace(AS));
597 Emitter.finalize(global: GV);
598 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: GVArrayQTy);
599 GV->setAlignment(Align.getAsAlign());
600 Address GVAddr(GV, GV->getValueType(), Align);
601 EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: GVArrayQTy));
602 return;
603 }
604 }
605
606 // Exception safety requires us to destroy all the
607 // already-constructed members if an initializer throws.
608 // For that, we'll need an EH cleanup.
609 QualType::DestructionKind dtorKind = elementType.isDestructedType();
610 Address endOfInit = Address::invalid();
611 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
612
613 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
614 if (dtorKind) {
615 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
616 // In principle we could tell the cleanup where we are more
617 // directly, but the control flow can get so varied here that it
618 // would actually be quite complex. Therefore we go through an
619 // alloca.
620 llvm::Instruction *dominatingIP =
621 Builder.CreateFlagLoad(Addr: llvm::ConstantInt::getNullValue(Ty: CGF.Int8PtrTy));
622 endOfInit = CGF.CreateTempAlloca(Ty: begin->getType(), align: CGF.getPointerAlign(),
623 Name: "arrayinit.endOfInit");
624 Builder.CreateStore(Val: begin, Addr: endOfInit);
625 CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
626 elementAlignment: elementAlign,
627 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
628 cast<EHCleanupScope>(Val&: *CGF.EHStack.find(sp: CGF.EHStack.stable_begin()))
629 .AddAuxAllocas(Allocas: allocaTracker.Take());
630
631 CGF.DeferredDeactivationCleanupStack.push_back(
632 Elt: {.Cleanup: CGF.EHStack.stable_begin(), .DominatingIP: dominatingIP});
633 }
634
635 llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
636
637 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
638 llvm::Value *element = begin;
639 if (ArrayIndex > 0) {
640 if (CGF.getLangOpts().EmitLogicalPointer)
641 element = Builder.CreateStructuredGEP(
642 BaseType: AType, PtrBase: begin, Indices: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
643 Name: "arrayinit.element");
644 else
645 element = Builder.CreateInBoundsGEP(
646 Ty: llvmElementType, Ptr: begin,
647 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
648 Name: "arrayinit.element");
649
650 // Tell the cleanup that it needs to destroy up to this
651 // element. TODO: some of these stores can be trivially
652 // observed to be unnecessary.
653 if (endOfInit.isValid())
654 Builder.CreateStore(Val: element, Addr: endOfInit);
655 }
656
657 LValue elementLV = CGF.MakeAddrLValue(
658 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
659 EmitInitializationToLValue(E: Init, Address: elementLV);
660 return true;
661 };
662
663 unsigned ArrayIndex = 0;
664 // Emit the explicit initializers.
665 for (uint64_t i = 0; i != NumInitElements; ++i) {
666 if (ArrayIndex >= NumInitElements)
667 break;
668 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Args[i]->IgnoreParenImpCasts())) {
669 EmbedS->doForEachDataElement(C&: Emit, StartingIndexInArray&: ArrayIndex);
670 } else {
671 Emit(Args[i], ArrayIndex);
672 ArrayIndex++;
673 }
674 }
675
676 // Check whether there's a non-trivial array-fill expression.
677 bool hasTrivialFiller = CodeGenUtils::isTrivialFiller(E: ArrayFiller);
678
679 // Any remaining elements need to be zero-initialized, possibly
680 // using the filler expression. We can skip this if the we're
681 // emitting to zeroed memory.
682 if (NumInitElements != NumArrayElements &&
683 !(Dest.isZeroed() && hasTrivialFiller &&
684 CGF.getTypes().isZeroInitializable(T: elementType))) {
685
686 // Use an actual loop. This is basically
687 // do { *array++ = filler; } while (array != end);
688
689 // Advance to the start of the rest of the array.
690 llvm::Value *element = begin;
691 if (NumInitElements) {
692 element = Builder.CreateInBoundsGEP(
693 Ty: llvmElementType, Ptr: element,
694 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumInitElements),
695 Name: "arrayinit.start");
696 if (endOfInit.isValid())
697 Builder.CreateStore(Val: element, Addr: endOfInit);
698 }
699
700 // Compute the end of the array.
701 llvm::Value *end = Builder.CreateInBoundsGEP(
702 Ty: llvmElementType, Ptr: begin,
703 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
704
705 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
706 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
707
708 // Jump into the body.
709 CGF.EmitBlock(BB: bodyBB);
710 llvm::PHINode *currentElement =
711 Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
712 currentElement->addIncoming(V: element, BB: entryBB);
713
714 if (CGF.CGM.shouldEmitConvergenceTokens())
715 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
716
717 // Emit the actual filler expression.
718 {
719 // C++1z [class.temporary]p5:
720 // when a default constructor is called to initialize an element of
721 // an array with no corresponding initializer [...] the destruction of
722 // every temporary created in a default argument is sequenced before
723 // the construction of the next array element, if any
724 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
725 LValue elementLV = CGF.MakeAddrLValue(
726 Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
727 if (ArrayFiller)
728 EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
729 else
730 EmitNullInitializationToLValue(Address: elementLV);
731 }
732
733 // Move on to the next element.
734 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
735 Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
736
737 // Tell the EH cleanup that we finished with the last element.
738 if (endOfInit.isValid())
739 Builder.CreateStore(Val: nextElement, Addr: endOfInit);
740
741 // Leave the loop if we're done.
742 llvm::Value *done =
743 Builder.CreateICmpEQ(LHS: nextElement, RHS: end, Name: "arrayinit.done");
744 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
745 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
746 currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
747
748 if (CGF.CGM.shouldEmitConvergenceTokens())
749 CGF.ConvergenceTokenStack.pop_back();
750
751 CGF.EmitBlock(BB: endBB);
752 }
753}
754
755//===----------------------------------------------------------------------===//
756// Visitor Methods
757//===----------------------------------------------------------------------===//
758
759void AggExprEmitter::VisitMaterializeTemporaryExpr(
760 MaterializeTemporaryExpr *E) {
761 Visit(E: E->getSubExpr());
762}
763
764void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
765 // If this is a unique OVE, just visit its source expression.
766 if (e->isUnique())
767 Visit(E: e->getSourceExpr());
768 else
769 EmitFinalDestCopy(type: e->getType(), src: CGF.getOrCreateOpaqueLValueMapping(e));
770}
771
772void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
773 if (Dest.isPotentiallyAliased()) {
774 // Just emit a load of the lvalue + a copy, because our compound literal
775 // might alias the destination.
776 EmitAggLoadOfLValue(E);
777 return;
778 }
779
780 AggValueSlot Slot = EnsureSlot(T: E->getType());
781
782 // Block-scope compound literals are destroyed at the end of the enclosing
783 // scope in C.
784 bool Destruct =
785 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
786 if (Destruct)
787 Slot.setExternallyDestructed();
788
789 CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
790
791 if (Destruct)
792 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
793 CGF.pushLifetimeExtendedDestroy(dtorKind: DtorKind, addr: Slot.getAddress(),
794 type: E->getType());
795}
796
797/// Attempt to look through various unimportant expressions to find a
798/// cast of the given kind.
799static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
800 op = op->IgnoreParenNoopCasts(Ctx: ctx);
801 if (auto castE = dyn_cast<CastExpr>(Val: op)) {
802 if (castE->getCastKind() == kind)
803 return castE->getSubExpr();
804 }
805 return nullptr;
806}
807
808void AggExprEmitter::VisitCastExpr(CastExpr *E) {
809 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
810 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
811 switch (E->getCastKind()) {
812 case CK_Dynamic: {
813 // FIXME: Can this actually happen? We have no test coverage for it.
814 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
815 LValue LV =
816 CGF.EmitCheckedLValue(E: E->getSubExpr(), TCK: CodeGenFunction::TCK_Load);
817 // FIXME: Do we also need to handle property references here?
818 if (LV.isSimple())
819 CGF.EmitDynamicCast(V: LV.getAddress(), DCE: cast<CXXDynamicCastExpr>(Val: E));
820 else
821 CGF.CGM.ErrorUnsupported(S: E, Type: "non-simple lvalue dynamic_cast");
822
823 if (!Dest.isIgnored())
824 CGF.CGM.ErrorUnsupported(S: E, Type: "lvalue dynamic_cast with a destination");
825 break;
826 }
827
828 case CK_ToUnion: {
829 // Evaluate even if the destination is ignored.
830 if (Dest.isIgnored()) {
831 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
832 /*ignoreResult=*/true);
833 break;
834 }
835
836 // GCC union extension
837 QualType Ty = E->getSubExpr()->getType();
838 Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
839 EmitInitializationToLValue(E: E->getSubExpr(),
840 Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
841 break;
842 }
843
844 case CK_LValueToRValueBitCast: {
845 if (Dest.isIgnored()) {
846 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
847 /*ignoreResult=*/true);
848 break;
849 }
850
851 LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
852 Address SourceAddress = SourceLV.getAddress().withElementType(ElemTy: CGF.Int8Ty);
853 Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
854 llvm::Value *SizeVal = llvm::ConstantInt::get(
855 Ty: CGF.SizeTy,
856 V: CGF.getContext().getTypeSizeInChars(T: E->getType()).getQuantity());
857 Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
858 break;
859 }
860
861 case CK_DerivedToBase: {
862 assert(CGF.getLangOpts().HLSL &&
863 "Derived/Base casts in EmitAggExpr are only supported in HLSL");
864
865 // Create a temporary for the derived record, switch it out with the current
866 // Dest slot, and emit the derived value.
867 QualType DerivedTy = E->getSubExpr()->getType();
868 RawAddress DerivedAddr = CGF.CreateMemTempWithoutCast(T: DerivedTy);
869 AggValueSlot DerivedTmpSlot = AggValueSlot::forAddr(
870 addr: DerivedAddr, quals: DerivedTy.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
871 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
872 mayOverlap: AggValueSlot::DoesNotOverlap);
873
874 AggValueSlot DestBaseSlot = Dest;
875 Dest = DerivedTmpSlot;
876
877 Visit(E: E->getSubExpr());
878
879 // Perform derived-to-base address conversion to get the address
880 // of the base record within the derived record. In HLSL this should
881 // always be same as the derived because of single inheritance, but let's
882 // do it properly.
883 Address BaseAddrInDerived = CGF.GetAddressOfBaseClass(
884 Value: DerivedTmpSlot.getAddress(), Derived: DerivedTy->castAsCXXRecordDecl(),
885 PathBegin: E->path_begin(), PathEnd: E->path_end(),
886 /*NullCheckValue=*/false, Loc: E->getExprLoc());
887
888 AggValueSlot SrcBaseSlot = AggValueSlot::forAddr(
889 addr: BaseAddrInDerived, quals: E->getType().getQualifiers(),
890 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
891 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
892
893 // Copy the base class to the original destination slot and restore it.
894 EmitCopy(type: E->getType(), dest: DestBaseSlot, src: SrcBaseSlot);
895 Dest = DestBaseSlot;
896 break;
897 }
898
899 case CK_BaseToDerived:
900 case CK_UncheckedDerivedToBase: {
901 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
902 "should have been unpacked before we got here");
903 }
904
905 case CK_NonAtomicToAtomic:
906 case CK_AtomicToNonAtomic: {
907 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
908
909 // Determine the atomic and value types.
910 QualType atomicType = E->getSubExpr()->getType();
911 QualType valueType = E->getType();
912 if (isToAtomic)
913 std::swap(a&: atomicType, b&: valueType);
914
915 assert(atomicType->isAtomicType());
916 assert(CGF.getContext().hasSameUnqualifiedType(
917 valueType, atomicType->castAs<AtomicType>()->getValueType()));
918
919 // Just recurse normally if we're ignoring the result or the
920 // atomic type doesn't change representation.
921 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
922 return Visit(E: E->getSubExpr());
923 }
924
925 CastKind peepholeTarget =
926 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
927
928 // These two cases are reverses of each other; try to peephole them.
929 if (Expr *op =
930 findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
931 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
932 E->getType()) &&
933 "peephole significantly changed types?");
934 return Visit(E: op);
935 }
936
937 // If we're converting an r-value of non-atomic type to an r-value
938 // of atomic type, just emit directly into the relevant sub-object.
939 if (isToAtomic) {
940 AggValueSlot valueDest = Dest;
941 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
942 // Zero-initialize. (Strictly speaking, we only need to initialize
943 // the padding at the end, but this is simpler.)
944 if (!Dest.isZeroed())
945 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
946
947 // Build a GEP to refer to the subobject.
948 Address valueAddr =
949 CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
950 valueDest = AggValueSlot::forAddr(
951 addr: valueAddr, quals: valueDest.getQualifiers(),
952 isDestructed: valueDest.isExternallyDestructed(), needsGC: valueDest.requiresGCollection(),
953 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
954 isZeroed: AggValueSlot::IsZeroed);
955 }
956
957 CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
958 return;
959 }
960
961 // Otherwise, we're converting an atomic type to a non-atomic type.
962 // Make an atomic temporary, emit into that, and then copy the value out.
963 AggValueSlot atomicSlot =
964 CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
965 CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
966
967 Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
968 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
969 return EmitFinalDestCopy(type: valueType, src: rvalue);
970 }
971 case CK_AddressSpaceConversion:
972 return Visit(E: E->getSubExpr());
973
974 case CK_LValueToRValue:
975 // If we're loading from a volatile type, force the destination
976 // into existence.
977 if (E->getSubExpr()->getType().isVolatileQualified()) {
978 bool Destruct =
979 !Dest.isExternallyDestructed() &&
980 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
981 if (Destruct)
982 Dest.setExternallyDestructed();
983 EnsureDest(T: E->getType());
984 Visit(E: E->getSubExpr());
985
986 if (Destruct)
987 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
988 type: E->getType());
989
990 return;
991 }
992
993 [[fallthrough]];
994
995 case CK_HLSLArrayRValue:
996 if (CGF.getLangOpts().HLSL &&
997 E->getSubExpr()->getType()->isHLSLResourceRecordArray())
998 if (CGF.CGM.getHLSLRuntime().emitGlobalResourceArray(CGF, E, DestSlot&: Dest))
999 break;
1000 Visit(E: E->getSubExpr());
1001 break;
1002 case CK_HLSLAggregateSplatCast: {
1003 Expr *Src = E->getSubExpr();
1004 QualType SrcTy = Src->getType();
1005 RValue RV = CGF.EmitAnyExpr(E: Src);
1006 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1007 SourceLocation Loc = E->getExprLoc();
1008
1009 assert(RV.isScalar() && SrcTy->isScalarType() &&
1010 "RHS of HLSL splat cast must be a scalar.");
1011 llvm::Value *SrcVal = RV.getScalarVal();
1012 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1013 break;
1014 }
1015 case CK_HLSLElementwiseCast: {
1016 Expr *Src = E->getSubExpr();
1017 QualType SrcTy = Src->getType();
1018 RValue RV = CGF.EmitAnyExpr(E: Src);
1019 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1020 SourceLocation Loc = E->getExprLoc();
1021
1022 if (RV.isScalar()) {
1023 llvm::Value *SrcVal = RV.getScalarVal();
1024 assert(SrcTy->isVectorType() &&
1025 "HLSL Elementwise cast doesn't handle splatting.");
1026 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1027 } else {
1028 assert(RV.isAggregate() &&
1029 "Can't perform HLSL Aggregate cast on a complex type.");
1030 Address SrcVal = RV.getAggregateAddress();
1031 EmitHLSLElementwiseCast(CGF, DestVal: DestLVal, SrcVal: CGF.MakeAddrLValue(Addr: SrcVal, T: SrcTy),
1032 Loc);
1033 }
1034 break;
1035 }
1036 case CK_NoOp:
1037 case CK_UserDefinedConversion:
1038 case CK_ConstructorConversion:
1039 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
1040 E->getType()) &&
1041 "Implicit cast types must be compatible");
1042 Visit(E: E->getSubExpr());
1043 break;
1044
1045 case CK_LValueBitCast:
1046 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
1047
1048 case CK_Dependent:
1049 case CK_BitCast:
1050 case CK_ArrayToPointerDecay:
1051 case CK_FunctionToPointerDecay:
1052 case CK_NullToPointer:
1053 case CK_NullToMemberPointer:
1054 case CK_BaseToDerivedMemberPointer:
1055 case CK_DerivedToBaseMemberPointer:
1056 case CK_MemberPointerToBoolean:
1057 case CK_ReinterpretMemberPointer:
1058 case CK_IntegralToPointer:
1059 case CK_PointerToIntegral:
1060 case CK_PointerToBoolean:
1061 case CK_ToVoid:
1062 case CK_VectorSplat:
1063 case CK_IntegralCast:
1064 case CK_BooleanToSignedIntegral:
1065 case CK_IntegralToBoolean:
1066 case CK_IntegralToFloating:
1067 case CK_FloatingToIntegral:
1068 case CK_FloatingToBoolean:
1069 case CK_FloatingCast:
1070 case CK_CPointerToObjCPointerCast:
1071 case CK_BlockPointerToObjCPointerCast:
1072 case CK_AnyPointerToBlockPointerCast:
1073 case CK_ObjCObjectLValueCast:
1074 case CK_FloatingRealToComplex:
1075 case CK_FloatingComplexToReal:
1076 case CK_FloatingComplexToBoolean:
1077 case CK_FloatingComplexCast:
1078 case CK_FloatingComplexToIntegralComplex:
1079 case CK_IntegralRealToComplex:
1080 case CK_IntegralComplexToReal:
1081 case CK_IntegralComplexToBoolean:
1082 case CK_IntegralComplexCast:
1083 case CK_IntegralComplexToFloatingComplex:
1084 case CK_ARCProduceObject:
1085 case CK_ARCConsumeObject:
1086 case CK_ARCReclaimReturnedObject:
1087 case CK_ARCExtendBlockObject:
1088 case CK_CopyAndAutoreleaseBlockObject:
1089 case CK_BuiltinFnToFnPtr:
1090 case CK_ZeroToOCLOpaqueType:
1091 case CK_MatrixCast:
1092 case CK_HLSLVectorTruncation:
1093 case CK_HLSLMatrixTruncation:
1094 case CK_IntToOCLSampler:
1095 case CK_FloatingToFixedPoint:
1096 case CK_FixedPointToFloating:
1097 case CK_FixedPointCast:
1098 case CK_FixedPointToBoolean:
1099 case CK_FixedPointToIntegral:
1100 case CK_IntegralToFixedPoint:
1101 llvm_unreachable("cast kind invalid for aggregate types");
1102 }
1103}
1104
1105void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1106 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
1107 EmitAggLoadOfLValue(E);
1108 return;
1109 }
1110
1111 withReturnValueSlot(
1112 E, EmitCall: [&](ReturnValueSlot Slot) { return CGF.EmitCallExpr(E, ReturnValue: Slot); });
1113}
1114
1115void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1116 withReturnValueSlot(E, EmitCall: [&](ReturnValueSlot Slot) {
1117 return CGF.EmitObjCMessageExpr(E, Return: Slot);
1118 });
1119}
1120
1121void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1122 CGF.EmitIgnoredExpr(E: E->getLHS());
1123 Visit(E: E->getRHS());
1124}
1125
1126void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1127 CodeGenFunction::StmtExprEvaluation eval(CGF);
1128 CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
1129}
1130
1131enum CompareKind {
1132 CK_Less,
1133 CK_Greater,
1134 CK_Equal,
1135};
1136
1137static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1138 const BinaryOperator *E, llvm::Value *LHS,
1139 llvm::Value *RHS, CompareKind Kind,
1140 const char *NameSuffix = "") {
1141 QualType ArgTy = E->getLHS()->getType();
1142 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1143 ArgTy = CT->getElementType();
1144
1145 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1146 assert(Kind == CK_Equal &&
1147 "member pointers may only be compared for equality");
1148 return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1149 CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1150 }
1151
1152 // Compute the comparison instructions for the specified comparison kind.
1153 struct CmpInstInfo {
1154 const char *Name;
1155 llvm::CmpInst::Predicate FCmp;
1156 llvm::CmpInst::Predicate SCmp;
1157 llvm::CmpInst::Predicate UCmp;
1158 };
1159 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1160 using FI = llvm::FCmpInst;
1161 using II = llvm::ICmpInst;
1162 switch (Kind) {
1163 case CK_Less:
1164 return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1165 case CK_Greater:
1166 return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1167 case CK_Equal:
1168 return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1169 }
1170 llvm_unreachable("Unrecognised CompareKind enum");
1171 }();
1172
1173 if (ArgTy->hasFloatingRepresentation())
1174 return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1175 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1176 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1177 auto Inst =
1178 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1179 return Builder.CreateICmp(P: Inst, LHS, RHS,
1180 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1181 }
1182
1183 llvm_unreachable("unsupported aggregate binary expression should have "
1184 "already been handled");
1185}
1186
1187void AggExprEmitter::EmitComparisonResult(const Expr *E,
1188 const ComparisonCategoryInfo &CmpInfo,
1189 llvm::Value *ResultValue) {
1190 // Create the return value in the destination slot.
1191 EnsureDest(T: E->getType());
1192 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1193
1194 // Emit the address of the first (and only) field in the comparison category
1195 // type, and initialize it from the constant integer value selected above.
1196 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1197 Base: DestLV, Field: *CmpInfo.Record->field_begin());
1198 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ResultValue), Dst: FieldLV,
1199 /*IsInit=*/isInit: true);
1200}
1201
1202void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1203 using llvm::BasicBlock;
1204 using llvm::PHINode;
1205 using llvm::Value;
1206 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1207 E->getRHS()->getType()));
1208 const ComparisonCategoryInfo &CmpInfo =
1209 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1210 assert(CmpInfo.Record->isTriviallyCopyable() &&
1211 "cannot copy non-trivially copyable aggregate");
1212
1213 QualType ArgTy = E->getLHS()->getType();
1214
1215 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1216 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1217 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1218 return CGF.ErrorUnsupported(S: E, Type: "aggregate three-way comparison");
1219 }
1220 bool IsComplex = ArgTy->isAnyComplexType();
1221
1222 // Evaluate the operands to the expression and extract their values.
1223 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1224 RValue RV = CGF.EmitAnyExpr(E);
1225 if (RV.isScalar())
1226 return {RV.getScalarVal(), nullptr};
1227 if (RV.isAggregate())
1228 return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1229 assert(RV.isComplex());
1230 return RV.getComplexVal();
1231 };
1232 auto LHSValues = EmitOperand(E->getLHS()),
1233 RHSValues = EmitOperand(E->getRHS());
1234
1235 auto EmitCmp = [&](CompareKind K) {
1236 Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1237 Kind: K, NameSuffix: IsComplex ? ".r" : "");
1238 if (!IsComplex)
1239 return Cmp;
1240 assert(K == CompareKind::CK_Equal);
1241 Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1242 RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1243 return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1244 };
1245 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1246 return Builder.getInt(AI: VInfo->getIntValue());
1247 };
1248
1249 Value *Select;
1250 if (ArgTy->isNullPtrType()) {
1251 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1252 } else if (!CmpInfo.isPartial()) {
1253 Value *SelectOne =
1254 Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1255 False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1256 Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1257 True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1258 False: SelectOne, Name: "sel.eq");
1259 } else {
1260 Value *SelectEq = Builder.CreateSelect(
1261 C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1262 False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1263 Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1264 True: EmitCmpRes(CmpInfo.getGreater()),
1265 False: SelectEq, Name: "sel.gt");
1266 Select = Builder.CreateSelect(
1267 C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1268 }
1269
1270 EmitComparisonResult(E, CmpInfo, ResultValue: Select);
1271}
1272
1273void AggExprEmitter::VisitTypeTraitExpr(const TypeTraitExpr *E) {
1274 assert(E->isStoredAsComparisonResult() &&
1275 "expected a strong_ordering type trait with a stored value");
1276
1277 const ComparisonCategoryInfo &CmpInfo =
1278 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1279 const auto Result =
1280 ComparisonCategoryResult(E->getAPValue().getInt().getZExtValue());
1281 llvm::Value *ResultValue =
1282 Builder.getInt(AI: CmpInfo.getValueInfo(ValueKind: Result)->getIntValue());
1283
1284 EmitComparisonResult(E, CmpInfo, ResultValue);
1285}
1286
1287void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1288 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1289 VisitPointerToDataMemberBinaryOperator(BO: E);
1290 else
1291 CGF.ErrorUnsupported(S: E, Type: "aggregate binary expression");
1292}
1293
1294void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1295 const BinaryOperator *E) {
1296 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1297 EmitFinalDestCopy(type: E->getType(), src: LV);
1298}
1299
1300void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1301 ApplyAtomGroup Grp(CGF.getDebugInfo());
1302 // For an assignment to work, the value on the right has
1303 // to be compatible with the value on the left.
1304 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1305 E->getRHS()->getType()) &&
1306 "Invalid assignment");
1307
1308 // If the LHS might be a __block variable, and the RHS can
1309 // potentially cause a block copy, we need to evaluate the RHS first
1310 // so that the assignment goes the right place.
1311 // This is pretty semantically fragile.
1312 if (CodeGenUtils::isBlockVarRef(E: E->getLHS()) &&
1313 E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1314 // Ensure that we have a destination, and evaluate the RHS into that.
1315 EnsureDest(T: E->getRHS()->getType());
1316 Visit(E: E->getRHS());
1317
1318 // Now emit the LHS and copy into it.
1319 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1320
1321 // That copy is an atomic copy if the LHS is atomic.
1322 if (LHS.getType()->isAtomicType() ||
1323 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1324 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1325 return;
1326 }
1327
1328 EmitCopy(type: E->getLHS()->getType(),
1329 dest: AggValueSlot::forLValue(LV: LHS, isDestructed: AggValueSlot::IsDestructed,
1330 needsGC: needsGC(T: E->getLHS()->getType()),
1331 isAliased: AggValueSlot::IsAliased,
1332 mayOverlap: AggValueSlot::MayOverlap),
1333 src: Dest);
1334 return;
1335 }
1336
1337 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1338
1339 // If we have an atomic type, evaluate into the destination and then
1340 // do an atomic copy.
1341 if (LHS.getType()->isAtomicType() ||
1342 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1343 EnsureDest(T: E->getRHS()->getType());
1344 Visit(E: E->getRHS());
1345 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1346 return;
1347 }
1348
1349 // Codegen the RHS so that it stores directly into the LHS.
1350 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1351 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1352 isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1353 // A non-volatile aggregate destination might have volatile member.
1354 if (!LHSSlot.isVolatile() && CGF.hasVolatileMember(T: E->getLHS()->getType()))
1355 LHSSlot.setVolatile(true);
1356
1357 CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1358
1359 // Copy into the destination if the assignment isn't ignored.
1360 EmitFinalDestCopy(type: E->getType(), src: LHS);
1361
1362 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1363 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1364 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1365 type: E->getType());
1366}
1367
1368void AggExprEmitter::VisitAbstractConditionalOperator(
1369 const AbstractConditionalOperator *E) {
1370 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1371 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1372 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1373
1374 // Bind the common expression if necessary.
1375 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1376
1377 CodeGenFunction::ConditionalEvaluation eval(CGF);
1378 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1379 TrueCount: CGF.getProfileCount(S: E));
1380
1381 // Save whether the destination's lifetime is externally managed.
1382 bool isExternallyDestructed = Dest.isExternallyDestructed();
1383 bool destructNonTrivialCStruct =
1384 !isExternallyDestructed &&
1385 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1386 isExternallyDestructed |= destructNonTrivialCStruct;
1387 Dest.setExternallyDestructed(isExternallyDestructed);
1388
1389 eval.begin(CGF);
1390 CGF.EmitBlock(BB: LHSBlock);
1391 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
1392 Visit(E: E->getTrueExpr());
1393 eval.end(CGF);
1394
1395 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1396 CGF.Builder.CreateBr(Dest: ContBlock);
1397
1398 // If the result of an agg expression is unused, then the emission
1399 // of the LHS might need to create a destination slot. That's fine
1400 // with us, and we can safely emit the RHS into the same slot, but
1401 // we shouldn't claim that it's already being destructed.
1402 Dest.setExternallyDestructed(isExternallyDestructed);
1403
1404 eval.begin(CGF);
1405 CGF.EmitBlock(BB: RHSBlock);
1406 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
1407 Visit(E: E->getFalseExpr());
1408 eval.end(CGF);
1409
1410 if (destructNonTrivialCStruct)
1411 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1412 type: E->getType());
1413
1414 CGF.EmitBlock(BB: ContBlock);
1415}
1416
1417void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1418 Visit(E: CE->getChosenSubExpr());
1419}
1420
1421void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1422 Address ArgValue = Address::invalid();
1423 CGF.EmitVAArg(VE, VAListAddr&: ArgValue, Slot: Dest);
1424
1425 // If EmitVAArg fails, emit an error.
1426 if (!ArgValue.isValid()) {
1427 CGF.ErrorUnsupported(S: VE, Type: "aggregate va_arg expression");
1428 return;
1429 }
1430}
1431
1432void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1433 // Ensure that we have a slot, but if we already do, remember
1434 // whether it was externally destructed.
1435 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1436 EnsureDest(T: E->getType());
1437
1438 // We're going to push a destructor if there isn't already one.
1439 Dest.setExternallyDestructed();
1440
1441 Visit(E: E->getSubExpr());
1442
1443 // Push that destructor we promised.
1444 if (!wasExternallyDestructed)
1445 CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1446}
1447
1448void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1449 AggValueSlot Slot = EnsureSlot(T: E->getType());
1450 CGF.EmitCXXConstructExpr(E, Dest: Slot);
1451}
1452
1453void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1454 const CXXInheritedCtorInitExpr *E) {
1455 AggValueSlot Slot = EnsureSlot(T: E->getType());
1456 CGF.EmitInheritedCXXConstructorCall(D: E->getConstructor(), ForVirtualBase: E->constructsVBase(),
1457 This: Slot.getAddress(),
1458 InheritedFromVBase: E->inheritedFromVBase(), E);
1459}
1460
1461void AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1462 AggValueSlot Slot = EnsureSlot(T: E->getType());
1463 LValue SlotLV = CGF.MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType());
1464
1465 // We'll need to enter cleanup scopes in case any of the element
1466 // initializers throws an exception or contains branch out of the expressions.
1467 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1468
1469 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1470 for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1471 e = E->capture_init_end();
1472 i != e; ++i, ++CurField) {
1473 // Emit initialization
1474 LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1475 if (CurField->hasCapturedVLAType()) {
1476 CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1477 continue;
1478 }
1479
1480 EmitInitializationToLValue(E: *i, Address: LV);
1481
1482 // Push a destructor if necessary.
1483 if (QualType::DestructionKind DtorKind =
1484 CurField->getType().isDestructedType()) {
1485 assert(LV.isSimple());
1486 if (DtorKind)
1487 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1488 type: CurField->getType(),
1489 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: false);
1490 }
1491 }
1492}
1493
1494void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1495 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1496 Visit(E: E->getSubExpr());
1497}
1498
1499void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1500 QualType T = E->getType();
1501 AggValueSlot Slot = EnsureSlot(T);
1502 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1503}
1504
1505void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1506 QualType T = E->getType();
1507 AggValueSlot Slot = EnsureSlot(T);
1508 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1509}
1510
1511/// Determine whether the given cast kind is known to always convert values
1512/// with all zero bits in their value representation to values with all zero
1513/// bits in their value representation.
1514static bool castPreservesZero(const CastExpr *CE) {
1515 switch (CE->getCastKind()) {
1516 // No-ops.
1517 case CK_NoOp:
1518 case CK_UserDefinedConversion:
1519 case CK_ConstructorConversion:
1520 case CK_BitCast:
1521 case CK_ToUnion:
1522 case CK_ToVoid:
1523 // Conversions between (possibly-complex) integral, (possibly-complex)
1524 // floating-point, and bool.
1525 case CK_BooleanToSignedIntegral:
1526 case CK_FloatingCast:
1527 case CK_FloatingComplexCast:
1528 case CK_FloatingComplexToBoolean:
1529 case CK_FloatingComplexToIntegralComplex:
1530 case CK_FloatingComplexToReal:
1531 case CK_FloatingRealToComplex:
1532 case CK_FloatingToBoolean:
1533 case CK_FloatingToIntegral:
1534 case CK_IntegralCast:
1535 case CK_IntegralComplexCast:
1536 case CK_IntegralComplexToBoolean:
1537 case CK_IntegralComplexToFloatingComplex:
1538 case CK_IntegralComplexToReal:
1539 case CK_IntegralRealToComplex:
1540 case CK_IntegralToBoolean:
1541 case CK_IntegralToFloating:
1542 // Reinterpreting integers as pointers and vice versa.
1543 case CK_IntegralToPointer:
1544 case CK_PointerToIntegral:
1545 // Language extensions.
1546 case CK_VectorSplat:
1547 case CK_MatrixCast:
1548 case CK_NonAtomicToAtomic:
1549 case CK_AtomicToNonAtomic:
1550 case CK_HLSLVectorTruncation:
1551 case CK_HLSLMatrixTruncation:
1552 case CK_HLSLElementwiseCast:
1553 case CK_HLSLAggregateSplatCast:
1554 return true;
1555
1556 case CK_BaseToDerivedMemberPointer:
1557 case CK_DerivedToBaseMemberPointer:
1558 case CK_MemberPointerToBoolean:
1559 case CK_NullToMemberPointer:
1560 case CK_ReinterpretMemberPointer:
1561 // FIXME: ABI-dependent.
1562 return false;
1563
1564 case CK_AnyPointerToBlockPointerCast:
1565 case CK_BlockPointerToObjCPointerCast:
1566 case CK_CPointerToObjCPointerCast:
1567 case CK_ObjCObjectLValueCast:
1568 case CK_IntToOCLSampler:
1569 case CK_ZeroToOCLOpaqueType:
1570 // FIXME: Check these.
1571 return false;
1572
1573 case CK_FixedPointCast:
1574 case CK_FixedPointToBoolean:
1575 case CK_FixedPointToFloating:
1576 case CK_FixedPointToIntegral:
1577 case CK_FloatingToFixedPoint:
1578 case CK_IntegralToFixedPoint:
1579 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1580 return false;
1581
1582 case CK_AddressSpaceConversion:
1583 case CK_BaseToDerived:
1584 case CK_DerivedToBase:
1585 case CK_Dynamic:
1586 case CK_NullToPointer:
1587 case CK_PointerToBoolean:
1588 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1589 // same representation in all involved address spaces.
1590 return false;
1591
1592 case CK_ARCConsumeObject:
1593 case CK_ARCExtendBlockObject:
1594 case CK_ARCProduceObject:
1595 case CK_ARCReclaimReturnedObject:
1596 case CK_CopyAndAutoreleaseBlockObject:
1597 case CK_ArrayToPointerDecay:
1598 case CK_FunctionToPointerDecay:
1599 case CK_BuiltinFnToFnPtr:
1600 case CK_Dependent:
1601 case CK_LValueBitCast:
1602 case CK_LValueToRValue:
1603 case CK_LValueToRValueBitCast:
1604 case CK_UncheckedDerivedToBase:
1605 case CK_HLSLArrayRValue:
1606 return false;
1607 }
1608 llvm_unreachable("Unhandled clang::CastKind enum");
1609}
1610
1611/// isSimpleZero - If emitting this value will obviously just cause a store of
1612/// zero to memory, return true. This can return false if uncertain, so it just
1613/// handles simple cases.
1614static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1615 E = E->IgnoreParens();
1616 while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1617 if (!castPreservesZero(CE))
1618 break;
1619 E = CE->getSubExpr()->IgnoreParens();
1620 }
1621
1622 // 0
1623 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1624 return IL->getValue() == 0;
1625 // +0.0
1626 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1627 return FL->getValue().isPosZero();
1628 // int()
1629 if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1630 CGF.getTypes().isZeroInitializable(T: E->getType()))
1631 return true;
1632 // (int*)0 - Null pointer expressions.
1633 if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1634 return ICE->getCastKind() == CK_NullToPointer &&
1635 CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1636 !E->HasSideEffects(Ctx: CGF.getContext());
1637 // '\0'
1638 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1639 return CL->getValue() == 0;
1640
1641 // Otherwise, hard case: conservatively return false.
1642 return false;
1643}
1644
1645void AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1646 QualType type = LV.getType();
1647 // FIXME: Ignore result?
1648 // FIXME: Are initializers affected by volatile?
1649 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1650 // Storing "i32 0" to a zero'd memory location is a noop.
1651 return;
1652 } else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1653 return EmitNullInitializationToLValue(Address: LV);
1654 } else if (isa<NoInitExpr>(Val: E)) {
1655 // Do nothing.
1656 return;
1657 } else if (type->isReferenceType()) {
1658 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1659 return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1660 }
1661
1662 CGF.EmitInitializationToLValue(E, LV, IsZeroed: Dest.isZeroed());
1663}
1664
1665void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1666 QualType type = lv.getType();
1667
1668 // If the destination slot is already zeroed out before the aggregate is
1669 // copied into it, we don't have to emit any zeros here.
1670 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1671 return;
1672
1673 if (CGF.hasScalarEvaluationKind(T: type)) {
1674 // For non-aggregates, we can store the appropriate null constant.
1675 llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1676 // Note that the following is not equivalent to
1677 // EmitStoreThroughBitfieldLValue for ARC types.
1678 if (lv.isBitField()) {
1679 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1680 } else {
1681 assert(lv.isSimple());
1682 CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1683 }
1684 } else {
1685 // There's a potential optimization opportunity in combining
1686 // memsets; that would be easy for arrays, but relatively
1687 // difficult for structures with the current code.
1688 CGF.EmitNullInitialization(DestPtr: lv.getAddress(), Ty: lv.getType());
1689 }
1690}
1691
1692void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1693 VisitCXXParenListOrInitListExpr(ExprToVisit: E, Args: E->getInitExprs(),
1694 InitializedFieldInUnion: E->getInitializedFieldInUnion(),
1695 ArrayFiller: E->getArrayFiller());
1696}
1697
1698void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1699 if (E->hadArrayRangeDesignator())
1700 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
1701
1702 if (E->isTransparent())
1703 return Visit(E: E->getInit(Init: 0));
1704
1705 VisitCXXParenListOrInitListExpr(
1706 ExprToVisit: E, Args: E->inits(), InitializedFieldInUnion: E->getInitializedFieldInUnion(), ArrayFiller: E->getArrayFiller());
1707}
1708
1709void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1710 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1711 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1712#if 0
1713 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1714 // (Length of globals? Chunks of zeroed-out space?).
1715 //
1716 // If we can, prefer a copy from a global; this is a lot less code for long
1717 // globals, and it's easier for the current optimizers to analyze.
1718 if (llvm::Constant *C =
1719 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1720 llvm::GlobalVariable* GV =
1721 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1722 llvm::GlobalValue::InternalLinkage, C, "");
1723 EmitFinalDestCopy(ExprToVisit->getType(),
1724 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1725 return;
1726 }
1727#endif
1728
1729 // HLSL initialization lists in the AST are an expansion which can contain
1730 // side-effecting expressions wrapped in opaque value expressions. To properly
1731 // emit these we need to emit the opaque values before we emit the argument
1732 // expressions themselves. This is a little hacky, but it prevents us needing
1733 // to do a bigger AST-level change for a language feature that we need
1734 // deprecate in the near future. See related HLSL language proposals:
1735 // * 0005-strict-initializer-lists.md
1736 // * https://github.com/microsoft/hlsl-specs/pull/325
1737 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(Val: ExprToVisit))
1738 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(
1739 CGF, E: cast<InitListExpr>(Val: ExprToVisit));
1740
1741 AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1742
1743 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1744
1745 // Handle initialization of an array.
1746 if (ExprToVisit->getType()->isConstantArrayType()) {
1747 auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1748 EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1749 Args: InitExprs, ArrayFiller);
1750 return;
1751 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1752 // A variable array type that has an initializer can only do empty
1753 // initialization. And because this feature is not exposed as an extension
1754 // in C++, we can safely memset the array memory to zero.
1755 assert(InitExprs.size() == 0 &&
1756 "you can only use an empty initializer with VLAs");
1757 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1758 return;
1759 }
1760
1761 assert(ExprToVisit->getType()->isRecordType() &&
1762 "Only support structs/unions here!");
1763
1764 // Do struct initialization; this code just sets each individual member
1765 // to the approprate value. This makes bitfield support automatic;
1766 // the disadvantage is that the generated code is more difficult for
1767 // the optimizer, especially with bitfields.
1768 unsigned NumInitElements = InitExprs.size();
1769 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1770
1771 // We'll need to enter cleanup scopes in case any of the element
1772 // initializers throws an exception.
1773 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1774
1775 unsigned curInitIndex = 0;
1776
1777 // Emit initialization of base classes.
1778 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1779 assert(NumInitElements >= CXXRD->getNumBases() &&
1780 "missing initializer for base class");
1781 for (auto &Base : CXXRD->bases()) {
1782 assert(!Base.isVirtual() && "should not see vbases here");
1783 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1784 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1785 Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1786 /*isBaseVirtual*/ BaseIsVirtual: false);
1787 AggValueSlot AggSlot = AggValueSlot::forAddr(
1788 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1789 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1790 mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1791 CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1792
1793 if (QualType::DestructionKind dtorKind =
1794 Base.getType().isDestructedType())
1795 CGF.pushDestroyAndDeferDeactivation(dtorKind, addr: V, type: Base.getType());
1796 }
1797 }
1798
1799 // Prepare a 'this' for CXXDefaultInitExprs.
1800 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1801
1802 const bool ZeroInitPadding =
1803 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1804
1805 if (record->isUnion()) {
1806 // Only initialize one field of a union. The field itself is
1807 // specified by the initializer list.
1808 if (!InitializedFieldInUnion) {
1809 // Empty union; we have nothing to do.
1810
1811#ifndef NDEBUG
1812 // Make sure that it's really an empty and not a failure of
1813 // semantic analysis.
1814 for (const auto *Field : record->fields())
1815 assert(
1816 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1817 "Only unnamed bitfields or anonymous class allowed");
1818#endif
1819 return;
1820 }
1821
1822 // FIXME: volatility
1823 FieldDecl *Field = InitializedFieldInUnion;
1824
1825 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1826 if (NumInitElements) {
1827 // Store the initializer into the field
1828 EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1829 if (ZeroInitPadding) {
1830 uint64_t TotalSize = CGF.getContext().toBits(
1831 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1832 uint64_t FieldSize = CGF.getContext().getTypeSize(T: FieldLoc.getType());
1833 DoZeroInitPadding(PaddingStart&: FieldSize, PaddingEnd: TotalSize, NextField: nullptr);
1834 }
1835 } else {
1836 // Default-initialize to null.
1837 if (ZeroInitPadding)
1838 EmitNullInitializationToLValue(lv: DestLV);
1839 else
1840 EmitNullInitializationToLValue(lv: FieldLoc);
1841 }
1842 return;
1843 }
1844
1845 // Here we iterate over the fields; this makes it simpler to both
1846 // default-initialize fields and skip over unnamed fields.
1847 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(D: record);
1848 uint64_t PaddingStart = 0;
1849
1850 for (const auto *field : record->fields()) {
1851 // We're done once we hit the flexible array member.
1852 if (field->getType()->isIncompleteArrayType())
1853 break;
1854
1855 // Always skip anonymous bitfields.
1856 if (field->isUnnamedBitField())
1857 continue;
1858
1859 // We're done if we reach the end of the explicit initializers, we
1860 // have a zeroed object, and the rest of the fields are
1861 // zero-initializable.
1862 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1863 CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1864 break;
1865
1866 if (ZeroInitPadding)
1867 DoZeroInitPadding(PaddingStart,
1868 PaddingEnd: Layout.getFieldOffset(FieldNo: field->getFieldIndex()), NextField: field);
1869
1870 LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1871 // We never generate write-barries for initialized fields.
1872 LV.setNonGC(true);
1873
1874 if (curInitIndex < NumInitElements) {
1875 // Store the initializer into the field.
1876 EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1877 } else {
1878 // We're out of initializers; default-initialize to null
1879 EmitNullInitializationToLValue(lv: LV);
1880 }
1881
1882 // Push a destructor if necessary.
1883 // FIXME: if we have an array of structures, all explicitly
1884 // initialized, we can end up pushing a linear number of cleanups.
1885 if (QualType::DestructionKind dtorKind =
1886 field->getType().isDestructedType()) {
1887 assert(LV.isSimple());
1888 if (dtorKind) {
1889 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1890 type: field->getType(),
1891 destroyer: CGF.getDestroyer(destructionKind: dtorKind), useEHCleanupForArray: false);
1892 }
1893 }
1894 }
1895 if (ZeroInitPadding) {
1896 uint64_t TotalSize = CGF.getContext().toBits(
1897 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1898 DoZeroInitPadding(PaddingStart, PaddingEnd: TotalSize, NextField: nullptr);
1899 }
1900}
1901
1902void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1903 uint64_t PaddingEnd,
1904 const FieldDecl *NextField) {
1905
1906 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1907 CharUnits Start = CGF.getContext().toCharUnitsFromBits(BitSize: StartBit);
1908 CharUnits End = CGF.getContext().toCharUnitsFromBits(BitSize: EndBit);
1909 Address Addr = Dest.getAddress().withElementType(ElemTy: CGF.CharTy);
1910 if (!Start.isZero())
1911 Addr = Builder.CreateConstGEP(Addr, Index: Start.getQuantity());
1912 llvm::Constant *SizeVal = Builder.getInt64(C: (End - Start).getQuantity());
1913 CGF.Builder.CreateMemSet(Dest: Addr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
1914 };
1915
1916 if (NextField != nullptr && NextField->isBitField()) {
1917 // For bitfield, zero init StorageSize before storing the bits. So we don't
1918 // need to handle big/little endian.
1919 const CGRecordLayout &RL =
1920 CGF.getTypes().getCGRecordLayout(NextField->getParent());
1921 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: NextField);
1922 uint64_t StorageStart = CGF.getContext().toBits(CharSize: Info.StorageOffset);
1923 if (StorageStart + Info.StorageSize > PaddingStart) {
1924 if (StorageStart > PaddingStart)
1925 InitBytes(PaddingStart, StorageStart);
1926 Address Addr = Dest.getAddress();
1927 if (!Info.StorageOffset.isZero())
1928 Addr = Builder.CreateConstGEP(Addr: Addr.withElementType(ElemTy: CGF.CharTy),
1929 Index: Info.StorageOffset.getQuantity());
1930 Addr = Addr.withElementType(
1931 ElemTy: llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: Info.StorageSize));
1932 Builder.CreateStore(Val: Builder.getIntN(N: Info.StorageSize, C: 0), Addr);
1933 PaddingStart = StorageStart + Info.StorageSize;
1934 }
1935 return;
1936 }
1937
1938 if (PaddingStart < PaddingEnd)
1939 InitBytes(PaddingStart, PaddingEnd);
1940 if (NextField != nullptr)
1941 PaddingStart =
1942 PaddingEnd + CGF.getContext().getTypeSize(T: NextField->getType());
1943}
1944
1945void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1946 llvm::Value *outerBegin) {
1947 // Emit the common subexpression.
1948 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1949
1950 Address destPtr = EnsureSlot(T: E->getType()).getAddress();
1951 uint64_t numElements = E->getArraySize().getZExtValue();
1952
1953 if (!numElements)
1954 return;
1955
1956 // destPtr is an array*. Construct an elementType* by drilling down a level.
1957 llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1958 llvm::Value *indices[] = {zero, zero};
1959 llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
1960 Ptr: destPtr.emitRawPointer(CGF),
1961 IdxList: indices, Name: "arrayinit.begin");
1962
1963 // Prepare to special-case multidimensional array initialization: we avoid
1964 // emitting multiple destructor loops in that case.
1965 if (!outerBegin)
1966 outerBegin = begin;
1967 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
1968
1969 QualType elementType =
1970 CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
1971 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
1972 CharUnits elementAlign =
1973 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1974 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
1975
1976 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1977 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
1978
1979 // Jump into the body.
1980 CGF.EmitBlock(BB: bodyBB);
1981 llvm::PHINode *index =
1982 Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
1983 index->addIncoming(V: zero, BB: entryBB);
1984 llvm::Value *element =
1985 Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
1986
1987 if (CGF.CGM.shouldEmitConvergenceTokens())
1988 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
1989
1990 // Prepare for a cleanup.
1991 QualType::DestructionKind dtorKind = elementType.isDestructedType();
1992 EHScopeStack::stable_iterator cleanup;
1993 if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
1994 if (outerBegin->getType() != element->getType())
1995 outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
1996 CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
1997 elementAlignment: elementAlign,
1998 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
1999 cleanup = CGF.EHStack.stable_begin();
2000 } else {
2001 dtorKind = QualType::DK_none;
2002 }
2003
2004 // Emit the actual filler expression.
2005 {
2006 // Temporaries created in an array initialization loop are destroyed
2007 // at the end of each iteration.
2008 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2009 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2010 LValue elementLV = CGF.MakeAddrLValue(
2011 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
2012
2013 if (InnerLoop) {
2014 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2015 auto elementSlot = AggValueSlot::forLValue(
2016 LV: elementLV, isDestructed: AggValueSlot::IsDestructed,
2017 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2018 mayOverlap: AggValueSlot::DoesNotOverlap);
2019 AggExprEmitter(CGF, elementSlot, false)
2020 .VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
2021 } else
2022 EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
2023 }
2024
2025 // Move on to the next element.
2026 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2027 LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
2028 index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
2029
2030 // Leave the loop if we're done.
2031 llvm::Value *done = Builder.CreateICmpEQ(
2032 LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
2033 Name: "arrayinit.done");
2034 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
2035 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
2036
2037 if (CGF.CGM.shouldEmitConvergenceTokens())
2038 CGF.ConvergenceTokenStack.pop_back();
2039
2040 CGF.EmitBlock(BB: endBB);
2041
2042 // Leave the partial-array cleanup if we entered one.
2043 if (dtorKind)
2044 CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
2045}
2046
2047void AggExprEmitter::VisitDesignatedInitUpdateExpr(
2048 DesignatedInitUpdateExpr *E) {
2049 AggValueSlot Dest = EnsureSlot(T: E->getType());
2050
2051 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
2052 EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
2053 VisitInitListExpr(E: E->getUpdater());
2054}
2055
2056//===----------------------------------------------------------------------===//
2057// Entry Points into this File
2058//===----------------------------------------------------------------------===//
2059
2060/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2061/// non-zero bytes that will be stored when outputting the initializer for the
2062/// specified initializer expression.
2063static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
2064 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
2065 E = MTE->getSubExpr();
2066 E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
2067
2068 // 0 and 0.0 won't require any non-zero stores!
2069 if (isSimpleZero(E, CGF))
2070 return CharUnits::Zero();
2071
2072 // If this is an initlist expr, sum up the size of sizes of the (present)
2073 // elements. If this is something weird, assume the whole thing is non-zero.
2074 const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
2075 while (ILE && ILE->isTransparent())
2076 ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
2077 if (!ILE || !CGF.getTypes().isZeroInitializable(T: ILE->getType()))
2078 return CGF.getContext().getTypeSizeInChars(T: E->getType());
2079
2080 // InitListExprs for structs have to be handled carefully. If there are
2081 // reference members, we need to consider the size of the reference, not the
2082 // referencee. InitListExprs for unions and arrays can't have references.
2083 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2084 if (!RT->isUnionType()) {
2085 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2086 CharUnits NumNonZeroBytes = CharUnits::Zero();
2087
2088 unsigned ILEElement = 0;
2089 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
2090 while (ILEElement != CXXRD->getNumBases())
2091 NumNonZeroBytes +=
2092 GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
2093 for (const auto *Field : SD->fields()) {
2094 // We're done once we hit the flexible array member or run out of
2095 // InitListExpr elements.
2096 if (Field->getType()->isIncompleteArrayType() ||
2097 ILEElement == ILE->getNumInits())
2098 break;
2099 if (Field->isUnnamedBitField())
2100 continue;
2101
2102 const Expr *E = ILE->getInit(Init: ILEElement++);
2103
2104 // Reference values are always non-null and have the width of a pointer.
2105 if (Field->getType()->isReferenceType())
2106 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2107 BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2108 else
2109 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2110 }
2111
2112 return NumNonZeroBytes;
2113 }
2114 }
2115
2116 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2117 CharUnits NumNonZeroBytes = CharUnits::Zero();
2118 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2119 NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2120 return NumNonZeroBytes;
2121}
2122
2123/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2124/// zeros in it, emit a memset and avoid storing the individual zeros.
2125///
2126static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2127 CodeGenFunction &CGF) {
2128 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2129 // volatile stores.
2130 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2131 return;
2132
2133 // C++ objects with a user-declared constructor don't need zero'ing.
2134 if (CGF.getLangOpts().CPlusPlus)
2135 if (const RecordType *RT = CGF.getContext()
2136 .getBaseElementType(QT: E->getType())
2137 ->getAsCanonical<RecordType>()) {
2138 const auto *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2139 if (RD->hasUserDeclaredConstructor())
2140 return;
2141 }
2142
2143 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2144 CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2145 if (Size <= CharUnits::fromQuantity(Quantity: 16))
2146 return;
2147
2148 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2149 // we prefer to emit memset + individual stores for the rest.
2150 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2151 if (NumNonZeroBytes * 4 > Size)
2152 return;
2153
2154 // Okay, it seems like a good idea to use an initial memset, emit the call.
2155 llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2156
2157 Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2158 CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2159
2160 // Tell the AggExprEmitter that the slot is known zero.
2161 Slot.setZeroed();
2162}
2163
2164/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2165/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2166/// the value of the aggregate expression is not needed. If VolatileDest is
2167/// true, DestPtr cannot be 0.
2168void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2169 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2170 "Invalid aggregate expression to emit");
2171 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2172 "slot has bits but no address");
2173
2174 // Optimize the slot if possible.
2175 CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2176
2177 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr *>(E));
2178}
2179
2180LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2181 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2182 Address Temp = CreateMemTempWithoutCast(T: E->getType());
2183 LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2184 EmitAggExpr(E, Slot: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsNotDestructed,
2185 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2186 isAliased: AggValueSlot::IsNotAliased,
2187 mayOverlap: AggValueSlot::DoesNotOverlap));
2188 return LV;
2189}
2190
2191void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,
2192 const LValue &Src,
2193 ExprValueKind SrcKind) {
2194 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2195 .EmitFinalDestCopy(type: Type, src: Src, SrcValueKind: SrcKind);
2196}
2197
2198AggValueSlot::Overlap_t
2199CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2200 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2201 return AggValueSlot::DoesNotOverlap;
2202
2203 // Empty fields can overlap earlier fields.
2204 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2205 return AggValueSlot::MayOverlap;
2206
2207 // If the field lies entirely within the enclosing class's nvsize, its tail
2208 // padding cannot overlap any already-initialized object. (The only subobjects
2209 // with greater addresses that might already be initialized are vbases.)
2210 const RecordDecl *ClassRD = FD->getParent();
2211 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2212 if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2213 getContext().getTypeSize(T: FD->getType()) <=
2214 (uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2215 return AggValueSlot::DoesNotOverlap;
2216
2217 // The tail padding may contain values we need to preserve.
2218 return AggValueSlot::MayOverlap;
2219}
2220
2221AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2222 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2223 // If the most-derived object is a field declared with [[no_unique_address]],
2224 // the tail padding of any virtual base could be reused for other subobjects
2225 // of that field's class.
2226 if (IsVirtual)
2227 return AggValueSlot::MayOverlap;
2228
2229 // Empty bases can overlap earlier bases.
2230 if (BaseRD->isEmpty())
2231 return AggValueSlot::MayOverlap;
2232
2233 // If the base class is laid out entirely within the nvsize of the derived
2234 // class, its tail padding cannot yet be initialized, so we can issue
2235 // stores at the full width of the base class.
2236 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2237 if (Layout.getBaseClassOffset(Base: BaseRD) +
2238 getContext().getASTRecordLayout(D: BaseRD).getSize() <=
2239 Layout.getNonVirtualSize())
2240 return AggValueSlot::DoesNotOverlap;
2241
2242 // The tail padding may contain values we need to preserve.
2243 return AggValueSlot::MayOverlap;
2244}
2245
2246void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2247 AggValueSlot::Overlap_t MayOverlap,
2248 bool isVolatile) {
2249 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2250
2251 Address DestPtr = Dest.getAddress();
2252 Address SrcPtr = Src.getAddress();
2253
2254 if (getLangOpts().CPlusPlus) {
2255 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2256 assert((Record->hasTrivialCopyConstructorForCall() ||
2257 Record->hasTrivialCopyAssignment() ||
2258 Record->hasTrivialMoveConstructorForCall() ||
2259 Record->hasTrivialMoveAssignment() || Record->isUnion() ||
2260 // HLSL uses aggregate-copy for user-defined record types.
2261 (getLangOpts().HLSL && !Record->isHLSLBuiltinRecord())) &&
2262 "Trying to aggregate-copy a type without a trivial copy/move "
2263 "constructor or assignment operator");
2264 // Ignore empty classes in C++.
2265 if (Record->isEmpty())
2266 return;
2267 }
2268 }
2269
2270 if (getLangOpts().CUDAIsDevice) {
2271 if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2272 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2273 Src))
2274 return;
2275 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2276 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2277 Src))
2278 return;
2279 }
2280 }
2281
2282 assert(Ty.getAddressSpace() != LangAS::hlsl_constant &&
2283 "copies of aggregates in hlsl_constant address space should be "
2284 "handled earlier by the HLSL runtime");
2285
2286 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2287 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2288 // read from another object that overlaps in anyway the storage of the first
2289 // object, then the overlap shall be exact and the two objects shall have
2290 // qualified or unqualified versions of a compatible type."
2291 //
2292 // memcpy is not defined if the source and destination pointers are exactly
2293 // equal, but other compilers do this optimization, and almost every memcpy
2294 // implementation handles this case safely. If there is a libc that does not
2295 // safely handle this, we can add a target hook.
2296
2297 // Get data size info for this aggregate. Don't copy the tail padding if this
2298 // might be a potentially-overlapping subobject, since the tail padding might
2299 // be occupied by a different object. Otherwise, copying it is fine.
2300 TypeInfoChars TypeInfo;
2301 if (MayOverlap)
2302 TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2303 else
2304 TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2305
2306 llvm::Value *SizeVal = nullptr;
2307 if (TypeInfo.Width.isZero()) {
2308 // But note that getTypeInfo returns 0 for a VLA.
2309 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2310 Val: getContext().getAsArrayType(T: Ty))) {
2311 QualType BaseEltTy;
2312 SizeVal = emitArrayLength(arrayType: VAT, baseType&: BaseEltTy, addr&: DestPtr);
2313 TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2314 assert(!TypeInfo.Width.isZero());
2315 SizeVal = Builder.CreateNUWMul(
2316 LHS: SizeVal,
2317 RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2318 }
2319 }
2320 if (!SizeVal) {
2321 SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2322 }
2323
2324 // FIXME: If we have a volatile struct, the optimizer can remove what might
2325 // appear to be `extra' memory ops:
2326 //
2327 // volatile struct { int i; } a, b;
2328 //
2329 // int main() {
2330 // a = b;
2331 // a = b;
2332 // }
2333 //
2334 // we need to use a different call here. We use isVolatile to indicate when
2335 // either the source or the destination is volatile.
2336
2337 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2338 SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2339
2340 // Don't do any of the memmove_collectable tests if GC isn't set.
2341 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2342 // fall through
2343 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2344 if (Record->hasObjectMember()) {
2345 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2346 Size: SizeVal);
2347 return;
2348 }
2349 } else if (Ty->isArrayType()) {
2350 QualType BaseType = getContext().getBaseElementType(QT: Ty);
2351 if (const auto *Record = BaseType->getAsRecordDecl()) {
2352 if (Record->hasObjectMember()) {
2353 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2354 Size: SizeVal);
2355 return;
2356 }
2357 }
2358 }
2359
2360 auto *Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2361 addInstToCurrentSourceAtom(KeyInstruction: Inst, Backup: nullptr);
2362 emitPFPPostCopyUpdates(DestPtr, SrcPtr, Ty);
2363
2364 // Determine the metadata to describe the position of any padding in this
2365 // memcpy, as well as the TBAA tags for the members of the struct, in case
2366 // the optimizer wishes to expand it in to scalar memory operations.
2367 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2368 Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2369
2370 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2371 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2372 DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2373 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2374 }
2375}
2376