1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
23#include "clang/AST/ASTContext.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
26#include "clang/AST/DeclTemplate.h"
27#include "clang/AST/StmtVisitor.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34using namespace clang;
35using namespace CodeGen;
36
37//===----------------------------------------------------------------------===//
38// Aggregate Expression Emitter
39//===----------------------------------------------------------------------===//
40
41namespace {
42class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
43 CodeGenFunction &CGF;
44 CGBuilderTy &Builder;
45 AggValueSlot Dest;
46 bool IsResultUnused;
47
48 AggValueSlot EnsureSlot(QualType T) {
49 if (!Dest.isIgnored())
50 return Dest;
51 return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
52 }
53 void EnsureDest(QualType T) {
54 if (!Dest.isIgnored())
55 return;
56 Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
57 }
58
59 // Calls `Fn` with a valid return value slot, potentially creating a temporary
60 // to do so. If a temporary is created, an appropriate copy into `Dest` will
61 // be emitted, as will lifetime markers.
62 //
63 // The given function should take a ReturnValueSlot, and return an RValue that
64 // points to said slot.
65 void withReturnValueSlot(const Expr *E,
66 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
67
68 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
69 const FieldDecl *NextField);
70
71public:
72 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
73 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
74 IsResultUnused(IsResultUnused) {}
75
76 //===--------------------------------------------------------------------===//
77 // Utilities
78 //===--------------------------------------------------------------------===//
79
80 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
81 /// represents a value lvalue, this method emits the address of the lvalue,
82 /// then loads the result into DestPtr.
83 void EmitAggLoadOfLValue(const Expr *E);
84
85 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
86 /// SrcIsRValue is true if source comes from an RValue.
87 void EmitFinalDestCopy(QualType type, const LValue &src,
88 CodeGenFunction::ExprValueKind SrcValueKind =
89 CodeGenFunction::EVK_NonRValue);
90 void EmitFinalDestCopy(QualType type, RValue src);
91 void EmitCopy(QualType type, const AggValueSlot &dest,
92 const AggValueSlot &src);
93
94 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
95 Expr *ExprToVisit, ArrayRef<Expr *> Args,
96 Expr *ArrayFiller);
97
98 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
99 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
100 return AggValueSlot::NeedsGCBarriers;
101 return AggValueSlot::DoesNotNeedGCBarriers;
102 }
103
104 bool TypeRequiresGCollection(QualType T);
105
106 //===--------------------------------------------------------------------===//
107 // Visitor Methods
108 //===--------------------------------------------------------------------===//
109
110 void Visit(Expr *E) {
111 ApplyDebugLocation DL(CGF, E);
112 StmtVisitor<AggExprEmitter>::Visit(S: E);
113 }
114
115 void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, Type: "aggregate expression"); }
116 void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
117 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
118 Visit(E: GE->getResultExpr());
119 }
120 void VisitCoawaitExpr(CoawaitExpr *E) {
121 CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
122 }
123 void VisitCoyieldExpr(CoyieldExpr *E) {
124 CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
125 }
126 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
127 void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
128 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
129 return Visit(E: E->getReplacement());
130 }
131
132 void VisitConstantExpr(ConstantExpr *E) {
133 EnsureDest(T: E->getType());
134
135 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
136 CGF.CreateCoercedStore(
137 Src: Result, SrcFETy: E->getType(), Dst: Dest.getAddress(),
138 DstSize: llvm::TypeSize::getFixed(
139 ExactSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType())
140 .getQuantity()),
141 DstIsVolatile: E->getType().isVolatileQualified());
142 return;
143 }
144 return Visit(E: E->getSubExpr());
145 }
146
147 // l-values.
148 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
149 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(E: ME); }
150 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
151 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
152 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
153 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
154 EmitAggLoadOfLValue(E);
155 }
156 void VisitPredefinedExpr(const PredefinedExpr *E) { EmitAggLoadOfLValue(E); }
157
158 // Operators.
159 void VisitCastExpr(CastExpr *E);
160 void VisitCallExpr(const CallExpr *E);
161 void VisitStmtExpr(const StmtExpr *E);
162 void VisitBinaryOperator(const BinaryOperator *BO);
163 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
164 void VisitBinAssign(const BinaryOperator *E);
165 void VisitBinComma(const BinaryOperator *E);
166 void VisitBinCmp(const BinaryOperator *E);
167 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
168 Visit(E: E->getSemanticForm());
169 }
170
171 void VisitObjCMessageExpr(ObjCMessageExpr *E);
172 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { EmitAggLoadOfLValue(E); }
173
174 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
175 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
176 void VisitChooseExpr(const ChooseExpr *CE);
177 void VisitInitListExpr(InitListExpr *E);
178 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
179 FieldDecl *InitializedFieldInUnion,
180 Expr *ArrayFiller);
181 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
182 llvm::Value *outerBegin = nullptr);
183 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
184 void VisitNoInitExpr(NoInitExpr *E) {} // Do nothing.
185 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
186 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
187 Visit(E: DAE->getExpr());
188 }
189 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
190 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
191 Visit(E: DIE->getExpr());
192 }
193 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
194 void VisitCXXConstructExpr(const CXXConstructExpr *E);
195 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
196 void VisitLambdaExpr(LambdaExpr *E);
197 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
198 void VisitExprWithCleanups(ExprWithCleanups *E);
199 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
200 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
201 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
202 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
203
204 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
205 if (E->isGLValue()) {
206 LValue LV = CGF.EmitPseudoObjectLValue(e: E);
207 return EmitFinalDestCopy(type: E->getType(), src: LV);
208 }
209
210 AggValueSlot Slot = EnsureSlot(T: E->getType());
211 bool NeedsDestruction =
212 !Slot.isExternallyDestructed() &&
213 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
214 if (NeedsDestruction)
215 Slot.setExternallyDestructed();
216 CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
217 if (NeedsDestruction)
218 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Slot.getAddress(),
219 type: E->getType());
220 }
221
222 void VisitVAArgExpr(VAArgExpr *E);
223 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
224 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
225 Expr *ArrayFiller);
226
227 void EmitInitializationToLValue(Expr *E, LValue Address);
228 void EmitNullInitializationToLValue(LValue Address);
229 // case Expr::ChooseExprClass:
230 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
231 void VisitAtomicExpr(AtomicExpr *E) {
232 RValue Res = CGF.EmitAtomicExpr(E);
233 EmitFinalDestCopy(type: E->getType(), src: Res);
234 }
235 void VisitPackIndexingExpr(PackIndexingExpr *E) {
236 Visit(E: E->getSelectedExpr());
237 }
238};
239} // end anonymous namespace.
240
241//===----------------------------------------------------------------------===//
242// Utilities
243//===----------------------------------------------------------------------===//
244
245/// EmitAggLoadOfLValue - Given an expression with aggregate type that
246/// represents a value lvalue, this method emits the address of the lvalue,
247/// then loads the result into DestPtr.
248void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
249 LValue LV = CGF.EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load);
250
251 // If the type of the l-value is atomic, then do an atomic load.
252 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
253 CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
254 return;
255 }
256
257 EmitFinalDestCopy(type: E->getType(), src: LV);
258}
259
260/// True if the given aggregate type requires special GC API calls.
261bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
262 // Only record types have members that might require garbage collection.
263 const auto *Record = T->getAsRecordDecl();
264 if (!Record)
265 return false;
266
267 // Don't mess with non-trivial C++ types.
268 if (isa<CXXRecordDecl>(Val: Record) &&
269 (cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
270 !cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
271 return false;
272
273 // Check whether the type has an object member.
274 return Record->hasObjectMember();
275}
276
277void AggExprEmitter::withReturnValueSlot(
278 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
279 QualType RetTy = E->getType();
280 bool RequiresDestruction =
281 !Dest.isExternallyDestructed() &&
282 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
283
284 // If it makes no observable difference, save a memcpy + temporary.
285 //
286 // We need to always provide our own temporary if destruction is required.
287 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
288 // its lifetime before we have the chance to emit a proper destructor call.
289 //
290 // We also need a temporary if the destination is in a different address space
291 // from the sret AS. Use the target hook to get the actual sret AS for this
292 // return type.
293 const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl();
294 LangAS SRetLangAS = CGF.CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
295 unsigned SRetAS = CGF.getContext().getTargetAddressSpace(AS: SRetLangAS);
296 bool CanAggregateCopy =
297 RD ? (RD->hasTrivialCopyConstructor() ||
298 RD->hasTrivialMoveConstructor() || RD->hasTrivialCopyAssignment() ||
299 RD->hasTrivialMoveAssignment() || RD->hasAttr<TrivialABIAttr>() ||
300 RD->isUnion())
301 : RetTy.isTriviallyCopyableType(Context: CGF.getContext());
302 bool DestASMismatch = !Dest.isIgnored() && CanAggregateCopy &&
303 Dest.getAddress()
304 .getBasePointer()
305 ->stripPointerCasts()
306 ->getType()
307 ->getPointerAddressSpace() != SRetAS;
308 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
309 (RequiresDestruction && Dest.isIgnored()) || DestASMismatch;
310
311 Address RetAddr = Address::invalid();
312
313 EHScopeStack::stable_iterator LifetimeEndBlock;
314 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
315 if (!UseTemp) {
316 RetAddr = Dest.getAddress();
317 if (RetAddr.isValid() && RetAddr.getAddressSpace() != SRetAS) {
318 llvm::Type *SRetPtrTy =
319 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: SRetAS);
320 RetAddr = RetAddr.withPointer(
321 NewPointer: CGF.performAddrSpaceCast(Src: RetAddr.getBasePointer(), DestTy: SRetPtrTy),
322 IsKnownNonNull: RetAddr.isKnownNonNull());
323 }
324 } else {
325 RetAddr = CGF.CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
326 if (CGF.EmitLifetimeStart(Addr: RetAddr.getBasePointer())) {
327 LifetimeStartInst =
328 cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
329 assert(LifetimeStartInst->getIntrinsicID() ==
330 llvm::Intrinsic::lifetime_start &&
331 "Last insertion wasn't a lifetime.start?");
332
333 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
334 kind: NormalEHLifetimeMarker, A: RetAddr);
335 LifetimeEndBlock = CGF.EHStack.stable_begin();
336 }
337 }
338
339 RValue Src =
340 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
341 Dest.isExternallyDestructed()));
342
343 if (!UseTemp)
344 return;
345
346 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
347 Src.getAggregatePointer(E->getType(), CGF));
348 EmitFinalDestCopy(type: E->getType(), src: Src);
349
350 if (!RequiresDestruction && LifetimeStartInst) {
351 // If there's no dtor to run, the copy was the last use of our temporary.
352 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
353 // eagerly.
354 CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
355 CGF.EmitLifetimeEnd(Addr: RetAddr.getBasePointer());
356 }
357}
358
359/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
360void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
361 assert(src.isAggregate() && "value must be aggregate value!");
362 LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
363 EmitFinalDestCopy(type, src: srcLV, SrcValueKind: CodeGenFunction::EVK_RValue);
364}
365
366/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
367void AggExprEmitter::EmitFinalDestCopy(
368 QualType type, const LValue &src,
369 CodeGenFunction::ExprValueKind SrcValueKind) {
370 // If Dest is ignored, then we're evaluating an aggregate expression
371 // in a context that doesn't care about the result. Note that loads
372 // from volatile l-values force the existence of a non-ignored
373 // destination.
374 if (Dest.isIgnored())
375 return;
376
377 // Copy non-trivial C structs here.
378 LValue DstLV = CGF.MakeAddrLValue(
379 Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
380
381 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
382 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
383 if (Dest.isPotentiallyAliased())
384 CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
385 else
386 CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
387 return;
388 }
389 } else {
390 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
391 if (Dest.isPotentiallyAliased())
392 CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
393 else
394 CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
395 return;
396 }
397 }
398
399 AggValueSlot srcAgg = AggValueSlot::forLValue(
400 LV: src, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type), isAliased: AggValueSlot::IsAliased,
401 mayOverlap: AggValueSlot::MayOverlap);
402 EmitCopy(type, dest: Dest, src: srcAgg);
403}
404
405/// Perform a copy from the source into the destination.
406///
407/// \param type - the type of the aggregate being copied; qualifiers are
408/// ignored
409void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
410 const AggValueSlot &src) {
411 if (dest.requiresGCollection()) {
412 CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
413 llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
414 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr: dest.getAddress(),
415 SrcPtr: src.getAddress(), Size: size);
416 return;
417 }
418
419 // If the result of the assignment is used, copy the LHS there also.
420 // It's volatile if either side is. Use the minimum alignment of
421 // the two sides.
422 LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
423 LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
424 CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
425 isVolatile: dest.isVolatile() || src.isVolatile());
426}
427
428/// Emit the initializer for a std::initializer_list initialized with a
429/// real initializer list.
430void AggExprEmitter::VisitCXXStdInitializerListExpr(
431 CXXStdInitializerListExpr *E) {
432 // Emit an array containing the elements. The array is externally destructed
433 // if the std::initializer_list object is.
434 ASTContext &Ctx = CGF.getContext();
435 LValue Array = CGF.EmitLValue(E: E->getSubExpr());
436 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
437 Address ArrayPtr = Array.getAddress();
438
439 const ConstantArrayType *ArrayType =
440 Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
441 assert(ArrayType && "std::initializer_list constructed from non-array");
442
443 auto *Record = E->getType()->castAsRecordDecl();
444 RecordDecl::field_iterator Field = Record->field_begin();
445 assert(Field != Record->field_end() &&
446 Ctx.hasSameType(Field->getType()->getPointeeType(),
447 ArrayType->getElementType()) &&
448 "Expected std::initializer_list first field to be const E *");
449
450 // Start pointer.
451 AggValueSlot Dest = EnsureSlot(T: E->getType());
452 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
453 LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
454 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
455 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
456 ++Field;
457 assert(Field != Record->field_end() &&
458 "Expected std::initializer_list to have two fields");
459
460 llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
461 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
462 if (Ctx.hasSameType(T1: Field->getType(), T2: Ctx.getSizeType())) {
463 // Length.
464 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
465
466 } else {
467 // End pointer.
468 assert(Field->getType()->isPointerType() &&
469 Ctx.hasSameType(Field->getType()->getPointeeType(),
470 ArrayType->getElementType()) &&
471 "Expected std::initializer_list second field to be const E *");
472 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
473 llvm::Value *IdxEnd[] = {Zero, Size};
474 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
475 Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
476 Name: "arrayend");
477 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
478 }
479
480 assert(++Field == Record->field_end() &&
481 "Expected std::initializer_list to only have two fields");
482}
483
484/// Determine if E is a trivial array filler, that is, one that is
485/// equivalent to zero-initialization.
486static bool isTrivialFiller(Expr *E) {
487 if (!E)
488 return true;
489
490 if (isa<ImplicitValueInitExpr>(Val: E))
491 return true;
492
493 if (auto *ILE = dyn_cast<InitListExpr>(Val: E)) {
494 if (ILE->getNumInits())
495 return false;
496 return isTrivialFiller(E: ILE->getArrayFiller());
497 }
498
499 if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(Val: E))
500 return Cons->getConstructor()->isDefaultConstructor() &&
501 Cons->getConstructor()->isTrivial();
502
503 // FIXME: Are there other cases where we can avoid emitting an initializer?
504 return false;
505}
506
507// emit an elementwise cast where the RHS is a scalar or vector
508// or emit an aggregate splat cast
509static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,
510 LValue DestVal,
511 llvm::Value *SrcVal,
512 QualType SrcTy,
513 SourceLocation Loc) {
514 // Flatten our destination
515 SmallVector<LValue, 16> StoreList;
516 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
517
518 bool isVector = false;
519 if (auto *VT = SrcTy->getAs<VectorType>()) {
520 isVector = true;
521 SrcTy = VT->getElementType();
522 assert(StoreList.size() <= VT->getNumElements() &&
523 "Cannot perform HLSL flat cast when vector source \
524 object has less elements than flattened destination \
525 object.");
526 }
527
528 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
529 LValue DestLVal = StoreList[I];
530 llvm::Value *Load =
531 isVector ? CGF.Builder.CreateExtractElement(Vec: SrcVal, Idx: I, Name: "vec.load")
532 : SrcVal;
533 llvm::Value *Cast =
534 CGF.EmitScalarConversion(Src: Load, SrcTy, DstTy: DestLVal.getType(), Loc);
535 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
536 }
537}
538
539// emit a flat cast where the RHS is an aggregate
540static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
541 LValue SrcVal, SourceLocation Loc) {
542 // Flatten our destination
543 SmallVector<LValue, 16> StoreList;
544 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
545 // Flatten our src
546 SmallVector<LValue, 16> LoadList;
547 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
548
549 assert(StoreList.size() <= LoadList.size() &&
550 "Cannot perform HLSL elementwise cast when flattened source object \
551 has less elements than flattened destination object.");
552 // apply casts to what we load from LoadList
553 // and store result in Dest
554 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
555 LValue DestLVal = StoreList[I];
556 LValue SrcLVal = LoadList[I];
557 RValue RVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc);
558 assert(RVal.isScalar() && "All flattened source values should be scalars");
559 llvm::Value *Val = RVal.getScalarVal();
560 llvm::Value *Cast = CGF.EmitScalarConversion(Src: Val, SrcTy: SrcLVal.getType(),
561 DstTy: DestLVal.getType(), Loc);
562 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
563 }
564}
565
566/// Emit initialization of an array from an initializer list. ExprToVisit must
567/// be either an InitListEpxr a CXXParenInitListExpr.
568void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
569 QualType ArrayQTy, Expr *ExprToVisit,
570 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
571 uint64_t NumInitElements = Args.size();
572
573 uint64_t NumArrayElements = AType->getNumElements();
574 for (const auto *Init : Args) {
575 if (const auto *Embed = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
576 NumInitElements += Embed->getDataElementCount() - 1;
577 if (NumInitElements > NumArrayElements) {
578 NumInitElements = NumArrayElements;
579 break;
580 }
581 }
582 }
583
584 assert(NumInitElements <= NumArrayElements);
585
586 QualType elementType =
587 CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
588 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
589 CharUnits elementAlign =
590 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
591 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
592
593 // Consider initializing the array by copying from a global. For this to be
594 // more efficient than per-element initialization, the size of the elements
595 // with explicit initializers should be large enough.
596 if (NumInitElements * elementSize.getQuantity() > 16 &&
597 elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
598 CodeGen::CodeGenModule &CGM = CGF.CGM;
599 ConstantEmitter Emitter(CGF);
600 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
601 T: CGM.getContext().removeAddrSpaceQualType(T: ArrayQTy),
602 AddressSpace: CGM.GetGlobalConstantAddressSpace());
603 LangAS AS = GVArrayQTy.getAddressSpace();
604 if (llvm::Constant *C =
605 Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: GVArrayQTy)) {
606 auto GV = new llvm::GlobalVariable(
607 CGM.getModule(), C->getType(),
608 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
609 "constinit",
610 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
611 CGM.getContext().getTargetAddressSpace(AS));
612 Emitter.finalize(global: GV);
613 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: GVArrayQTy);
614 GV->setAlignment(Align.getAsAlign());
615 Address GVAddr(GV, GV->getValueType(), Align);
616 EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: GVArrayQTy));
617 return;
618 }
619 }
620
621 // Exception safety requires us to destroy all the
622 // already-constructed members if an initializer throws.
623 // For that, we'll need an EH cleanup.
624 QualType::DestructionKind dtorKind = elementType.isDestructedType();
625 Address endOfInit = Address::invalid();
626 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
627
628 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
629 if (dtorKind) {
630 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
631 // In principle we could tell the cleanup where we are more
632 // directly, but the control flow can get so varied here that it
633 // would actually be quite complex. Therefore we go through an
634 // alloca.
635 llvm::Instruction *dominatingIP =
636 Builder.CreateFlagLoad(Addr: llvm::ConstantInt::getNullValue(Ty: CGF.Int8PtrTy));
637 endOfInit = CGF.CreateTempAlloca(Ty: begin->getType(), align: CGF.getPointerAlign(),
638 Name: "arrayinit.endOfInit");
639 Builder.CreateStore(Val: begin, Addr: endOfInit);
640 CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
641 elementAlignment: elementAlign,
642 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
643 cast<EHCleanupScope>(Val&: *CGF.EHStack.find(sp: CGF.EHStack.stable_begin()))
644 .AddAuxAllocas(Allocas: allocaTracker.Take());
645
646 CGF.DeferredDeactivationCleanupStack.push_back(
647 Elt: {.Cleanup: CGF.EHStack.stable_begin(), .DominatingIP: dominatingIP});
648 }
649
650 llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
651
652 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
653 llvm::Value *element = begin;
654 if (ArrayIndex > 0) {
655 if (CGF.getLangOpts().EmitLogicalPointer)
656 element = Builder.CreateStructuredGEP(
657 BaseType: AType, PtrBase: begin, Indices: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
658 Name: "arrayinit.element");
659 else
660 element = Builder.CreateInBoundsGEP(
661 Ty: llvmElementType, Ptr: begin,
662 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
663 Name: "arrayinit.element");
664
665 // Tell the cleanup that it needs to destroy up to this
666 // element. TODO: some of these stores can be trivially
667 // observed to be unnecessary.
668 if (endOfInit.isValid())
669 Builder.CreateStore(Val: element, Addr: endOfInit);
670 }
671
672 LValue elementLV = CGF.MakeAddrLValue(
673 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
674 EmitInitializationToLValue(E: Init, Address: elementLV);
675 return true;
676 };
677
678 unsigned ArrayIndex = 0;
679 // Emit the explicit initializers.
680 for (uint64_t i = 0; i != NumInitElements; ++i) {
681 if (ArrayIndex >= NumInitElements)
682 break;
683 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Args[i]->IgnoreParenImpCasts())) {
684 EmbedS->doForEachDataElement(C&: Emit, StartingIndexInArray&: ArrayIndex);
685 } else {
686 Emit(Args[i], ArrayIndex);
687 ArrayIndex++;
688 }
689 }
690
691 // Check whether there's a non-trivial array-fill expression.
692 bool hasTrivialFiller = isTrivialFiller(E: ArrayFiller);
693
694 // Any remaining elements need to be zero-initialized, possibly
695 // using the filler expression. We can skip this if the we're
696 // emitting to zeroed memory.
697 if (NumInitElements != NumArrayElements &&
698 !(Dest.isZeroed() && hasTrivialFiller &&
699 CGF.getTypes().isZeroInitializable(T: elementType))) {
700
701 // Use an actual loop. This is basically
702 // do { *array++ = filler; } while (array != end);
703
704 // Advance to the start of the rest of the array.
705 llvm::Value *element = begin;
706 if (NumInitElements) {
707 element = Builder.CreateInBoundsGEP(
708 Ty: llvmElementType, Ptr: element,
709 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumInitElements),
710 Name: "arrayinit.start");
711 if (endOfInit.isValid())
712 Builder.CreateStore(Val: element, Addr: endOfInit);
713 }
714
715 // Compute the end of the array.
716 llvm::Value *end = Builder.CreateInBoundsGEP(
717 Ty: llvmElementType, Ptr: begin,
718 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
719
720 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
721 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
722
723 // Jump into the body.
724 CGF.EmitBlock(BB: bodyBB);
725 llvm::PHINode *currentElement =
726 Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
727 currentElement->addIncoming(V: element, BB: entryBB);
728
729 if (CGF.CGM.shouldEmitConvergenceTokens())
730 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
731
732 // Emit the actual filler expression.
733 {
734 // C++1z [class.temporary]p5:
735 // when a default constructor is called to initialize an element of
736 // an array with no corresponding initializer [...] the destruction of
737 // every temporary created in a default argument is sequenced before
738 // the construction of the next array element, if any
739 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
740 LValue elementLV = CGF.MakeAddrLValue(
741 Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
742 if (ArrayFiller)
743 EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
744 else
745 EmitNullInitializationToLValue(Address: elementLV);
746 }
747
748 // Move on to the next element.
749 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
750 Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
751
752 // Tell the EH cleanup that we finished with the last element.
753 if (endOfInit.isValid())
754 Builder.CreateStore(Val: nextElement, Addr: endOfInit);
755
756 // Leave the loop if we're done.
757 llvm::Value *done =
758 Builder.CreateICmpEQ(LHS: nextElement, RHS: end, Name: "arrayinit.done");
759 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
760 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
761 currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
762
763 if (CGF.CGM.shouldEmitConvergenceTokens())
764 CGF.ConvergenceTokenStack.pop_back();
765
766 CGF.EmitBlock(BB: endBB);
767 }
768}
769
770//===----------------------------------------------------------------------===//
771// Visitor Methods
772//===----------------------------------------------------------------------===//
773
774void AggExprEmitter::VisitMaterializeTemporaryExpr(
775 MaterializeTemporaryExpr *E) {
776 Visit(E: E->getSubExpr());
777}
778
779void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
780 // If this is a unique OVE, just visit its source expression.
781 if (e->isUnique())
782 Visit(E: e->getSourceExpr());
783 else
784 EmitFinalDestCopy(type: e->getType(), src: CGF.getOrCreateOpaqueLValueMapping(e));
785}
786
787void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
788 if (Dest.isPotentiallyAliased()) {
789 // Just emit a load of the lvalue + a copy, because our compound literal
790 // might alias the destination.
791 EmitAggLoadOfLValue(E);
792 return;
793 }
794
795 AggValueSlot Slot = EnsureSlot(T: E->getType());
796
797 // Block-scope compound literals are destroyed at the end of the enclosing
798 // scope in C.
799 bool Destruct =
800 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
801 if (Destruct)
802 Slot.setExternallyDestructed();
803
804 CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
805
806 if (Destruct)
807 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
808 CGF.pushLifetimeExtendedDestroy(
809 kind: CGF.getCleanupKind(kind: DtorKind), addr: Slot.getAddress(), type: E->getType(),
810 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: DtorKind & EHCleanup);
811}
812
813/// Attempt to look through various unimportant expressions to find a
814/// cast of the given kind.
815static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
816 op = op->IgnoreParenNoopCasts(Ctx: ctx);
817 if (auto castE = dyn_cast<CastExpr>(Val: op)) {
818 if (castE->getCastKind() == kind)
819 return castE->getSubExpr();
820 }
821 return nullptr;
822}
823
824void AggExprEmitter::VisitCastExpr(CastExpr *E) {
825 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
826 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
827 switch (E->getCastKind()) {
828 case CK_Dynamic: {
829 // FIXME: Can this actually happen? We have no test coverage for it.
830 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
831 LValue LV =
832 CGF.EmitCheckedLValue(E: E->getSubExpr(), TCK: CodeGenFunction::TCK_Load);
833 // FIXME: Do we also need to handle property references here?
834 if (LV.isSimple())
835 CGF.EmitDynamicCast(V: LV.getAddress(), DCE: cast<CXXDynamicCastExpr>(Val: E));
836 else
837 CGF.CGM.ErrorUnsupported(S: E, Type: "non-simple lvalue dynamic_cast");
838
839 if (!Dest.isIgnored())
840 CGF.CGM.ErrorUnsupported(S: E, Type: "lvalue dynamic_cast with a destination");
841 break;
842 }
843
844 case CK_ToUnion: {
845 // Evaluate even if the destination is ignored.
846 if (Dest.isIgnored()) {
847 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
848 /*ignoreResult=*/true);
849 break;
850 }
851
852 // GCC union extension
853 QualType Ty = E->getSubExpr()->getType();
854 Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
855 EmitInitializationToLValue(E: E->getSubExpr(),
856 Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
857 break;
858 }
859
860 case CK_LValueToRValueBitCast: {
861 if (Dest.isIgnored()) {
862 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
863 /*ignoreResult=*/true);
864 break;
865 }
866
867 LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
868 Address SourceAddress = SourceLV.getAddress().withElementType(ElemTy: CGF.Int8Ty);
869 Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
870 llvm::Value *SizeVal = llvm::ConstantInt::get(
871 Ty: CGF.SizeTy,
872 V: CGF.getContext().getTypeSizeInChars(T: E->getType()).getQuantity());
873 Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
874 break;
875 }
876
877 case CK_DerivedToBase: {
878 assert(CGF.getLangOpts().HLSL &&
879 "Derived/Base casts in EmitAggExpr are only supported in HLSL");
880
881 // Create a temporary for the derived record, switch it out with the current
882 // Dest slot, and emit the derived value.
883 QualType DerivedTy = E->getSubExpr()->getType();
884 RawAddress DerivedAddr = CGF.CreateMemTempWithoutCast(T: DerivedTy);
885 AggValueSlot DerivedTmpSlot = AggValueSlot::forAddr(
886 addr: DerivedAddr, quals: DerivedTy.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
887 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
888 mayOverlap: AggValueSlot::DoesNotOverlap);
889
890 AggValueSlot DestBaseSlot = Dest;
891 Dest = DerivedTmpSlot;
892
893 Visit(E: E->getSubExpr());
894
895 // Perform derived-to-base address conversion to get the address
896 // of the base record within the derived record. In HLSL this should
897 // always be same as the derived because of single inheritance, but let's
898 // do it properly.
899 Address BaseAddrInDerived = CGF.GetAddressOfBaseClass(
900 Value: DerivedTmpSlot.getAddress(), Derived: DerivedTy->castAsCXXRecordDecl(),
901 PathBegin: E->path_begin(), PathEnd: E->path_end(),
902 /*NullCheckValue=*/false, Loc: E->getExprLoc());
903
904 AggValueSlot SrcBaseSlot = AggValueSlot::forAddr(
905 addr: BaseAddrInDerived, quals: E->getType().getQualifiers(),
906 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
907 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
908
909 // Copy the base class to the original destination slot and restore it.
910 EmitCopy(type: E->getType(), dest: DestBaseSlot, src: SrcBaseSlot);
911 Dest = DestBaseSlot;
912 break;
913 }
914
915 case CK_BaseToDerived:
916 case CK_UncheckedDerivedToBase: {
917 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
918 "should have been unpacked before we got here");
919 }
920
921 case CK_NonAtomicToAtomic:
922 case CK_AtomicToNonAtomic: {
923 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
924
925 // Determine the atomic and value types.
926 QualType atomicType = E->getSubExpr()->getType();
927 QualType valueType = E->getType();
928 if (isToAtomic)
929 std::swap(a&: atomicType, b&: valueType);
930
931 assert(atomicType->isAtomicType());
932 assert(CGF.getContext().hasSameUnqualifiedType(
933 valueType, atomicType->castAs<AtomicType>()->getValueType()));
934
935 // Just recurse normally if we're ignoring the result or the
936 // atomic type doesn't change representation.
937 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
938 return Visit(E: E->getSubExpr());
939 }
940
941 CastKind peepholeTarget =
942 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
943
944 // These two cases are reverses of each other; try to peephole them.
945 if (Expr *op =
946 findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
947 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
948 E->getType()) &&
949 "peephole significantly changed types?");
950 return Visit(E: op);
951 }
952
953 // If we're converting an r-value of non-atomic type to an r-value
954 // of atomic type, just emit directly into the relevant sub-object.
955 if (isToAtomic) {
956 AggValueSlot valueDest = Dest;
957 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
958 // Zero-initialize. (Strictly speaking, we only need to initialize
959 // the padding at the end, but this is simpler.)
960 if (!Dest.isZeroed())
961 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
962
963 // Build a GEP to refer to the subobject.
964 Address valueAddr =
965 CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
966 valueDest = AggValueSlot::forAddr(
967 addr: valueAddr, quals: valueDest.getQualifiers(),
968 isDestructed: valueDest.isExternallyDestructed(), needsGC: valueDest.requiresGCollection(),
969 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
970 isZeroed: AggValueSlot::IsZeroed);
971 }
972
973 CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
974 return;
975 }
976
977 // Otherwise, we're converting an atomic type to a non-atomic type.
978 // Make an atomic temporary, emit into that, and then copy the value out.
979 AggValueSlot atomicSlot =
980 CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
981 CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
982
983 Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
984 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
985 return EmitFinalDestCopy(type: valueType, src: rvalue);
986 }
987 case CK_AddressSpaceConversion:
988 return Visit(E: E->getSubExpr());
989
990 case CK_LValueToRValue:
991 // If we're loading from a volatile type, force the destination
992 // into existence.
993 if (E->getSubExpr()->getType().isVolatileQualified()) {
994 bool Destruct =
995 !Dest.isExternallyDestructed() &&
996 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
997 if (Destruct)
998 Dest.setExternallyDestructed();
999 EnsureDest(T: E->getType());
1000 Visit(E: E->getSubExpr());
1001
1002 if (Destruct)
1003 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1004 type: E->getType());
1005
1006 return;
1007 }
1008
1009 [[fallthrough]];
1010
1011 case CK_HLSLArrayRValue:
1012 if (CGF.getLangOpts().HLSL &&
1013 E->getSubExpr()->getType()->isHLSLResourceRecordArray())
1014 if (CGF.CGM.getHLSLRuntime().emitGlobalResourceArray(CGF, E, DestSlot&: Dest))
1015 break;
1016 Visit(E: E->getSubExpr());
1017 break;
1018 case CK_HLSLAggregateSplatCast: {
1019 Expr *Src = E->getSubExpr();
1020 QualType SrcTy = Src->getType();
1021 RValue RV = CGF.EmitAnyExpr(E: Src);
1022 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1023 SourceLocation Loc = E->getExprLoc();
1024
1025 assert(RV.isScalar() && SrcTy->isScalarType() &&
1026 "RHS of HLSL splat cast must be a scalar.");
1027 llvm::Value *SrcVal = RV.getScalarVal();
1028 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1029 break;
1030 }
1031 case CK_HLSLElementwiseCast: {
1032 Expr *Src = E->getSubExpr();
1033 QualType SrcTy = Src->getType();
1034 RValue RV = CGF.EmitAnyExpr(E: Src);
1035 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1036 SourceLocation Loc = E->getExprLoc();
1037
1038 if (RV.isScalar()) {
1039 llvm::Value *SrcVal = RV.getScalarVal();
1040 assert(SrcTy->isVectorType() &&
1041 "HLSL Elementwise cast doesn't handle splatting.");
1042 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1043 } else {
1044 assert(RV.isAggregate() &&
1045 "Can't perform HLSL Aggregate cast on a complex type.");
1046 Address SrcVal = RV.getAggregateAddress();
1047 EmitHLSLElementwiseCast(CGF, DestVal: DestLVal, SrcVal: CGF.MakeAddrLValue(Addr: SrcVal, T: SrcTy),
1048 Loc);
1049 }
1050 break;
1051 }
1052 case CK_NoOp:
1053 case CK_UserDefinedConversion:
1054 case CK_ConstructorConversion:
1055 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
1056 E->getType()) &&
1057 "Implicit cast types must be compatible");
1058 Visit(E: E->getSubExpr());
1059 break;
1060
1061 case CK_LValueBitCast:
1062 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
1063
1064 case CK_Dependent:
1065 case CK_BitCast:
1066 case CK_ArrayToPointerDecay:
1067 case CK_FunctionToPointerDecay:
1068 case CK_NullToPointer:
1069 case CK_NullToMemberPointer:
1070 case CK_BaseToDerivedMemberPointer:
1071 case CK_DerivedToBaseMemberPointer:
1072 case CK_MemberPointerToBoolean:
1073 case CK_ReinterpretMemberPointer:
1074 case CK_IntegralToPointer:
1075 case CK_PointerToIntegral:
1076 case CK_PointerToBoolean:
1077 case CK_ToVoid:
1078 case CK_VectorSplat:
1079 case CK_IntegralCast:
1080 case CK_BooleanToSignedIntegral:
1081 case CK_IntegralToBoolean:
1082 case CK_IntegralToFloating:
1083 case CK_FloatingToIntegral:
1084 case CK_FloatingToBoolean:
1085 case CK_FloatingCast:
1086 case CK_CPointerToObjCPointerCast:
1087 case CK_BlockPointerToObjCPointerCast:
1088 case CK_AnyPointerToBlockPointerCast:
1089 case CK_ObjCObjectLValueCast:
1090 case CK_FloatingRealToComplex:
1091 case CK_FloatingComplexToReal:
1092 case CK_FloatingComplexToBoolean:
1093 case CK_FloatingComplexCast:
1094 case CK_FloatingComplexToIntegralComplex:
1095 case CK_IntegralRealToComplex:
1096 case CK_IntegralComplexToReal:
1097 case CK_IntegralComplexToBoolean:
1098 case CK_IntegralComplexCast:
1099 case CK_IntegralComplexToFloatingComplex:
1100 case CK_ARCProduceObject:
1101 case CK_ARCConsumeObject:
1102 case CK_ARCReclaimReturnedObject:
1103 case CK_ARCExtendBlockObject:
1104 case CK_CopyAndAutoreleaseBlockObject:
1105 case CK_BuiltinFnToFnPtr:
1106 case CK_ZeroToOCLOpaqueType:
1107 case CK_MatrixCast:
1108 case CK_HLSLVectorTruncation:
1109 case CK_HLSLMatrixTruncation:
1110 case CK_IntToOCLSampler:
1111 case CK_FloatingToFixedPoint:
1112 case CK_FixedPointToFloating:
1113 case CK_FixedPointCast:
1114 case CK_FixedPointToBoolean:
1115 case CK_FixedPointToIntegral:
1116 case CK_IntegralToFixedPoint:
1117 llvm_unreachable("cast kind invalid for aggregate types");
1118 }
1119}
1120
1121void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1122 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
1123 EmitAggLoadOfLValue(E);
1124 return;
1125 }
1126
1127 withReturnValueSlot(
1128 E, EmitCall: [&](ReturnValueSlot Slot) { return CGF.EmitCallExpr(E, ReturnValue: Slot); });
1129}
1130
1131void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1132 withReturnValueSlot(E, EmitCall: [&](ReturnValueSlot Slot) {
1133 return CGF.EmitObjCMessageExpr(E, Return: Slot);
1134 });
1135}
1136
1137void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1138 CGF.EmitIgnoredExpr(E: E->getLHS());
1139 Visit(E: E->getRHS());
1140}
1141
1142void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1143 CodeGenFunction::StmtExprEvaluation eval(CGF);
1144 CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
1145}
1146
1147enum CompareKind {
1148 CK_Less,
1149 CK_Greater,
1150 CK_Equal,
1151};
1152
1153static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1154 const BinaryOperator *E, llvm::Value *LHS,
1155 llvm::Value *RHS, CompareKind Kind,
1156 const char *NameSuffix = "") {
1157 QualType ArgTy = E->getLHS()->getType();
1158 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1159 ArgTy = CT->getElementType();
1160
1161 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1162 assert(Kind == CK_Equal &&
1163 "member pointers may only be compared for equality");
1164 return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1165 CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1166 }
1167
1168 // Compute the comparison instructions for the specified comparison kind.
1169 struct CmpInstInfo {
1170 const char *Name;
1171 llvm::CmpInst::Predicate FCmp;
1172 llvm::CmpInst::Predicate SCmp;
1173 llvm::CmpInst::Predicate UCmp;
1174 };
1175 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1176 using FI = llvm::FCmpInst;
1177 using II = llvm::ICmpInst;
1178 switch (Kind) {
1179 case CK_Less:
1180 return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1181 case CK_Greater:
1182 return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1183 case CK_Equal:
1184 return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1185 }
1186 llvm_unreachable("Unrecognised CompareKind enum");
1187 }();
1188
1189 if (ArgTy->hasFloatingRepresentation())
1190 return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1191 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1192 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1193 auto Inst =
1194 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1195 return Builder.CreateICmp(P: Inst, LHS, RHS,
1196 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1197 }
1198
1199 llvm_unreachable("unsupported aggregate binary expression should have "
1200 "already been handled");
1201}
1202
1203void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1204 using llvm::BasicBlock;
1205 using llvm::PHINode;
1206 using llvm::Value;
1207 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1208 E->getRHS()->getType()));
1209 const ComparisonCategoryInfo &CmpInfo =
1210 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1211 assert(CmpInfo.Record->isTriviallyCopyable() &&
1212 "cannot copy non-trivially copyable aggregate");
1213
1214 QualType ArgTy = E->getLHS()->getType();
1215
1216 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1217 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1218 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1219 return CGF.ErrorUnsupported(S: E, Type: "aggregate three-way comparison");
1220 }
1221 bool IsComplex = ArgTy->isAnyComplexType();
1222
1223 // Evaluate the operands to the expression and extract their values.
1224 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1225 RValue RV = CGF.EmitAnyExpr(E);
1226 if (RV.isScalar())
1227 return {RV.getScalarVal(), nullptr};
1228 if (RV.isAggregate())
1229 return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1230 assert(RV.isComplex());
1231 return RV.getComplexVal();
1232 };
1233 auto LHSValues = EmitOperand(E->getLHS()),
1234 RHSValues = EmitOperand(E->getRHS());
1235
1236 auto EmitCmp = [&](CompareKind K) {
1237 Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1238 Kind: K, NameSuffix: IsComplex ? ".r" : "");
1239 if (!IsComplex)
1240 return Cmp;
1241 assert(K == CompareKind::CK_Equal);
1242 Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1243 RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1244 return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1245 };
1246 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1247 return Builder.getInt(AI: VInfo->getIntValue());
1248 };
1249
1250 Value *Select;
1251 if (ArgTy->isNullPtrType()) {
1252 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1253 } else if (!CmpInfo.isPartial()) {
1254 Value *SelectOne =
1255 Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1256 False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1257 Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1258 True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1259 False: SelectOne, Name: "sel.eq");
1260 } else {
1261 Value *SelectEq = Builder.CreateSelect(
1262 C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1263 False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1264 Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1265 True: EmitCmpRes(CmpInfo.getGreater()),
1266 False: SelectEq, Name: "sel.gt");
1267 Select = Builder.CreateSelect(
1268 C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1269 }
1270 // Create the return value in the destination slot.
1271 EnsureDest(T: E->getType());
1272 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1273
1274 // Emit the address of the first (and only) field in the comparison category
1275 // type, and initialize it from the constant integer value selected above.
1276 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1277 Base: DestLV, Field: *CmpInfo.Record->field_begin());
1278 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Select), Dst: FieldLV, /*IsInit*/ isInit: true);
1279
1280 // All done! The result is in the Dest slot.
1281}
1282
1283void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1284 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1285 VisitPointerToDataMemberBinaryOperator(BO: E);
1286 else
1287 CGF.ErrorUnsupported(S: E, Type: "aggregate binary expression");
1288}
1289
1290void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1291 const BinaryOperator *E) {
1292 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1293 EmitFinalDestCopy(type: E->getType(), src: LV);
1294}
1295
1296/// Is the value of the given expression possibly a reference to or
1297/// into a __block variable?
1298static bool isBlockVarRef(const Expr *E) {
1299 // Make sure we look through parens.
1300 E = E->IgnoreParens();
1301
1302 // Check for a direct reference to a __block variable.
1303 if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1304 const VarDecl *var = dyn_cast<VarDecl>(Val: DRE->getDecl());
1305 return (var && var->hasAttr<BlocksAttr>());
1306 }
1307
1308 // More complicated stuff.
1309
1310 // Binary operators.
1311 if (const BinaryOperator *op = dyn_cast<BinaryOperator>(Val: E)) {
1312 // For an assignment or pointer-to-member operation, just care
1313 // about the LHS.
1314 if (op->isAssignmentOp() || op->isPtrMemOp())
1315 return isBlockVarRef(E: op->getLHS());
1316
1317 // For a comma, just care about the RHS.
1318 if (op->getOpcode() == BO_Comma)
1319 return isBlockVarRef(E: op->getRHS());
1320
1321 // FIXME: pointer arithmetic?
1322 return false;
1323
1324 // Check both sides of a conditional operator.
1325 } else if (const AbstractConditionalOperator *op =
1326 dyn_cast<AbstractConditionalOperator>(Val: E)) {
1327 return isBlockVarRef(E: op->getTrueExpr()) ||
1328 isBlockVarRef(E: op->getFalseExpr());
1329
1330 // OVEs are required to support BinaryConditionalOperators.
1331 } else if (const OpaqueValueExpr *op = dyn_cast<OpaqueValueExpr>(Val: E)) {
1332 if (const Expr *src = op->getSourceExpr())
1333 return isBlockVarRef(E: src);
1334
1335 // Casts are necessary to get things like (*(int*)&var) = foo().
1336 // We don't really care about the kind of cast here, except
1337 // we don't want to look through l2r casts, because it's okay
1338 // to get the *value* in a __block variable.
1339 } else if (const CastExpr *cast = dyn_cast<CastExpr>(Val: E)) {
1340 if (cast->getCastKind() == CK_LValueToRValue)
1341 return false;
1342 return isBlockVarRef(E: cast->getSubExpr());
1343
1344 // Handle unary operators. Again, just aggressively look through
1345 // it, ignoring the operation.
1346 } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E)) {
1347 return isBlockVarRef(E: uop->getSubExpr());
1348
1349 // Look into the base of a field access.
1350 } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(Val: E)) {
1351 return isBlockVarRef(E: mem->getBase());
1352
1353 // Look into the base of a subscript.
1354 } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(Val: E)) {
1355 return isBlockVarRef(E: sub->getBase());
1356 }
1357
1358 return false;
1359}
1360
1361void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1362 ApplyAtomGroup Grp(CGF.getDebugInfo());
1363 // For an assignment to work, the value on the right has
1364 // to be compatible with the value on the left.
1365 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1366 E->getRHS()->getType()) &&
1367 "Invalid assignment");
1368
1369 // If the LHS might be a __block variable, and the RHS can
1370 // potentially cause a block copy, we need to evaluate the RHS first
1371 // so that the assignment goes the right place.
1372 // This is pretty semantically fragile.
1373 if (isBlockVarRef(E: E->getLHS()) &&
1374 E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1375 // Ensure that we have a destination, and evaluate the RHS into that.
1376 EnsureDest(T: E->getRHS()->getType());
1377 Visit(E: E->getRHS());
1378
1379 // Now emit the LHS and copy into it.
1380 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1381
1382 // That copy is an atomic copy if the LHS is atomic.
1383 if (LHS.getType()->isAtomicType() ||
1384 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1385 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1386 return;
1387 }
1388
1389 EmitCopy(type: E->getLHS()->getType(),
1390 dest: AggValueSlot::forLValue(LV: LHS, isDestructed: AggValueSlot::IsDestructed,
1391 needsGC: needsGC(T: E->getLHS()->getType()),
1392 isAliased: AggValueSlot::IsAliased,
1393 mayOverlap: AggValueSlot::MayOverlap),
1394 src: Dest);
1395 return;
1396 }
1397
1398 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1399
1400 // If we have an atomic type, evaluate into the destination and then
1401 // do an atomic copy.
1402 if (LHS.getType()->isAtomicType() ||
1403 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1404 EnsureDest(T: E->getRHS()->getType());
1405 Visit(E: E->getRHS());
1406 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1407 return;
1408 }
1409
1410 // Codegen the RHS so that it stores directly into the LHS.
1411 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1412 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1413 isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1414 // A non-volatile aggregate destination might have volatile member.
1415 if (!LHSSlot.isVolatile() && CGF.hasVolatileMember(T: E->getLHS()->getType()))
1416 LHSSlot.setVolatile(true);
1417
1418 CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1419
1420 // Copy into the destination if the assignment isn't ignored.
1421 EmitFinalDestCopy(type: E->getType(), src: LHS);
1422
1423 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1424 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1425 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1426 type: E->getType());
1427}
1428
1429void AggExprEmitter::VisitAbstractConditionalOperator(
1430 const AbstractConditionalOperator *E) {
1431 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1432 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1433 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1434
1435 // Bind the common expression if necessary.
1436 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1437
1438 CodeGenFunction::ConditionalEvaluation eval(CGF);
1439 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1440 TrueCount: CGF.getProfileCount(S: E));
1441
1442 // Save whether the destination's lifetime is externally managed.
1443 bool isExternallyDestructed = Dest.isExternallyDestructed();
1444 bool destructNonTrivialCStruct =
1445 !isExternallyDestructed &&
1446 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1447 isExternallyDestructed |= destructNonTrivialCStruct;
1448 Dest.setExternallyDestructed(isExternallyDestructed);
1449
1450 eval.begin(CGF);
1451 CGF.EmitBlock(BB: LHSBlock);
1452 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
1453 Visit(E: E->getTrueExpr());
1454 eval.end(CGF);
1455
1456 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1457 CGF.Builder.CreateBr(Dest: ContBlock);
1458
1459 // If the result of an agg expression is unused, then the emission
1460 // of the LHS might need to create a destination slot. That's fine
1461 // with us, and we can safely emit the RHS into the same slot, but
1462 // we shouldn't claim that it's already being destructed.
1463 Dest.setExternallyDestructed(isExternallyDestructed);
1464
1465 eval.begin(CGF);
1466 CGF.EmitBlock(BB: RHSBlock);
1467 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
1468 Visit(E: E->getFalseExpr());
1469 eval.end(CGF);
1470
1471 if (destructNonTrivialCStruct)
1472 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1473 type: E->getType());
1474
1475 CGF.EmitBlock(BB: ContBlock);
1476}
1477
1478void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1479 Visit(E: CE->getChosenSubExpr());
1480}
1481
1482void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1483 Address ArgValue = Address::invalid();
1484 CGF.EmitVAArg(VE, VAListAddr&: ArgValue, Slot: Dest);
1485
1486 // If EmitVAArg fails, emit an error.
1487 if (!ArgValue.isValid()) {
1488 CGF.ErrorUnsupported(S: VE, Type: "aggregate va_arg expression");
1489 return;
1490 }
1491}
1492
1493void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1494 // Ensure that we have a slot, but if we already do, remember
1495 // whether it was externally destructed.
1496 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1497 EnsureDest(T: E->getType());
1498
1499 // We're going to push a destructor if there isn't already one.
1500 Dest.setExternallyDestructed();
1501
1502 Visit(E: E->getSubExpr());
1503
1504 // Push that destructor we promised.
1505 if (!wasExternallyDestructed)
1506 CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1507}
1508
1509void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1510 AggValueSlot Slot = EnsureSlot(T: E->getType());
1511 CGF.EmitCXXConstructExpr(E, Dest: Slot);
1512}
1513
1514void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1515 const CXXInheritedCtorInitExpr *E) {
1516 AggValueSlot Slot = EnsureSlot(T: E->getType());
1517 CGF.EmitInheritedCXXConstructorCall(D: E->getConstructor(), ForVirtualBase: E->constructsVBase(),
1518 This: Slot.getAddress(),
1519 InheritedFromVBase: E->inheritedFromVBase(), E);
1520}
1521
1522void AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1523 AggValueSlot Slot = EnsureSlot(T: E->getType());
1524 LValue SlotLV = CGF.MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType());
1525
1526 // We'll need to enter cleanup scopes in case any of the element
1527 // initializers throws an exception or contains branch out of the expressions.
1528 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1529
1530 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1531 for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1532 e = E->capture_init_end();
1533 i != e; ++i, ++CurField) {
1534 // Emit initialization
1535 LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1536 if (CurField->hasCapturedVLAType()) {
1537 CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1538 continue;
1539 }
1540
1541 EmitInitializationToLValue(E: *i, Address: LV);
1542
1543 // Push a destructor if necessary.
1544 if (QualType::DestructionKind DtorKind =
1545 CurField->getType().isDestructedType()) {
1546 assert(LV.isSimple());
1547 if (DtorKind)
1548 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1549 type: CurField->getType(),
1550 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: false);
1551 }
1552 }
1553}
1554
1555void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1556 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1557 Visit(E: E->getSubExpr());
1558}
1559
1560void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1561 QualType T = E->getType();
1562 AggValueSlot Slot = EnsureSlot(T);
1563 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1564}
1565
1566void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1567 QualType T = E->getType();
1568 AggValueSlot Slot = EnsureSlot(T);
1569 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1570}
1571
1572/// Determine whether the given cast kind is known to always convert values
1573/// with all zero bits in their value representation to values with all zero
1574/// bits in their value representation.
1575static bool castPreservesZero(const CastExpr *CE) {
1576 switch (CE->getCastKind()) {
1577 // No-ops.
1578 case CK_NoOp:
1579 case CK_UserDefinedConversion:
1580 case CK_ConstructorConversion:
1581 case CK_BitCast:
1582 case CK_ToUnion:
1583 case CK_ToVoid:
1584 // Conversions between (possibly-complex) integral, (possibly-complex)
1585 // floating-point, and bool.
1586 case CK_BooleanToSignedIntegral:
1587 case CK_FloatingCast:
1588 case CK_FloatingComplexCast:
1589 case CK_FloatingComplexToBoolean:
1590 case CK_FloatingComplexToIntegralComplex:
1591 case CK_FloatingComplexToReal:
1592 case CK_FloatingRealToComplex:
1593 case CK_FloatingToBoolean:
1594 case CK_FloatingToIntegral:
1595 case CK_IntegralCast:
1596 case CK_IntegralComplexCast:
1597 case CK_IntegralComplexToBoolean:
1598 case CK_IntegralComplexToFloatingComplex:
1599 case CK_IntegralComplexToReal:
1600 case CK_IntegralRealToComplex:
1601 case CK_IntegralToBoolean:
1602 case CK_IntegralToFloating:
1603 // Reinterpreting integers as pointers and vice versa.
1604 case CK_IntegralToPointer:
1605 case CK_PointerToIntegral:
1606 // Language extensions.
1607 case CK_VectorSplat:
1608 case CK_MatrixCast:
1609 case CK_NonAtomicToAtomic:
1610 case CK_AtomicToNonAtomic:
1611 case CK_HLSLVectorTruncation:
1612 case CK_HLSLMatrixTruncation:
1613 case CK_HLSLElementwiseCast:
1614 case CK_HLSLAggregateSplatCast:
1615 return true;
1616
1617 case CK_BaseToDerivedMemberPointer:
1618 case CK_DerivedToBaseMemberPointer:
1619 case CK_MemberPointerToBoolean:
1620 case CK_NullToMemberPointer:
1621 case CK_ReinterpretMemberPointer:
1622 // FIXME: ABI-dependent.
1623 return false;
1624
1625 case CK_AnyPointerToBlockPointerCast:
1626 case CK_BlockPointerToObjCPointerCast:
1627 case CK_CPointerToObjCPointerCast:
1628 case CK_ObjCObjectLValueCast:
1629 case CK_IntToOCLSampler:
1630 case CK_ZeroToOCLOpaqueType:
1631 // FIXME: Check these.
1632 return false;
1633
1634 case CK_FixedPointCast:
1635 case CK_FixedPointToBoolean:
1636 case CK_FixedPointToFloating:
1637 case CK_FixedPointToIntegral:
1638 case CK_FloatingToFixedPoint:
1639 case CK_IntegralToFixedPoint:
1640 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1641 return false;
1642
1643 case CK_AddressSpaceConversion:
1644 case CK_BaseToDerived:
1645 case CK_DerivedToBase:
1646 case CK_Dynamic:
1647 case CK_NullToPointer:
1648 case CK_PointerToBoolean:
1649 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1650 // same representation in all involved address spaces.
1651 return false;
1652
1653 case CK_ARCConsumeObject:
1654 case CK_ARCExtendBlockObject:
1655 case CK_ARCProduceObject:
1656 case CK_ARCReclaimReturnedObject:
1657 case CK_CopyAndAutoreleaseBlockObject:
1658 case CK_ArrayToPointerDecay:
1659 case CK_FunctionToPointerDecay:
1660 case CK_BuiltinFnToFnPtr:
1661 case CK_Dependent:
1662 case CK_LValueBitCast:
1663 case CK_LValueToRValue:
1664 case CK_LValueToRValueBitCast:
1665 case CK_UncheckedDerivedToBase:
1666 case CK_HLSLArrayRValue:
1667 return false;
1668 }
1669 llvm_unreachable("Unhandled clang::CastKind enum");
1670}
1671
1672/// isSimpleZero - If emitting this value will obviously just cause a store of
1673/// zero to memory, return true. This can return false if uncertain, so it just
1674/// handles simple cases.
1675static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1676 E = E->IgnoreParens();
1677 while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1678 if (!castPreservesZero(CE))
1679 break;
1680 E = CE->getSubExpr()->IgnoreParens();
1681 }
1682
1683 // 0
1684 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1685 return IL->getValue() == 0;
1686 // +0.0
1687 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1688 return FL->getValue().isPosZero();
1689 // int()
1690 if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1691 CGF.getTypes().isZeroInitializable(T: E->getType()))
1692 return true;
1693 // (int*)0 - Null pointer expressions.
1694 if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1695 return ICE->getCastKind() == CK_NullToPointer &&
1696 CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1697 !E->HasSideEffects(Ctx: CGF.getContext());
1698 // '\0'
1699 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1700 return CL->getValue() == 0;
1701
1702 // Otherwise, hard case: conservatively return false.
1703 return false;
1704}
1705
1706void AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1707 QualType type = LV.getType();
1708 // FIXME: Ignore result?
1709 // FIXME: Are initializers affected by volatile?
1710 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1711 // Storing "i32 0" to a zero'd memory location is a noop.
1712 return;
1713 } else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1714 return EmitNullInitializationToLValue(Address: LV);
1715 } else if (isa<NoInitExpr>(Val: E)) {
1716 // Do nothing.
1717 return;
1718 } else if (type->isReferenceType()) {
1719 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1720 return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1721 }
1722
1723 CGF.EmitInitializationToLValue(E, LV, IsZeroed: Dest.isZeroed());
1724}
1725
1726void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1727 QualType type = lv.getType();
1728
1729 // If the destination slot is already zeroed out before the aggregate is
1730 // copied into it, we don't have to emit any zeros here.
1731 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1732 return;
1733
1734 if (CGF.hasScalarEvaluationKind(T: type)) {
1735 // For non-aggregates, we can store the appropriate null constant.
1736 llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1737 // Note that the following is not equivalent to
1738 // EmitStoreThroughBitfieldLValue for ARC types.
1739 if (lv.isBitField()) {
1740 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1741 } else {
1742 assert(lv.isSimple());
1743 CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1744 }
1745 } else {
1746 // There's a potential optimization opportunity in combining
1747 // memsets; that would be easy for arrays, but relatively
1748 // difficult for structures with the current code.
1749 CGF.EmitNullInitialization(DestPtr: lv.getAddress(), Ty: lv.getType());
1750 }
1751}
1752
1753void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1754 VisitCXXParenListOrInitListExpr(ExprToVisit: E, Args: E->getInitExprs(),
1755 InitializedFieldInUnion: E->getInitializedFieldInUnion(),
1756 ArrayFiller: E->getArrayFiller());
1757}
1758
1759void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1760 if (E->hadArrayRangeDesignator())
1761 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
1762
1763 if (E->isTransparent())
1764 return Visit(E: E->getInit(Init: 0));
1765
1766 VisitCXXParenListOrInitListExpr(
1767 ExprToVisit: E, Args: E->inits(), InitializedFieldInUnion: E->getInitializedFieldInUnion(), ArrayFiller: E->getArrayFiller());
1768}
1769
1770void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1771 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1772 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1773#if 0
1774 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1775 // (Length of globals? Chunks of zeroed-out space?).
1776 //
1777 // If we can, prefer a copy from a global; this is a lot less code for long
1778 // globals, and it's easier for the current optimizers to analyze.
1779 if (llvm::Constant *C =
1780 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1781 llvm::GlobalVariable* GV =
1782 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1783 llvm::GlobalValue::InternalLinkage, C, "");
1784 EmitFinalDestCopy(ExprToVisit->getType(),
1785 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1786 return;
1787 }
1788#endif
1789
1790 // HLSL initialization lists in the AST are an expansion which can contain
1791 // side-effecting expressions wrapped in opaque value expressions. To properly
1792 // emit these we need to emit the opaque values before we emit the argument
1793 // expressions themselves. This is a little hacky, but it prevents us needing
1794 // to do a bigger AST-level change for a language feature that we need
1795 // deprecate in the near future. See related HLSL language proposals:
1796 // * 0005-strict-initializer-lists.md
1797 // * https://github.com/microsoft/hlsl-specs/pull/325
1798 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(Val: ExprToVisit))
1799 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(
1800 CGF, E: cast<InitListExpr>(Val: ExprToVisit));
1801
1802 AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1803
1804 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1805
1806 // Handle initialization of an array.
1807 if (ExprToVisit->getType()->isConstantArrayType()) {
1808 auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1809 EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1810 Args: InitExprs, ArrayFiller);
1811 return;
1812 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1813 // A variable array type that has an initializer can only do empty
1814 // initialization. And because this feature is not exposed as an extension
1815 // in C++, we can safely memset the array memory to zero.
1816 assert(InitExprs.size() == 0 &&
1817 "you can only use an empty initializer with VLAs");
1818 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1819 return;
1820 }
1821
1822 assert(ExprToVisit->getType()->isRecordType() &&
1823 "Only support structs/unions here!");
1824
1825 // Do struct initialization; this code just sets each individual member
1826 // to the approprate value. This makes bitfield support automatic;
1827 // the disadvantage is that the generated code is more difficult for
1828 // the optimizer, especially with bitfields.
1829 unsigned NumInitElements = InitExprs.size();
1830 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1831
1832 // We'll need to enter cleanup scopes in case any of the element
1833 // initializers throws an exception.
1834 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1835
1836 unsigned curInitIndex = 0;
1837
1838 // Emit initialization of base classes.
1839 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1840 assert(NumInitElements >= CXXRD->getNumBases() &&
1841 "missing initializer for base class");
1842 for (auto &Base : CXXRD->bases()) {
1843 assert(!Base.isVirtual() && "should not see vbases here");
1844 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1845 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1846 Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1847 /*isBaseVirtual*/ BaseIsVirtual: false);
1848 AggValueSlot AggSlot = AggValueSlot::forAddr(
1849 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1850 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1851 mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1852 CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1853
1854 if (QualType::DestructionKind dtorKind =
1855 Base.getType().isDestructedType())
1856 CGF.pushDestroyAndDeferDeactivation(dtorKind, addr: V, type: Base.getType());
1857 }
1858 }
1859
1860 // Prepare a 'this' for CXXDefaultInitExprs.
1861 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1862
1863 const bool ZeroInitPadding =
1864 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1865
1866 if (record->isUnion()) {
1867 // Only initialize one field of a union. The field itself is
1868 // specified by the initializer list.
1869 if (!InitializedFieldInUnion) {
1870 // Empty union; we have nothing to do.
1871
1872#ifndef NDEBUG
1873 // Make sure that it's really an empty and not a failure of
1874 // semantic analysis.
1875 for (const auto *Field : record->fields())
1876 assert(
1877 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1878 "Only unnamed bitfields or anonymous class allowed");
1879#endif
1880 return;
1881 }
1882
1883 // FIXME: volatility
1884 FieldDecl *Field = InitializedFieldInUnion;
1885
1886 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1887 if (NumInitElements) {
1888 // Store the initializer into the field
1889 EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1890 if (ZeroInitPadding) {
1891 uint64_t TotalSize = CGF.getContext().toBits(
1892 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1893 uint64_t FieldSize = CGF.getContext().getTypeSize(T: FieldLoc.getType());
1894 DoZeroInitPadding(PaddingStart&: FieldSize, PaddingEnd: TotalSize, NextField: nullptr);
1895 }
1896 } else {
1897 // Default-initialize to null.
1898 if (ZeroInitPadding)
1899 EmitNullInitializationToLValue(lv: DestLV);
1900 else
1901 EmitNullInitializationToLValue(lv: FieldLoc);
1902 }
1903 return;
1904 }
1905
1906 // Here we iterate over the fields; this makes it simpler to both
1907 // default-initialize fields and skip over unnamed fields.
1908 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(D: record);
1909 uint64_t PaddingStart = 0;
1910
1911 for (const auto *field : record->fields()) {
1912 // We're done once we hit the flexible array member.
1913 if (field->getType()->isIncompleteArrayType())
1914 break;
1915
1916 // Always skip anonymous bitfields.
1917 if (field->isUnnamedBitField())
1918 continue;
1919
1920 // We're done if we reach the end of the explicit initializers, we
1921 // have a zeroed object, and the rest of the fields are
1922 // zero-initializable.
1923 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1924 CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1925 break;
1926
1927 if (ZeroInitPadding)
1928 DoZeroInitPadding(PaddingStart,
1929 PaddingEnd: Layout.getFieldOffset(FieldNo: field->getFieldIndex()), NextField: field);
1930
1931 LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1932 // We never generate write-barries for initialized fields.
1933 LV.setNonGC(true);
1934
1935 if (curInitIndex < NumInitElements) {
1936 // Store the initializer into the field.
1937 EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1938 } else {
1939 // We're out of initializers; default-initialize to null
1940 EmitNullInitializationToLValue(lv: LV);
1941 }
1942
1943 // Push a destructor if necessary.
1944 // FIXME: if we have an array of structures, all explicitly
1945 // initialized, we can end up pushing a linear number of cleanups.
1946 if (QualType::DestructionKind dtorKind =
1947 field->getType().isDestructedType()) {
1948 assert(LV.isSimple());
1949 if (dtorKind) {
1950 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1951 type: field->getType(),
1952 destroyer: CGF.getDestroyer(destructionKind: dtorKind), useEHCleanupForArray: false);
1953 }
1954 }
1955 }
1956 if (ZeroInitPadding) {
1957 uint64_t TotalSize = CGF.getContext().toBits(
1958 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1959 DoZeroInitPadding(PaddingStart, PaddingEnd: TotalSize, NextField: nullptr);
1960 }
1961}
1962
1963void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1964 uint64_t PaddingEnd,
1965 const FieldDecl *NextField) {
1966
1967 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1968 CharUnits Start = CGF.getContext().toCharUnitsFromBits(BitSize: StartBit);
1969 CharUnits End = CGF.getContext().toCharUnitsFromBits(BitSize: EndBit);
1970 Address Addr = Dest.getAddress().withElementType(ElemTy: CGF.CharTy);
1971 if (!Start.isZero())
1972 Addr = Builder.CreateConstGEP(Addr, Index: Start.getQuantity());
1973 llvm::Constant *SizeVal = Builder.getInt64(C: (End - Start).getQuantity());
1974 CGF.Builder.CreateMemSet(Dest: Addr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
1975 };
1976
1977 if (NextField != nullptr && NextField->isBitField()) {
1978 // For bitfield, zero init StorageSize before storing the bits. So we don't
1979 // need to handle big/little endian.
1980 const CGRecordLayout &RL =
1981 CGF.getTypes().getCGRecordLayout(NextField->getParent());
1982 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: NextField);
1983 uint64_t StorageStart = CGF.getContext().toBits(CharSize: Info.StorageOffset);
1984 if (StorageStart + Info.StorageSize > PaddingStart) {
1985 if (StorageStart > PaddingStart)
1986 InitBytes(PaddingStart, StorageStart);
1987 Address Addr = Dest.getAddress();
1988 if (!Info.StorageOffset.isZero())
1989 Addr = Builder.CreateConstGEP(Addr: Addr.withElementType(ElemTy: CGF.CharTy),
1990 Index: Info.StorageOffset.getQuantity());
1991 Addr = Addr.withElementType(
1992 ElemTy: llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: Info.StorageSize));
1993 Builder.CreateStore(Val: Builder.getIntN(N: Info.StorageSize, C: 0), Addr);
1994 PaddingStart = StorageStart + Info.StorageSize;
1995 }
1996 return;
1997 }
1998
1999 if (PaddingStart < PaddingEnd)
2000 InitBytes(PaddingStart, PaddingEnd);
2001 if (NextField != nullptr)
2002 PaddingStart =
2003 PaddingEnd + CGF.getContext().getTypeSize(T: NextField->getType());
2004}
2005
2006void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
2007 llvm::Value *outerBegin) {
2008 // Emit the common subexpression.
2009 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
2010
2011 Address destPtr = EnsureSlot(T: E->getType()).getAddress();
2012 uint64_t numElements = E->getArraySize().getZExtValue();
2013
2014 if (!numElements)
2015 return;
2016
2017 // destPtr is an array*. Construct an elementType* by drilling down a level.
2018 llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
2019 llvm::Value *indices[] = {zero, zero};
2020 llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
2021 Ptr: destPtr.emitRawPointer(CGF),
2022 IdxList: indices, Name: "arrayinit.begin");
2023
2024 // Prepare to special-case multidimensional array initialization: we avoid
2025 // emitting multiple destructor loops in that case.
2026 if (!outerBegin)
2027 outerBegin = begin;
2028 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
2029
2030 QualType elementType =
2031 CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
2032 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
2033 CharUnits elementAlign =
2034 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
2035 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
2036
2037 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2038 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
2039
2040 // Jump into the body.
2041 CGF.EmitBlock(BB: bodyBB);
2042 llvm::PHINode *index =
2043 Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
2044 index->addIncoming(V: zero, BB: entryBB);
2045 llvm::Value *element =
2046 Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
2047
2048 if (CGF.CGM.shouldEmitConvergenceTokens())
2049 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
2050
2051 // Prepare for a cleanup.
2052 QualType::DestructionKind dtorKind = elementType.isDestructedType();
2053 EHScopeStack::stable_iterator cleanup;
2054 if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
2055 if (outerBegin->getType() != element->getType())
2056 outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
2057 CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
2058 elementAlignment: elementAlign,
2059 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
2060 cleanup = CGF.EHStack.stable_begin();
2061 } else {
2062 dtorKind = QualType::DK_none;
2063 }
2064
2065 // Emit the actual filler expression.
2066 {
2067 // Temporaries created in an array initialization loop are destroyed
2068 // at the end of each iteration.
2069 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2070 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2071 LValue elementLV = CGF.MakeAddrLValue(
2072 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
2073
2074 if (InnerLoop) {
2075 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2076 auto elementSlot = AggValueSlot::forLValue(
2077 LV: elementLV, isDestructed: AggValueSlot::IsDestructed,
2078 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2079 mayOverlap: AggValueSlot::DoesNotOverlap);
2080 AggExprEmitter(CGF, elementSlot, false)
2081 .VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
2082 } else
2083 EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
2084 }
2085
2086 // Move on to the next element.
2087 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2088 LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
2089 index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
2090
2091 // Leave the loop if we're done.
2092 llvm::Value *done = Builder.CreateICmpEQ(
2093 LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
2094 Name: "arrayinit.done");
2095 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
2096 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
2097
2098 if (CGF.CGM.shouldEmitConvergenceTokens())
2099 CGF.ConvergenceTokenStack.pop_back();
2100
2101 CGF.EmitBlock(BB: endBB);
2102
2103 // Leave the partial-array cleanup if we entered one.
2104 if (dtorKind)
2105 CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
2106}
2107
2108void AggExprEmitter::VisitDesignatedInitUpdateExpr(
2109 DesignatedInitUpdateExpr *E) {
2110 AggValueSlot Dest = EnsureSlot(T: E->getType());
2111
2112 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
2113 EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
2114 VisitInitListExpr(E: E->getUpdater());
2115}
2116
2117//===----------------------------------------------------------------------===//
2118// Entry Points into this File
2119//===----------------------------------------------------------------------===//
2120
2121/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2122/// non-zero bytes that will be stored when outputting the initializer for the
2123/// specified initializer expression.
2124static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
2125 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
2126 E = MTE->getSubExpr();
2127 E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
2128
2129 // 0 and 0.0 won't require any non-zero stores!
2130 if (isSimpleZero(E, CGF))
2131 return CharUnits::Zero();
2132
2133 // If this is an initlist expr, sum up the size of sizes of the (present)
2134 // elements. If this is something weird, assume the whole thing is non-zero.
2135 const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
2136 while (ILE && ILE->isTransparent())
2137 ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
2138 if (!ILE || !CGF.getTypes().isZeroInitializable(T: ILE->getType()))
2139 return CGF.getContext().getTypeSizeInChars(T: E->getType());
2140
2141 // InitListExprs for structs have to be handled carefully. If there are
2142 // reference members, we need to consider the size of the reference, not the
2143 // referencee. InitListExprs for unions and arrays can't have references.
2144 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2145 if (!RT->isUnionType()) {
2146 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2147 CharUnits NumNonZeroBytes = CharUnits::Zero();
2148
2149 unsigned ILEElement = 0;
2150 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
2151 while (ILEElement != CXXRD->getNumBases())
2152 NumNonZeroBytes +=
2153 GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
2154 for (const auto *Field : SD->fields()) {
2155 // We're done once we hit the flexible array member or run out of
2156 // InitListExpr elements.
2157 if (Field->getType()->isIncompleteArrayType() ||
2158 ILEElement == ILE->getNumInits())
2159 break;
2160 if (Field->isUnnamedBitField())
2161 continue;
2162
2163 const Expr *E = ILE->getInit(Init: ILEElement++);
2164
2165 // Reference values are always non-null and have the width of a pointer.
2166 if (Field->getType()->isReferenceType())
2167 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2168 BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2169 else
2170 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2171 }
2172
2173 return NumNonZeroBytes;
2174 }
2175 }
2176
2177 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2178 CharUnits NumNonZeroBytes = CharUnits::Zero();
2179 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2180 NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2181 return NumNonZeroBytes;
2182}
2183
2184/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2185/// zeros in it, emit a memset and avoid storing the individual zeros.
2186///
2187static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2188 CodeGenFunction &CGF) {
2189 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2190 // volatile stores.
2191 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2192 return;
2193
2194 // C++ objects with a user-declared constructor don't need zero'ing.
2195 if (CGF.getLangOpts().CPlusPlus)
2196 if (const RecordType *RT = CGF.getContext()
2197 .getBaseElementType(QT: E->getType())
2198 ->getAsCanonical<RecordType>()) {
2199 const auto *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2200 if (RD->hasUserDeclaredConstructor())
2201 return;
2202 }
2203
2204 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2205 CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2206 if (Size <= CharUnits::fromQuantity(Quantity: 16))
2207 return;
2208
2209 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2210 // we prefer to emit memset + individual stores for the rest.
2211 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2212 if (NumNonZeroBytes * 4 > Size)
2213 return;
2214
2215 // Okay, it seems like a good idea to use an initial memset, emit the call.
2216 llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2217
2218 Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2219 CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2220
2221 // Tell the AggExprEmitter that the slot is known zero.
2222 Slot.setZeroed();
2223}
2224
2225/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2226/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2227/// the value of the aggregate expression is not needed. If VolatileDest is
2228/// true, DestPtr cannot be 0.
2229void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2230 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2231 "Invalid aggregate expression to emit");
2232 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2233 "slot has bits but no address");
2234
2235 // Optimize the slot if possible.
2236 CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2237
2238 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr *>(E));
2239}
2240
2241LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2242 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2243 Address Temp = CreateMemTempWithoutCast(T: E->getType());
2244 LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2245 EmitAggExpr(E, Slot: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsNotDestructed,
2246 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2247 isAliased: AggValueSlot::IsNotAliased,
2248 mayOverlap: AggValueSlot::DoesNotOverlap));
2249 return LV;
2250}
2251
2252void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,
2253 const LValue &Src,
2254 ExprValueKind SrcKind) {
2255 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2256 .EmitFinalDestCopy(type: Type, src: Src, SrcValueKind: SrcKind);
2257}
2258
2259AggValueSlot::Overlap_t
2260CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2261 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2262 return AggValueSlot::DoesNotOverlap;
2263
2264 // Empty fields can overlap earlier fields.
2265 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2266 return AggValueSlot::MayOverlap;
2267
2268 // If the field lies entirely within the enclosing class's nvsize, its tail
2269 // padding cannot overlap any already-initialized object. (The only subobjects
2270 // with greater addresses that might already be initialized are vbases.)
2271 const RecordDecl *ClassRD = FD->getParent();
2272 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2273 if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2274 getContext().getTypeSize(T: FD->getType()) <=
2275 (uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2276 return AggValueSlot::DoesNotOverlap;
2277
2278 // The tail padding may contain values we need to preserve.
2279 return AggValueSlot::MayOverlap;
2280}
2281
2282AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2283 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2284 // If the most-derived object is a field declared with [[no_unique_address]],
2285 // the tail padding of any virtual base could be reused for other subobjects
2286 // of that field's class.
2287 if (IsVirtual)
2288 return AggValueSlot::MayOverlap;
2289
2290 // Empty bases can overlap earlier bases.
2291 if (BaseRD->isEmpty())
2292 return AggValueSlot::MayOverlap;
2293
2294 // If the base class is laid out entirely within the nvsize of the derived
2295 // class, its tail padding cannot yet be initialized, so we can issue
2296 // stores at the full width of the base class.
2297 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2298 if (Layout.getBaseClassOffset(Base: BaseRD) +
2299 getContext().getASTRecordLayout(D: BaseRD).getSize() <=
2300 Layout.getNonVirtualSize())
2301 return AggValueSlot::DoesNotOverlap;
2302
2303 // The tail padding may contain values we need to preserve.
2304 return AggValueSlot::MayOverlap;
2305}
2306
2307void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2308 AggValueSlot::Overlap_t MayOverlap,
2309 bool isVolatile) {
2310 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2311
2312 Address DestPtr = Dest.getAddress();
2313 Address SrcPtr = Src.getAddress();
2314
2315 if (getLangOpts().CPlusPlus) {
2316 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2317 assert((Record->hasTrivialCopyConstructor() ||
2318 Record->hasTrivialCopyAssignment() ||
2319 Record->hasTrivialMoveConstructor() ||
2320 Record->hasTrivialMoveAssignment() ||
2321 Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&
2322 "Trying to aggregate-copy a type without a trivial copy/move "
2323 "constructor or assignment operator");
2324 // Ignore empty classes in C++.
2325 if (Record->isEmpty())
2326 return;
2327 }
2328 }
2329
2330 if (getLangOpts().CUDAIsDevice) {
2331 if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2332 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2333 Src))
2334 return;
2335 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2336 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2337 Src))
2338 return;
2339 }
2340 }
2341
2342 if (getLangOpts().HLSL && Ty.getAddressSpace() == LangAS::hlsl_constant)
2343 if (CGM.getHLSLRuntime().emitBufferCopy(CGF&: *this, DestPtr, SrcPtr, CType: Ty))
2344 return;
2345
2346 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2347 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2348 // read from another object that overlaps in anyway the storage of the first
2349 // object, then the overlap shall be exact and the two objects shall have
2350 // qualified or unqualified versions of a compatible type."
2351 //
2352 // memcpy is not defined if the source and destination pointers are exactly
2353 // equal, but other compilers do this optimization, and almost every memcpy
2354 // implementation handles this case safely. If there is a libc that does not
2355 // safely handle this, we can add a target hook.
2356
2357 // Get data size info for this aggregate. Don't copy the tail padding if this
2358 // might be a potentially-overlapping subobject, since the tail padding might
2359 // be occupied by a different object. Otherwise, copying it is fine.
2360 TypeInfoChars TypeInfo;
2361 if (MayOverlap)
2362 TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2363 else
2364 TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2365
2366 llvm::Value *SizeVal = nullptr;
2367 if (TypeInfo.Width.isZero()) {
2368 // But note that getTypeInfo returns 0 for a VLA.
2369 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2370 Val: getContext().getAsArrayType(T: Ty))) {
2371 QualType BaseEltTy;
2372 SizeVal = emitArrayLength(arrayType: VAT, baseType&: BaseEltTy, addr&: DestPtr);
2373 TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2374 assert(!TypeInfo.Width.isZero());
2375 SizeVal = Builder.CreateNUWMul(
2376 LHS: SizeVal,
2377 RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2378 }
2379 }
2380 if (!SizeVal) {
2381 SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2382 }
2383
2384 // FIXME: If we have a volatile struct, the optimizer can remove what might
2385 // appear to be `extra' memory ops:
2386 //
2387 // volatile struct { int i; } a, b;
2388 //
2389 // int main() {
2390 // a = b;
2391 // a = b;
2392 // }
2393 //
2394 // we need to use a different call here. We use isVolatile to indicate when
2395 // either the source or the destination is volatile.
2396
2397 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2398 SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2399
2400 // Don't do any of the memmove_collectable tests if GC isn't set.
2401 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2402 // fall through
2403 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2404 if (Record->hasObjectMember()) {
2405 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2406 Size: SizeVal);
2407 return;
2408 }
2409 } else if (Ty->isArrayType()) {
2410 QualType BaseType = getContext().getBaseElementType(QT: Ty);
2411 if (const auto *Record = BaseType->getAsRecordDecl()) {
2412 if (Record->hasObjectMember()) {
2413 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2414 Size: SizeVal);
2415 return;
2416 }
2417 }
2418 }
2419
2420 auto *Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2421 addInstToCurrentSourceAtom(KeyInstruction: Inst, Backup: nullptr);
2422 emitPFPPostCopyUpdates(DestPtr, SrcPtr, Ty);
2423
2424 // Determine the metadata to describe the position of any padding in this
2425 // memcpy, as well as the TBAA tags for the members of the struct, in case
2426 // the optimizer wishes to expand it in to scalar memory operations.
2427 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2428 Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2429
2430 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2431 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2432 DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2433 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2434 }
2435}
2436