1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
23#include "clang/AST/ASTContext.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
26#include "clang/AST/DeclTemplate.h"
27#include "clang/AST/StmtVisitor.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34using namespace clang;
35using namespace CodeGen;
36
37//===----------------------------------------------------------------------===//
38// Aggregate Expression Emitter
39//===----------------------------------------------------------------------===//
40
41namespace {
42class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
43 CodeGenFunction &CGF;
44 CGBuilderTy &Builder;
45 AggValueSlot Dest;
46 bool IsResultUnused;
47
48 AggValueSlot EnsureSlot(QualType T) {
49 if (!Dest.isIgnored())
50 return Dest;
51 return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
52 }
53 void EnsureDest(QualType T) {
54 if (!Dest.isIgnored())
55 return;
56 Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
57 }
58
59 // Calls `Fn` with a valid return value slot, potentially creating a temporary
60 // to do so. If a temporary is created, an appropriate copy into `Dest` will
61 // be emitted, as will lifetime markers.
62 //
63 // The given function should take a ReturnValueSlot, and return an RValue that
64 // points to said slot.
65 void withReturnValueSlot(const Expr *E,
66 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
67
68 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
69 const FieldDecl *NextField);
70
71public:
72 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
73 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
74 IsResultUnused(IsResultUnused) {}
75
76 //===--------------------------------------------------------------------===//
77 // Utilities
78 //===--------------------------------------------------------------------===//
79
80 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
81 /// represents a value lvalue, this method emits the address of the lvalue,
82 /// then loads the result into DestPtr.
83 void EmitAggLoadOfLValue(const Expr *E);
84
85 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
86 /// SrcIsRValue is true if source comes from an RValue.
87 void EmitFinalDestCopy(QualType type, const LValue &src,
88 CodeGenFunction::ExprValueKind SrcValueKind =
89 CodeGenFunction::EVK_NonRValue);
90 void EmitFinalDestCopy(QualType type, RValue src);
91 void EmitCopy(QualType type, const AggValueSlot &dest,
92 const AggValueSlot &src);
93
94 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
95 Expr *ExprToVisit, ArrayRef<Expr *> Args,
96 Expr *ArrayFiller);
97
98 void EmitComparisonResult(const Expr *E,
99 const ComparisonCategoryInfo &CmpInfo,
100 llvm::Value *ResultValue);
101
102 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
103 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
104 return AggValueSlot::NeedsGCBarriers;
105 return AggValueSlot::DoesNotNeedGCBarriers;
106 }
107
108 bool TypeRequiresGCollection(QualType T);
109
110 //===--------------------------------------------------------------------===//
111 // Visitor Methods
112 //===--------------------------------------------------------------------===//
113
114 void Visit(Expr *E) {
115 ApplyDebugLocation DL(CGF, E);
116 StmtVisitor<AggExprEmitter>::Visit(S: E);
117 }
118
119 void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, Type: "aggregate expression"); }
120 void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
121 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
122 Visit(E: GE->getResultExpr());
123 }
124 void VisitCoawaitExpr(CoawaitExpr *E) {
125 CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
126 }
127 void VisitCoyieldExpr(CoyieldExpr *E) {
128 CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
129 }
130 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
131 void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
132 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
133 return Visit(E: E->getReplacement());
134 }
135
136 void VisitConstantExpr(ConstantExpr *E) {
137 EnsureDest(T: E->getType());
138
139 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
140 CGF.CreateCoercedStore(
141 Src: Result, SrcFETy: E->getType(), Dst: Dest.getAddress(),
142 DstSize: llvm::TypeSize::getFixed(
143 ExactSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType())
144 .getQuantity()),
145 DstIsVolatile: E->getType().isVolatileQualified());
146 return;
147 }
148 return Visit(E: E->getSubExpr());
149 }
150
151 // l-values.
152 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
153 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(E: ME); }
154 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
155 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
156 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
157 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
158 EmitAggLoadOfLValue(E);
159 }
160 void VisitPredefinedExpr(const PredefinedExpr *E) { EmitAggLoadOfLValue(E); }
161
162 // Operators.
163 void VisitCastExpr(CastExpr *E);
164 void VisitCallExpr(const CallExpr *E);
165 void VisitStmtExpr(const StmtExpr *E);
166 void VisitBinaryOperator(const BinaryOperator *BO);
167 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
168 void VisitBinAssign(const BinaryOperator *E);
169 void VisitBinComma(const BinaryOperator *E);
170 void VisitBinCmp(const BinaryOperator *E);
171 void VisitTypeTraitExpr(const TypeTraitExpr *E);
172 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
173 Visit(E: E->getSemanticForm());
174 }
175
176 void VisitObjCMessageExpr(ObjCMessageExpr *E);
177 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { EmitAggLoadOfLValue(E); }
178
179 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
180 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
181 void VisitChooseExpr(const ChooseExpr *CE);
182 void VisitInitListExpr(InitListExpr *E);
183 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
184 FieldDecl *InitializedFieldInUnion,
185 Expr *ArrayFiller);
186 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
187 llvm::Value *outerBegin = nullptr);
188 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
189 void VisitNoInitExpr(NoInitExpr *E) {} // Do nothing.
190 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
191 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
192 Visit(E: DAE->getExpr());
193 }
194 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
195 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
196 Visit(E: DIE->getExpr());
197 }
198 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
199 void VisitCXXConstructExpr(const CXXConstructExpr *E);
200 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
201 void VisitLambdaExpr(LambdaExpr *E);
202 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
203 void VisitExprWithCleanups(ExprWithCleanups *E);
204 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
205 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
206 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
207 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
208
209 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
210 if (E->isGLValue()) {
211 LValue LV = CGF.EmitPseudoObjectLValue(e: E);
212 return EmitFinalDestCopy(type: E->getType(), src: LV);
213 }
214
215 AggValueSlot Slot = EnsureSlot(T: E->getType());
216 bool NeedsDestruction =
217 !Slot.isExternallyDestructed() &&
218 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
219 if (NeedsDestruction)
220 Slot.setExternallyDestructed();
221 CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
222 if (NeedsDestruction)
223 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Slot.getAddress(),
224 type: E->getType());
225 }
226
227 void VisitVAArgExpr(VAArgExpr *E);
228 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
229 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
230 Expr *ArrayFiller);
231
232 void EmitInitializationToLValue(Expr *E, LValue Address);
233 void EmitNullInitializationToLValue(LValue Address);
234 // case Expr::ChooseExprClass:
235 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
236 void VisitAtomicExpr(AtomicExpr *E) {
237 RValue Res = CGF.EmitAtomicExpr(E);
238 EmitFinalDestCopy(type: E->getType(), src: Res);
239 }
240 void VisitPackIndexingExpr(PackIndexingExpr *E) {
241 Visit(E: E->getSelectedExpr());
242 }
243};
244} // end anonymous namespace.
245
246//===----------------------------------------------------------------------===//
247// Utilities
248//===----------------------------------------------------------------------===//
249
250/// EmitAggLoadOfLValue - Given an expression with aggregate type that
251/// represents a value lvalue, this method emits the address of the lvalue,
252/// then loads the result into DestPtr.
253void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
254 LValue LV = CGF.EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load);
255
256 // If the type of the l-value is atomic, then do an atomic load.
257 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
258 CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
259 return;
260 }
261
262 if (E->getType().getAddressSpace() == LangAS::hlsl_constant)
263 if (CGF.CGM.getHLSLRuntime().emitBufferCopy(CGF, E, SrcLV: LV, DestSlot&: Dest))
264 return;
265
266 EmitFinalDestCopy(type: E->getType(), src: LV);
267}
268
269/// True if the given aggregate type requires special GC API calls.
270bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
271 // Only record types have members that might require garbage collection.
272 const auto *Record = T->getAsRecordDecl();
273 if (!Record)
274 return false;
275
276 // Don't mess with non-trivial C++ types.
277 if (isa<CXXRecordDecl>(Val: Record) &&
278 (cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
279 !cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
280 return false;
281
282 // Check whether the type has an object member.
283 return Record->hasObjectMember();
284}
285
286void AggExprEmitter::withReturnValueSlot(
287 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
288 QualType RetTy = E->getType();
289 bool RequiresDestruction =
290 !Dest.isExternallyDestructed() &&
291 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
292
293 // If it makes no observable difference, save a memcpy + temporary.
294 //
295 // We need to always provide our own temporary if destruction is required.
296 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
297 // its lifetime before we have the chance to emit a proper destructor call.
298 //
299 // We also need a temporary if the destination is in a different address space
300 // from the sret AS. Use the target hook to get the actual sret AS for this
301 // return type.
302 const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl();
303 LangAS SRetLangAS = CGF.CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
304 unsigned SRetAS = CGF.getContext().getTargetAddressSpace(AS: SRetLangAS);
305 bool CanAggregateCopy =
306 RD ? (RD->hasTrivialCopyConstructor() ||
307 RD->hasTrivialMoveConstructor() || RD->hasTrivialCopyAssignment() ||
308 RD->hasTrivialMoveAssignment() || RD->hasAttr<TrivialABIAttr>() ||
309 RD->isUnion())
310 : RetTy.isTriviallyCopyableType(Context: CGF.getContext());
311 bool DestASMismatch = !Dest.isIgnored() && CanAggregateCopy &&
312 Dest.getAddress()
313 .getBasePointer()
314 ->stripPointerCasts()
315 ->getType()
316 ->getPointerAddressSpace() != SRetAS;
317 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
318 (RequiresDestruction && Dest.isIgnored()) || DestASMismatch;
319
320 Address RetAddr = Address::invalid();
321
322 EHScopeStack::stable_iterator LifetimeEndBlock;
323 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
324 if (!UseTemp) {
325 RetAddr = Dest.getAddress();
326 if (RetAddr.isValid() && RetAddr.getAddressSpace() != SRetAS) {
327 llvm::Type *SRetPtrTy =
328 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: SRetAS);
329 RetAddr = RetAddr.withPointer(
330 NewPointer: CGF.performAddrSpaceCast(Src: RetAddr.getBasePointer(), DestTy: SRetPtrTy),
331 IsKnownNonNull: RetAddr.isKnownNonNull());
332 }
333 } else {
334 RetAddr = CGF.CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
335 if (CGF.EmitLifetimeStart(Addr: RetAddr.getBasePointer())) {
336 LifetimeStartInst =
337 cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
338 assert(LifetimeStartInst->getIntrinsicID() ==
339 llvm::Intrinsic::lifetime_start &&
340 "Last insertion wasn't a lifetime.start?");
341
342 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
343 kind: NormalEHLifetimeMarker, A: RetAddr);
344 LifetimeEndBlock = CGF.EHStack.stable_begin();
345 }
346 }
347
348 RValue Src =
349 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
350 Dest.isExternallyDestructed()));
351
352 if (!UseTemp)
353 return;
354
355 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
356 Src.getAggregatePointer(E->getType(), CGF));
357 EmitFinalDestCopy(type: E->getType(), src: Src);
358
359 if (!RequiresDestruction && LifetimeStartInst) {
360 // If there's no dtor to run, the copy was the last use of our temporary.
361 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
362 // eagerly.
363 CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
364 CGF.EmitLifetimeEnd(Addr: RetAddr.getBasePointer());
365 }
366}
367
368/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
369void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
370 assert(src.isAggregate() && "value must be aggregate value!");
371 LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
372 EmitFinalDestCopy(type, src: srcLV, SrcValueKind: CodeGenFunction::EVK_RValue);
373}
374
375/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
376void AggExprEmitter::EmitFinalDestCopy(
377 QualType type, const LValue &src,
378 CodeGenFunction::ExprValueKind SrcValueKind) {
379 // If Dest is ignored, then we're evaluating an aggregate expression
380 // in a context that doesn't care about the result. Note that loads
381 // from volatile l-values force the existence of a non-ignored
382 // destination.
383 if (Dest.isIgnored())
384 return;
385
386 // Copy non-trivial C structs here.
387 LValue DstLV = CGF.MakeAddrLValue(
388 Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
389
390 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
391 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
392 if (Dest.isPotentiallyAliased())
393 CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
394 else
395 CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
396 return;
397 }
398 } else {
399 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
400 if (Dest.isPotentiallyAliased())
401 CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
402 else
403 CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
404 return;
405 }
406 }
407
408 AggValueSlot srcAgg = AggValueSlot::forLValue(
409 LV: src, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type), isAliased: AggValueSlot::IsAliased,
410 mayOverlap: AggValueSlot::MayOverlap);
411 EmitCopy(type, dest: Dest, src: srcAgg);
412}
413
414/// Perform a copy from the source into the destination.
415///
416/// \param type - the type of the aggregate being copied; qualifiers are
417/// ignored
418void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
419 const AggValueSlot &src) {
420 if (dest.requiresGCollection()) {
421 CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
422 llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
423 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr: dest.getAddress(),
424 SrcPtr: src.getAddress(), Size: size);
425 return;
426 }
427
428 // If the result of the assignment is used, copy the LHS there also.
429 // It's volatile if either side is. Use the minimum alignment of
430 // the two sides.
431 LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
432 LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
433 CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
434 isVolatile: dest.isVolatile() || src.isVolatile());
435}
436
437/// Emit the initializer for a std::initializer_list initialized with a
438/// real initializer list.
439void AggExprEmitter::VisitCXXStdInitializerListExpr(
440 CXXStdInitializerListExpr *E) {
441 // Emit an array containing the elements. The array is externally destructed
442 // if the std::initializer_list object is.
443 ASTContext &Ctx = CGF.getContext();
444 LValue Array = CGF.EmitLValue(E: E->getSubExpr());
445 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
446 Address ArrayPtr = Array.getAddress();
447
448 const ConstantArrayType *ArrayType =
449 Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
450 assert(ArrayType && "std::initializer_list constructed from non-array");
451
452 auto *Record = E->getType()->castAsRecordDecl();
453 RecordDecl::field_iterator Field = Record->field_begin();
454 assert(Field != Record->field_end() &&
455 Ctx.hasSameType(Field->getType()->getPointeeType(),
456 ArrayType->getElementType()) &&
457 "Expected std::initializer_list first field to be const E *");
458
459 // Start pointer.
460 AggValueSlot Dest = EnsureSlot(T: E->getType());
461 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
462 LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
463 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
464 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
465 ++Field;
466 assert(Field != Record->field_end() &&
467 "Expected std::initializer_list to have two fields");
468
469 llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
470 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
471 if (Ctx.hasSameType(T1: Field->getType(), T2: Ctx.getSizeType())) {
472 // Length.
473 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
474
475 } else {
476 // End pointer.
477 assert(Field->getType()->isPointerType() &&
478 Ctx.hasSameType(Field->getType()->getPointeeType(),
479 ArrayType->getElementType()) &&
480 "Expected std::initializer_list second field to be const E *");
481 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
482 llvm::Value *IdxEnd[] = {Zero, Size};
483 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
484 Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
485 Name: "arrayend");
486 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
487 }
488
489 assert(++Field == Record->field_end() &&
490 "Expected std::initializer_list to only have two fields");
491}
492
493/// Determine if E is a trivial array filler, that is, one that is
494/// equivalent to zero-initialization.
495static bool isTrivialFiller(Expr *E) {
496 if (!E)
497 return true;
498
499 if (isa<ImplicitValueInitExpr>(Val: E))
500 return true;
501
502 if (auto *ILE = dyn_cast<InitListExpr>(Val: E)) {
503 if (ILE->getNumInits())
504 return false;
505 return isTrivialFiller(E: ILE->getArrayFiller());
506 }
507
508 if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(Val: E))
509 return Cons->getConstructor()->isDefaultConstructor() &&
510 Cons->getConstructor()->isTrivial();
511
512 // FIXME: Are there other cases where we can avoid emitting an initializer?
513 return false;
514}
515
516// emit an elementwise cast where the RHS is a scalar or vector
517// or emit an aggregate splat cast
518static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,
519 LValue DestVal,
520 llvm::Value *SrcVal,
521 QualType SrcTy,
522 SourceLocation Loc) {
523 // Flatten our destination
524 SmallVector<LValue, 16> StoreList;
525 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
526
527 bool isVector = false;
528 if (auto *VT = SrcTy->getAs<VectorType>()) {
529 isVector = true;
530 SrcTy = VT->getElementType();
531 assert(StoreList.size() <= VT->getNumElements() &&
532 "Cannot perform HLSL flat cast when vector source \
533 object has less elements than flattened destination \
534 object.");
535 }
536
537 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
538 LValue DestLVal = StoreList[I];
539 llvm::Value *Load =
540 isVector ? CGF.Builder.CreateExtractElement(Vec: SrcVal, Idx: I, Name: "vec.load")
541 : SrcVal;
542 llvm::Value *Cast =
543 CGF.EmitScalarConversion(Src: Load, SrcTy, DstTy: DestLVal.getType(), Loc);
544 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
545 }
546}
547
548// emit a flat cast where the RHS is an aggregate
549static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
550 LValue SrcVal, SourceLocation Loc) {
551 // Flatten our destination
552 SmallVector<LValue, 16> StoreList;
553 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
554 // Flatten our src
555 SmallVector<LValue, 16> LoadList;
556 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
557
558 assert(StoreList.size() <= LoadList.size() &&
559 "Cannot perform HLSL elementwise cast when flattened source object \
560 has less elements than flattened destination object.");
561 // apply casts to what we load from LoadList
562 // and store result in Dest
563 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
564 LValue DestLVal = StoreList[I];
565 LValue SrcLVal = LoadList[I];
566 RValue RVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc);
567 assert(RVal.isScalar() && "All flattened source values should be scalars");
568 llvm::Value *Val = RVal.getScalarVal();
569 llvm::Value *Cast = CGF.EmitScalarConversion(Src: Val, SrcTy: SrcLVal.getType(),
570 DstTy: DestLVal.getType(), Loc);
571 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
572 }
573}
574
575/// Emit initialization of an array from an initializer list. ExprToVisit must
576/// be either an InitListEpxr a CXXParenInitListExpr.
577void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
578 QualType ArrayQTy, Expr *ExprToVisit,
579 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
580 uint64_t NumInitElements = Args.size();
581
582 uint64_t NumArrayElements = AType->getNumElements();
583 for (const auto *Init : Args) {
584 if (const auto *Embed = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
585 NumInitElements += Embed->getDataElementCount() - 1;
586 if (NumInitElements > NumArrayElements) {
587 NumInitElements = NumArrayElements;
588 break;
589 }
590 }
591 }
592
593 assert(NumInitElements <= NumArrayElements);
594
595 QualType elementType =
596 CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
597 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
598 CharUnits elementAlign =
599 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
600 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
601
602 // Consider initializing the array by copying from a global. For this to be
603 // more efficient than per-element initialization, the size of the elements
604 // with explicit initializers should be large enough.
605 if (NumInitElements * elementSize.getQuantity() > 16 &&
606 elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
607 CodeGen::CodeGenModule &CGM = CGF.CGM;
608 ConstantEmitter Emitter(CGF);
609 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
610 T: CGM.getContext().removeAddrSpaceQualType(T: ArrayQTy),
611 AddressSpace: CGM.GetGlobalConstantAddressSpace());
612 LangAS AS = GVArrayQTy.getAddressSpace();
613 if (llvm::Constant *C =
614 Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: GVArrayQTy)) {
615 auto GV = new llvm::GlobalVariable(
616 CGM.getModule(), C->getType(),
617 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
618 "constinit",
619 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
620 CGM.getContext().getTargetAddressSpace(AS));
621 Emitter.finalize(global: GV);
622 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: GVArrayQTy);
623 GV->setAlignment(Align.getAsAlign());
624 Address GVAddr(GV, GV->getValueType(), Align);
625 EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: GVArrayQTy));
626 return;
627 }
628 }
629
630 // Exception safety requires us to destroy all the
631 // already-constructed members if an initializer throws.
632 // For that, we'll need an EH cleanup.
633 QualType::DestructionKind dtorKind = elementType.isDestructedType();
634 Address endOfInit = Address::invalid();
635 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
636
637 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
638 if (dtorKind) {
639 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
640 // In principle we could tell the cleanup where we are more
641 // directly, but the control flow can get so varied here that it
642 // would actually be quite complex. Therefore we go through an
643 // alloca.
644 llvm::Instruction *dominatingIP =
645 Builder.CreateFlagLoad(Addr: llvm::ConstantInt::getNullValue(Ty: CGF.Int8PtrTy));
646 endOfInit = CGF.CreateTempAlloca(Ty: begin->getType(), align: CGF.getPointerAlign(),
647 Name: "arrayinit.endOfInit");
648 Builder.CreateStore(Val: begin, Addr: endOfInit);
649 CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
650 elementAlignment: elementAlign,
651 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
652 cast<EHCleanupScope>(Val&: *CGF.EHStack.find(sp: CGF.EHStack.stable_begin()))
653 .AddAuxAllocas(Allocas: allocaTracker.Take());
654
655 CGF.DeferredDeactivationCleanupStack.push_back(
656 Elt: {.Cleanup: CGF.EHStack.stable_begin(), .DominatingIP: dominatingIP});
657 }
658
659 llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
660
661 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
662 llvm::Value *element = begin;
663 if (ArrayIndex > 0) {
664 if (CGF.getLangOpts().EmitLogicalPointer)
665 element = Builder.CreateStructuredGEP(
666 BaseType: AType, PtrBase: begin, Indices: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
667 Name: "arrayinit.element");
668 else
669 element = Builder.CreateInBoundsGEP(
670 Ty: llvmElementType, Ptr: begin,
671 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
672 Name: "arrayinit.element");
673
674 // Tell the cleanup that it needs to destroy up to this
675 // element. TODO: some of these stores can be trivially
676 // observed to be unnecessary.
677 if (endOfInit.isValid())
678 Builder.CreateStore(Val: element, Addr: endOfInit);
679 }
680
681 LValue elementLV = CGF.MakeAddrLValue(
682 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
683 EmitInitializationToLValue(E: Init, Address: elementLV);
684 return true;
685 };
686
687 unsigned ArrayIndex = 0;
688 // Emit the explicit initializers.
689 for (uint64_t i = 0; i != NumInitElements; ++i) {
690 if (ArrayIndex >= NumInitElements)
691 break;
692 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Args[i]->IgnoreParenImpCasts())) {
693 EmbedS->doForEachDataElement(C&: Emit, StartingIndexInArray&: ArrayIndex);
694 } else {
695 Emit(Args[i], ArrayIndex);
696 ArrayIndex++;
697 }
698 }
699
700 // Check whether there's a non-trivial array-fill expression.
701 bool hasTrivialFiller = isTrivialFiller(E: ArrayFiller);
702
703 // Any remaining elements need to be zero-initialized, possibly
704 // using the filler expression. We can skip this if the we're
705 // emitting to zeroed memory.
706 if (NumInitElements != NumArrayElements &&
707 !(Dest.isZeroed() && hasTrivialFiller &&
708 CGF.getTypes().isZeroInitializable(T: elementType))) {
709
710 // Use an actual loop. This is basically
711 // do { *array++ = filler; } while (array != end);
712
713 // Advance to the start of the rest of the array.
714 llvm::Value *element = begin;
715 if (NumInitElements) {
716 element = Builder.CreateInBoundsGEP(
717 Ty: llvmElementType, Ptr: element,
718 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumInitElements),
719 Name: "arrayinit.start");
720 if (endOfInit.isValid())
721 Builder.CreateStore(Val: element, Addr: endOfInit);
722 }
723
724 // Compute the end of the array.
725 llvm::Value *end = Builder.CreateInBoundsGEP(
726 Ty: llvmElementType, Ptr: begin,
727 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
728
729 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
730 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
731
732 // Jump into the body.
733 CGF.EmitBlock(BB: bodyBB);
734 llvm::PHINode *currentElement =
735 Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
736 currentElement->addIncoming(V: element, BB: entryBB);
737
738 if (CGF.CGM.shouldEmitConvergenceTokens())
739 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
740
741 // Emit the actual filler expression.
742 {
743 // C++1z [class.temporary]p5:
744 // when a default constructor is called to initialize an element of
745 // an array with no corresponding initializer [...] the destruction of
746 // every temporary created in a default argument is sequenced before
747 // the construction of the next array element, if any
748 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
749 LValue elementLV = CGF.MakeAddrLValue(
750 Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
751 if (ArrayFiller)
752 EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
753 else
754 EmitNullInitializationToLValue(Address: elementLV);
755 }
756
757 // Move on to the next element.
758 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
759 Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
760
761 // Tell the EH cleanup that we finished with the last element.
762 if (endOfInit.isValid())
763 Builder.CreateStore(Val: nextElement, Addr: endOfInit);
764
765 // Leave the loop if we're done.
766 llvm::Value *done =
767 Builder.CreateICmpEQ(LHS: nextElement, RHS: end, Name: "arrayinit.done");
768 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
769 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
770 currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
771
772 if (CGF.CGM.shouldEmitConvergenceTokens())
773 CGF.ConvergenceTokenStack.pop_back();
774
775 CGF.EmitBlock(BB: endBB);
776 }
777}
778
779//===----------------------------------------------------------------------===//
780// Visitor Methods
781//===----------------------------------------------------------------------===//
782
783void AggExprEmitter::VisitMaterializeTemporaryExpr(
784 MaterializeTemporaryExpr *E) {
785 Visit(E: E->getSubExpr());
786}
787
788void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
789 // If this is a unique OVE, just visit its source expression.
790 if (e->isUnique())
791 Visit(E: e->getSourceExpr());
792 else
793 EmitFinalDestCopy(type: e->getType(), src: CGF.getOrCreateOpaqueLValueMapping(e));
794}
795
796void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
797 if (Dest.isPotentiallyAliased()) {
798 // Just emit a load of the lvalue + a copy, because our compound literal
799 // might alias the destination.
800 EmitAggLoadOfLValue(E);
801 return;
802 }
803
804 AggValueSlot Slot = EnsureSlot(T: E->getType());
805
806 // Block-scope compound literals are destroyed at the end of the enclosing
807 // scope in C.
808 bool Destruct =
809 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
810 if (Destruct)
811 Slot.setExternallyDestructed();
812
813 CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
814
815 if (Destruct)
816 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
817 CGF.pushLifetimeExtendedDestroy(
818 kind: CGF.getCleanupKind(kind: DtorKind), addr: Slot.getAddress(), type: E->getType(),
819 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: DtorKind & EHCleanup);
820}
821
822/// Attempt to look through various unimportant expressions to find a
823/// cast of the given kind.
824static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
825 op = op->IgnoreParenNoopCasts(Ctx: ctx);
826 if (auto castE = dyn_cast<CastExpr>(Val: op)) {
827 if (castE->getCastKind() == kind)
828 return castE->getSubExpr();
829 }
830 return nullptr;
831}
832
833void AggExprEmitter::VisitCastExpr(CastExpr *E) {
834 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
835 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
836 switch (E->getCastKind()) {
837 case CK_Dynamic: {
838 // FIXME: Can this actually happen? We have no test coverage for it.
839 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
840 LValue LV =
841 CGF.EmitCheckedLValue(E: E->getSubExpr(), TCK: CodeGenFunction::TCK_Load);
842 // FIXME: Do we also need to handle property references here?
843 if (LV.isSimple())
844 CGF.EmitDynamicCast(V: LV.getAddress(), DCE: cast<CXXDynamicCastExpr>(Val: E));
845 else
846 CGF.CGM.ErrorUnsupported(S: E, Type: "non-simple lvalue dynamic_cast");
847
848 if (!Dest.isIgnored())
849 CGF.CGM.ErrorUnsupported(S: E, Type: "lvalue dynamic_cast with a destination");
850 break;
851 }
852
853 case CK_ToUnion: {
854 // Evaluate even if the destination is ignored.
855 if (Dest.isIgnored()) {
856 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
857 /*ignoreResult=*/true);
858 break;
859 }
860
861 // GCC union extension
862 QualType Ty = E->getSubExpr()->getType();
863 Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
864 EmitInitializationToLValue(E: E->getSubExpr(),
865 Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
866 break;
867 }
868
869 case CK_LValueToRValueBitCast: {
870 if (Dest.isIgnored()) {
871 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
872 /*ignoreResult=*/true);
873 break;
874 }
875
876 LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
877 Address SourceAddress = SourceLV.getAddress().withElementType(ElemTy: CGF.Int8Ty);
878 Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
879 llvm::Value *SizeVal = llvm::ConstantInt::get(
880 Ty: CGF.SizeTy,
881 V: CGF.getContext().getTypeSizeInChars(T: E->getType()).getQuantity());
882 Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
883 break;
884 }
885
886 case CK_DerivedToBase: {
887 assert(CGF.getLangOpts().HLSL &&
888 "Derived/Base casts in EmitAggExpr are only supported in HLSL");
889
890 // Create a temporary for the derived record, switch it out with the current
891 // Dest slot, and emit the derived value.
892 QualType DerivedTy = E->getSubExpr()->getType();
893 RawAddress DerivedAddr = CGF.CreateMemTempWithoutCast(T: DerivedTy);
894 AggValueSlot DerivedTmpSlot = AggValueSlot::forAddr(
895 addr: DerivedAddr, quals: DerivedTy.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
896 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
897 mayOverlap: AggValueSlot::DoesNotOverlap);
898
899 AggValueSlot DestBaseSlot = Dest;
900 Dest = DerivedTmpSlot;
901
902 Visit(E: E->getSubExpr());
903
904 // Perform derived-to-base address conversion to get the address
905 // of the base record within the derived record. In HLSL this should
906 // always be same as the derived because of single inheritance, but let's
907 // do it properly.
908 Address BaseAddrInDerived = CGF.GetAddressOfBaseClass(
909 Value: DerivedTmpSlot.getAddress(), Derived: DerivedTy->castAsCXXRecordDecl(),
910 PathBegin: E->path_begin(), PathEnd: E->path_end(),
911 /*NullCheckValue=*/false, Loc: E->getExprLoc());
912
913 AggValueSlot SrcBaseSlot = AggValueSlot::forAddr(
914 addr: BaseAddrInDerived, quals: E->getType().getQualifiers(),
915 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
916 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
917
918 // Copy the base class to the original destination slot and restore it.
919 EmitCopy(type: E->getType(), dest: DestBaseSlot, src: SrcBaseSlot);
920 Dest = DestBaseSlot;
921 break;
922 }
923
924 case CK_BaseToDerived:
925 case CK_UncheckedDerivedToBase: {
926 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
927 "should have been unpacked before we got here");
928 }
929
930 case CK_NonAtomicToAtomic:
931 case CK_AtomicToNonAtomic: {
932 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
933
934 // Determine the atomic and value types.
935 QualType atomicType = E->getSubExpr()->getType();
936 QualType valueType = E->getType();
937 if (isToAtomic)
938 std::swap(a&: atomicType, b&: valueType);
939
940 assert(atomicType->isAtomicType());
941 assert(CGF.getContext().hasSameUnqualifiedType(
942 valueType, atomicType->castAs<AtomicType>()->getValueType()));
943
944 // Just recurse normally if we're ignoring the result or the
945 // atomic type doesn't change representation.
946 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
947 return Visit(E: E->getSubExpr());
948 }
949
950 CastKind peepholeTarget =
951 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
952
953 // These two cases are reverses of each other; try to peephole them.
954 if (Expr *op =
955 findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
956 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
957 E->getType()) &&
958 "peephole significantly changed types?");
959 return Visit(E: op);
960 }
961
962 // If we're converting an r-value of non-atomic type to an r-value
963 // of atomic type, just emit directly into the relevant sub-object.
964 if (isToAtomic) {
965 AggValueSlot valueDest = Dest;
966 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
967 // Zero-initialize. (Strictly speaking, we only need to initialize
968 // the padding at the end, but this is simpler.)
969 if (!Dest.isZeroed())
970 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
971
972 // Build a GEP to refer to the subobject.
973 Address valueAddr =
974 CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
975 valueDest = AggValueSlot::forAddr(
976 addr: valueAddr, quals: valueDest.getQualifiers(),
977 isDestructed: valueDest.isExternallyDestructed(), needsGC: valueDest.requiresGCollection(),
978 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
979 isZeroed: AggValueSlot::IsZeroed);
980 }
981
982 CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
983 return;
984 }
985
986 // Otherwise, we're converting an atomic type to a non-atomic type.
987 // Make an atomic temporary, emit into that, and then copy the value out.
988 AggValueSlot atomicSlot =
989 CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
990 CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
991
992 Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
993 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
994 return EmitFinalDestCopy(type: valueType, src: rvalue);
995 }
996 case CK_AddressSpaceConversion:
997 return Visit(E: E->getSubExpr());
998
999 case CK_LValueToRValue:
1000 // If we're loading from a volatile type, force the destination
1001 // into existence.
1002 if (E->getSubExpr()->getType().isVolatileQualified()) {
1003 bool Destruct =
1004 !Dest.isExternallyDestructed() &&
1005 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1006 if (Destruct)
1007 Dest.setExternallyDestructed();
1008 EnsureDest(T: E->getType());
1009 Visit(E: E->getSubExpr());
1010
1011 if (Destruct)
1012 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1013 type: E->getType());
1014
1015 return;
1016 }
1017
1018 [[fallthrough]];
1019
1020 case CK_HLSLArrayRValue:
1021 if (CGF.getLangOpts().HLSL &&
1022 E->getSubExpr()->getType()->isHLSLResourceRecordArray())
1023 if (CGF.CGM.getHLSLRuntime().emitGlobalResourceArray(CGF, E, DestSlot&: Dest))
1024 break;
1025 Visit(E: E->getSubExpr());
1026 break;
1027 case CK_HLSLAggregateSplatCast: {
1028 Expr *Src = E->getSubExpr();
1029 QualType SrcTy = Src->getType();
1030 RValue RV = CGF.EmitAnyExpr(E: Src);
1031 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1032 SourceLocation Loc = E->getExprLoc();
1033
1034 assert(RV.isScalar() && SrcTy->isScalarType() &&
1035 "RHS of HLSL splat cast must be a scalar.");
1036 llvm::Value *SrcVal = RV.getScalarVal();
1037 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1038 break;
1039 }
1040 case CK_HLSLElementwiseCast: {
1041 Expr *Src = E->getSubExpr();
1042 QualType SrcTy = Src->getType();
1043 RValue RV = CGF.EmitAnyExpr(E: Src);
1044 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1045 SourceLocation Loc = E->getExprLoc();
1046
1047 if (RV.isScalar()) {
1048 llvm::Value *SrcVal = RV.getScalarVal();
1049 assert(SrcTy->isVectorType() &&
1050 "HLSL Elementwise cast doesn't handle splatting.");
1051 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1052 } else {
1053 assert(RV.isAggregate() &&
1054 "Can't perform HLSL Aggregate cast on a complex type.");
1055 Address SrcVal = RV.getAggregateAddress();
1056 EmitHLSLElementwiseCast(CGF, DestVal: DestLVal, SrcVal: CGF.MakeAddrLValue(Addr: SrcVal, T: SrcTy),
1057 Loc);
1058 }
1059 break;
1060 }
1061 case CK_NoOp:
1062 case CK_UserDefinedConversion:
1063 case CK_ConstructorConversion:
1064 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
1065 E->getType()) &&
1066 "Implicit cast types must be compatible");
1067 Visit(E: E->getSubExpr());
1068 break;
1069
1070 case CK_LValueBitCast:
1071 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
1072
1073 case CK_Dependent:
1074 case CK_BitCast:
1075 case CK_ArrayToPointerDecay:
1076 case CK_FunctionToPointerDecay:
1077 case CK_NullToPointer:
1078 case CK_NullToMemberPointer:
1079 case CK_BaseToDerivedMemberPointer:
1080 case CK_DerivedToBaseMemberPointer:
1081 case CK_MemberPointerToBoolean:
1082 case CK_ReinterpretMemberPointer:
1083 case CK_IntegralToPointer:
1084 case CK_PointerToIntegral:
1085 case CK_PointerToBoolean:
1086 case CK_ToVoid:
1087 case CK_VectorSplat:
1088 case CK_IntegralCast:
1089 case CK_BooleanToSignedIntegral:
1090 case CK_IntegralToBoolean:
1091 case CK_IntegralToFloating:
1092 case CK_FloatingToIntegral:
1093 case CK_FloatingToBoolean:
1094 case CK_FloatingCast:
1095 case CK_CPointerToObjCPointerCast:
1096 case CK_BlockPointerToObjCPointerCast:
1097 case CK_AnyPointerToBlockPointerCast:
1098 case CK_ObjCObjectLValueCast:
1099 case CK_FloatingRealToComplex:
1100 case CK_FloatingComplexToReal:
1101 case CK_FloatingComplexToBoolean:
1102 case CK_FloatingComplexCast:
1103 case CK_FloatingComplexToIntegralComplex:
1104 case CK_IntegralRealToComplex:
1105 case CK_IntegralComplexToReal:
1106 case CK_IntegralComplexToBoolean:
1107 case CK_IntegralComplexCast:
1108 case CK_IntegralComplexToFloatingComplex:
1109 case CK_ARCProduceObject:
1110 case CK_ARCConsumeObject:
1111 case CK_ARCReclaimReturnedObject:
1112 case CK_ARCExtendBlockObject:
1113 case CK_CopyAndAutoreleaseBlockObject:
1114 case CK_BuiltinFnToFnPtr:
1115 case CK_ZeroToOCLOpaqueType:
1116 case CK_MatrixCast:
1117 case CK_HLSLVectorTruncation:
1118 case CK_HLSLMatrixTruncation:
1119 case CK_IntToOCLSampler:
1120 case CK_FloatingToFixedPoint:
1121 case CK_FixedPointToFloating:
1122 case CK_FixedPointCast:
1123 case CK_FixedPointToBoolean:
1124 case CK_FixedPointToIntegral:
1125 case CK_IntegralToFixedPoint:
1126 llvm_unreachable("cast kind invalid for aggregate types");
1127 }
1128}
1129
1130void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1131 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
1132 EmitAggLoadOfLValue(E);
1133 return;
1134 }
1135
1136 withReturnValueSlot(
1137 E, EmitCall: [&](ReturnValueSlot Slot) { return CGF.EmitCallExpr(E, ReturnValue: Slot); });
1138}
1139
1140void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1141 withReturnValueSlot(E, EmitCall: [&](ReturnValueSlot Slot) {
1142 return CGF.EmitObjCMessageExpr(E, Return: Slot);
1143 });
1144}
1145
1146void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1147 CGF.EmitIgnoredExpr(E: E->getLHS());
1148 Visit(E: E->getRHS());
1149}
1150
1151void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1152 CodeGenFunction::StmtExprEvaluation eval(CGF);
1153 CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
1154}
1155
1156enum CompareKind {
1157 CK_Less,
1158 CK_Greater,
1159 CK_Equal,
1160};
1161
1162static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1163 const BinaryOperator *E, llvm::Value *LHS,
1164 llvm::Value *RHS, CompareKind Kind,
1165 const char *NameSuffix = "") {
1166 QualType ArgTy = E->getLHS()->getType();
1167 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1168 ArgTy = CT->getElementType();
1169
1170 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1171 assert(Kind == CK_Equal &&
1172 "member pointers may only be compared for equality");
1173 return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1174 CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1175 }
1176
1177 // Compute the comparison instructions for the specified comparison kind.
1178 struct CmpInstInfo {
1179 const char *Name;
1180 llvm::CmpInst::Predicate FCmp;
1181 llvm::CmpInst::Predicate SCmp;
1182 llvm::CmpInst::Predicate UCmp;
1183 };
1184 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1185 using FI = llvm::FCmpInst;
1186 using II = llvm::ICmpInst;
1187 switch (Kind) {
1188 case CK_Less:
1189 return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1190 case CK_Greater:
1191 return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1192 case CK_Equal:
1193 return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1194 }
1195 llvm_unreachable("Unrecognised CompareKind enum");
1196 }();
1197
1198 if (ArgTy->hasFloatingRepresentation())
1199 return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1200 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1201 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1202 auto Inst =
1203 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1204 return Builder.CreateICmp(P: Inst, LHS, RHS,
1205 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1206 }
1207
1208 llvm_unreachable("unsupported aggregate binary expression should have "
1209 "already been handled");
1210}
1211
1212void AggExprEmitter::EmitComparisonResult(const Expr *E,
1213 const ComparisonCategoryInfo &CmpInfo,
1214 llvm::Value *ResultValue) {
1215 // Create the return value in the destination slot.
1216 EnsureDest(T: E->getType());
1217 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1218
1219 // Emit the address of the first (and only) field in the comparison category
1220 // type, and initialize it from the constant integer value selected above.
1221 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1222 Base: DestLV, Field: *CmpInfo.Record->field_begin());
1223 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ResultValue), Dst: FieldLV,
1224 /*IsInit=*/isInit: true);
1225}
1226
1227void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1228 using llvm::BasicBlock;
1229 using llvm::PHINode;
1230 using llvm::Value;
1231 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1232 E->getRHS()->getType()));
1233 const ComparisonCategoryInfo &CmpInfo =
1234 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1235 assert(CmpInfo.Record->isTriviallyCopyable() &&
1236 "cannot copy non-trivially copyable aggregate");
1237
1238 QualType ArgTy = E->getLHS()->getType();
1239
1240 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1241 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1242 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1243 return CGF.ErrorUnsupported(S: E, Type: "aggregate three-way comparison");
1244 }
1245 bool IsComplex = ArgTy->isAnyComplexType();
1246
1247 // Evaluate the operands to the expression and extract their values.
1248 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1249 RValue RV = CGF.EmitAnyExpr(E);
1250 if (RV.isScalar())
1251 return {RV.getScalarVal(), nullptr};
1252 if (RV.isAggregate())
1253 return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1254 assert(RV.isComplex());
1255 return RV.getComplexVal();
1256 };
1257 auto LHSValues = EmitOperand(E->getLHS()),
1258 RHSValues = EmitOperand(E->getRHS());
1259
1260 auto EmitCmp = [&](CompareKind K) {
1261 Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1262 Kind: K, NameSuffix: IsComplex ? ".r" : "");
1263 if (!IsComplex)
1264 return Cmp;
1265 assert(K == CompareKind::CK_Equal);
1266 Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1267 RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1268 return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1269 };
1270 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1271 return Builder.getInt(AI: VInfo->getIntValue());
1272 };
1273
1274 Value *Select;
1275 if (ArgTy->isNullPtrType()) {
1276 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1277 } else if (!CmpInfo.isPartial()) {
1278 Value *SelectOne =
1279 Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1280 False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1281 Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1282 True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1283 False: SelectOne, Name: "sel.eq");
1284 } else {
1285 Value *SelectEq = Builder.CreateSelect(
1286 C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1287 False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1288 Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1289 True: EmitCmpRes(CmpInfo.getGreater()),
1290 False: SelectEq, Name: "sel.gt");
1291 Select = Builder.CreateSelect(
1292 C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1293 }
1294
1295 EmitComparisonResult(E, CmpInfo, ResultValue: Select);
1296}
1297
1298void AggExprEmitter::VisitTypeTraitExpr(const TypeTraitExpr *E) {
1299 assert(E->isStoredAsComparisonResult() &&
1300 "expected a strong_ordering type trait with a stored value");
1301
1302 const ComparisonCategoryInfo &CmpInfo =
1303 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1304 const auto Result =
1305 ComparisonCategoryResult(E->getAPValue().getInt().getZExtValue());
1306 llvm::Value *ResultValue =
1307 Builder.getInt(AI: CmpInfo.getValueInfo(ValueKind: Result)->getIntValue());
1308
1309 EmitComparisonResult(E, CmpInfo, ResultValue);
1310}
1311
1312void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1313 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1314 VisitPointerToDataMemberBinaryOperator(BO: E);
1315 else
1316 CGF.ErrorUnsupported(S: E, Type: "aggregate binary expression");
1317}
1318
1319void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1320 const BinaryOperator *E) {
1321 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1322 EmitFinalDestCopy(type: E->getType(), src: LV);
1323}
1324
1325/// Is the value of the given expression possibly a reference to or
1326/// into a __block variable?
1327static bool isBlockVarRef(const Expr *E) {
1328 // Make sure we look through parens.
1329 E = E->IgnoreParens();
1330
1331 // Check for a direct reference to a __block variable.
1332 if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1333 const VarDecl *var = dyn_cast<VarDecl>(Val: DRE->getDecl());
1334 return (var && var->hasAttr<BlocksAttr>());
1335 }
1336
1337 // More complicated stuff.
1338
1339 // Binary operators.
1340 if (const BinaryOperator *op = dyn_cast<BinaryOperator>(Val: E)) {
1341 // For an assignment or pointer-to-member operation, just care
1342 // about the LHS.
1343 if (op->isAssignmentOp() || op->isPtrMemOp())
1344 return isBlockVarRef(E: op->getLHS());
1345
1346 // For a comma, just care about the RHS.
1347 if (op->getOpcode() == BO_Comma)
1348 return isBlockVarRef(E: op->getRHS());
1349
1350 // FIXME: pointer arithmetic?
1351 return false;
1352
1353 // Check both sides of a conditional operator.
1354 } else if (const AbstractConditionalOperator *op =
1355 dyn_cast<AbstractConditionalOperator>(Val: E)) {
1356 return isBlockVarRef(E: op->getTrueExpr()) ||
1357 isBlockVarRef(E: op->getFalseExpr());
1358
1359 // OVEs are required to support BinaryConditionalOperators.
1360 } else if (const OpaqueValueExpr *op = dyn_cast<OpaqueValueExpr>(Val: E)) {
1361 if (const Expr *src = op->getSourceExpr())
1362 return isBlockVarRef(E: src);
1363
1364 // Casts are necessary to get things like (*(int*)&var) = foo().
1365 // We don't really care about the kind of cast here, except
1366 // we don't want to look through l2r casts, because it's okay
1367 // to get the *value* in a __block variable.
1368 } else if (const CastExpr *cast = dyn_cast<CastExpr>(Val: E)) {
1369 if (cast->getCastKind() == CK_LValueToRValue)
1370 return false;
1371 return isBlockVarRef(E: cast->getSubExpr());
1372
1373 // Handle unary operators. Again, just aggressively look through
1374 // it, ignoring the operation.
1375 } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E)) {
1376 return isBlockVarRef(E: uop->getSubExpr());
1377
1378 // Look into the base of a field access.
1379 } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(Val: E)) {
1380 return isBlockVarRef(E: mem->getBase());
1381
1382 // Look into the base of a subscript.
1383 } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(Val: E)) {
1384 return isBlockVarRef(E: sub->getBase());
1385 }
1386
1387 return false;
1388}
1389
1390void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1391 ApplyAtomGroup Grp(CGF.getDebugInfo());
1392 // For an assignment to work, the value on the right has
1393 // to be compatible with the value on the left.
1394 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1395 E->getRHS()->getType()) &&
1396 "Invalid assignment");
1397
1398 // If the LHS might be a __block variable, and the RHS can
1399 // potentially cause a block copy, we need to evaluate the RHS first
1400 // so that the assignment goes the right place.
1401 // This is pretty semantically fragile.
1402 if (isBlockVarRef(E: E->getLHS()) &&
1403 E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1404 // Ensure that we have a destination, and evaluate the RHS into that.
1405 EnsureDest(T: E->getRHS()->getType());
1406 Visit(E: E->getRHS());
1407
1408 // Now emit the LHS and copy into it.
1409 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1410
1411 // That copy is an atomic copy if the LHS is atomic.
1412 if (LHS.getType()->isAtomicType() ||
1413 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1414 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1415 return;
1416 }
1417
1418 EmitCopy(type: E->getLHS()->getType(),
1419 dest: AggValueSlot::forLValue(LV: LHS, isDestructed: AggValueSlot::IsDestructed,
1420 needsGC: needsGC(T: E->getLHS()->getType()),
1421 isAliased: AggValueSlot::IsAliased,
1422 mayOverlap: AggValueSlot::MayOverlap),
1423 src: Dest);
1424 return;
1425 }
1426
1427 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1428
1429 // If we have an atomic type, evaluate into the destination and then
1430 // do an atomic copy.
1431 if (LHS.getType()->isAtomicType() ||
1432 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1433 EnsureDest(T: E->getRHS()->getType());
1434 Visit(E: E->getRHS());
1435 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1436 return;
1437 }
1438
1439 // Codegen the RHS so that it stores directly into the LHS.
1440 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1441 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1442 isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1443 // A non-volatile aggregate destination might have volatile member.
1444 if (!LHSSlot.isVolatile() && CGF.hasVolatileMember(T: E->getLHS()->getType()))
1445 LHSSlot.setVolatile(true);
1446
1447 CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1448
1449 // Copy into the destination if the assignment isn't ignored.
1450 EmitFinalDestCopy(type: E->getType(), src: LHS);
1451
1452 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1453 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1454 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1455 type: E->getType());
1456}
1457
1458void AggExprEmitter::VisitAbstractConditionalOperator(
1459 const AbstractConditionalOperator *E) {
1460 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1461 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1462 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1463
1464 // Bind the common expression if necessary.
1465 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1466
1467 CodeGenFunction::ConditionalEvaluation eval(CGF);
1468 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1469 TrueCount: CGF.getProfileCount(S: E));
1470
1471 // Save whether the destination's lifetime is externally managed.
1472 bool isExternallyDestructed = Dest.isExternallyDestructed();
1473 bool destructNonTrivialCStruct =
1474 !isExternallyDestructed &&
1475 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1476 isExternallyDestructed |= destructNonTrivialCStruct;
1477 Dest.setExternallyDestructed(isExternallyDestructed);
1478
1479 eval.begin(CGF);
1480 CGF.EmitBlock(BB: LHSBlock);
1481 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
1482 Visit(E: E->getTrueExpr());
1483 eval.end(CGF);
1484
1485 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1486 CGF.Builder.CreateBr(Dest: ContBlock);
1487
1488 // If the result of an agg expression is unused, then the emission
1489 // of the LHS might need to create a destination slot. That's fine
1490 // with us, and we can safely emit the RHS into the same slot, but
1491 // we shouldn't claim that it's already being destructed.
1492 Dest.setExternallyDestructed(isExternallyDestructed);
1493
1494 eval.begin(CGF);
1495 CGF.EmitBlock(BB: RHSBlock);
1496 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
1497 Visit(E: E->getFalseExpr());
1498 eval.end(CGF);
1499
1500 if (destructNonTrivialCStruct)
1501 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1502 type: E->getType());
1503
1504 CGF.EmitBlock(BB: ContBlock);
1505}
1506
1507void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1508 Visit(E: CE->getChosenSubExpr());
1509}
1510
1511void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1512 Address ArgValue = Address::invalid();
1513 CGF.EmitVAArg(VE, VAListAddr&: ArgValue, Slot: Dest);
1514
1515 // If EmitVAArg fails, emit an error.
1516 if (!ArgValue.isValid()) {
1517 CGF.ErrorUnsupported(S: VE, Type: "aggregate va_arg expression");
1518 return;
1519 }
1520}
1521
1522void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1523 // Ensure that we have a slot, but if we already do, remember
1524 // whether it was externally destructed.
1525 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1526 EnsureDest(T: E->getType());
1527
1528 // We're going to push a destructor if there isn't already one.
1529 Dest.setExternallyDestructed();
1530
1531 Visit(E: E->getSubExpr());
1532
1533 // Push that destructor we promised.
1534 if (!wasExternallyDestructed)
1535 CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1536}
1537
1538void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1539 AggValueSlot Slot = EnsureSlot(T: E->getType());
1540 CGF.EmitCXXConstructExpr(E, Dest: Slot);
1541}
1542
1543void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1544 const CXXInheritedCtorInitExpr *E) {
1545 AggValueSlot Slot = EnsureSlot(T: E->getType());
1546 CGF.EmitInheritedCXXConstructorCall(D: E->getConstructor(), ForVirtualBase: E->constructsVBase(),
1547 This: Slot.getAddress(),
1548 InheritedFromVBase: E->inheritedFromVBase(), E);
1549}
1550
1551void AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1552 AggValueSlot Slot = EnsureSlot(T: E->getType());
1553 LValue SlotLV = CGF.MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType());
1554
1555 // We'll need to enter cleanup scopes in case any of the element
1556 // initializers throws an exception or contains branch out of the expressions.
1557 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1558
1559 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1560 for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1561 e = E->capture_init_end();
1562 i != e; ++i, ++CurField) {
1563 // Emit initialization
1564 LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1565 if (CurField->hasCapturedVLAType()) {
1566 CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1567 continue;
1568 }
1569
1570 EmitInitializationToLValue(E: *i, Address: LV);
1571
1572 // Push a destructor if necessary.
1573 if (QualType::DestructionKind DtorKind =
1574 CurField->getType().isDestructedType()) {
1575 assert(LV.isSimple());
1576 if (DtorKind)
1577 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1578 type: CurField->getType(),
1579 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: false);
1580 }
1581 }
1582}
1583
1584void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1585 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1586 Visit(E: E->getSubExpr());
1587}
1588
1589void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1590 QualType T = E->getType();
1591 AggValueSlot Slot = EnsureSlot(T);
1592 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1593}
1594
1595void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1596 QualType T = E->getType();
1597 AggValueSlot Slot = EnsureSlot(T);
1598 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1599}
1600
1601/// Determine whether the given cast kind is known to always convert values
1602/// with all zero bits in their value representation to values with all zero
1603/// bits in their value representation.
1604static bool castPreservesZero(const CastExpr *CE) {
1605 switch (CE->getCastKind()) {
1606 // No-ops.
1607 case CK_NoOp:
1608 case CK_UserDefinedConversion:
1609 case CK_ConstructorConversion:
1610 case CK_BitCast:
1611 case CK_ToUnion:
1612 case CK_ToVoid:
1613 // Conversions between (possibly-complex) integral, (possibly-complex)
1614 // floating-point, and bool.
1615 case CK_BooleanToSignedIntegral:
1616 case CK_FloatingCast:
1617 case CK_FloatingComplexCast:
1618 case CK_FloatingComplexToBoolean:
1619 case CK_FloatingComplexToIntegralComplex:
1620 case CK_FloatingComplexToReal:
1621 case CK_FloatingRealToComplex:
1622 case CK_FloatingToBoolean:
1623 case CK_FloatingToIntegral:
1624 case CK_IntegralCast:
1625 case CK_IntegralComplexCast:
1626 case CK_IntegralComplexToBoolean:
1627 case CK_IntegralComplexToFloatingComplex:
1628 case CK_IntegralComplexToReal:
1629 case CK_IntegralRealToComplex:
1630 case CK_IntegralToBoolean:
1631 case CK_IntegralToFloating:
1632 // Reinterpreting integers as pointers and vice versa.
1633 case CK_IntegralToPointer:
1634 case CK_PointerToIntegral:
1635 // Language extensions.
1636 case CK_VectorSplat:
1637 case CK_MatrixCast:
1638 case CK_NonAtomicToAtomic:
1639 case CK_AtomicToNonAtomic:
1640 case CK_HLSLVectorTruncation:
1641 case CK_HLSLMatrixTruncation:
1642 case CK_HLSLElementwiseCast:
1643 case CK_HLSLAggregateSplatCast:
1644 return true;
1645
1646 case CK_BaseToDerivedMemberPointer:
1647 case CK_DerivedToBaseMemberPointer:
1648 case CK_MemberPointerToBoolean:
1649 case CK_NullToMemberPointer:
1650 case CK_ReinterpretMemberPointer:
1651 // FIXME: ABI-dependent.
1652 return false;
1653
1654 case CK_AnyPointerToBlockPointerCast:
1655 case CK_BlockPointerToObjCPointerCast:
1656 case CK_CPointerToObjCPointerCast:
1657 case CK_ObjCObjectLValueCast:
1658 case CK_IntToOCLSampler:
1659 case CK_ZeroToOCLOpaqueType:
1660 // FIXME: Check these.
1661 return false;
1662
1663 case CK_FixedPointCast:
1664 case CK_FixedPointToBoolean:
1665 case CK_FixedPointToFloating:
1666 case CK_FixedPointToIntegral:
1667 case CK_FloatingToFixedPoint:
1668 case CK_IntegralToFixedPoint:
1669 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1670 return false;
1671
1672 case CK_AddressSpaceConversion:
1673 case CK_BaseToDerived:
1674 case CK_DerivedToBase:
1675 case CK_Dynamic:
1676 case CK_NullToPointer:
1677 case CK_PointerToBoolean:
1678 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1679 // same representation in all involved address spaces.
1680 return false;
1681
1682 case CK_ARCConsumeObject:
1683 case CK_ARCExtendBlockObject:
1684 case CK_ARCProduceObject:
1685 case CK_ARCReclaimReturnedObject:
1686 case CK_CopyAndAutoreleaseBlockObject:
1687 case CK_ArrayToPointerDecay:
1688 case CK_FunctionToPointerDecay:
1689 case CK_BuiltinFnToFnPtr:
1690 case CK_Dependent:
1691 case CK_LValueBitCast:
1692 case CK_LValueToRValue:
1693 case CK_LValueToRValueBitCast:
1694 case CK_UncheckedDerivedToBase:
1695 case CK_HLSLArrayRValue:
1696 return false;
1697 }
1698 llvm_unreachable("Unhandled clang::CastKind enum");
1699}
1700
1701/// isSimpleZero - If emitting this value will obviously just cause a store of
1702/// zero to memory, return true. This can return false if uncertain, so it just
1703/// handles simple cases.
1704static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1705 E = E->IgnoreParens();
1706 while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1707 if (!castPreservesZero(CE))
1708 break;
1709 E = CE->getSubExpr()->IgnoreParens();
1710 }
1711
1712 // 0
1713 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1714 return IL->getValue() == 0;
1715 // +0.0
1716 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1717 return FL->getValue().isPosZero();
1718 // int()
1719 if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1720 CGF.getTypes().isZeroInitializable(T: E->getType()))
1721 return true;
1722 // (int*)0 - Null pointer expressions.
1723 if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1724 return ICE->getCastKind() == CK_NullToPointer &&
1725 CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1726 !E->HasSideEffects(Ctx: CGF.getContext());
1727 // '\0'
1728 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1729 return CL->getValue() == 0;
1730
1731 // Otherwise, hard case: conservatively return false.
1732 return false;
1733}
1734
1735void AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1736 QualType type = LV.getType();
1737 // FIXME: Ignore result?
1738 // FIXME: Are initializers affected by volatile?
1739 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1740 // Storing "i32 0" to a zero'd memory location is a noop.
1741 return;
1742 } else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1743 return EmitNullInitializationToLValue(Address: LV);
1744 } else if (isa<NoInitExpr>(Val: E)) {
1745 // Do nothing.
1746 return;
1747 } else if (type->isReferenceType()) {
1748 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1749 return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1750 }
1751
1752 CGF.EmitInitializationToLValue(E, LV, IsZeroed: Dest.isZeroed());
1753}
1754
1755void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1756 QualType type = lv.getType();
1757
1758 // If the destination slot is already zeroed out before the aggregate is
1759 // copied into it, we don't have to emit any zeros here.
1760 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1761 return;
1762
1763 if (CGF.hasScalarEvaluationKind(T: type)) {
1764 // For non-aggregates, we can store the appropriate null constant.
1765 llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1766 // Note that the following is not equivalent to
1767 // EmitStoreThroughBitfieldLValue for ARC types.
1768 if (lv.isBitField()) {
1769 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1770 } else {
1771 assert(lv.isSimple());
1772 CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1773 }
1774 } else {
1775 // There's a potential optimization opportunity in combining
1776 // memsets; that would be easy for arrays, but relatively
1777 // difficult for structures with the current code.
1778 CGF.EmitNullInitialization(DestPtr: lv.getAddress(), Ty: lv.getType());
1779 }
1780}
1781
1782void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1783 VisitCXXParenListOrInitListExpr(ExprToVisit: E, Args: E->getInitExprs(),
1784 InitializedFieldInUnion: E->getInitializedFieldInUnion(),
1785 ArrayFiller: E->getArrayFiller());
1786}
1787
1788void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1789 if (E->hadArrayRangeDesignator())
1790 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
1791
1792 if (E->isTransparent())
1793 return Visit(E: E->getInit(Init: 0));
1794
1795 VisitCXXParenListOrInitListExpr(
1796 ExprToVisit: E, Args: E->inits(), InitializedFieldInUnion: E->getInitializedFieldInUnion(), ArrayFiller: E->getArrayFiller());
1797}
1798
1799void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1800 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1801 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1802#if 0
1803 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1804 // (Length of globals? Chunks of zeroed-out space?).
1805 //
1806 // If we can, prefer a copy from a global; this is a lot less code for long
1807 // globals, and it's easier for the current optimizers to analyze.
1808 if (llvm::Constant *C =
1809 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1810 llvm::GlobalVariable* GV =
1811 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1812 llvm::GlobalValue::InternalLinkage, C, "");
1813 EmitFinalDestCopy(ExprToVisit->getType(),
1814 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1815 return;
1816 }
1817#endif
1818
1819 // HLSL initialization lists in the AST are an expansion which can contain
1820 // side-effecting expressions wrapped in opaque value expressions. To properly
1821 // emit these we need to emit the opaque values before we emit the argument
1822 // expressions themselves. This is a little hacky, but it prevents us needing
1823 // to do a bigger AST-level change for a language feature that we need
1824 // deprecate in the near future. See related HLSL language proposals:
1825 // * 0005-strict-initializer-lists.md
1826 // * https://github.com/microsoft/hlsl-specs/pull/325
1827 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(Val: ExprToVisit))
1828 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(
1829 CGF, E: cast<InitListExpr>(Val: ExprToVisit));
1830
1831 AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1832
1833 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1834
1835 // Handle initialization of an array.
1836 if (ExprToVisit->getType()->isConstantArrayType()) {
1837 auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1838 EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1839 Args: InitExprs, ArrayFiller);
1840 return;
1841 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1842 // A variable array type that has an initializer can only do empty
1843 // initialization. And because this feature is not exposed as an extension
1844 // in C++, we can safely memset the array memory to zero.
1845 assert(InitExprs.size() == 0 &&
1846 "you can only use an empty initializer with VLAs");
1847 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1848 return;
1849 }
1850
1851 assert(ExprToVisit->getType()->isRecordType() &&
1852 "Only support structs/unions here!");
1853
1854 // Do struct initialization; this code just sets each individual member
1855 // to the approprate value. This makes bitfield support automatic;
1856 // the disadvantage is that the generated code is more difficult for
1857 // the optimizer, especially with bitfields.
1858 unsigned NumInitElements = InitExprs.size();
1859 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1860
1861 // We'll need to enter cleanup scopes in case any of the element
1862 // initializers throws an exception.
1863 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1864
1865 unsigned curInitIndex = 0;
1866
1867 // Emit initialization of base classes.
1868 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1869 assert(NumInitElements >= CXXRD->getNumBases() &&
1870 "missing initializer for base class");
1871 for (auto &Base : CXXRD->bases()) {
1872 assert(!Base.isVirtual() && "should not see vbases here");
1873 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1874 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1875 Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1876 /*isBaseVirtual*/ BaseIsVirtual: false);
1877 AggValueSlot AggSlot = AggValueSlot::forAddr(
1878 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1879 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1880 mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1881 CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1882
1883 if (QualType::DestructionKind dtorKind =
1884 Base.getType().isDestructedType())
1885 CGF.pushDestroyAndDeferDeactivation(dtorKind, addr: V, type: Base.getType());
1886 }
1887 }
1888
1889 // Prepare a 'this' for CXXDefaultInitExprs.
1890 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1891
1892 const bool ZeroInitPadding =
1893 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1894
1895 if (record->isUnion()) {
1896 // Only initialize one field of a union. The field itself is
1897 // specified by the initializer list.
1898 if (!InitializedFieldInUnion) {
1899 // Empty union; we have nothing to do.
1900
1901#ifndef NDEBUG
1902 // Make sure that it's really an empty and not a failure of
1903 // semantic analysis.
1904 for (const auto *Field : record->fields())
1905 assert(
1906 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1907 "Only unnamed bitfields or anonymous class allowed");
1908#endif
1909 return;
1910 }
1911
1912 // FIXME: volatility
1913 FieldDecl *Field = InitializedFieldInUnion;
1914
1915 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1916 if (NumInitElements) {
1917 // Store the initializer into the field
1918 EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1919 if (ZeroInitPadding) {
1920 uint64_t TotalSize = CGF.getContext().toBits(
1921 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1922 uint64_t FieldSize = CGF.getContext().getTypeSize(T: FieldLoc.getType());
1923 DoZeroInitPadding(PaddingStart&: FieldSize, PaddingEnd: TotalSize, NextField: nullptr);
1924 }
1925 } else {
1926 // Default-initialize to null.
1927 if (ZeroInitPadding)
1928 EmitNullInitializationToLValue(lv: DestLV);
1929 else
1930 EmitNullInitializationToLValue(lv: FieldLoc);
1931 }
1932 return;
1933 }
1934
1935 // Here we iterate over the fields; this makes it simpler to both
1936 // default-initialize fields and skip over unnamed fields.
1937 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(D: record);
1938 uint64_t PaddingStart = 0;
1939
1940 for (const auto *field : record->fields()) {
1941 // We're done once we hit the flexible array member.
1942 if (field->getType()->isIncompleteArrayType())
1943 break;
1944
1945 // Always skip anonymous bitfields.
1946 if (field->isUnnamedBitField())
1947 continue;
1948
1949 // We're done if we reach the end of the explicit initializers, we
1950 // have a zeroed object, and the rest of the fields are
1951 // zero-initializable.
1952 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1953 CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1954 break;
1955
1956 if (ZeroInitPadding)
1957 DoZeroInitPadding(PaddingStart,
1958 PaddingEnd: Layout.getFieldOffset(FieldNo: field->getFieldIndex()), NextField: field);
1959
1960 LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1961 // We never generate write-barries for initialized fields.
1962 LV.setNonGC(true);
1963
1964 if (curInitIndex < NumInitElements) {
1965 // Store the initializer into the field.
1966 EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1967 } else {
1968 // We're out of initializers; default-initialize to null
1969 EmitNullInitializationToLValue(lv: LV);
1970 }
1971
1972 // Push a destructor if necessary.
1973 // FIXME: if we have an array of structures, all explicitly
1974 // initialized, we can end up pushing a linear number of cleanups.
1975 if (QualType::DestructionKind dtorKind =
1976 field->getType().isDestructedType()) {
1977 assert(LV.isSimple());
1978 if (dtorKind) {
1979 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1980 type: field->getType(),
1981 destroyer: CGF.getDestroyer(destructionKind: dtorKind), useEHCleanupForArray: false);
1982 }
1983 }
1984 }
1985 if (ZeroInitPadding) {
1986 uint64_t TotalSize = CGF.getContext().toBits(
1987 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1988 DoZeroInitPadding(PaddingStart, PaddingEnd: TotalSize, NextField: nullptr);
1989 }
1990}
1991
1992void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1993 uint64_t PaddingEnd,
1994 const FieldDecl *NextField) {
1995
1996 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1997 CharUnits Start = CGF.getContext().toCharUnitsFromBits(BitSize: StartBit);
1998 CharUnits End = CGF.getContext().toCharUnitsFromBits(BitSize: EndBit);
1999 Address Addr = Dest.getAddress().withElementType(ElemTy: CGF.CharTy);
2000 if (!Start.isZero())
2001 Addr = Builder.CreateConstGEP(Addr, Index: Start.getQuantity());
2002 llvm::Constant *SizeVal = Builder.getInt64(C: (End - Start).getQuantity());
2003 CGF.Builder.CreateMemSet(Dest: Addr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2004 };
2005
2006 if (NextField != nullptr && NextField->isBitField()) {
2007 // For bitfield, zero init StorageSize before storing the bits. So we don't
2008 // need to handle big/little endian.
2009 const CGRecordLayout &RL =
2010 CGF.getTypes().getCGRecordLayout(NextField->getParent());
2011 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: NextField);
2012 uint64_t StorageStart = CGF.getContext().toBits(CharSize: Info.StorageOffset);
2013 if (StorageStart + Info.StorageSize > PaddingStart) {
2014 if (StorageStart > PaddingStart)
2015 InitBytes(PaddingStart, StorageStart);
2016 Address Addr = Dest.getAddress();
2017 if (!Info.StorageOffset.isZero())
2018 Addr = Builder.CreateConstGEP(Addr: Addr.withElementType(ElemTy: CGF.CharTy),
2019 Index: Info.StorageOffset.getQuantity());
2020 Addr = Addr.withElementType(
2021 ElemTy: llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: Info.StorageSize));
2022 Builder.CreateStore(Val: Builder.getIntN(N: Info.StorageSize, C: 0), Addr);
2023 PaddingStart = StorageStart + Info.StorageSize;
2024 }
2025 return;
2026 }
2027
2028 if (PaddingStart < PaddingEnd)
2029 InitBytes(PaddingStart, PaddingEnd);
2030 if (NextField != nullptr)
2031 PaddingStart =
2032 PaddingEnd + CGF.getContext().getTypeSize(T: NextField->getType());
2033}
2034
2035void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
2036 llvm::Value *outerBegin) {
2037 // Emit the common subexpression.
2038 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
2039
2040 Address destPtr = EnsureSlot(T: E->getType()).getAddress();
2041 uint64_t numElements = E->getArraySize().getZExtValue();
2042
2043 if (!numElements)
2044 return;
2045
2046 // destPtr is an array*. Construct an elementType* by drilling down a level.
2047 llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
2048 llvm::Value *indices[] = {zero, zero};
2049 llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
2050 Ptr: destPtr.emitRawPointer(CGF),
2051 IdxList: indices, Name: "arrayinit.begin");
2052
2053 // Prepare to special-case multidimensional array initialization: we avoid
2054 // emitting multiple destructor loops in that case.
2055 if (!outerBegin)
2056 outerBegin = begin;
2057 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
2058
2059 QualType elementType =
2060 CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
2061 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
2062 CharUnits elementAlign =
2063 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
2064 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
2065
2066 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2067 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
2068
2069 // Jump into the body.
2070 CGF.EmitBlock(BB: bodyBB);
2071 llvm::PHINode *index =
2072 Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
2073 index->addIncoming(V: zero, BB: entryBB);
2074 llvm::Value *element =
2075 Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
2076
2077 if (CGF.CGM.shouldEmitConvergenceTokens())
2078 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
2079
2080 // Prepare for a cleanup.
2081 QualType::DestructionKind dtorKind = elementType.isDestructedType();
2082 EHScopeStack::stable_iterator cleanup;
2083 if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
2084 if (outerBegin->getType() != element->getType())
2085 outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
2086 CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
2087 elementAlignment: elementAlign,
2088 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
2089 cleanup = CGF.EHStack.stable_begin();
2090 } else {
2091 dtorKind = QualType::DK_none;
2092 }
2093
2094 // Emit the actual filler expression.
2095 {
2096 // Temporaries created in an array initialization loop are destroyed
2097 // at the end of each iteration.
2098 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2099 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2100 LValue elementLV = CGF.MakeAddrLValue(
2101 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
2102
2103 if (InnerLoop) {
2104 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2105 auto elementSlot = AggValueSlot::forLValue(
2106 LV: elementLV, isDestructed: AggValueSlot::IsDestructed,
2107 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2108 mayOverlap: AggValueSlot::DoesNotOverlap);
2109 AggExprEmitter(CGF, elementSlot, false)
2110 .VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
2111 } else
2112 EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
2113 }
2114
2115 // Move on to the next element.
2116 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2117 LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
2118 index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
2119
2120 // Leave the loop if we're done.
2121 llvm::Value *done = Builder.CreateICmpEQ(
2122 LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
2123 Name: "arrayinit.done");
2124 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
2125 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
2126
2127 if (CGF.CGM.shouldEmitConvergenceTokens())
2128 CGF.ConvergenceTokenStack.pop_back();
2129
2130 CGF.EmitBlock(BB: endBB);
2131
2132 // Leave the partial-array cleanup if we entered one.
2133 if (dtorKind)
2134 CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
2135}
2136
2137void AggExprEmitter::VisitDesignatedInitUpdateExpr(
2138 DesignatedInitUpdateExpr *E) {
2139 AggValueSlot Dest = EnsureSlot(T: E->getType());
2140
2141 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
2142 EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
2143 VisitInitListExpr(E: E->getUpdater());
2144}
2145
2146//===----------------------------------------------------------------------===//
2147// Entry Points into this File
2148//===----------------------------------------------------------------------===//
2149
2150/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2151/// non-zero bytes that will be stored when outputting the initializer for the
2152/// specified initializer expression.
2153static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
2154 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
2155 E = MTE->getSubExpr();
2156 E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
2157
2158 // 0 and 0.0 won't require any non-zero stores!
2159 if (isSimpleZero(E, CGF))
2160 return CharUnits::Zero();
2161
2162 // If this is an initlist expr, sum up the size of sizes of the (present)
2163 // elements. If this is something weird, assume the whole thing is non-zero.
2164 const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
2165 while (ILE && ILE->isTransparent())
2166 ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
2167 if (!ILE || !CGF.getTypes().isZeroInitializable(T: ILE->getType()))
2168 return CGF.getContext().getTypeSizeInChars(T: E->getType());
2169
2170 // InitListExprs for structs have to be handled carefully. If there are
2171 // reference members, we need to consider the size of the reference, not the
2172 // referencee. InitListExprs for unions and arrays can't have references.
2173 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2174 if (!RT->isUnionType()) {
2175 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2176 CharUnits NumNonZeroBytes = CharUnits::Zero();
2177
2178 unsigned ILEElement = 0;
2179 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
2180 while (ILEElement != CXXRD->getNumBases())
2181 NumNonZeroBytes +=
2182 GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
2183 for (const auto *Field : SD->fields()) {
2184 // We're done once we hit the flexible array member or run out of
2185 // InitListExpr elements.
2186 if (Field->getType()->isIncompleteArrayType() ||
2187 ILEElement == ILE->getNumInits())
2188 break;
2189 if (Field->isUnnamedBitField())
2190 continue;
2191
2192 const Expr *E = ILE->getInit(Init: ILEElement++);
2193
2194 // Reference values are always non-null and have the width of a pointer.
2195 if (Field->getType()->isReferenceType())
2196 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2197 BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2198 else
2199 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2200 }
2201
2202 return NumNonZeroBytes;
2203 }
2204 }
2205
2206 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2207 CharUnits NumNonZeroBytes = CharUnits::Zero();
2208 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2209 NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2210 return NumNonZeroBytes;
2211}
2212
2213/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2214/// zeros in it, emit a memset and avoid storing the individual zeros.
2215///
2216static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2217 CodeGenFunction &CGF) {
2218 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2219 // volatile stores.
2220 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2221 return;
2222
2223 // C++ objects with a user-declared constructor don't need zero'ing.
2224 if (CGF.getLangOpts().CPlusPlus)
2225 if (const RecordType *RT = CGF.getContext()
2226 .getBaseElementType(QT: E->getType())
2227 ->getAsCanonical<RecordType>()) {
2228 const auto *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2229 if (RD->hasUserDeclaredConstructor())
2230 return;
2231 }
2232
2233 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2234 CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2235 if (Size <= CharUnits::fromQuantity(Quantity: 16))
2236 return;
2237
2238 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2239 // we prefer to emit memset + individual stores for the rest.
2240 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2241 if (NumNonZeroBytes * 4 > Size)
2242 return;
2243
2244 // Okay, it seems like a good idea to use an initial memset, emit the call.
2245 llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2246
2247 Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2248 CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2249
2250 // Tell the AggExprEmitter that the slot is known zero.
2251 Slot.setZeroed();
2252}
2253
2254/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2255/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2256/// the value of the aggregate expression is not needed. If VolatileDest is
2257/// true, DestPtr cannot be 0.
2258void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2259 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2260 "Invalid aggregate expression to emit");
2261 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2262 "slot has bits but no address");
2263
2264 // Optimize the slot if possible.
2265 CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2266
2267 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr *>(E));
2268}
2269
2270LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2271 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2272 Address Temp = CreateMemTempWithoutCast(T: E->getType());
2273 LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2274 EmitAggExpr(E, Slot: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsNotDestructed,
2275 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2276 isAliased: AggValueSlot::IsNotAliased,
2277 mayOverlap: AggValueSlot::DoesNotOverlap));
2278 return LV;
2279}
2280
2281void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,
2282 const LValue &Src,
2283 ExprValueKind SrcKind) {
2284 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2285 .EmitFinalDestCopy(type: Type, src: Src, SrcValueKind: SrcKind);
2286}
2287
2288AggValueSlot::Overlap_t
2289CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2290 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2291 return AggValueSlot::DoesNotOverlap;
2292
2293 // Empty fields can overlap earlier fields.
2294 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2295 return AggValueSlot::MayOverlap;
2296
2297 // If the field lies entirely within the enclosing class's nvsize, its tail
2298 // padding cannot overlap any already-initialized object. (The only subobjects
2299 // with greater addresses that might already be initialized are vbases.)
2300 const RecordDecl *ClassRD = FD->getParent();
2301 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2302 if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2303 getContext().getTypeSize(T: FD->getType()) <=
2304 (uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2305 return AggValueSlot::DoesNotOverlap;
2306
2307 // The tail padding may contain values we need to preserve.
2308 return AggValueSlot::MayOverlap;
2309}
2310
2311AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2312 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2313 // If the most-derived object is a field declared with [[no_unique_address]],
2314 // the tail padding of any virtual base could be reused for other subobjects
2315 // of that field's class.
2316 if (IsVirtual)
2317 return AggValueSlot::MayOverlap;
2318
2319 // Empty bases can overlap earlier bases.
2320 if (BaseRD->isEmpty())
2321 return AggValueSlot::MayOverlap;
2322
2323 // If the base class is laid out entirely within the nvsize of the derived
2324 // class, its tail padding cannot yet be initialized, so we can issue
2325 // stores at the full width of the base class.
2326 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2327 if (Layout.getBaseClassOffset(Base: BaseRD) +
2328 getContext().getASTRecordLayout(D: BaseRD).getSize() <=
2329 Layout.getNonVirtualSize())
2330 return AggValueSlot::DoesNotOverlap;
2331
2332 // The tail padding may contain values we need to preserve.
2333 return AggValueSlot::MayOverlap;
2334}
2335
2336void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2337 AggValueSlot::Overlap_t MayOverlap,
2338 bool isVolatile) {
2339 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2340
2341 Address DestPtr = Dest.getAddress();
2342 Address SrcPtr = Src.getAddress();
2343
2344 if (getLangOpts().CPlusPlus) {
2345 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2346 assert((Record->hasTrivialCopyConstructor() ||
2347 Record->hasTrivialCopyAssignment() ||
2348 Record->hasTrivialMoveConstructor() ||
2349 Record->hasTrivialMoveAssignment() ||
2350 Record->hasAttr<TrivialABIAttr>() || Record->isUnion() ||
2351 // HLSL uses aggregate-copy for user-defined record types.
2352 (getLangOpts().HLSL && !Record->isHLSLBuiltinRecord())) &&
2353 "Trying to aggregate-copy a type without a trivial copy/move "
2354 "constructor or assignment operator");
2355 // Ignore empty classes in C++.
2356 if (Record->isEmpty())
2357 return;
2358 }
2359 }
2360
2361 if (getLangOpts().CUDAIsDevice) {
2362 if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2363 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2364 Src))
2365 return;
2366 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2367 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2368 Src))
2369 return;
2370 }
2371 }
2372
2373 assert(Ty.getAddressSpace() != LangAS::hlsl_constant &&
2374 "copies of aggregates in hlsl_constant address space should be "
2375 "handled earlier by the HLSL runtime");
2376
2377 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2378 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2379 // read from another object that overlaps in anyway the storage of the first
2380 // object, then the overlap shall be exact and the two objects shall have
2381 // qualified or unqualified versions of a compatible type."
2382 //
2383 // memcpy is not defined if the source and destination pointers are exactly
2384 // equal, but other compilers do this optimization, and almost every memcpy
2385 // implementation handles this case safely. If there is a libc that does not
2386 // safely handle this, we can add a target hook.
2387
2388 // Get data size info for this aggregate. Don't copy the tail padding if this
2389 // might be a potentially-overlapping subobject, since the tail padding might
2390 // be occupied by a different object. Otherwise, copying it is fine.
2391 TypeInfoChars TypeInfo;
2392 if (MayOverlap)
2393 TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2394 else
2395 TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2396
2397 llvm::Value *SizeVal = nullptr;
2398 if (TypeInfo.Width.isZero()) {
2399 // But note that getTypeInfo returns 0 for a VLA.
2400 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2401 Val: getContext().getAsArrayType(T: Ty))) {
2402 QualType BaseEltTy;
2403 SizeVal = emitArrayLength(arrayType: VAT, baseType&: BaseEltTy, addr&: DestPtr);
2404 TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2405 assert(!TypeInfo.Width.isZero());
2406 SizeVal = Builder.CreateNUWMul(
2407 LHS: SizeVal,
2408 RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2409 }
2410 }
2411 if (!SizeVal) {
2412 SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2413 }
2414
2415 // FIXME: If we have a volatile struct, the optimizer can remove what might
2416 // appear to be `extra' memory ops:
2417 //
2418 // volatile struct { int i; } a, b;
2419 //
2420 // int main() {
2421 // a = b;
2422 // a = b;
2423 // }
2424 //
2425 // we need to use a different call here. We use isVolatile to indicate when
2426 // either the source or the destination is volatile.
2427
2428 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2429 SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2430
2431 // Don't do any of the memmove_collectable tests if GC isn't set.
2432 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2433 // fall through
2434 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2435 if (Record->hasObjectMember()) {
2436 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2437 Size: SizeVal);
2438 return;
2439 }
2440 } else if (Ty->isArrayType()) {
2441 QualType BaseType = getContext().getBaseElementType(QT: Ty);
2442 if (const auto *Record = BaseType->getAsRecordDecl()) {
2443 if (Record->hasObjectMember()) {
2444 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2445 Size: SizeVal);
2446 return;
2447 }
2448 }
2449 }
2450
2451 auto *Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2452 addInstToCurrentSourceAtom(KeyInstruction: Inst, Backup: nullptr);
2453 emitPFPPostCopyUpdates(DestPtr, SrcPtr, Ty);
2454
2455 // Determine the metadata to describe the position of any padding in this
2456 // memcpy, as well as the TBAA tags for the members of the struct, in case
2457 // the optimizer wishes to expand it in to scalar memory operations.
2458 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2459 Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2460
2461 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2462 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2463 DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2464 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2465 }
2466}
2467