1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
23#include "clang/AST/ASTContext.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
26#include "clang/AST/DeclTemplate.h"
27#include "clang/AST/StmtVisitor.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34using namespace clang;
35using namespace CodeGen;
36
37//===----------------------------------------------------------------------===//
38// Aggregate Expression Emitter
39//===----------------------------------------------------------------------===//
40
41namespace {
42class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
43 CodeGenFunction &CGF;
44 CGBuilderTy &Builder;
45 AggValueSlot Dest;
46 bool IsResultUnused;
47
48 AggValueSlot EnsureSlot(QualType T) {
49 if (!Dest.isIgnored())
50 return Dest;
51 return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
52 }
53 void EnsureDest(QualType T) {
54 if (!Dest.isIgnored())
55 return;
56 Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
57 }
58
59 // Calls `Fn` with a valid return value slot, potentially creating a temporary
60 // to do so. If a temporary is created, an appropriate copy into `Dest` will
61 // be emitted, as will lifetime markers.
62 //
63 // The given function should take a ReturnValueSlot, and return an RValue that
64 // points to said slot.
65 void withReturnValueSlot(const Expr *E,
66 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
67
68 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
69 const FieldDecl *NextField);
70
71public:
72 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
73 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
74 IsResultUnused(IsResultUnused) {}
75
76 //===--------------------------------------------------------------------===//
77 // Utilities
78 //===--------------------------------------------------------------------===//
79
80 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
81 /// represents a value lvalue, this method emits the address of the lvalue,
82 /// then loads the result into DestPtr.
83 void EmitAggLoadOfLValue(const Expr *E);
84
85 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
86 /// SrcIsRValue is true if source comes from an RValue.
87 void EmitFinalDestCopy(QualType type, const LValue &src,
88 CodeGenFunction::ExprValueKind SrcValueKind =
89 CodeGenFunction::EVK_NonRValue);
90 void EmitFinalDestCopy(QualType type, RValue src);
91 void EmitCopy(QualType type, const AggValueSlot &dest,
92 const AggValueSlot &src);
93
94 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
95 Expr *ExprToVisit, ArrayRef<Expr *> Args,
96 Expr *ArrayFiller);
97
98 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
99 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
100 return AggValueSlot::NeedsGCBarriers;
101 return AggValueSlot::DoesNotNeedGCBarriers;
102 }
103
104 bool TypeRequiresGCollection(QualType T);
105
106 //===--------------------------------------------------------------------===//
107 // Visitor Methods
108 //===--------------------------------------------------------------------===//
109
110 void Visit(Expr *E) {
111 ApplyDebugLocation DL(CGF, E);
112 StmtVisitor<AggExprEmitter>::Visit(S: E);
113 }
114
115 void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, Type: "aggregate expression"); }
116 void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
117 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
118 Visit(E: GE->getResultExpr());
119 }
120 void VisitCoawaitExpr(CoawaitExpr *E) {
121 CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
122 }
123 void VisitCoyieldExpr(CoyieldExpr *E) {
124 CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
125 }
126 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
127 void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
128 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
129 return Visit(E: E->getReplacement());
130 }
131
132 void VisitConstantExpr(ConstantExpr *E) {
133 EnsureDest(T: E->getType());
134
135 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
136 CGF.CreateCoercedStore(
137 Src: Result, SrcFETy: E->getType(), Dst: Dest.getAddress(),
138 DstSize: llvm::TypeSize::getFixed(
139 ExactSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType())
140 .getQuantity()),
141 DstIsVolatile: E->getType().isVolatileQualified());
142 return;
143 }
144 return Visit(E: E->getSubExpr());
145 }
146
147 // l-values.
148 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
149 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(E: ME); }
150 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
151 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
152 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
153 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
154 EmitAggLoadOfLValue(E);
155 }
156 void VisitPredefinedExpr(const PredefinedExpr *E) { EmitAggLoadOfLValue(E); }
157
158 // Operators.
159 void VisitCastExpr(CastExpr *E);
160 void VisitCallExpr(const CallExpr *E);
161 void VisitStmtExpr(const StmtExpr *E);
162 void VisitBinaryOperator(const BinaryOperator *BO);
163 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
164 void VisitBinAssign(const BinaryOperator *E);
165 void VisitBinComma(const BinaryOperator *E);
166 void VisitBinCmp(const BinaryOperator *E);
167 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
168 Visit(E: E->getSemanticForm());
169 }
170
171 void VisitObjCMessageExpr(ObjCMessageExpr *E);
172 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { EmitAggLoadOfLValue(E); }
173
174 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
175 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
176 void VisitChooseExpr(const ChooseExpr *CE);
177 void VisitInitListExpr(InitListExpr *E);
178 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
179 FieldDecl *InitializedFieldInUnion,
180 Expr *ArrayFiller);
181 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
182 llvm::Value *outerBegin = nullptr);
183 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
184 void VisitNoInitExpr(NoInitExpr *E) {} // Do nothing.
185 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
186 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
187 Visit(E: DAE->getExpr());
188 }
189 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
190 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
191 Visit(E: DIE->getExpr());
192 }
193 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
194 void VisitCXXConstructExpr(const CXXConstructExpr *E);
195 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
196 void VisitLambdaExpr(LambdaExpr *E);
197 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
198 void VisitExprWithCleanups(ExprWithCleanups *E);
199 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
200 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
201 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
202 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
203
204 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
205 if (E->isGLValue()) {
206 LValue LV = CGF.EmitPseudoObjectLValue(e: E);
207 return EmitFinalDestCopy(type: E->getType(), src: LV);
208 }
209
210 AggValueSlot Slot = EnsureSlot(T: E->getType());
211 bool NeedsDestruction =
212 !Slot.isExternallyDestructed() &&
213 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
214 if (NeedsDestruction)
215 Slot.setExternallyDestructed();
216 CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
217 if (NeedsDestruction)
218 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Slot.getAddress(),
219 type: E->getType());
220 }
221
222 void VisitVAArgExpr(VAArgExpr *E);
223 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
224 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
225 Expr *ArrayFiller);
226
227 void EmitInitializationToLValue(Expr *E, LValue Address);
228 void EmitNullInitializationToLValue(LValue Address);
229 // case Expr::ChooseExprClass:
230 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
231 void VisitAtomicExpr(AtomicExpr *E) {
232 RValue Res = CGF.EmitAtomicExpr(E);
233 EmitFinalDestCopy(type: E->getType(), src: Res);
234 }
235 void VisitPackIndexingExpr(PackIndexingExpr *E) {
236 Visit(E: E->getSelectedExpr());
237 }
238};
239} // end anonymous namespace.
240
241//===----------------------------------------------------------------------===//
242// Utilities
243//===----------------------------------------------------------------------===//
244
245/// EmitAggLoadOfLValue - Given an expression with aggregate type that
246/// represents a value lvalue, this method emits the address of the lvalue,
247/// then loads the result into DestPtr.
248void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
249 LValue LV = CGF.EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load);
250
251 // If the type of the l-value is atomic, then do an atomic load.
252 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
253 CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
254 return;
255 }
256
257 if (E->getType().getAddressSpace() == LangAS::hlsl_constant)
258 if (CGF.CGM.getHLSLRuntime().emitBufferCopy(CGF, E, SrcLV: LV, DestSlot&: Dest))
259 return;
260
261 EmitFinalDestCopy(type: E->getType(), src: LV);
262}
263
264/// True if the given aggregate type requires special GC API calls.
265bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
266 // Only record types have members that might require garbage collection.
267 const auto *Record = T->getAsRecordDecl();
268 if (!Record)
269 return false;
270
271 // Don't mess with non-trivial C++ types.
272 if (isa<CXXRecordDecl>(Val: Record) &&
273 (cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
274 !cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
275 return false;
276
277 // Check whether the type has an object member.
278 return Record->hasObjectMember();
279}
280
281void AggExprEmitter::withReturnValueSlot(
282 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
283 QualType RetTy = E->getType();
284 bool RequiresDestruction =
285 !Dest.isExternallyDestructed() &&
286 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
287
288 // If it makes no observable difference, save a memcpy + temporary.
289 //
290 // We need to always provide our own temporary if destruction is required.
291 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
292 // its lifetime before we have the chance to emit a proper destructor call.
293 //
294 // We also need a temporary if the destination is in a different address space
295 // from the sret AS. Use the target hook to get the actual sret AS for this
296 // return type.
297 const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl();
298 LangAS SRetLangAS = CGF.CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
299 unsigned SRetAS = CGF.getContext().getTargetAddressSpace(AS: SRetLangAS);
300 bool CanAggregateCopy =
301 RD ? (RD->hasTrivialCopyConstructor() ||
302 RD->hasTrivialMoveConstructor() || RD->hasTrivialCopyAssignment() ||
303 RD->hasTrivialMoveAssignment() || RD->hasAttr<TrivialABIAttr>() ||
304 RD->isUnion())
305 : RetTy.isTriviallyCopyableType(Context: CGF.getContext());
306 bool DestASMismatch = !Dest.isIgnored() && CanAggregateCopy &&
307 Dest.getAddress()
308 .getBasePointer()
309 ->stripPointerCasts()
310 ->getType()
311 ->getPointerAddressSpace() != SRetAS;
312 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
313 (RequiresDestruction && Dest.isIgnored()) || DestASMismatch;
314
315 Address RetAddr = Address::invalid();
316
317 EHScopeStack::stable_iterator LifetimeEndBlock;
318 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
319 if (!UseTemp) {
320 RetAddr = Dest.getAddress();
321 if (RetAddr.isValid() && RetAddr.getAddressSpace() != SRetAS) {
322 llvm::Type *SRetPtrTy =
323 llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: SRetAS);
324 RetAddr = RetAddr.withPointer(
325 NewPointer: CGF.performAddrSpaceCast(Src: RetAddr.getBasePointer(), DestTy: SRetPtrTy),
326 IsKnownNonNull: RetAddr.isKnownNonNull());
327 }
328 } else {
329 RetAddr = CGF.CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
330 if (CGF.EmitLifetimeStart(Addr: RetAddr.getBasePointer())) {
331 LifetimeStartInst =
332 cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
333 assert(LifetimeStartInst->getIntrinsicID() ==
334 llvm::Intrinsic::lifetime_start &&
335 "Last insertion wasn't a lifetime.start?");
336
337 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
338 kind: NormalEHLifetimeMarker, A: RetAddr);
339 LifetimeEndBlock = CGF.EHStack.stable_begin();
340 }
341 }
342
343 RValue Src =
344 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
345 Dest.isExternallyDestructed()));
346
347 if (!UseTemp)
348 return;
349
350 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
351 Src.getAggregatePointer(E->getType(), CGF));
352 EmitFinalDestCopy(type: E->getType(), src: Src);
353
354 if (!RequiresDestruction && LifetimeStartInst) {
355 // If there's no dtor to run, the copy was the last use of our temporary.
356 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
357 // eagerly.
358 CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
359 CGF.EmitLifetimeEnd(Addr: RetAddr.getBasePointer());
360 }
361}
362
363/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
364void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
365 assert(src.isAggregate() && "value must be aggregate value!");
366 LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
367 EmitFinalDestCopy(type, src: srcLV, SrcValueKind: CodeGenFunction::EVK_RValue);
368}
369
370/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
371void AggExprEmitter::EmitFinalDestCopy(
372 QualType type, const LValue &src,
373 CodeGenFunction::ExprValueKind SrcValueKind) {
374 // If Dest is ignored, then we're evaluating an aggregate expression
375 // in a context that doesn't care about the result. Note that loads
376 // from volatile l-values force the existence of a non-ignored
377 // destination.
378 if (Dest.isIgnored())
379 return;
380
381 // Copy non-trivial C structs here.
382 LValue DstLV = CGF.MakeAddrLValue(
383 Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
384
385 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
386 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
387 if (Dest.isPotentiallyAliased())
388 CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
389 else
390 CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
391 return;
392 }
393 } else {
394 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
395 if (Dest.isPotentiallyAliased())
396 CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
397 else
398 CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
399 return;
400 }
401 }
402
403 AggValueSlot srcAgg = AggValueSlot::forLValue(
404 LV: src, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type), isAliased: AggValueSlot::IsAliased,
405 mayOverlap: AggValueSlot::MayOverlap);
406 EmitCopy(type, dest: Dest, src: srcAgg);
407}
408
409/// Perform a copy from the source into the destination.
410///
411/// \param type - the type of the aggregate being copied; qualifiers are
412/// ignored
413void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
414 const AggValueSlot &src) {
415 if (dest.requiresGCollection()) {
416 CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
417 llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
418 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr: dest.getAddress(),
419 SrcPtr: src.getAddress(), Size: size);
420 return;
421 }
422
423 // If the result of the assignment is used, copy the LHS there also.
424 // It's volatile if either side is. Use the minimum alignment of
425 // the two sides.
426 LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
427 LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
428 CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
429 isVolatile: dest.isVolatile() || src.isVolatile());
430}
431
432/// Emit the initializer for a std::initializer_list initialized with a
433/// real initializer list.
434void AggExprEmitter::VisitCXXStdInitializerListExpr(
435 CXXStdInitializerListExpr *E) {
436 // Emit an array containing the elements. The array is externally destructed
437 // if the std::initializer_list object is.
438 ASTContext &Ctx = CGF.getContext();
439 LValue Array = CGF.EmitLValue(E: E->getSubExpr());
440 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
441 Address ArrayPtr = Array.getAddress();
442
443 const ConstantArrayType *ArrayType =
444 Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
445 assert(ArrayType && "std::initializer_list constructed from non-array");
446
447 auto *Record = E->getType()->castAsRecordDecl();
448 RecordDecl::field_iterator Field = Record->field_begin();
449 assert(Field != Record->field_end() &&
450 Ctx.hasSameType(Field->getType()->getPointeeType(),
451 ArrayType->getElementType()) &&
452 "Expected std::initializer_list first field to be const E *");
453
454 // Start pointer.
455 AggValueSlot Dest = EnsureSlot(T: E->getType());
456 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
457 LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
458 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
459 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
460 ++Field;
461 assert(Field != Record->field_end() &&
462 "Expected std::initializer_list to have two fields");
463
464 llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
465 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
466 if (Ctx.hasSameType(T1: Field->getType(), T2: Ctx.getSizeType())) {
467 // Length.
468 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
469
470 } else {
471 // End pointer.
472 assert(Field->getType()->isPointerType() &&
473 Ctx.hasSameType(Field->getType()->getPointeeType(),
474 ArrayType->getElementType()) &&
475 "Expected std::initializer_list second field to be const E *");
476 llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
477 llvm::Value *IdxEnd[] = {Zero, Size};
478 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
479 Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
480 Name: "arrayend");
481 CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
482 }
483
484 assert(++Field == Record->field_end() &&
485 "Expected std::initializer_list to only have two fields");
486}
487
488/// Determine if E is a trivial array filler, that is, one that is
489/// equivalent to zero-initialization.
490static bool isTrivialFiller(Expr *E) {
491 if (!E)
492 return true;
493
494 if (isa<ImplicitValueInitExpr>(Val: E))
495 return true;
496
497 if (auto *ILE = dyn_cast<InitListExpr>(Val: E)) {
498 if (ILE->getNumInits())
499 return false;
500 return isTrivialFiller(E: ILE->getArrayFiller());
501 }
502
503 if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(Val: E))
504 return Cons->getConstructor()->isDefaultConstructor() &&
505 Cons->getConstructor()->isTrivial();
506
507 // FIXME: Are there other cases where we can avoid emitting an initializer?
508 return false;
509}
510
511// emit an elementwise cast where the RHS is a scalar or vector
512// or emit an aggregate splat cast
513static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,
514 LValue DestVal,
515 llvm::Value *SrcVal,
516 QualType SrcTy,
517 SourceLocation Loc) {
518 // Flatten our destination
519 SmallVector<LValue, 16> StoreList;
520 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
521
522 bool isVector = false;
523 if (auto *VT = SrcTy->getAs<VectorType>()) {
524 isVector = true;
525 SrcTy = VT->getElementType();
526 assert(StoreList.size() <= VT->getNumElements() &&
527 "Cannot perform HLSL flat cast when vector source \
528 object has less elements than flattened destination \
529 object.");
530 }
531
532 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
533 LValue DestLVal = StoreList[I];
534 llvm::Value *Load =
535 isVector ? CGF.Builder.CreateExtractElement(Vec: SrcVal, Idx: I, Name: "vec.load")
536 : SrcVal;
537 llvm::Value *Cast =
538 CGF.EmitScalarConversion(Src: Load, SrcTy, DstTy: DestLVal.getType(), Loc);
539 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
540 }
541}
542
543// emit a flat cast where the RHS is an aggregate
544static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
545 LValue SrcVal, SourceLocation Loc) {
546 // Flatten our destination
547 SmallVector<LValue, 16> StoreList;
548 CGF.FlattenAccessAndTypeLValue(LVal: DestVal, AccessList&: StoreList);
549 // Flatten our src
550 SmallVector<LValue, 16> LoadList;
551 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
552
553 assert(StoreList.size() <= LoadList.size() &&
554 "Cannot perform HLSL elementwise cast when flattened source object \
555 has less elements than flattened destination object.");
556 // apply casts to what we load from LoadList
557 // and store result in Dest
558 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
559 LValue DestLVal = StoreList[I];
560 LValue SrcLVal = LoadList[I];
561 RValue RVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc);
562 assert(RVal.isScalar() && "All flattened source values should be scalars");
563 llvm::Value *Val = RVal.getScalarVal();
564 llvm::Value *Cast = CGF.EmitScalarConversion(Src: Val, SrcTy: SrcLVal.getType(),
565 DstTy: DestLVal.getType(), Loc);
566 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Cast), Dst: DestLVal);
567 }
568}
569
570/// Emit initialization of an array from an initializer list. ExprToVisit must
571/// be either an InitListEpxr a CXXParenInitListExpr.
572void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
573 QualType ArrayQTy, Expr *ExprToVisit,
574 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
575 uint64_t NumInitElements = Args.size();
576
577 uint64_t NumArrayElements = AType->getNumElements();
578 for (const auto *Init : Args) {
579 if (const auto *Embed = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
580 NumInitElements += Embed->getDataElementCount() - 1;
581 if (NumInitElements > NumArrayElements) {
582 NumInitElements = NumArrayElements;
583 break;
584 }
585 }
586 }
587
588 assert(NumInitElements <= NumArrayElements);
589
590 QualType elementType =
591 CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
592 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
593 CharUnits elementAlign =
594 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
595 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
596
597 // Consider initializing the array by copying from a global. For this to be
598 // more efficient than per-element initialization, the size of the elements
599 // with explicit initializers should be large enough.
600 if (NumInitElements * elementSize.getQuantity() > 16 &&
601 elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
602 CodeGen::CodeGenModule &CGM = CGF.CGM;
603 ConstantEmitter Emitter(CGF);
604 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
605 T: CGM.getContext().removeAddrSpaceQualType(T: ArrayQTy),
606 AddressSpace: CGM.GetGlobalConstantAddressSpace());
607 LangAS AS = GVArrayQTy.getAddressSpace();
608 if (llvm::Constant *C =
609 Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: GVArrayQTy)) {
610 auto GV = new llvm::GlobalVariable(
611 CGM.getModule(), C->getType(),
612 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
613 "constinit",
614 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
615 CGM.getContext().getTargetAddressSpace(AS));
616 Emitter.finalize(global: GV);
617 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: GVArrayQTy);
618 GV->setAlignment(Align.getAsAlign());
619 Address GVAddr(GV, GV->getValueType(), Align);
620 EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: GVArrayQTy));
621 return;
622 }
623 }
624
625 // Exception safety requires us to destroy all the
626 // already-constructed members if an initializer throws.
627 // For that, we'll need an EH cleanup.
628 QualType::DestructionKind dtorKind = elementType.isDestructedType();
629 Address endOfInit = Address::invalid();
630 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
631
632 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
633 if (dtorKind) {
634 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
635 // In principle we could tell the cleanup where we are more
636 // directly, but the control flow can get so varied here that it
637 // would actually be quite complex. Therefore we go through an
638 // alloca.
639 llvm::Instruction *dominatingIP =
640 Builder.CreateFlagLoad(Addr: llvm::ConstantInt::getNullValue(Ty: CGF.Int8PtrTy));
641 endOfInit = CGF.CreateTempAlloca(Ty: begin->getType(), align: CGF.getPointerAlign(),
642 Name: "arrayinit.endOfInit");
643 Builder.CreateStore(Val: begin, Addr: endOfInit);
644 CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
645 elementAlignment: elementAlign,
646 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
647 cast<EHCleanupScope>(Val&: *CGF.EHStack.find(sp: CGF.EHStack.stable_begin()))
648 .AddAuxAllocas(Allocas: allocaTracker.Take());
649
650 CGF.DeferredDeactivationCleanupStack.push_back(
651 Elt: {.Cleanup: CGF.EHStack.stable_begin(), .DominatingIP: dominatingIP});
652 }
653
654 llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
655
656 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
657 llvm::Value *element = begin;
658 if (ArrayIndex > 0) {
659 if (CGF.getLangOpts().EmitLogicalPointer)
660 element = Builder.CreateStructuredGEP(
661 BaseType: AType, PtrBase: begin, Indices: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
662 Name: "arrayinit.element");
663 else
664 element = Builder.CreateInBoundsGEP(
665 Ty: llvmElementType, Ptr: begin,
666 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: ArrayIndex),
667 Name: "arrayinit.element");
668
669 // Tell the cleanup that it needs to destroy up to this
670 // element. TODO: some of these stores can be trivially
671 // observed to be unnecessary.
672 if (endOfInit.isValid())
673 Builder.CreateStore(Val: element, Addr: endOfInit);
674 }
675
676 LValue elementLV = CGF.MakeAddrLValue(
677 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
678 EmitInitializationToLValue(E: Init, Address: elementLV);
679 return true;
680 };
681
682 unsigned ArrayIndex = 0;
683 // Emit the explicit initializers.
684 for (uint64_t i = 0; i != NumInitElements; ++i) {
685 if (ArrayIndex >= NumInitElements)
686 break;
687 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Args[i]->IgnoreParenImpCasts())) {
688 EmbedS->doForEachDataElement(C&: Emit, StartingIndexInArray&: ArrayIndex);
689 } else {
690 Emit(Args[i], ArrayIndex);
691 ArrayIndex++;
692 }
693 }
694
695 // Check whether there's a non-trivial array-fill expression.
696 bool hasTrivialFiller = isTrivialFiller(E: ArrayFiller);
697
698 // Any remaining elements need to be zero-initialized, possibly
699 // using the filler expression. We can skip this if the we're
700 // emitting to zeroed memory.
701 if (NumInitElements != NumArrayElements &&
702 !(Dest.isZeroed() && hasTrivialFiller &&
703 CGF.getTypes().isZeroInitializable(T: elementType))) {
704
705 // Use an actual loop. This is basically
706 // do { *array++ = filler; } while (array != end);
707
708 // Advance to the start of the rest of the array.
709 llvm::Value *element = begin;
710 if (NumInitElements) {
711 element = Builder.CreateInBoundsGEP(
712 Ty: llvmElementType, Ptr: element,
713 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumInitElements),
714 Name: "arrayinit.start");
715 if (endOfInit.isValid())
716 Builder.CreateStore(Val: element, Addr: endOfInit);
717 }
718
719 // Compute the end of the array.
720 llvm::Value *end = Builder.CreateInBoundsGEP(
721 Ty: llvmElementType, Ptr: begin,
722 IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
723
724 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
725 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
726
727 // Jump into the body.
728 CGF.EmitBlock(BB: bodyBB);
729 llvm::PHINode *currentElement =
730 Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
731 currentElement->addIncoming(V: element, BB: entryBB);
732
733 if (CGF.CGM.shouldEmitConvergenceTokens())
734 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
735
736 // Emit the actual filler expression.
737 {
738 // C++1z [class.temporary]p5:
739 // when a default constructor is called to initialize an element of
740 // an array with no corresponding initializer [...] the destruction of
741 // every temporary created in a default argument is sequenced before
742 // the construction of the next array element, if any
743 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
744 LValue elementLV = CGF.MakeAddrLValue(
745 Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
746 if (ArrayFiller)
747 EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
748 else
749 EmitNullInitializationToLValue(Address: elementLV);
750 }
751
752 // Move on to the next element.
753 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
754 Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
755
756 // Tell the EH cleanup that we finished with the last element.
757 if (endOfInit.isValid())
758 Builder.CreateStore(Val: nextElement, Addr: endOfInit);
759
760 // Leave the loop if we're done.
761 llvm::Value *done =
762 Builder.CreateICmpEQ(LHS: nextElement, RHS: end, Name: "arrayinit.done");
763 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
764 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
765 currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
766
767 if (CGF.CGM.shouldEmitConvergenceTokens())
768 CGF.ConvergenceTokenStack.pop_back();
769
770 CGF.EmitBlock(BB: endBB);
771 }
772}
773
774//===----------------------------------------------------------------------===//
775// Visitor Methods
776//===----------------------------------------------------------------------===//
777
778void AggExprEmitter::VisitMaterializeTemporaryExpr(
779 MaterializeTemporaryExpr *E) {
780 Visit(E: E->getSubExpr());
781}
782
783void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
784 // If this is a unique OVE, just visit its source expression.
785 if (e->isUnique())
786 Visit(E: e->getSourceExpr());
787 else
788 EmitFinalDestCopy(type: e->getType(), src: CGF.getOrCreateOpaqueLValueMapping(e));
789}
790
791void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
792 if (Dest.isPotentiallyAliased()) {
793 // Just emit a load of the lvalue + a copy, because our compound literal
794 // might alias the destination.
795 EmitAggLoadOfLValue(E);
796 return;
797 }
798
799 AggValueSlot Slot = EnsureSlot(T: E->getType());
800
801 // Block-scope compound literals are destroyed at the end of the enclosing
802 // scope in C.
803 bool Destruct =
804 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
805 if (Destruct)
806 Slot.setExternallyDestructed();
807
808 CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
809
810 if (Destruct)
811 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
812 CGF.pushLifetimeExtendedDestroy(
813 kind: CGF.getCleanupKind(kind: DtorKind), addr: Slot.getAddress(), type: E->getType(),
814 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: DtorKind & EHCleanup);
815}
816
817/// Attempt to look through various unimportant expressions to find a
818/// cast of the given kind.
819static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
820 op = op->IgnoreParenNoopCasts(Ctx: ctx);
821 if (auto castE = dyn_cast<CastExpr>(Val: op)) {
822 if (castE->getCastKind() == kind)
823 return castE->getSubExpr();
824 }
825 return nullptr;
826}
827
828void AggExprEmitter::VisitCastExpr(CastExpr *E) {
829 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
830 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
831 switch (E->getCastKind()) {
832 case CK_Dynamic: {
833 // FIXME: Can this actually happen? We have no test coverage for it.
834 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
835 LValue LV =
836 CGF.EmitCheckedLValue(E: E->getSubExpr(), TCK: CodeGenFunction::TCK_Load);
837 // FIXME: Do we also need to handle property references here?
838 if (LV.isSimple())
839 CGF.EmitDynamicCast(V: LV.getAddress(), DCE: cast<CXXDynamicCastExpr>(Val: E));
840 else
841 CGF.CGM.ErrorUnsupported(S: E, Type: "non-simple lvalue dynamic_cast");
842
843 if (!Dest.isIgnored())
844 CGF.CGM.ErrorUnsupported(S: E, Type: "lvalue dynamic_cast with a destination");
845 break;
846 }
847
848 case CK_ToUnion: {
849 // Evaluate even if the destination is ignored.
850 if (Dest.isIgnored()) {
851 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
852 /*ignoreResult=*/true);
853 break;
854 }
855
856 // GCC union extension
857 QualType Ty = E->getSubExpr()->getType();
858 Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
859 EmitInitializationToLValue(E: E->getSubExpr(),
860 Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
861 break;
862 }
863
864 case CK_LValueToRValueBitCast: {
865 if (Dest.isIgnored()) {
866 CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
867 /*ignoreResult=*/true);
868 break;
869 }
870
871 LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
872 Address SourceAddress = SourceLV.getAddress().withElementType(ElemTy: CGF.Int8Ty);
873 Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
874 llvm::Value *SizeVal = llvm::ConstantInt::get(
875 Ty: CGF.SizeTy,
876 V: CGF.getContext().getTypeSizeInChars(T: E->getType()).getQuantity());
877 Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
878 break;
879 }
880
881 case CK_DerivedToBase: {
882 assert(CGF.getLangOpts().HLSL &&
883 "Derived/Base casts in EmitAggExpr are only supported in HLSL");
884
885 // Create a temporary for the derived record, switch it out with the current
886 // Dest slot, and emit the derived value.
887 QualType DerivedTy = E->getSubExpr()->getType();
888 RawAddress DerivedAddr = CGF.CreateMemTempWithoutCast(T: DerivedTy);
889 AggValueSlot DerivedTmpSlot = AggValueSlot::forAddr(
890 addr: DerivedAddr, quals: DerivedTy.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
891 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
892 mayOverlap: AggValueSlot::DoesNotOverlap);
893
894 AggValueSlot DestBaseSlot = Dest;
895 Dest = DerivedTmpSlot;
896
897 Visit(E: E->getSubExpr());
898
899 // Perform derived-to-base address conversion to get the address
900 // of the base record within the derived record. In HLSL this should
901 // always be same as the derived because of single inheritance, but let's
902 // do it properly.
903 Address BaseAddrInDerived = CGF.GetAddressOfBaseClass(
904 Value: DerivedTmpSlot.getAddress(), Derived: DerivedTy->castAsCXXRecordDecl(),
905 PathBegin: E->path_begin(), PathEnd: E->path_end(),
906 /*NullCheckValue=*/false, Loc: E->getExprLoc());
907
908 AggValueSlot SrcBaseSlot = AggValueSlot::forAddr(
909 addr: BaseAddrInDerived, quals: E->getType().getQualifiers(),
910 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
911 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
912
913 // Copy the base class to the original destination slot and restore it.
914 EmitCopy(type: E->getType(), dest: DestBaseSlot, src: SrcBaseSlot);
915 Dest = DestBaseSlot;
916 break;
917 }
918
919 case CK_BaseToDerived:
920 case CK_UncheckedDerivedToBase: {
921 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
922 "should have been unpacked before we got here");
923 }
924
925 case CK_NonAtomicToAtomic:
926 case CK_AtomicToNonAtomic: {
927 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
928
929 // Determine the atomic and value types.
930 QualType atomicType = E->getSubExpr()->getType();
931 QualType valueType = E->getType();
932 if (isToAtomic)
933 std::swap(a&: atomicType, b&: valueType);
934
935 assert(atomicType->isAtomicType());
936 assert(CGF.getContext().hasSameUnqualifiedType(
937 valueType, atomicType->castAs<AtomicType>()->getValueType()));
938
939 // Just recurse normally if we're ignoring the result or the
940 // atomic type doesn't change representation.
941 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
942 return Visit(E: E->getSubExpr());
943 }
944
945 CastKind peepholeTarget =
946 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
947
948 // These two cases are reverses of each other; try to peephole them.
949 if (Expr *op =
950 findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
951 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
952 E->getType()) &&
953 "peephole significantly changed types?");
954 return Visit(E: op);
955 }
956
957 // If we're converting an r-value of non-atomic type to an r-value
958 // of atomic type, just emit directly into the relevant sub-object.
959 if (isToAtomic) {
960 AggValueSlot valueDest = Dest;
961 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
962 // Zero-initialize. (Strictly speaking, we only need to initialize
963 // the padding at the end, but this is simpler.)
964 if (!Dest.isZeroed())
965 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
966
967 // Build a GEP to refer to the subobject.
968 Address valueAddr =
969 CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
970 valueDest = AggValueSlot::forAddr(
971 addr: valueAddr, quals: valueDest.getQualifiers(),
972 isDestructed: valueDest.isExternallyDestructed(), needsGC: valueDest.requiresGCollection(),
973 isAliased: valueDest.isPotentiallyAliased(), mayOverlap: AggValueSlot::DoesNotOverlap,
974 isZeroed: AggValueSlot::IsZeroed);
975 }
976
977 CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
978 return;
979 }
980
981 // Otherwise, we're converting an atomic type to a non-atomic type.
982 // Make an atomic temporary, emit into that, and then copy the value out.
983 AggValueSlot atomicSlot =
984 CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
985 CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
986
987 Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
988 RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
989 return EmitFinalDestCopy(type: valueType, src: rvalue);
990 }
991 case CK_AddressSpaceConversion:
992 return Visit(E: E->getSubExpr());
993
994 case CK_LValueToRValue:
995 // If we're loading from a volatile type, force the destination
996 // into existence.
997 if (E->getSubExpr()->getType().isVolatileQualified()) {
998 bool Destruct =
999 !Dest.isExternallyDestructed() &&
1000 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1001 if (Destruct)
1002 Dest.setExternallyDestructed();
1003 EnsureDest(T: E->getType());
1004 Visit(E: E->getSubExpr());
1005
1006 if (Destruct)
1007 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1008 type: E->getType());
1009
1010 return;
1011 }
1012
1013 [[fallthrough]];
1014
1015 case CK_HLSLArrayRValue:
1016 if (CGF.getLangOpts().HLSL &&
1017 E->getSubExpr()->getType()->isHLSLResourceRecordArray())
1018 if (CGF.CGM.getHLSLRuntime().emitGlobalResourceArray(CGF, E, DestSlot&: Dest))
1019 break;
1020 Visit(E: E->getSubExpr());
1021 break;
1022 case CK_HLSLAggregateSplatCast: {
1023 Expr *Src = E->getSubExpr();
1024 QualType SrcTy = Src->getType();
1025 RValue RV = CGF.EmitAnyExpr(E: Src);
1026 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1027 SourceLocation Loc = E->getExprLoc();
1028
1029 assert(RV.isScalar() && SrcTy->isScalarType() &&
1030 "RHS of HLSL splat cast must be a scalar.");
1031 llvm::Value *SrcVal = RV.getScalarVal();
1032 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1033 break;
1034 }
1035 case CK_HLSLElementwiseCast: {
1036 Expr *Src = E->getSubExpr();
1037 QualType SrcTy = Src->getType();
1038 RValue RV = CGF.EmitAnyExpr(E: Src);
1039 LValue DestLVal = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1040 SourceLocation Loc = E->getExprLoc();
1041
1042 if (RV.isScalar()) {
1043 llvm::Value *SrcVal = RV.getScalarVal();
1044 assert(SrcTy->isVectorType() &&
1045 "HLSL Elementwise cast doesn't handle splatting.");
1046 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestVal: DestLVal, SrcVal, SrcTy, Loc);
1047 } else {
1048 assert(RV.isAggregate() &&
1049 "Can't perform HLSL Aggregate cast on a complex type.");
1050 Address SrcVal = RV.getAggregateAddress();
1051 EmitHLSLElementwiseCast(CGF, DestVal: DestLVal, SrcVal: CGF.MakeAddrLValue(Addr: SrcVal, T: SrcTy),
1052 Loc);
1053 }
1054 break;
1055 }
1056 case CK_NoOp:
1057 case CK_UserDefinedConversion:
1058 case CK_ConstructorConversion:
1059 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
1060 E->getType()) &&
1061 "Implicit cast types must be compatible");
1062 Visit(E: E->getSubExpr());
1063 break;
1064
1065 case CK_LValueBitCast:
1066 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
1067
1068 case CK_Dependent:
1069 case CK_BitCast:
1070 case CK_ArrayToPointerDecay:
1071 case CK_FunctionToPointerDecay:
1072 case CK_NullToPointer:
1073 case CK_NullToMemberPointer:
1074 case CK_BaseToDerivedMemberPointer:
1075 case CK_DerivedToBaseMemberPointer:
1076 case CK_MemberPointerToBoolean:
1077 case CK_ReinterpretMemberPointer:
1078 case CK_IntegralToPointer:
1079 case CK_PointerToIntegral:
1080 case CK_PointerToBoolean:
1081 case CK_ToVoid:
1082 case CK_VectorSplat:
1083 case CK_IntegralCast:
1084 case CK_BooleanToSignedIntegral:
1085 case CK_IntegralToBoolean:
1086 case CK_IntegralToFloating:
1087 case CK_FloatingToIntegral:
1088 case CK_FloatingToBoolean:
1089 case CK_FloatingCast:
1090 case CK_CPointerToObjCPointerCast:
1091 case CK_BlockPointerToObjCPointerCast:
1092 case CK_AnyPointerToBlockPointerCast:
1093 case CK_ObjCObjectLValueCast:
1094 case CK_FloatingRealToComplex:
1095 case CK_FloatingComplexToReal:
1096 case CK_FloatingComplexToBoolean:
1097 case CK_FloatingComplexCast:
1098 case CK_FloatingComplexToIntegralComplex:
1099 case CK_IntegralRealToComplex:
1100 case CK_IntegralComplexToReal:
1101 case CK_IntegralComplexToBoolean:
1102 case CK_IntegralComplexCast:
1103 case CK_IntegralComplexToFloatingComplex:
1104 case CK_ARCProduceObject:
1105 case CK_ARCConsumeObject:
1106 case CK_ARCReclaimReturnedObject:
1107 case CK_ARCExtendBlockObject:
1108 case CK_CopyAndAutoreleaseBlockObject:
1109 case CK_BuiltinFnToFnPtr:
1110 case CK_ZeroToOCLOpaqueType:
1111 case CK_MatrixCast:
1112 case CK_HLSLVectorTruncation:
1113 case CK_HLSLMatrixTruncation:
1114 case CK_IntToOCLSampler:
1115 case CK_FloatingToFixedPoint:
1116 case CK_FixedPointToFloating:
1117 case CK_FixedPointCast:
1118 case CK_FixedPointToBoolean:
1119 case CK_FixedPointToIntegral:
1120 case CK_IntegralToFixedPoint:
1121 llvm_unreachable("cast kind invalid for aggregate types");
1122 }
1123}
1124
1125void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1126 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
1127 EmitAggLoadOfLValue(E);
1128 return;
1129 }
1130
1131 withReturnValueSlot(
1132 E, EmitCall: [&](ReturnValueSlot Slot) { return CGF.EmitCallExpr(E, ReturnValue: Slot); });
1133}
1134
1135void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1136 withReturnValueSlot(E, EmitCall: [&](ReturnValueSlot Slot) {
1137 return CGF.EmitObjCMessageExpr(E, Return: Slot);
1138 });
1139}
1140
1141void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1142 CGF.EmitIgnoredExpr(E: E->getLHS());
1143 Visit(E: E->getRHS());
1144}
1145
1146void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1147 CodeGenFunction::StmtExprEvaluation eval(CGF);
1148 CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
1149}
1150
1151enum CompareKind {
1152 CK_Less,
1153 CK_Greater,
1154 CK_Equal,
1155};
1156
1157static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1158 const BinaryOperator *E, llvm::Value *LHS,
1159 llvm::Value *RHS, CompareKind Kind,
1160 const char *NameSuffix = "") {
1161 QualType ArgTy = E->getLHS()->getType();
1162 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1163 ArgTy = CT->getElementType();
1164
1165 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1166 assert(Kind == CK_Equal &&
1167 "member pointers may only be compared for equality");
1168 return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1169 CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1170 }
1171
1172 // Compute the comparison instructions for the specified comparison kind.
1173 struct CmpInstInfo {
1174 const char *Name;
1175 llvm::CmpInst::Predicate FCmp;
1176 llvm::CmpInst::Predicate SCmp;
1177 llvm::CmpInst::Predicate UCmp;
1178 };
1179 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1180 using FI = llvm::FCmpInst;
1181 using II = llvm::ICmpInst;
1182 switch (Kind) {
1183 case CK_Less:
1184 return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1185 case CK_Greater:
1186 return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1187 case CK_Equal:
1188 return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1189 }
1190 llvm_unreachable("Unrecognised CompareKind enum");
1191 }();
1192
1193 if (ArgTy->hasFloatingRepresentation())
1194 return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1195 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1196 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1197 auto Inst =
1198 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1199 return Builder.CreateICmp(P: Inst, LHS, RHS,
1200 Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1201 }
1202
1203 llvm_unreachable("unsupported aggregate binary expression should have "
1204 "already been handled");
1205}
1206
1207void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1208 using llvm::BasicBlock;
1209 using llvm::PHINode;
1210 using llvm::Value;
1211 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1212 E->getRHS()->getType()));
1213 const ComparisonCategoryInfo &CmpInfo =
1214 CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1215 assert(CmpInfo.Record->isTriviallyCopyable() &&
1216 "cannot copy non-trivially copyable aggregate");
1217
1218 QualType ArgTy = E->getLHS()->getType();
1219
1220 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1221 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1222 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1223 return CGF.ErrorUnsupported(S: E, Type: "aggregate three-way comparison");
1224 }
1225 bool IsComplex = ArgTy->isAnyComplexType();
1226
1227 // Evaluate the operands to the expression and extract their values.
1228 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1229 RValue RV = CGF.EmitAnyExpr(E);
1230 if (RV.isScalar())
1231 return {RV.getScalarVal(), nullptr};
1232 if (RV.isAggregate())
1233 return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1234 assert(RV.isComplex());
1235 return RV.getComplexVal();
1236 };
1237 auto LHSValues = EmitOperand(E->getLHS()),
1238 RHSValues = EmitOperand(E->getRHS());
1239
1240 auto EmitCmp = [&](CompareKind K) {
1241 Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1242 Kind: K, NameSuffix: IsComplex ? ".r" : "");
1243 if (!IsComplex)
1244 return Cmp;
1245 assert(K == CompareKind::CK_Equal);
1246 Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1247 RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1248 return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1249 };
1250 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1251 return Builder.getInt(AI: VInfo->getIntValue());
1252 };
1253
1254 Value *Select;
1255 if (ArgTy->isNullPtrType()) {
1256 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1257 } else if (!CmpInfo.isPartial()) {
1258 Value *SelectOne =
1259 Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1260 False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1261 Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1262 True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1263 False: SelectOne, Name: "sel.eq");
1264 } else {
1265 Value *SelectEq = Builder.CreateSelect(
1266 C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1267 False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1268 Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1269 True: EmitCmpRes(CmpInfo.getGreater()),
1270 False: SelectEq, Name: "sel.gt");
1271 Select = Builder.CreateSelect(
1272 C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1273 }
1274 // Create the return value in the destination slot.
1275 EnsureDest(T: E->getType());
1276 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
1277
1278 // Emit the address of the first (and only) field in the comparison category
1279 // type, and initialize it from the constant integer value selected above.
1280 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1281 Base: DestLV, Field: *CmpInfo.Record->field_begin());
1282 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Select), Dst: FieldLV, /*IsInit*/ isInit: true);
1283
1284 // All done! The result is in the Dest slot.
1285}
1286
1287void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1288 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1289 VisitPointerToDataMemberBinaryOperator(BO: E);
1290 else
1291 CGF.ErrorUnsupported(S: E, Type: "aggregate binary expression");
1292}
1293
1294void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1295 const BinaryOperator *E) {
1296 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1297 EmitFinalDestCopy(type: E->getType(), src: LV);
1298}
1299
1300/// Is the value of the given expression possibly a reference to or
1301/// into a __block variable?
1302static bool isBlockVarRef(const Expr *E) {
1303 // Make sure we look through parens.
1304 E = E->IgnoreParens();
1305
1306 // Check for a direct reference to a __block variable.
1307 if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1308 const VarDecl *var = dyn_cast<VarDecl>(Val: DRE->getDecl());
1309 return (var && var->hasAttr<BlocksAttr>());
1310 }
1311
1312 // More complicated stuff.
1313
1314 // Binary operators.
1315 if (const BinaryOperator *op = dyn_cast<BinaryOperator>(Val: E)) {
1316 // For an assignment or pointer-to-member operation, just care
1317 // about the LHS.
1318 if (op->isAssignmentOp() || op->isPtrMemOp())
1319 return isBlockVarRef(E: op->getLHS());
1320
1321 // For a comma, just care about the RHS.
1322 if (op->getOpcode() == BO_Comma)
1323 return isBlockVarRef(E: op->getRHS());
1324
1325 // FIXME: pointer arithmetic?
1326 return false;
1327
1328 // Check both sides of a conditional operator.
1329 } else if (const AbstractConditionalOperator *op =
1330 dyn_cast<AbstractConditionalOperator>(Val: E)) {
1331 return isBlockVarRef(E: op->getTrueExpr()) ||
1332 isBlockVarRef(E: op->getFalseExpr());
1333
1334 // OVEs are required to support BinaryConditionalOperators.
1335 } else if (const OpaqueValueExpr *op = dyn_cast<OpaqueValueExpr>(Val: E)) {
1336 if (const Expr *src = op->getSourceExpr())
1337 return isBlockVarRef(E: src);
1338
1339 // Casts are necessary to get things like (*(int*)&var) = foo().
1340 // We don't really care about the kind of cast here, except
1341 // we don't want to look through l2r casts, because it's okay
1342 // to get the *value* in a __block variable.
1343 } else if (const CastExpr *cast = dyn_cast<CastExpr>(Val: E)) {
1344 if (cast->getCastKind() == CK_LValueToRValue)
1345 return false;
1346 return isBlockVarRef(E: cast->getSubExpr());
1347
1348 // Handle unary operators. Again, just aggressively look through
1349 // it, ignoring the operation.
1350 } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E)) {
1351 return isBlockVarRef(E: uop->getSubExpr());
1352
1353 // Look into the base of a field access.
1354 } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(Val: E)) {
1355 return isBlockVarRef(E: mem->getBase());
1356
1357 // Look into the base of a subscript.
1358 } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(Val: E)) {
1359 return isBlockVarRef(E: sub->getBase());
1360 }
1361
1362 return false;
1363}
1364
1365void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1366 ApplyAtomGroup Grp(CGF.getDebugInfo());
1367 // For an assignment to work, the value on the right has
1368 // to be compatible with the value on the left.
1369 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1370 E->getRHS()->getType()) &&
1371 "Invalid assignment");
1372
1373 // If the LHS might be a __block variable, and the RHS can
1374 // potentially cause a block copy, we need to evaluate the RHS first
1375 // so that the assignment goes the right place.
1376 // This is pretty semantically fragile.
1377 if (isBlockVarRef(E: E->getLHS()) &&
1378 E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1379 // Ensure that we have a destination, and evaluate the RHS into that.
1380 EnsureDest(T: E->getRHS()->getType());
1381 Visit(E: E->getRHS());
1382
1383 // Now emit the LHS and copy into it.
1384 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1385
1386 // That copy is an atomic copy if the LHS is atomic.
1387 if (LHS.getType()->isAtomicType() ||
1388 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1389 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1390 return;
1391 }
1392
1393 EmitCopy(type: E->getLHS()->getType(),
1394 dest: AggValueSlot::forLValue(LV: LHS, isDestructed: AggValueSlot::IsDestructed,
1395 needsGC: needsGC(T: E->getLHS()->getType()),
1396 isAliased: AggValueSlot::IsAliased,
1397 mayOverlap: AggValueSlot::MayOverlap),
1398 src: Dest);
1399 return;
1400 }
1401
1402 LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1403
1404 // If we have an atomic type, evaluate into the destination and then
1405 // do an atomic copy.
1406 if (LHS.getType()->isAtomicType() ||
1407 CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1408 EnsureDest(T: E->getRHS()->getType());
1409 Visit(E: E->getRHS());
1410 CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1411 return;
1412 }
1413
1414 // Codegen the RHS so that it stores directly into the LHS.
1415 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1416 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1417 isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1418 // A non-volatile aggregate destination might have volatile member.
1419 if (!LHSSlot.isVolatile() && CGF.hasVolatileMember(T: E->getLHS()->getType()))
1420 LHSSlot.setVolatile(true);
1421
1422 CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1423
1424 // Copy into the destination if the assignment isn't ignored.
1425 EmitFinalDestCopy(type: E->getType(), src: LHS);
1426
1427 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1428 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1429 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1430 type: E->getType());
1431}
1432
1433void AggExprEmitter::VisitAbstractConditionalOperator(
1434 const AbstractConditionalOperator *E) {
1435 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1436 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1437 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1438
1439 // Bind the common expression if necessary.
1440 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1441
1442 CodeGenFunction::ConditionalEvaluation eval(CGF);
1443 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1444 TrueCount: CGF.getProfileCount(S: E));
1445
1446 // Save whether the destination's lifetime is externally managed.
1447 bool isExternallyDestructed = Dest.isExternallyDestructed();
1448 bool destructNonTrivialCStruct =
1449 !isExternallyDestructed &&
1450 E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1451 isExternallyDestructed |= destructNonTrivialCStruct;
1452 Dest.setExternallyDestructed(isExternallyDestructed);
1453
1454 eval.begin(CGF);
1455 CGF.EmitBlock(BB: LHSBlock);
1456 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
1457 Visit(E: E->getTrueExpr());
1458 eval.end(CGF);
1459
1460 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1461 CGF.Builder.CreateBr(Dest: ContBlock);
1462
1463 // If the result of an agg expression is unused, then the emission
1464 // of the LHS might need to create a destination slot. That's fine
1465 // with us, and we can safely emit the RHS into the same slot, but
1466 // we shouldn't claim that it's already being destructed.
1467 Dest.setExternallyDestructed(isExternallyDestructed);
1468
1469 eval.begin(CGF);
1470 CGF.EmitBlock(BB: RHSBlock);
1471 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
1472 Visit(E: E->getFalseExpr());
1473 eval.end(CGF);
1474
1475 if (destructNonTrivialCStruct)
1476 CGF.pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Dest.getAddress(),
1477 type: E->getType());
1478
1479 CGF.EmitBlock(BB: ContBlock);
1480}
1481
1482void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1483 Visit(E: CE->getChosenSubExpr());
1484}
1485
1486void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1487 Address ArgValue = Address::invalid();
1488 CGF.EmitVAArg(VE, VAListAddr&: ArgValue, Slot: Dest);
1489
1490 // If EmitVAArg fails, emit an error.
1491 if (!ArgValue.isValid()) {
1492 CGF.ErrorUnsupported(S: VE, Type: "aggregate va_arg expression");
1493 return;
1494 }
1495}
1496
1497void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1498 // Ensure that we have a slot, but if we already do, remember
1499 // whether it was externally destructed.
1500 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1501 EnsureDest(T: E->getType());
1502
1503 // We're going to push a destructor if there isn't already one.
1504 Dest.setExternallyDestructed();
1505
1506 Visit(E: E->getSubExpr());
1507
1508 // Push that destructor we promised.
1509 if (!wasExternallyDestructed)
1510 CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1511}
1512
1513void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1514 AggValueSlot Slot = EnsureSlot(T: E->getType());
1515 CGF.EmitCXXConstructExpr(E, Dest: Slot);
1516}
1517
1518void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1519 const CXXInheritedCtorInitExpr *E) {
1520 AggValueSlot Slot = EnsureSlot(T: E->getType());
1521 CGF.EmitInheritedCXXConstructorCall(D: E->getConstructor(), ForVirtualBase: E->constructsVBase(),
1522 This: Slot.getAddress(),
1523 InheritedFromVBase: E->inheritedFromVBase(), E);
1524}
1525
1526void AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1527 AggValueSlot Slot = EnsureSlot(T: E->getType());
1528 LValue SlotLV = CGF.MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType());
1529
1530 // We'll need to enter cleanup scopes in case any of the element
1531 // initializers throws an exception or contains branch out of the expressions.
1532 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1533
1534 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1535 for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1536 e = E->capture_init_end();
1537 i != e; ++i, ++CurField) {
1538 // Emit initialization
1539 LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1540 if (CurField->hasCapturedVLAType()) {
1541 CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1542 continue;
1543 }
1544
1545 EmitInitializationToLValue(E: *i, Address: LV);
1546
1547 // Push a destructor if necessary.
1548 if (QualType::DestructionKind DtorKind =
1549 CurField->getType().isDestructedType()) {
1550 assert(LV.isSimple());
1551 if (DtorKind)
1552 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1553 type: CurField->getType(),
1554 destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: false);
1555 }
1556 }
1557}
1558
1559void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1560 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1561 Visit(E: E->getSubExpr());
1562}
1563
1564void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1565 QualType T = E->getType();
1566 AggValueSlot Slot = EnsureSlot(T);
1567 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1568}
1569
1570void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1571 QualType T = E->getType();
1572 AggValueSlot Slot = EnsureSlot(T);
1573 EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1574}
1575
1576/// Determine whether the given cast kind is known to always convert values
1577/// with all zero bits in their value representation to values with all zero
1578/// bits in their value representation.
1579static bool castPreservesZero(const CastExpr *CE) {
1580 switch (CE->getCastKind()) {
1581 // No-ops.
1582 case CK_NoOp:
1583 case CK_UserDefinedConversion:
1584 case CK_ConstructorConversion:
1585 case CK_BitCast:
1586 case CK_ToUnion:
1587 case CK_ToVoid:
1588 // Conversions between (possibly-complex) integral, (possibly-complex)
1589 // floating-point, and bool.
1590 case CK_BooleanToSignedIntegral:
1591 case CK_FloatingCast:
1592 case CK_FloatingComplexCast:
1593 case CK_FloatingComplexToBoolean:
1594 case CK_FloatingComplexToIntegralComplex:
1595 case CK_FloatingComplexToReal:
1596 case CK_FloatingRealToComplex:
1597 case CK_FloatingToBoolean:
1598 case CK_FloatingToIntegral:
1599 case CK_IntegralCast:
1600 case CK_IntegralComplexCast:
1601 case CK_IntegralComplexToBoolean:
1602 case CK_IntegralComplexToFloatingComplex:
1603 case CK_IntegralComplexToReal:
1604 case CK_IntegralRealToComplex:
1605 case CK_IntegralToBoolean:
1606 case CK_IntegralToFloating:
1607 // Reinterpreting integers as pointers and vice versa.
1608 case CK_IntegralToPointer:
1609 case CK_PointerToIntegral:
1610 // Language extensions.
1611 case CK_VectorSplat:
1612 case CK_MatrixCast:
1613 case CK_NonAtomicToAtomic:
1614 case CK_AtomicToNonAtomic:
1615 case CK_HLSLVectorTruncation:
1616 case CK_HLSLMatrixTruncation:
1617 case CK_HLSLElementwiseCast:
1618 case CK_HLSLAggregateSplatCast:
1619 return true;
1620
1621 case CK_BaseToDerivedMemberPointer:
1622 case CK_DerivedToBaseMemberPointer:
1623 case CK_MemberPointerToBoolean:
1624 case CK_NullToMemberPointer:
1625 case CK_ReinterpretMemberPointer:
1626 // FIXME: ABI-dependent.
1627 return false;
1628
1629 case CK_AnyPointerToBlockPointerCast:
1630 case CK_BlockPointerToObjCPointerCast:
1631 case CK_CPointerToObjCPointerCast:
1632 case CK_ObjCObjectLValueCast:
1633 case CK_IntToOCLSampler:
1634 case CK_ZeroToOCLOpaqueType:
1635 // FIXME: Check these.
1636 return false;
1637
1638 case CK_FixedPointCast:
1639 case CK_FixedPointToBoolean:
1640 case CK_FixedPointToFloating:
1641 case CK_FixedPointToIntegral:
1642 case CK_FloatingToFixedPoint:
1643 case CK_IntegralToFixedPoint:
1644 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1645 return false;
1646
1647 case CK_AddressSpaceConversion:
1648 case CK_BaseToDerived:
1649 case CK_DerivedToBase:
1650 case CK_Dynamic:
1651 case CK_NullToPointer:
1652 case CK_PointerToBoolean:
1653 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1654 // same representation in all involved address spaces.
1655 return false;
1656
1657 case CK_ARCConsumeObject:
1658 case CK_ARCExtendBlockObject:
1659 case CK_ARCProduceObject:
1660 case CK_ARCReclaimReturnedObject:
1661 case CK_CopyAndAutoreleaseBlockObject:
1662 case CK_ArrayToPointerDecay:
1663 case CK_FunctionToPointerDecay:
1664 case CK_BuiltinFnToFnPtr:
1665 case CK_Dependent:
1666 case CK_LValueBitCast:
1667 case CK_LValueToRValue:
1668 case CK_LValueToRValueBitCast:
1669 case CK_UncheckedDerivedToBase:
1670 case CK_HLSLArrayRValue:
1671 return false;
1672 }
1673 llvm_unreachable("Unhandled clang::CastKind enum");
1674}
1675
1676/// isSimpleZero - If emitting this value will obviously just cause a store of
1677/// zero to memory, return true. This can return false if uncertain, so it just
1678/// handles simple cases.
1679static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1680 E = E->IgnoreParens();
1681 while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1682 if (!castPreservesZero(CE))
1683 break;
1684 E = CE->getSubExpr()->IgnoreParens();
1685 }
1686
1687 // 0
1688 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1689 return IL->getValue() == 0;
1690 // +0.0
1691 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1692 return FL->getValue().isPosZero();
1693 // int()
1694 if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1695 CGF.getTypes().isZeroInitializable(T: E->getType()))
1696 return true;
1697 // (int*)0 - Null pointer expressions.
1698 if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1699 return ICE->getCastKind() == CK_NullToPointer &&
1700 CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1701 !E->HasSideEffects(Ctx: CGF.getContext());
1702 // '\0'
1703 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1704 return CL->getValue() == 0;
1705
1706 // Otherwise, hard case: conservatively return false.
1707 return false;
1708}
1709
1710void AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1711 QualType type = LV.getType();
1712 // FIXME: Ignore result?
1713 // FIXME: Are initializers affected by volatile?
1714 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1715 // Storing "i32 0" to a zero'd memory location is a noop.
1716 return;
1717 } else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1718 return EmitNullInitializationToLValue(Address: LV);
1719 } else if (isa<NoInitExpr>(Val: E)) {
1720 // Do nothing.
1721 return;
1722 } else if (type->isReferenceType()) {
1723 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1724 return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1725 }
1726
1727 CGF.EmitInitializationToLValue(E, LV, IsZeroed: Dest.isZeroed());
1728}
1729
1730void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1731 QualType type = lv.getType();
1732
1733 // If the destination slot is already zeroed out before the aggregate is
1734 // copied into it, we don't have to emit any zeros here.
1735 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1736 return;
1737
1738 if (CGF.hasScalarEvaluationKind(T: type)) {
1739 // For non-aggregates, we can store the appropriate null constant.
1740 llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1741 // Note that the following is not equivalent to
1742 // EmitStoreThroughBitfieldLValue for ARC types.
1743 if (lv.isBitField()) {
1744 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1745 } else {
1746 assert(lv.isSimple());
1747 CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1748 }
1749 } else {
1750 // There's a potential optimization opportunity in combining
1751 // memsets; that would be easy for arrays, but relatively
1752 // difficult for structures with the current code.
1753 CGF.EmitNullInitialization(DestPtr: lv.getAddress(), Ty: lv.getType());
1754 }
1755}
1756
1757void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1758 VisitCXXParenListOrInitListExpr(ExprToVisit: E, Args: E->getInitExprs(),
1759 InitializedFieldInUnion: E->getInitializedFieldInUnion(),
1760 ArrayFiller: E->getArrayFiller());
1761}
1762
1763void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1764 if (E->hadArrayRangeDesignator())
1765 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
1766
1767 if (E->isTransparent())
1768 return Visit(E: E->getInit(Init: 0));
1769
1770 VisitCXXParenListOrInitListExpr(
1771 ExprToVisit: E, Args: E->inits(), InitializedFieldInUnion: E->getInitializedFieldInUnion(), ArrayFiller: E->getArrayFiller());
1772}
1773
1774void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1775 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1776 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1777#if 0
1778 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1779 // (Length of globals? Chunks of zeroed-out space?).
1780 //
1781 // If we can, prefer a copy from a global; this is a lot less code for long
1782 // globals, and it's easier for the current optimizers to analyze.
1783 if (llvm::Constant *C =
1784 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1785 llvm::GlobalVariable* GV =
1786 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1787 llvm::GlobalValue::InternalLinkage, C, "");
1788 EmitFinalDestCopy(ExprToVisit->getType(),
1789 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1790 return;
1791 }
1792#endif
1793
1794 // HLSL initialization lists in the AST are an expansion which can contain
1795 // side-effecting expressions wrapped in opaque value expressions. To properly
1796 // emit these we need to emit the opaque values before we emit the argument
1797 // expressions themselves. This is a little hacky, but it prevents us needing
1798 // to do a bigger AST-level change for a language feature that we need
1799 // deprecate in the near future. See related HLSL language proposals:
1800 // * 0005-strict-initializer-lists.md
1801 // * https://github.com/microsoft/hlsl-specs/pull/325
1802 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(Val: ExprToVisit))
1803 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(
1804 CGF, E: cast<InitListExpr>(Val: ExprToVisit));
1805
1806 AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1807
1808 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1809
1810 // Handle initialization of an array.
1811 if (ExprToVisit->getType()->isConstantArrayType()) {
1812 auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1813 EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1814 Args: InitExprs, ArrayFiller);
1815 return;
1816 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1817 // A variable array type that has an initializer can only do empty
1818 // initialization. And because this feature is not exposed as an extension
1819 // in C++, we can safely memset the array memory to zero.
1820 assert(InitExprs.size() == 0 &&
1821 "you can only use an empty initializer with VLAs");
1822 CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1823 return;
1824 }
1825
1826 assert(ExprToVisit->getType()->isRecordType() &&
1827 "Only support structs/unions here!");
1828
1829 // Do struct initialization; this code just sets each individual member
1830 // to the approprate value. This makes bitfield support automatic;
1831 // the disadvantage is that the generated code is more difficult for
1832 // the optimizer, especially with bitfields.
1833 unsigned NumInitElements = InitExprs.size();
1834 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1835
1836 // We'll need to enter cleanup scopes in case any of the element
1837 // initializers throws an exception.
1838 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1839
1840 unsigned curInitIndex = 0;
1841
1842 // Emit initialization of base classes.
1843 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1844 assert(NumInitElements >= CXXRD->getNumBases() &&
1845 "missing initializer for base class");
1846 for (auto &Base : CXXRD->bases()) {
1847 assert(!Base.isVirtual() && "should not see vbases here");
1848 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1849 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1850 Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1851 /*isBaseVirtual*/ BaseIsVirtual: false);
1852 AggValueSlot AggSlot = AggValueSlot::forAddr(
1853 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
1854 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1855 mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1856 CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1857
1858 if (QualType::DestructionKind dtorKind =
1859 Base.getType().isDestructedType())
1860 CGF.pushDestroyAndDeferDeactivation(dtorKind, addr: V, type: Base.getType());
1861 }
1862 }
1863
1864 // Prepare a 'this' for CXXDefaultInitExprs.
1865 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1866
1867 const bool ZeroInitPadding =
1868 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1869
1870 if (record->isUnion()) {
1871 // Only initialize one field of a union. The field itself is
1872 // specified by the initializer list.
1873 if (!InitializedFieldInUnion) {
1874 // Empty union; we have nothing to do.
1875
1876#ifndef NDEBUG
1877 // Make sure that it's really an empty and not a failure of
1878 // semantic analysis.
1879 for (const auto *Field : record->fields())
1880 assert(
1881 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1882 "Only unnamed bitfields or anonymous class allowed");
1883#endif
1884 return;
1885 }
1886
1887 // FIXME: volatility
1888 FieldDecl *Field = InitializedFieldInUnion;
1889
1890 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1891 if (NumInitElements) {
1892 // Store the initializer into the field
1893 EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1894 if (ZeroInitPadding) {
1895 uint64_t TotalSize = CGF.getContext().toBits(
1896 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1897 uint64_t FieldSize = CGF.getContext().getTypeSize(T: FieldLoc.getType());
1898 DoZeroInitPadding(PaddingStart&: FieldSize, PaddingEnd: TotalSize, NextField: nullptr);
1899 }
1900 } else {
1901 // Default-initialize to null.
1902 if (ZeroInitPadding)
1903 EmitNullInitializationToLValue(lv: DestLV);
1904 else
1905 EmitNullInitializationToLValue(lv: FieldLoc);
1906 }
1907 return;
1908 }
1909
1910 // Here we iterate over the fields; this makes it simpler to both
1911 // default-initialize fields and skip over unnamed fields.
1912 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(D: record);
1913 uint64_t PaddingStart = 0;
1914
1915 for (const auto *field : record->fields()) {
1916 // We're done once we hit the flexible array member.
1917 if (field->getType()->isIncompleteArrayType())
1918 break;
1919
1920 // Always skip anonymous bitfields.
1921 if (field->isUnnamedBitField())
1922 continue;
1923
1924 // We're done if we reach the end of the explicit initializers, we
1925 // have a zeroed object, and the rest of the fields are
1926 // zero-initializable.
1927 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1928 CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1929 break;
1930
1931 if (ZeroInitPadding)
1932 DoZeroInitPadding(PaddingStart,
1933 PaddingEnd: Layout.getFieldOffset(FieldNo: field->getFieldIndex()), NextField: field);
1934
1935 LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1936 // We never generate write-barries for initialized fields.
1937 LV.setNonGC(true);
1938
1939 if (curInitIndex < NumInitElements) {
1940 // Store the initializer into the field.
1941 EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1942 } else {
1943 // We're out of initializers; default-initialize to null
1944 EmitNullInitializationToLValue(lv: LV);
1945 }
1946
1947 // Push a destructor if necessary.
1948 // FIXME: if we have an array of structures, all explicitly
1949 // initialized, we can end up pushing a linear number of cleanups.
1950 if (QualType::DestructionKind dtorKind =
1951 field->getType().isDestructedType()) {
1952 assert(LV.isSimple());
1953 if (dtorKind) {
1954 CGF.pushDestroyAndDeferDeactivation(cleanupKind: NormalAndEHCleanup, addr: LV.getAddress(),
1955 type: field->getType(),
1956 destroyer: CGF.getDestroyer(destructionKind: dtorKind), useEHCleanupForArray: false);
1957 }
1958 }
1959 }
1960 if (ZeroInitPadding) {
1961 uint64_t TotalSize = CGF.getContext().toBits(
1962 CharSize: Dest.getPreferredSize(Ctx&: CGF.getContext(), Type: DestLV.getType()));
1963 DoZeroInitPadding(PaddingStart, PaddingEnd: TotalSize, NextField: nullptr);
1964 }
1965}
1966
1967void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1968 uint64_t PaddingEnd,
1969 const FieldDecl *NextField) {
1970
1971 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1972 CharUnits Start = CGF.getContext().toCharUnitsFromBits(BitSize: StartBit);
1973 CharUnits End = CGF.getContext().toCharUnitsFromBits(BitSize: EndBit);
1974 Address Addr = Dest.getAddress().withElementType(ElemTy: CGF.CharTy);
1975 if (!Start.isZero())
1976 Addr = Builder.CreateConstGEP(Addr, Index: Start.getQuantity());
1977 llvm::Constant *SizeVal = Builder.getInt64(C: (End - Start).getQuantity());
1978 CGF.Builder.CreateMemSet(Dest: Addr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
1979 };
1980
1981 if (NextField != nullptr && NextField->isBitField()) {
1982 // For bitfield, zero init StorageSize before storing the bits. So we don't
1983 // need to handle big/little endian.
1984 const CGRecordLayout &RL =
1985 CGF.getTypes().getCGRecordLayout(NextField->getParent());
1986 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: NextField);
1987 uint64_t StorageStart = CGF.getContext().toBits(CharSize: Info.StorageOffset);
1988 if (StorageStart + Info.StorageSize > PaddingStart) {
1989 if (StorageStart > PaddingStart)
1990 InitBytes(PaddingStart, StorageStart);
1991 Address Addr = Dest.getAddress();
1992 if (!Info.StorageOffset.isZero())
1993 Addr = Builder.CreateConstGEP(Addr: Addr.withElementType(ElemTy: CGF.CharTy),
1994 Index: Info.StorageOffset.getQuantity());
1995 Addr = Addr.withElementType(
1996 ElemTy: llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: Info.StorageSize));
1997 Builder.CreateStore(Val: Builder.getIntN(N: Info.StorageSize, C: 0), Addr);
1998 PaddingStart = StorageStart + Info.StorageSize;
1999 }
2000 return;
2001 }
2002
2003 if (PaddingStart < PaddingEnd)
2004 InitBytes(PaddingStart, PaddingEnd);
2005 if (NextField != nullptr)
2006 PaddingStart =
2007 PaddingEnd + CGF.getContext().getTypeSize(T: NextField->getType());
2008}
2009
2010void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
2011 llvm::Value *outerBegin) {
2012 // Emit the common subexpression.
2013 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
2014
2015 Address destPtr = EnsureSlot(T: E->getType()).getAddress();
2016 uint64_t numElements = E->getArraySize().getZExtValue();
2017
2018 if (!numElements)
2019 return;
2020
2021 // destPtr is an array*. Construct an elementType* by drilling down a level.
2022 llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
2023 llvm::Value *indices[] = {zero, zero};
2024 llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
2025 Ptr: destPtr.emitRawPointer(CGF),
2026 IdxList: indices, Name: "arrayinit.begin");
2027
2028 // Prepare to special-case multidimensional array initialization: we avoid
2029 // emitting multiple destructor loops in that case.
2030 if (!outerBegin)
2031 outerBegin = begin;
2032 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
2033
2034 QualType elementType =
2035 CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
2036 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
2037 CharUnits elementAlign =
2038 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
2039 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
2040
2041 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2042 llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
2043
2044 // Jump into the body.
2045 CGF.EmitBlock(BB: bodyBB);
2046 llvm::PHINode *index =
2047 Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
2048 index->addIncoming(V: zero, BB: entryBB);
2049 llvm::Value *element =
2050 Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
2051
2052 if (CGF.CGM.shouldEmitConvergenceTokens())
2053 CGF.ConvergenceTokenStack.push_back(Elt: CGF.emitConvergenceLoopToken(BB: bodyBB));
2054
2055 // Prepare for a cleanup.
2056 QualType::DestructionKind dtorKind = elementType.isDestructedType();
2057 EHScopeStack::stable_iterator cleanup;
2058 if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
2059 if (outerBegin->getType() != element->getType())
2060 outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
2061 CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
2062 elementAlignment: elementAlign,
2063 destroyer: CGF.getDestroyer(destructionKind: dtorKind));
2064 cleanup = CGF.EHStack.stable_begin();
2065 } else {
2066 dtorKind = QualType::DK_none;
2067 }
2068
2069 // Emit the actual filler expression.
2070 {
2071 // Temporaries created in an array initialization loop are destroyed
2072 // at the end of each iteration.
2073 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2074 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2075 LValue elementLV = CGF.MakeAddrLValue(
2076 Addr: Address(element, llvmElementType, elementAlign), T: elementType);
2077
2078 if (InnerLoop) {
2079 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2080 auto elementSlot = AggValueSlot::forLValue(
2081 LV: elementLV, isDestructed: AggValueSlot::IsDestructed,
2082 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2083 mayOverlap: AggValueSlot::DoesNotOverlap);
2084 AggExprEmitter(CGF, elementSlot, false)
2085 .VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
2086 } else
2087 EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
2088 }
2089
2090 // Move on to the next element.
2091 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2092 LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
2093 index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
2094
2095 // Leave the loop if we're done.
2096 llvm::Value *done = Builder.CreateICmpEQ(
2097 LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
2098 Name: "arrayinit.done");
2099 llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
2100 Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
2101
2102 if (CGF.CGM.shouldEmitConvergenceTokens())
2103 CGF.ConvergenceTokenStack.pop_back();
2104
2105 CGF.EmitBlock(BB: endBB);
2106
2107 // Leave the partial-array cleanup if we entered one.
2108 if (dtorKind)
2109 CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
2110}
2111
2112void AggExprEmitter::VisitDesignatedInitUpdateExpr(
2113 DesignatedInitUpdateExpr *E) {
2114 AggValueSlot Dest = EnsureSlot(T: E->getType());
2115
2116 LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: E->getType());
2117 EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
2118 VisitInitListExpr(E: E->getUpdater());
2119}
2120
2121//===----------------------------------------------------------------------===//
2122// Entry Points into this File
2123//===----------------------------------------------------------------------===//
2124
2125/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2126/// non-zero bytes that will be stored when outputting the initializer for the
2127/// specified initializer expression.
2128static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
2129 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
2130 E = MTE->getSubExpr();
2131 E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
2132
2133 // 0 and 0.0 won't require any non-zero stores!
2134 if (isSimpleZero(E, CGF))
2135 return CharUnits::Zero();
2136
2137 // If this is an initlist expr, sum up the size of sizes of the (present)
2138 // elements. If this is something weird, assume the whole thing is non-zero.
2139 const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
2140 while (ILE && ILE->isTransparent())
2141 ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
2142 if (!ILE || !CGF.getTypes().isZeroInitializable(T: ILE->getType()))
2143 return CGF.getContext().getTypeSizeInChars(T: E->getType());
2144
2145 // InitListExprs for structs have to be handled carefully. If there are
2146 // reference members, we need to consider the size of the reference, not the
2147 // referencee. InitListExprs for unions and arrays can't have references.
2148 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2149 if (!RT->isUnionType()) {
2150 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2151 CharUnits NumNonZeroBytes = CharUnits::Zero();
2152
2153 unsigned ILEElement = 0;
2154 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
2155 while (ILEElement != CXXRD->getNumBases())
2156 NumNonZeroBytes +=
2157 GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
2158 for (const auto *Field : SD->fields()) {
2159 // We're done once we hit the flexible array member or run out of
2160 // InitListExpr elements.
2161 if (Field->getType()->isIncompleteArrayType() ||
2162 ILEElement == ILE->getNumInits())
2163 break;
2164 if (Field->isUnnamedBitField())
2165 continue;
2166
2167 const Expr *E = ILE->getInit(Init: ILEElement++);
2168
2169 // Reference values are always non-null and have the width of a pointer.
2170 if (Field->getType()->isReferenceType())
2171 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2172 BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2173 else
2174 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2175 }
2176
2177 return NumNonZeroBytes;
2178 }
2179 }
2180
2181 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2182 CharUnits NumNonZeroBytes = CharUnits::Zero();
2183 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2184 NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2185 return NumNonZeroBytes;
2186}
2187
2188/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2189/// zeros in it, emit a memset and avoid storing the individual zeros.
2190///
2191static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2192 CodeGenFunction &CGF) {
2193 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2194 // volatile stores.
2195 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2196 return;
2197
2198 // C++ objects with a user-declared constructor don't need zero'ing.
2199 if (CGF.getLangOpts().CPlusPlus)
2200 if (const RecordType *RT = CGF.getContext()
2201 .getBaseElementType(QT: E->getType())
2202 ->getAsCanonical<RecordType>()) {
2203 const auto *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2204 if (RD->hasUserDeclaredConstructor())
2205 return;
2206 }
2207
2208 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2209 CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2210 if (Size <= CharUnits::fromQuantity(Quantity: 16))
2211 return;
2212
2213 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2214 // we prefer to emit memset + individual stores for the rest.
2215 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2216 if (NumNonZeroBytes * 4 > Size)
2217 return;
2218
2219 // Okay, it seems like a good idea to use an initial memset, emit the call.
2220 llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2221
2222 Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2223 CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2224
2225 // Tell the AggExprEmitter that the slot is known zero.
2226 Slot.setZeroed();
2227}
2228
2229/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2230/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2231/// the value of the aggregate expression is not needed. If VolatileDest is
2232/// true, DestPtr cannot be 0.
2233void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2234 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2235 "Invalid aggregate expression to emit");
2236 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2237 "slot has bits but no address");
2238
2239 // Optimize the slot if possible.
2240 CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2241
2242 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr *>(E));
2243}
2244
2245LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2246 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2247 Address Temp = CreateMemTempWithoutCast(T: E->getType());
2248 LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2249 EmitAggExpr(E, Slot: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsNotDestructed,
2250 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2251 isAliased: AggValueSlot::IsNotAliased,
2252 mayOverlap: AggValueSlot::DoesNotOverlap));
2253 return LV;
2254}
2255
2256void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,
2257 const LValue &Src,
2258 ExprValueKind SrcKind) {
2259 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2260 .EmitFinalDestCopy(type: Type, src: Src, SrcValueKind: SrcKind);
2261}
2262
2263AggValueSlot::Overlap_t
2264CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2265 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2266 return AggValueSlot::DoesNotOverlap;
2267
2268 // Empty fields can overlap earlier fields.
2269 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2270 return AggValueSlot::MayOverlap;
2271
2272 // If the field lies entirely within the enclosing class's nvsize, its tail
2273 // padding cannot overlap any already-initialized object. (The only subobjects
2274 // with greater addresses that might already be initialized are vbases.)
2275 const RecordDecl *ClassRD = FD->getParent();
2276 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2277 if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2278 getContext().getTypeSize(T: FD->getType()) <=
2279 (uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2280 return AggValueSlot::DoesNotOverlap;
2281
2282 // The tail padding may contain values we need to preserve.
2283 return AggValueSlot::MayOverlap;
2284}
2285
2286AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2287 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2288 // If the most-derived object is a field declared with [[no_unique_address]],
2289 // the tail padding of any virtual base could be reused for other subobjects
2290 // of that field's class.
2291 if (IsVirtual)
2292 return AggValueSlot::MayOverlap;
2293
2294 // Empty bases can overlap earlier bases.
2295 if (BaseRD->isEmpty())
2296 return AggValueSlot::MayOverlap;
2297
2298 // If the base class is laid out entirely within the nvsize of the derived
2299 // class, its tail padding cannot yet be initialized, so we can issue
2300 // stores at the full width of the base class.
2301 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2302 if (Layout.getBaseClassOffset(Base: BaseRD) +
2303 getContext().getASTRecordLayout(D: BaseRD).getSize() <=
2304 Layout.getNonVirtualSize())
2305 return AggValueSlot::DoesNotOverlap;
2306
2307 // The tail padding may contain values we need to preserve.
2308 return AggValueSlot::MayOverlap;
2309}
2310
2311void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2312 AggValueSlot::Overlap_t MayOverlap,
2313 bool isVolatile) {
2314 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2315
2316 Address DestPtr = Dest.getAddress();
2317 Address SrcPtr = Src.getAddress();
2318
2319 if (getLangOpts().CPlusPlus) {
2320 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2321 assert((Record->hasTrivialCopyConstructor() ||
2322 Record->hasTrivialCopyAssignment() ||
2323 Record->hasTrivialMoveConstructor() ||
2324 Record->hasTrivialMoveAssignment() ||
2325 Record->hasAttr<TrivialABIAttr>() || Record->isUnion() ||
2326 // HLSL uses aggregate-copy for user-defined record types.
2327 (getLangOpts().HLSL && !Record->isHLSLBuiltinRecord())) &&
2328 "Trying to aggregate-copy a type without a trivial copy/move "
2329 "constructor or assignment operator");
2330 // Ignore empty classes in C++.
2331 if (Record->isEmpty())
2332 return;
2333 }
2334 }
2335
2336 if (getLangOpts().CUDAIsDevice) {
2337 if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2338 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2339 Src))
2340 return;
2341 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2342 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2343 Src))
2344 return;
2345 }
2346 }
2347
2348 assert(Ty.getAddressSpace() != LangAS::hlsl_constant &&
2349 "copies of aggregates in hlsl_constant address space should be "
2350 "handled earlier by the HLSL runtime");
2351
2352 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2353 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2354 // read from another object that overlaps in anyway the storage of the first
2355 // object, then the overlap shall be exact and the two objects shall have
2356 // qualified or unqualified versions of a compatible type."
2357 //
2358 // memcpy is not defined if the source and destination pointers are exactly
2359 // equal, but other compilers do this optimization, and almost every memcpy
2360 // implementation handles this case safely. If there is a libc that does not
2361 // safely handle this, we can add a target hook.
2362
2363 // Get data size info for this aggregate. Don't copy the tail padding if this
2364 // might be a potentially-overlapping subobject, since the tail padding might
2365 // be occupied by a different object. Otherwise, copying it is fine.
2366 TypeInfoChars TypeInfo;
2367 if (MayOverlap)
2368 TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2369 else
2370 TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2371
2372 llvm::Value *SizeVal = nullptr;
2373 if (TypeInfo.Width.isZero()) {
2374 // But note that getTypeInfo returns 0 for a VLA.
2375 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2376 Val: getContext().getAsArrayType(T: Ty))) {
2377 QualType BaseEltTy;
2378 SizeVal = emitArrayLength(arrayType: VAT, baseType&: BaseEltTy, addr&: DestPtr);
2379 TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2380 assert(!TypeInfo.Width.isZero());
2381 SizeVal = Builder.CreateNUWMul(
2382 LHS: SizeVal,
2383 RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2384 }
2385 }
2386 if (!SizeVal) {
2387 SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2388 }
2389
2390 // FIXME: If we have a volatile struct, the optimizer can remove what might
2391 // appear to be `extra' memory ops:
2392 //
2393 // volatile struct { int i; } a, b;
2394 //
2395 // int main() {
2396 // a = b;
2397 // a = b;
2398 // }
2399 //
2400 // we need to use a different call here. We use isVolatile to indicate when
2401 // either the source or the destination is volatile.
2402
2403 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2404 SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2405
2406 // Don't do any of the memmove_collectable tests if GC isn't set.
2407 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2408 // fall through
2409 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2410 if (Record->hasObjectMember()) {
2411 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2412 Size: SizeVal);
2413 return;
2414 }
2415 } else if (Ty->isArrayType()) {
2416 QualType BaseType = getContext().getBaseElementType(QT: Ty);
2417 if (const auto *Record = BaseType->getAsRecordDecl()) {
2418 if (Record->hasObjectMember()) {
2419 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2420 Size: SizeVal);
2421 return;
2422 }
2423 }
2424 }
2425
2426 auto *Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2427 addInstToCurrentSourceAtom(KeyInstruction: Inst, Backup: nullptr);
2428 emitPFPPostCopyUpdates(DestPtr, SrcPtr, Ty);
2429
2430 // Determine the metadata to describe the position of any padding in this
2431 // memcpy, as well as the TBAA tags for the members of the struct, in case
2432 // the optimizer wishes to expand it in to scalar memory operations.
2433 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2434 Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2435
2436 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2437 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2438 DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2439 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2440 }
2441}
2442