1//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with C++ code generation of classes
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGBlocks.h"
15#include "CGCXXABI.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "TargetInfo.h"
20#include "clang/AST/Attr.h"
21#include "clang/AST/CXXInheritance.h"
22#include "clang/AST/CharUnits.h"
23#include "clang/AST/DeclTemplate.h"
24#include "clang/AST/RecordLayout.h"
25#include "clang/AST/StmtCXX.h"
26#include "clang/Basic/CodeGenOptions.h"
27#include "clang/CodeGen/CGFunctionInfo.h"
28#include "clang/CodeGenUtils/ClassUtils.h"
29#include "clang/CodeGenUtils/RecordLayoutUtils.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/Support/SaveAndRestore.h"
33#include "llvm/Transforms/Utils/ModuleUtils.h"
34#include "llvm/Transforms/Utils/SanitizerStats.h"
35#include <optional>
36
37using namespace clang;
38using namespace CodeGen;
39
40/// Return the best known alignment for an unknown pointer to a
41/// particular class.
42CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
43 if (!RD->hasDefinition())
44 return CharUnits::One(); // Hopefully won't be used anywhere.
45
46 auto &layout = getContext().getASTRecordLayout(D: RD);
47
48 // If the class is final, then we know that the pointer points to an
49 // object of that type and can use the full alignment.
50 if (RD->isEffectivelyFinal())
51 return layout.getAlignment();
52
53 // Otherwise, we have to assume it could be a subclass.
54 return layout.getNonVirtualAlignment();
55}
56
57/// Return the smallest possible amount of storage that might be allocated
58/// starting from the beginning of an object of a particular class.
59///
60/// This may be smaller than sizeof(RD) if RD has virtual base classes.
61CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
62 if (!RD->hasDefinition())
63 return CharUnits::One();
64
65 auto &layout = getContext().getASTRecordLayout(D: RD);
66
67 // If the class is final, then we know that the pointer points to an
68 // object of that type and can use the full alignment.
69 if (RD->isEffectivelyFinal())
70 return layout.getSize();
71
72 // Otherwise, we have to assume it could be a subclass.
73 return std::max(a: layout.getNonVirtualSize(), b: CharUnits::One());
74}
75
76/// Return the best known alignment for a pointer to a virtual base,
77/// given the alignment of a pointer to the derived class.
78CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
79 const CXXRecordDecl *derivedClass,
80 const CXXRecordDecl *vbaseClass) {
81 // The basic idea here is that an underaligned derived pointer might
82 // indicate an underaligned base pointer.
83
84 assert(vbaseClass->isCompleteDefinition());
85 auto &baseLayout = getContext().getASTRecordLayout(D: vbaseClass);
86 CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
87
88 return getDynamicOffsetAlignment(ActualAlign: actualDerivedAlign, Class: derivedClass,
89 ExpectedTargetAlign: expectedVBaseAlign);
90}
91
92CharUnits
93CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
94 const CXXRecordDecl *baseDecl,
95 CharUnits expectedTargetAlign) {
96 // If the base is an incomplete type (which is, alas, possible with
97 // member pointers), be pessimistic.
98 if (!baseDecl->isCompleteDefinition())
99 return std::min(a: actualBaseAlign, b: expectedTargetAlign);
100
101 auto &baseLayout = getContext().getASTRecordLayout(D: baseDecl);
102 CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
103
104 // If the class is properly aligned, assume the target offset is, too.
105 //
106 // This actually isn't necessarily the right thing to do --- if the
107 // class is a complete object, but it's only properly aligned for a
108 // base subobject, then the alignments of things relative to it are
109 // probably off as well. (Note that this requires the alignment of
110 // the target to be greater than the NV alignment of the derived
111 // class.)
112 //
113 // However, our approach to this kind of under-alignment can only
114 // ever be best effort; after all, we're never going to propagate
115 // alignments through variables or parameters. Note, in particular,
116 // that constructing a polymorphic type in an address that's less
117 // than pointer-aligned will generally trap in the constructor,
118 // unless we someday add some sort of attribute to change the
119 // assumed alignment of 'this'. So our goal here is pretty much
120 // just to allow the user to explicitly say that a pointer is
121 // under-aligned and then safely access its fields and vtables.
122 if (actualBaseAlign >= expectedBaseAlign) {
123 return expectedTargetAlign;
124 }
125
126 // Otherwise, we might be offset by an arbitrary multiple of the
127 // actual alignment. The correct adjustment is to take the min of
128 // the two alignments.
129 return std::min(a: actualBaseAlign, b: expectedTargetAlign);
130}
131
132Address CodeGenFunction::LoadCXXThisAddress() {
133 assert(CurFuncDecl && "loading 'this' without a func declaration?");
134 auto *MD = cast<CXXMethodDecl>(Val: CurFuncDecl);
135
136 // Lazily compute CXXThisAlignment.
137 if (CXXThisAlignment.isZero()) {
138 // Just use the best known alignment for the parent.
139 // TODO: if we're currently emitting a complete-object ctor/dtor,
140 // we can always use the complete-object alignment.
141 CXXThisAlignment = CGM.getClassPointerAlignment(RD: MD->getParent());
142 }
143
144 return makeNaturalAddressForPointer(
145 Ptr: LoadCXXThis(), T: MD->getFunctionObjectParameterType(), Alignment: CXXThisAlignment,
146 ForPointeeType: false, BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
147}
148
149/// Emit the address of a field using a member data pointer.
150///
151/// \param E Only used for emergency diagnostics
152Address CodeGenFunction::EmitCXXMemberDataPointerAddress(
153 const Expr *E, Address base, llvm::Value *memberPtr,
154 const MemberPointerType *memberPtrType, bool IsInBounds,
155 LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
156 // Ask the ABI to compute the actual address.
157 llvm::Value *ptr = CGM.getCXXABI().EmitMemberDataPointerAddress(
158 CGF&: *this, E, Base: base, MemPtr: memberPtr, MPT: memberPtrType, IsInBounds);
159
160 QualType memberType = memberPtrType->getPointeeType();
161 CharUnits memberAlign =
162 CGM.getNaturalTypeAlignment(T: memberType, BaseInfo, TBAAInfo);
163 memberAlign = CGM.getDynamicOffsetAlignment(
164 actualBaseAlign: base.getAlignment(), baseDecl: memberPtrType->getMostRecentCXXRecordDecl(),
165 expectedTargetAlign: memberAlign);
166 return Address(ptr, ConvertTypeForMem(T: memberPtrType->getPointeeType()),
167 memberAlign);
168}
169
170CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
171 const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
172 CastExpr::path_const_iterator End) {
173 CharUnits Offset = CharUnits::Zero();
174
175 const ASTContext &Context = getContext();
176 const CXXRecordDecl *RD = DerivedClass;
177
178 for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
179 const CXXBaseSpecifier *Base = *I;
180 assert(!Base->isVirtual() && "Should not see virtual bases here!");
181
182 // Get the layout.
183 const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
184
185 const auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
186 // Add the offset.
187 Offset += Layout.getBaseClassOffset(Base: BaseDecl);
188
189 RD = BaseDecl;
190 }
191
192 return Offset;
193}
194
195llvm::Constant *CodeGenModule::GetNonVirtualBaseClassOffset(
196 const CXXRecordDecl *ClassDecl, CastExpr::path_const_iterator PathBegin,
197 CastExpr::path_const_iterator PathEnd) {
198 assert(PathBegin != PathEnd && "Base path should not be empty!");
199
200 CharUnits Offset =
201 computeNonVirtualBaseClassOffset(DerivedClass: ClassDecl, Start: PathBegin, End: PathEnd);
202 if (Offset.isZero())
203 return nullptr;
204
205 llvm::Type *PtrDiffTy =
206 getTypes().ConvertType(T: getContext().getPointerDiffType());
207
208 return llvm::ConstantInt::get(Ty: PtrDiffTy, V: Offset.getQuantity());
209}
210
211/// Gets the address of a direct base class within a complete object.
212/// This should only be used for (1) non-virtual bases or (2) virtual bases
213/// when the type is known to be complete (e.g. in complete destructors).
214///
215/// The object pointed to by 'This' is assumed to be non-null.
216Address CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(
217 Address This, const CXXRecordDecl *Derived, const CXXRecordDecl *Base,
218 bool BaseIsVirtual) {
219 // 'this' must be a pointer (in some address space) to Derived.
220 assert(This.getElementType() == ConvertType(Derived));
221
222 // Compute the offset of the virtual base.
223 CharUnits Offset;
224 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: Derived);
225 if (BaseIsVirtual)
226 Offset = Layout.getVBaseClassOffset(VBase: Base);
227 else
228 Offset = Layout.getBaseClassOffset(Base);
229
230 // Shift and cast down to the base type.
231 // TODO: for complete types, this should be possible with a GEP.
232 Address V = This;
233 if (!Offset.isZero()) {
234 V = V.withElementType(ElemTy: Int8Ty);
235 V = Builder.CreateConstInBoundsByteGEP(Addr: V, Offset);
236 }
237 return V.withElementType(ElemTy: ConvertType(T: Base));
238}
239
240static Address ApplyNonVirtualAndVirtualOffset(
241 CodeGenFunction &CGF, Address addr, CharUnits nonVirtualOffset,
242 llvm::Value *virtualOffset, const CXXRecordDecl *derivedClass,
243 const CXXRecordDecl *nearestVBase) {
244 // Assert that we have something to do.
245 assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
246
247 // Compute the offset from the static and dynamic components.
248 llvm::Value *baseOffset;
249 if (!nonVirtualOffset.isZero()) {
250 llvm::Type *OffsetType =
251 (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
252 CGF.CGM.getLangOpts().RelativeCXXABIVTables)
253 ? CGF.Int32Ty
254 : CGF.PtrDiffTy;
255 baseOffset =
256 llvm::ConstantInt::get(Ty: OffsetType, V: nonVirtualOffset.getQuantity());
257 if (virtualOffset) {
258 baseOffset = CGF.Builder.CreateAdd(LHS: virtualOffset, RHS: baseOffset);
259 }
260 } else {
261 baseOffset = virtualOffset;
262 }
263
264 // Apply the base offset.
265 llvm::Value *ptr = addr.emitRawPointer(CGF);
266 ptr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ptr, IdxList: baseOffset, Name: "add.ptr");
267
268 // If we have a virtual component, the alignment of the result will
269 // be relative only to the known alignment of that vbase.
270 CharUnits alignment;
271 if (virtualOffset) {
272 assert(nearestVBase && "virtual offset without vbase?");
273 alignment = CGF.CGM.getVBaseAlignment(actualDerivedAlign: addr.getAlignment(), derivedClass,
274 vbaseClass: nearestVBase);
275 } else {
276 alignment = addr.getAlignment();
277 }
278 alignment = alignment.alignmentAtOffset(offset: nonVirtualOffset);
279
280 return Address(ptr, CGF.Int8Ty, alignment);
281}
282
283Address CodeGenFunction::GetAddressOfBaseClass(
284 Address Value, const CXXRecordDecl *Derived,
285 CastExpr::path_const_iterator PathBegin,
286 CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
287 SourceLocation Loc) {
288 assert(PathBegin != PathEnd && "Base path should not be empty!");
289
290 CastExpr::path_const_iterator Start = PathBegin;
291 const CXXRecordDecl *VBase = nullptr;
292
293 // Sema has done some convenient canonicalization here: if the
294 // access path involved any virtual steps, the conversion path will
295 // *start* with a step down to the correct virtual base subobject,
296 // and hence will not require any further steps.
297 if ((*Start)->isVirtual()) {
298 VBase = (*Start)->getType()->castAsCXXRecordDecl();
299 ++Start;
300 }
301
302 // Compute the static offset of the ultimate destination within its
303 // allocating subobject (the virtual base, if there is one, or else
304 // the "complete" object that we see).
305 CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
306 DerivedClass: VBase ? VBase : Derived, Start, End: PathEnd);
307
308 // If there's a virtual step, we can sometimes "devirtualize" it.
309 // For now, that's limited to when the derived type is final.
310 // TODO: "devirtualize" this for accesses to known-complete objects.
311 if (VBase && Derived->hasAttr<FinalAttr>()) {
312 const ASTRecordLayout &layout = getContext().getASTRecordLayout(D: Derived);
313 CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
314 NonVirtualOffset += vBaseOffset;
315 VBase = nullptr; // we no longer have a virtual step
316 }
317
318 // Get the base pointer type.
319 llvm::Type *BaseValueTy = ConvertType(T: (PathEnd[-1])->getType());
320 llvm::Type *PtrTy = llvm::PointerType::get(
321 C&: CGM.getLLVMContext(), AddressSpace: Value.getType()->getPointerAddressSpace());
322
323 CanQualType DerivedTy = getContext().getCanonicalTagType(TD: Derived);
324 CharUnits DerivedAlign = CGM.getClassPointerAlignment(RD: Derived);
325
326 // If the static offset is zero and we don't have a virtual step,
327 // just do a bitcast; null checks are unnecessary.
328 if (NonVirtualOffset.isZero() && !VBase) {
329 if (sanitizePerformTypeCheck()) {
330 SanitizerSet SkippedChecks;
331 SkippedChecks.set(K: SanitizerKind::Null, Value: !NullCheckValue);
332 EmitTypeCheck(TCK: TCK_Upcast, Loc, V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy,
333 Alignment: DerivedAlign, SkippedChecks);
334 }
335 return Value.withElementType(ElemTy: BaseValueTy);
336 }
337
338 llvm::BasicBlock *origBB = nullptr;
339 llvm::BasicBlock *endBB = nullptr;
340
341 // Skip over the offset (and the vtable load) if we're supposed to
342 // null-check the pointer.
343 if (NullCheckValue) {
344 origBB = Builder.GetInsertBlock();
345 llvm::BasicBlock *notNullBB = createBasicBlock(name: "cast.notnull");
346 endBB = createBasicBlock(name: "cast.end");
347
348 llvm::Value *isNull = Builder.CreateIsNull(Addr: Value);
349 Builder.CreateCondBr(Cond: isNull, True: endBB, False: notNullBB);
350 EmitBlock(BB: notNullBB);
351 }
352
353 if (sanitizePerformTypeCheck()) {
354 SanitizerSet SkippedChecks;
355 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
356 EmitTypeCheck(TCK: VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
357 V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy, Alignment: DerivedAlign,
358 SkippedChecks);
359 }
360
361 // Compute the virtual offset.
362 llvm::Value *VirtualOffset = nullptr;
363 if (VBase) {
364 VirtualOffset =
365 CGM.getCXXABI().GetVirtualBaseClassOffset(CGF&: *this, This: Value, ClassDecl: Derived, BaseClassDecl: VBase);
366 }
367
368 // Apply both offsets.
369 Value = ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: Value, nonVirtualOffset: NonVirtualOffset,
370 virtualOffset: VirtualOffset, derivedClass: Derived, nearestVBase: VBase);
371
372 // Cast to the destination type.
373 Value = Value.withElementType(ElemTy: BaseValueTy);
374
375 // Build a phi if we needed a null check.
376 if (NullCheckValue) {
377 llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
378 Builder.CreateBr(Dest: endBB);
379 EmitBlock(BB: endBB);
380
381 llvm::PHINode *PHI = Builder.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "cast.result");
382 PHI->addIncoming(V: Value.emitRawPointer(CGF&: *this), BB: notNullBB);
383 PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: PtrTy), BB: origBB);
384 Value = Value.withPointer(NewPointer: PHI, IsKnownNonNull: NotKnownNonNull);
385 }
386
387 return Value;
388}
389
390Address CodeGenFunction::GetAddressOfDerivedClass(
391 Address BaseAddr, const CXXRecordDecl *Derived,
392 CastExpr::path_const_iterator PathBegin,
393 CastExpr::path_const_iterator PathEnd, bool NullCheckValue) {
394 assert(PathBegin != PathEnd && "Base path should not be empty!");
395
396 CanQualType DerivedTy = getContext().getCanonicalTagType(TD: Derived);
397 llvm::Type *DerivedValueTy = ConvertType(T: DerivedTy);
398
399 llvm::Value *NonVirtualOffset =
400 CGM.GetNonVirtualBaseClassOffset(ClassDecl: Derived, PathBegin, PathEnd);
401
402 if (!NonVirtualOffset) {
403 // No offset, we can just cast back.
404 return BaseAddr.withElementType(ElemTy: DerivedValueTy);
405 }
406
407 llvm::BasicBlock *CastNull = nullptr;
408 llvm::BasicBlock *CastNotNull = nullptr;
409 llvm::BasicBlock *CastEnd = nullptr;
410
411 if (NullCheckValue) {
412 CastNull = createBasicBlock(name: "cast.null");
413 CastNotNull = createBasicBlock(name: "cast.notnull");
414 CastEnd = createBasicBlock(name: "cast.end");
415
416 llvm::Value *IsNull = Builder.CreateIsNull(Addr: BaseAddr);
417 Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
418 EmitBlock(BB: CastNotNull);
419 }
420
421 // Apply the offset.
422 Address Addr = BaseAddr.withElementType(ElemTy: Int8Ty);
423 Addr = Builder.CreateInBoundsGEP(
424 Addr, IdxList: Builder.CreateNeg(V: NonVirtualOffset), ElementType: Int8Ty,
425 Align: CGM.getClassPointerAlignment(RD: Derived), Name: "sub.ptr");
426
427 // Just cast.
428 Addr = Addr.withElementType(ElemTy: DerivedValueTy);
429
430 // Produce a PHI if we had a null-check.
431 if (NullCheckValue) {
432 Builder.CreateBr(Dest: CastEnd);
433 EmitBlock(BB: CastNull);
434 Builder.CreateBr(Dest: CastEnd);
435 EmitBlock(BB: CastEnd);
436
437 llvm::Value *Value = Addr.emitRawPointer(CGF&: *this);
438 llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
439 PHI->addIncoming(V: Value, BB: CastNotNull);
440 PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: Value->getType()), BB: CastNull);
441 return Address(PHI, Addr.getElementType(),
442 CGM.getClassPointerAlignment(RD: Derived));
443 }
444
445 return Addr;
446}
447
448llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
449 bool ForVirtualBase,
450 bool Delegating) {
451 if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
452 // This constructor/destructor does not need a VTT parameter.
453 return nullptr;
454 }
455
456 const CXXRecordDecl *RD = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParent();
457 const CXXRecordDecl *Base = cast<CXXMethodDecl>(Val: GD.getDecl())->getParent();
458
459 uint64_t SubVTTIndex;
460
461 if (Delegating) {
462 // If this is a delegating constructor call, just load the VTT.
463 return LoadCXXVTT();
464 } else if (RD == Base) {
465 // If the record matches the base, this is the complete ctor/dtor
466 // variant calling the base variant in a class with virtual bases.
467 assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
468 "doing no-op VTT offset in base dtor/ctor?");
469 assert(!ForVirtualBase && "Can't have same class as virtual base!");
470 SubVTTIndex = 0;
471 } else {
472 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
473 CharUnits BaseOffset = ForVirtualBase ? Layout.getVBaseClassOffset(VBase: Base)
474 : Layout.getBaseClassOffset(Base);
475
476 SubVTTIndex =
477 CGM.getVTables().getSubVTTIndex(RD, Base: BaseSubobject(Base, BaseOffset));
478 assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
479 }
480
481 llvm::Value *VTT;
482 if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
483 // A VTT parameter was passed to the constructor, use it.
484 VTT = LoadCXXVTT();
485 } else {
486 // We're the complete constructor, so get the VTT by name.
487 VTT = CGM.getVTables().GetAddrOfVTT(RD);
488 }
489 return Builder.CreateConstInBoundsGEP1_64(Ty: CGM.GlobalsInt8PtrTy, Ptr: VTT,
490 Idx0: SubVTTIndex);
491}
492
493namespace {
494/// Call the destructor for a direct base class.
495struct CallBaseDtor final : EHScopeStack::Cleanup {
496 const CXXRecordDecl *BaseClass;
497 bool BaseIsVirtual;
498 CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
499 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
500
501 void Emit(CodeGenFunction &CGF, Flags flags) override {
502 const CXXRecordDecl *DerivedClass =
503 cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
504
505 const CXXDestructorDecl *D = BaseClass->getDestructor();
506 // We are already inside a destructor, so presumably the object being
507 // destroyed should have the expected type.
508 QualType ThisTy = D->getFunctionObjectParameterType();
509 Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
510 This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
511 CGF.EmitCXXDestructorCall(D, Type: Dtor_Base, ForVirtualBase: BaseIsVirtual,
512 /*Delegating=*/false, This: Addr, ThisTy);
513 }
514};
515
516} // end anonymous namespace
517
518static void EmitBaseInitializer(CodeGenFunction &CGF,
519 const CXXRecordDecl *ClassDecl,
520 CXXCtorInitializer *BaseInit) {
521 assert(BaseInit->isBaseInitializer() && "Must have base initializer!");
522
523 Address ThisPtr = CGF.LoadCXXThisAddress();
524
525 const auto *BaseClassDecl = BaseInit->getBaseClass()->castAsCXXRecordDecl();
526
527 bool isBaseVirtual = BaseInit->isBaseVirtual();
528
529 // If the initializer for the base (other than the constructor
530 // itself) accesses 'this' in any way, we need to initialize the
531 // vtables.
532 if (CodeGenUtils::baseInitializerUsesThis(Ctx&: CGF.getContext(),
533 Init: BaseInit->getInit()))
534 CGF.InitializeVTablePointers(ClassDecl);
535
536 // We can pretend to be a complete class because it only matters for
537 // virtual bases, and we only do virtual bases for complete ctors.
538 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
539 This: ThisPtr, Derived: ClassDecl, Base: BaseClassDecl, BaseIsVirtual: isBaseVirtual);
540 AggValueSlot AggSlot = AggValueSlot::forAddr(
541 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
542 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
543 mayOverlap: CGF.getOverlapForBaseInit(RD: ClassDecl, BaseRD: BaseClassDecl, IsVirtual: isBaseVirtual));
544
545 CGF.EmitAggExpr(E: BaseInit->getInit(), AS: AggSlot);
546
547 if (CGF.CGM.getLangOpts().Exceptions &&
548 !BaseClassDecl->hasTrivialDestructor())
549 CGF.EHStack.pushCleanup<CallBaseDtor>(Kind: EHCleanup, A: BaseClassDecl,
550 A: isBaseVirtual);
551}
552
553static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
554 CXXCtorInitializer *MemberInit,
555 LValue &LHS) {
556 FieldDecl *Field = MemberInit->getAnyMember();
557 if (MemberInit->isIndirectMemberInitializer()) {
558 // If we are initializing an anonymous union field, drill down to the field.
559 IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
560 for (const auto *I : IndirectField->chain())
561 LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field: cast<FieldDecl>(Val: I));
562 } else {
563 LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field);
564 }
565}
566
567static void EmitMemberInitializer(CodeGenFunction &CGF,
568 const CXXRecordDecl *ClassDecl,
569 CXXCtorInitializer *MemberInit,
570 const CXXConstructorDecl *Constructor,
571 FunctionArgList &Args) {
572 ApplyAtomGroup Grp(CGF.getDebugInfo());
573 ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
574 assert(MemberInit->isAnyMemberInitializer() &&
575 "Must have member initializer!");
576 assert(MemberInit->getInit() && "Must have initializer!");
577
578 // non-static data member initializers.
579 FieldDecl *Field = MemberInit->getAnyMember();
580 QualType FieldType = Field->getType();
581
582 llvm::Value *ThisPtr = CGF.LoadCXXThis();
583 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
584 LValue LHS;
585
586 // If a base constructor is being emitted, create an LValue that has the
587 // non-virtual alignment.
588 if (CGF.CurGD.getCtorType() == Ctor_Base)
589 LHS = CGF.MakeNaturalAlignPointeeAddrLValue(V: ThisPtr, T: RecordTy);
590 else
591 LHS = CGF.MakeNaturalAlignAddrLValue(V: ThisPtr, T: RecordTy);
592
593 EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
594
595 // Special case: if we are in a copy or move constructor, and we are copying
596 // an array of PODs or classes with trivial copy constructors, ignore the
597 // AST and perform the copy we know is equivalent.
598 // FIXME: This is hacky at best... if we had a bit more explicit information
599 // in the AST, we could generalize it more easily.
600 const ConstantArrayType *Array =
601 CGF.getContext().getAsConstantArrayType(T: FieldType);
602 if (Array && Constructor->isDefaulted() &&
603 Constructor->isCopyOrMoveConstructor()) {
604 QualType BaseElementTy = CGF.getContext().getBaseElementType(VAT: Array);
605 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
606 if (BaseElementTy.isPODType(Context: CGF.getContext()) ||
607 (CE && CE->getConstructor()->isMemcpyEquivalentSpecialMember(
608 Ctx: CGF.getContext()))) {
609 unsigned SrcArgIndex =
610 CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
611 llvm::Value *SrcPtr =
612 CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: Args[SrcArgIndex]));
613 LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
614 LValue Src = CGF.EmitLValueForFieldInitialization(Base: ThisRHSLV, Field);
615
616 // Copy the aggregate.
617 CGF.EmitAggregateCopy(Dest: LHS, Src, EltTy: FieldType,
618 MayOverlap: CGF.getOverlapForFieldInit(FD: Field),
619 isVolatile: LHS.isVolatileQualified());
620 // Ensure that we destroy the objects if an exception is thrown later in
621 // the constructor.
622 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
623 if (CGF.needsEHCleanup(kind: dtorKind))
624 CGF.pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
625 return;
626 }
627 }
628
629 CGF.EmitInitializerForField(Field, LHS, Init: MemberInit->getInit());
630}
631
632void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
633 Expr *Init) {
634 QualType FieldType = Field->getType();
635 switch (getEvaluationKind(T: FieldType)) {
636 case TEK_Scalar:
637 if (LHS.isSimple()) {
638 EmitExprAsInit(init: Init, D: Field, lvalue: LHS, capturedByInit: false);
639 } else {
640 RValue RHS = RValue::get(V: EmitScalarExpr(E: Init));
641 EmitStoreThroughLValue(Src: RHS, Dst: LHS);
642 }
643 break;
644 case TEK_Complex:
645 EmitComplexExprIntoLValue(E: Init, dest: LHS, /*isInit*/ true);
646 break;
647 case TEK_Aggregate: {
648 AggValueSlot Slot = AggValueSlot::forLValue(
649 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
650 isAliased: AggValueSlot::IsNotAliased, mayOverlap: getOverlapForFieldInit(FD: Field),
651 isZeroed: AggValueSlot::IsNotZeroed,
652 // Checks are made by the code that calls constructor.
653 isChecked: AggValueSlot::IsSanitizerChecked);
654 EmitAggExpr(E: Init, AS: Slot);
655 break;
656 }
657 }
658
659 // Ensure that we destroy this object if an exception is thrown
660 // later in the constructor.
661 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
662 if (needsEHCleanup(kind: dtorKind))
663 pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
664}
665
666/// Checks whether the given constructor is a valid subject for the
667/// complete-to-base constructor delegation optimization, i.e.
668/// emitting the complete constructor as a simple call to the base
669/// constructor.
670bool CodeGenFunction::IsConstructorDelegationValid(
671 const CXXConstructorDecl *Ctor) {
672
673 // Currently we disable the optimization for classes with virtual
674 // bases because (1) the addresses of parameter variables need to be
675 // consistent across all initializers but (2) the delegate function
676 // call necessarily creates a second copy of the parameter variable.
677 //
678 // The limiting example (purely theoretical AFAIK):
679 // struct A { A(int &c) { c++; } };
680 // struct B : virtual A {
681 // B(int count) : A(count) { printf("%d\n", count); }
682 // };
683 // ...although even this example could in principle be emitted as a
684 // delegation since the address of the parameter doesn't escape.
685 if (Ctor->getParent()->getNumVBases()) {
686 // TODO: white-list trivial vbase initializers. This case wouldn't
687 // be subject to the restrictions below.
688
689 // TODO: white-list cases where:
690 // - there are no non-reference parameters to the constructor
691 // - the initializers don't access any non-reference parameters
692 // - the initializers don't take the address of non-reference
693 // parameters
694 // - etc.
695 // If we ever add any of the above cases, remember that:
696 // - function-try-blocks will always exclude this optimization
697 // - we need to perform the constructor prologue and cleanup in
698 // EmitConstructorBody.
699
700 return false;
701 }
702
703 // We also disable the optimization for variadic functions because
704 // it's impossible to "re-pass" varargs.
705 if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
706 return false;
707
708 // FIXME: Decide if we can do a delegation of a delegating constructor.
709 if (Ctor->isDelegatingConstructor())
710 return false;
711
712 return true;
713}
714
715// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
716// to poison the extra field paddings inserted under
717// -fsanitize-address-field-padding=1|2.
718void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
719 ASTContext &Context = getContext();
720 const CXXRecordDecl *ClassDecl =
721 Prologue ? cast<CXXConstructorDecl>(Val: CurGD.getDecl())->getParent()
722 : cast<CXXDestructorDecl>(Val: CurGD.getDecl())->getParent();
723 if (!ClassDecl->mayInsertExtraPadding())
724 return;
725
726 struct SizeAndOffset {
727 uint64_t Size;
728 uint64_t Offset;
729 };
730
731 unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
732 const ASTRecordLayout &Info = Context.getASTRecordLayout(D: ClassDecl);
733
734 // Populate sizes and offsets of fields.
735 SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
736 for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
737 SSV[i].Offset =
738 Context.toCharUnitsFromBits(BitSize: Info.getFieldOffset(FieldNo: i)).getQuantity();
739
740 size_t NumFields = 0;
741 for (const auto *Field : ClassDecl->fields()) {
742 const FieldDecl *D = Field;
743 auto FieldInfo = Context.getTypeInfoInChars(T: D->getType());
744 CharUnits FieldSize = FieldInfo.Width;
745 assert(NumFields < SSV.size());
746 SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
747 NumFields++;
748 }
749 assert(NumFields == SSV.size());
750 if (SSV.size() <= 1)
751 return;
752
753 // We will insert calls to __asan_* run-time functions.
754 // LLVM AddressSanitizer pass may decide to inline them later.
755 llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
756 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, isVarArg: false);
757 llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
758 Ty: FTy, Name: Prologue ? "__asan_poison_intra_object_redzone"
759 : "__asan_unpoison_intra_object_redzone");
760
761 llvm::Value *ThisPtr = LoadCXXThis();
762 ThisPtr = Builder.CreatePtrToInt(V: ThisPtr, DestTy: IntPtrTy);
763 uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
764 // For each field check if it has sufficient padding,
765 // if so (un)poison it with a call.
766 for (size_t i = 0; i < SSV.size(); i++) {
767 uint64_t AsanAlignment = 8;
768 uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
769 uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
770 uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
771 if (PoisonSize < AsanAlignment || !SSV[i].Size ||
772 (NextField % AsanAlignment) != 0)
773 continue;
774 Builder.CreateCall(
775 Callee: F, Args: {Builder.CreateAdd(LHS: ThisPtr, RHS: Builder.getIntN(N: PtrSize, C: EndOffset)),
776 Builder.getIntN(N: PtrSize, C: PoisonSize)});
777 }
778}
779
780/// EmitConstructorBody - Emits the body of the current constructor.
781void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
782 EmitAsanPrologueOrEpilogue(Prologue: true);
783 const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(Val: CurGD.getDecl());
784 CXXCtorType CtorType = CurGD.getCtorType();
785
786 assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
787 CtorType == Ctor_Complete) &&
788 "can only generate complete ctor for this ABI");
789
790 // Before we go any further, try the complete->base constructor
791 // delegation optimization.
792 if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
793 CGM.getTarget().getCXXABI().hasConstructorVariants()) {
794 EmitDelegateCXXConstructorCall(Ctor, CtorType: Ctor_Base, Args, Loc: Ctor->getEndLoc());
795 return;
796 }
797
798 const FunctionDecl *Definition = nullptr;
799 Stmt *Body = Ctor->getBody(Definition);
800 assert(Definition == Ctor && "emitting wrong constructor body");
801
802 // Enter the function-try-block before the constructor prologue if
803 // applicable.
804 bool IsTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
805 if (IsTryBody)
806 EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
807
808 incrementProfileCounter(S: Body);
809 maybeCreateMCDCCondBitmap();
810
811 RunCleanupsScope RunCleanups(*this);
812
813 // TODO: in restricted cases, we can emit the vbase initializers of
814 // a complete ctor and then delegate to the base ctor.
815
816 // Emit the constructor prologue, i.e. the base and member
817 // initializers.
818 EmitCtorPrologue(CD: Ctor, Type: CtorType, Args);
819
820 // Emit the body of the statement.
821 if (IsTryBody)
822 EmitStmt(S: cast<CXXTryStmt>(Val: Body)->getTryBlock());
823 else if (Body)
824 EmitStmt(S: Body);
825
826 // Emit any cleanup blocks associated with the member or base
827 // initializers, which includes (along the exceptional path) the
828 // destructors for those members and bases that were fully
829 // constructed.
830 RunCleanups.ForceCleanup();
831
832 if (IsTryBody)
833 ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
834}
835
836namespace {
837/// RAII object to indicate that codegen is copying the value representation
838/// instead of the object representation. Useful when copying a struct or
839/// class which has uninitialized members and we're only performing
840/// lvalue-to-rvalue conversion on the object but not its members.
841class CopyingValueRepresentation {
842public:
843 explicit CopyingValueRepresentation(CodeGenFunction &CGF)
844 : CGF(CGF), OldSanOpts(CGF.SanOpts) {
845 CGF.SanOpts.set(K: SanitizerKind::Bool, Value: false);
846 CGF.SanOpts.set(K: SanitizerKind::Enum, Value: false);
847 }
848 ~CopyingValueRepresentation() { CGF.SanOpts = OldSanOpts; }
849
850private:
851 CodeGenFunction &CGF;
852 SanitizerSet OldSanOpts;
853};
854} // end anonymous namespace
855
856namespace {
857class FieldMemcpyizer {
858public:
859 FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
860 const VarDecl *SrcRec)
861 : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
862 RecLayout(CGF.getContext().getASTRecordLayout(D: ClassDecl)),
863 FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
864 LastFieldOffset(0), LastAddedFieldIndex(0) {}
865
866 bool isMemcpyableField(FieldDecl *F) const {
867 // Never memcpy fields when we are adding poisoned paddings.
868 if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
869 return false;
870 Qualifiers Qual = F->getType().getQualifiers();
871 if (Qual.hasVolatile() || Qual.hasObjCLifetime())
872 return false;
873 if (PointerAuthQualifier Q = F->getType().getPointerAuth();
874 Q && Q.isAddressDiscriminated())
875 return false;
876 // Non-trivially-copyable fields with pointer field protection need to be
877 // copied one by one.
878 if (!CGF.getContext().arePFPFieldsTriviallyCopyable(RD: ClassDecl) &&
879 CGF.getContext().isPFPField(Field: F))
880 return false;
881 return true;
882 }
883
884 void addMemcpyableField(FieldDecl *F) {
885 if (CodeGenUtils::isEmptyFieldForLayout(Ctx: CGF.getContext(), FD: F))
886 return;
887 if (!FirstField)
888 addInitialField(F);
889 else
890 addNextField(F);
891 }
892
893 CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
894 ASTContext &Ctx = CGF.getContext();
895 unsigned LastFieldSize =
896 LastField->isBitField()
897 ? LastField->getBitWidthValue()
898 : Ctx.toBits(
899 CharSize: Ctx.getTypeInfoDataSizeInChars(T: LastField->getType()).Width);
900 uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
901 FirstByteOffset + Ctx.getCharWidth() - 1;
902 CharUnits MemcpySize = Ctx.toCharUnitsFromBits(BitSize: MemcpySizeBits);
903 return MemcpySize;
904 }
905
906 void emitMemcpy() {
907 // Give the subclass a chance to bail out if it feels the memcpy isn't
908 // worth it (e.g. Hasn't aggregated enough data).
909 if (!FirstField) {
910 return;
911 }
912
913 // Unreachable, e.g. after a noreturn member copy; drop it like EmitStmt.
914 if (!CGF.HaveInsertPoint()) {
915 reset();
916 return;
917 }
918
919 uint64_t FirstByteOffset;
920 if (FirstField->isBitField()) {
921 const CGRecordLayout &RL =
922 CGF.getTypes().getCGRecordLayout(FirstField->getParent());
923 const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FD: FirstField);
924 // FirstFieldOffset is not appropriate for bitfields,
925 // we need to use the storage offset instead.
926 FirstByteOffset = CGF.getContext().toBits(CharSize: BFInfo.StorageOffset);
927 } else {
928 FirstByteOffset = FirstFieldOffset;
929 }
930
931 CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
932 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
933 Address ThisPtr = CGF.LoadCXXThisAddress();
934 LValue DestLV = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
935 LValue Dest = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: FirstField);
936 llvm::Value *SrcPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: SrcRec));
937 LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
938 LValue Src = CGF.EmitLValueForFieldInitialization(Base: SrcLV, Field: FirstField);
939
940 emitMemcpyIR(DestPtr: Dest.isBitField() ? Dest.getBitFieldAddress()
941 : Dest.getAddress(),
942 SrcPtr: Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),
943 Size: MemcpySize);
944 reset();
945 }
946
947 void reset() { FirstField = nullptr; }
948
949protected:
950 CodeGenFunction &CGF;
951 const CXXRecordDecl *ClassDecl;
952
953private:
954 void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
955 DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
956 SrcPtr = SrcPtr.withElementType(ElemTy: CGF.Int8Ty);
957 auto *I = CGF.Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: Size.getQuantity());
958 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: nullptr);
959 }
960
961 void addInitialField(FieldDecl *F) {
962 FirstField = F;
963 LastField = F;
964 FirstFieldOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
965 LastFieldOffset = FirstFieldOffset;
966 LastAddedFieldIndex = F->getFieldIndex();
967 }
968
969 void addNextField(FieldDecl *F) {
970 // For the most part, the following invariant will hold:
971 // F->getFieldIndex() == LastAddedFieldIndex + 1
972 // The one exception is that Sema won't add a copy-initializer for an
973 // unnamed bitfield, which will show up here as a gap in the sequence.
974 assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
975 "Cannot aggregate fields out of order.");
976 LastAddedFieldIndex = F->getFieldIndex();
977
978 // The 'first' and 'last' fields are chosen by offset, rather than field
979 // index. This allows the code to support bitfields, as well as regular
980 // fields.
981 uint64_t FOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
982 if (FOffset < FirstFieldOffset) {
983 FirstField = F;
984 FirstFieldOffset = FOffset;
985 } else if (FOffset >= LastFieldOffset) {
986 LastField = F;
987 LastFieldOffset = FOffset;
988 }
989 }
990
991 const VarDecl *SrcRec;
992 const ASTRecordLayout &RecLayout;
993 FieldDecl *FirstField;
994 FieldDecl *LastField;
995 uint64_t FirstFieldOffset, LastFieldOffset;
996 unsigned LastAddedFieldIndex;
997};
998
999class ConstructorMemcpyizer : public FieldMemcpyizer {
1000private:
1001 /// Get source argument for copy constructor. Returns null if not a copy
1002 /// constructor.
1003 static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1004 const CXXConstructorDecl *CD,
1005 FunctionArgList &Args) {
1006 if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1007 return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1008 return nullptr;
1009 }
1010
1011 // Returns true if a CXXCtorInitializer represents a member initialization
1012 // that can be rolled into a memcpy.
1013 bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1014 if (!MemcpyableCtor)
1015 return false;
1016 FieldDecl *Field = MemberInit->getMember();
1017 assert(Field && "No field for member init.");
1018 QualType FieldType = Field->getType();
1019 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
1020
1021 // Bail out on non-memcpyable, not-trivially-copyable members.
1022 if (!(CE && CE->getConstructor()->isMemcpyEquivalentSpecialMember(
1023 Ctx: CGF.getContext())) &&
1024 !(FieldType.isTriviallyCopyableType(Context: CGF.getContext()) ||
1025 FieldType->isReferenceType()))
1026 return false;
1027
1028 // Bail out on volatile fields.
1029 if (!isMemcpyableField(F: Field))
1030 return false;
1031
1032 // Otherwise we're good.
1033 return true;
1034 }
1035
1036public:
1037 ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1038 FunctionArgList &Args)
1039 : FieldMemcpyizer(CGF, CD->getParent(),
1040 getTrivialCopySource(CGF, CD, Args)),
1041 ConstructorDecl(CD),
1042 MemcpyableCtor(CD->isDefaulted() && CD->isCopyOrMoveConstructor() &&
1043 CGF.getLangOpts().getGC() == LangOptions::NonGC),
1044 Args(Args) {}
1045
1046 void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1047 if (isMemberInitMemcpyable(MemberInit)) {
1048 AggregatedInits.push_back(Elt: MemberInit);
1049 addMemcpyableField(F: MemberInit->getMember());
1050 } else {
1051 emitAggregatedInits();
1052 EmitMemberInitializer(CGF, ClassDecl: ConstructorDecl->getParent(), MemberInit,
1053 Constructor: ConstructorDecl, Args);
1054 }
1055 }
1056
1057 void emitAggregatedInits() {
1058 if (AggregatedInits.size() <= 1) {
1059 // This memcpy is too small to be worthwhile. Fall back on default
1060 // codegen.
1061 if (!AggregatedInits.empty()) {
1062 CopyingValueRepresentation CVR(CGF);
1063 EmitMemberInitializer(CGF, ClassDecl: ConstructorDecl->getParent(),
1064 MemberInit: AggregatedInits[0], Constructor: ConstructorDecl, Args);
1065 AggregatedInits.clear();
1066 }
1067 reset();
1068 return;
1069 }
1070
1071 pushEHDestructors();
1072 ApplyAtomGroup Grp(CGF.getDebugInfo());
1073 emitMemcpy();
1074 AggregatedInits.clear();
1075 }
1076
1077 void pushEHDestructors() {
1078 Address ThisPtr = CGF.LoadCXXThisAddress();
1079 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
1080 LValue LHS = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
1081
1082 for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1083 CXXCtorInitializer *MemberInit = AggregatedInits[i];
1084 QualType FieldType = MemberInit->getAnyMember()->getType();
1085 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1086 if (!CGF.needsEHCleanup(kind: dtorKind))
1087 continue;
1088 LValue FieldLHS = LHS;
1089 EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS&: FieldLHS);
1090 CGF.pushEHDestroy(dtorKind, addr: FieldLHS.getAddress(), type: FieldType);
1091 }
1092 }
1093
1094 void finish() { emitAggregatedInits(); }
1095
1096private:
1097 const CXXConstructorDecl *ConstructorDecl;
1098 bool MemcpyableCtor;
1099 FunctionArgList &Args;
1100 SmallVector<CXXCtorInitializer *, 16> AggregatedInits;
1101};
1102
1103class AssignmentMemcpyizer : public FieldMemcpyizer {
1104private:
1105 // Returns the memcpyable field copied by the given statement, if one
1106 // exists. Otherwise returns null.
1107 FieldDecl *getMemcpyableField(Stmt *S) {
1108 if (!AssignmentsMemcpyable)
1109 return nullptr;
1110 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: S)) {
1111 // Recognise trivial assignments.
1112 if (BO->getOpcode() != BO_Assign)
1113 return nullptr;
1114 MemberExpr *ME = dyn_cast<MemberExpr>(Val: BO->getLHS());
1115 if (!ME)
1116 return nullptr;
1117 FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1118 if (!Field || !isMemcpyableField(F: Field))
1119 return nullptr;
1120 Stmt *RHS = BO->getRHS();
1121 if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(Val: RHS))
1122 RHS = EC->getSubExpr();
1123 if (!RHS)
1124 return nullptr;
1125 if (MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: RHS)) {
1126 if (ME2->getMemberDecl() == Field)
1127 return Field;
1128 }
1129 return nullptr;
1130 } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(Val: S)) {
1131 CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MCE->getCalleeDecl());
1132 if (!(MD && MD->isMemcpyEquivalentSpecialMember(Ctx: CGF.getContext())))
1133 return nullptr;
1134 MemberExpr *IOA = dyn_cast<MemberExpr>(Val: MCE->getImplicitObjectArgument());
1135 if (!IOA)
1136 return nullptr;
1137 FieldDecl *Field = dyn_cast<FieldDecl>(Val: IOA->getMemberDecl());
1138 if (!Field || !isMemcpyableField(F: Field))
1139 return nullptr;
1140 MemberExpr *Arg0 = dyn_cast<MemberExpr>(Val: MCE->getArg(Arg: 0));
1141 if (!Arg0 || Field != dyn_cast<FieldDecl>(Val: Arg0->getMemberDecl()))
1142 return nullptr;
1143 return Field;
1144 } else if (CallExpr *CE = dyn_cast<CallExpr>(Val: S)) {
1145 FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: CE->getCalleeDecl());
1146 if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1147 return nullptr;
1148 Expr *DstPtr = CE->getArg(Arg: 0);
1149 if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(Val: DstPtr))
1150 DstPtr = DC->getSubExpr();
1151 UnaryOperator *DUO = dyn_cast<UnaryOperator>(Val: DstPtr);
1152 if (!DUO || DUO->getOpcode() != UO_AddrOf)
1153 return nullptr;
1154 MemberExpr *ME = dyn_cast<MemberExpr>(Val: DUO->getSubExpr());
1155 if (!ME)
1156 return nullptr;
1157 FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1158 if (!Field || !isMemcpyableField(F: Field))
1159 return nullptr;
1160 Expr *SrcPtr = CE->getArg(Arg: 1);
1161 if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(Val: SrcPtr))
1162 SrcPtr = SC->getSubExpr();
1163 UnaryOperator *SUO = dyn_cast<UnaryOperator>(Val: SrcPtr);
1164 if (!SUO || SUO->getOpcode() != UO_AddrOf)
1165 return nullptr;
1166 MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: SUO->getSubExpr());
1167 if (!ME2 || Field != dyn_cast<FieldDecl>(Val: ME2->getMemberDecl()))
1168 return nullptr;
1169 return Field;
1170 }
1171
1172 return nullptr;
1173 }
1174
1175 bool AssignmentsMemcpyable;
1176 SmallVector<Stmt *, 16> AggregatedStmts;
1177
1178public:
1179 AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1180 FunctionArgList &Args)
1181 : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1182 AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1183 assert(Args.size() == 2);
1184 }
1185
1186 void emitAssignment(Stmt *S) {
1187 FieldDecl *F = getMemcpyableField(S);
1188 if (F) {
1189 addMemcpyableField(F);
1190 AggregatedStmts.push_back(Elt: S);
1191 } else {
1192 emitAggregatedStmts();
1193 CGF.EmitStmt(S);
1194 }
1195 }
1196
1197 void emitAggregatedStmts() {
1198 if (AggregatedStmts.size() <= 1) {
1199 if (!AggregatedStmts.empty()) {
1200 CopyingValueRepresentation CVR(CGF);
1201 CGF.EmitStmt(S: AggregatedStmts[0]);
1202 }
1203 reset();
1204 }
1205
1206 ApplyAtomGroup Grp(CGF.getDebugInfo());
1207 emitMemcpy();
1208 AggregatedStmts.clear();
1209 }
1210
1211 void finish() { emitAggregatedStmts(); }
1212};
1213
1214} // end anonymous namespace
1215
1216/// EmitCtorPrologue - This routine generates necessary code to initialize
1217/// base classes and non-static data members belonging to this constructor.
1218void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1219 CXXCtorType CtorType,
1220 FunctionArgList &Args) {
1221 if (CD->isDelegatingConstructor())
1222 return EmitDelegatingCXXConstructorCall(Ctor: CD, Args);
1223
1224 const CXXRecordDecl *ClassDecl = CD->getParent();
1225
1226 // Virtual base initializers aren't needed if:
1227 // - This is a base ctor variant
1228 // - There are no vbases
1229 // - The class is abstract, so a complete object of it cannot be constructed
1230 //
1231 // The check for an abstract class is necessary because sema may not have
1232 // marked virtual base destructors referenced.
1233 bool ConstructVBases = CtorType != Ctor_Base &&
1234 ClassDecl->getNumVBases() != 0 &&
1235 !ClassDecl->isAbstract();
1236
1237 // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1238 // constructor of a class with virtual bases takes an additional parameter to
1239 // conditionally construct the virtual bases. Emit that check here.
1240 llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1241 if (ConstructVBases &&
1242 !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1243 BaseCtorContinueBB =
1244 CGM.getCXXABI().EmitCtorCompleteObjectHandler(CGF&: *this, RD: ClassDecl);
1245 assert(BaseCtorContinueBB);
1246 }
1247
1248 // Create three separate ranges for the different types of initializers.
1249 auto AllInits = CD->inits();
1250
1251 // Find the boundaries between the three groups.
1252 auto VirtualBaseEnd = std::find_if(
1253 first: AllInits.begin(), last: AllInits.end(), pred: [](const CXXCtorInitializer *Init) {
1254 return !(Init->isBaseInitializer() && Init->isBaseVirtual());
1255 });
1256
1257 auto NonVirtualBaseEnd = std::find_if(first: VirtualBaseEnd, last: AllInits.end(),
1258 pred: [](const CXXCtorInitializer *Init) {
1259 return !Init->isBaseInitializer();
1260 });
1261
1262 // Create the three ranges.
1263 auto VirtualBaseInits = llvm::make_range(x: AllInits.begin(), y: VirtualBaseEnd);
1264 auto NonVirtualBaseInits =
1265 llvm::make_range(x: VirtualBaseEnd, y: NonVirtualBaseEnd);
1266 auto MemberInits = llvm::make_range(x: NonVirtualBaseEnd, y: AllInits.end());
1267
1268 // Process virtual base initializers, if necessary.
1269 if (ConstructVBases) {
1270 for (CXXCtorInitializer *Initializer : VirtualBaseInits) {
1271 SaveAndRestore ThisRAII(CXXThisValue);
1272 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1273 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1274 CodeGenUtils::isInitializerOfDynamicClass(BaseInit: Initializer))
1275 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1276 EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: Initializer);
1277 }
1278 }
1279
1280 if (BaseCtorContinueBB) {
1281 // Complete object handler should continue to the remaining initializers.
1282 Builder.CreateBr(Dest: BaseCtorContinueBB);
1283 EmitBlock(BB: BaseCtorContinueBB);
1284 }
1285
1286 // Then, non-virtual base initializers.
1287 for (CXXCtorInitializer *Initializer : NonVirtualBaseInits) {
1288 assert(!Initializer->isBaseVirtual());
1289 SaveAndRestore ThisRAII(CXXThisValue);
1290 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1291 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1292 CodeGenUtils::isInitializerOfDynamicClass(BaseInit: Initializer))
1293 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1294 EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: Initializer);
1295 }
1296
1297 InitializeVTablePointers(ClassDecl);
1298
1299 // And finally, initialize class members.
1300 FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1301 ConstructorMemcpyizer CM(*this, CD, Args);
1302 for (CXXCtorInitializer *Member : MemberInits) {
1303 assert(!Member->isBaseInitializer());
1304 assert(Member->isAnyMemberInitializer() &&
1305 "Delegating initializer on non-delegating constructor");
1306 CM.addMemberInitializer(MemberInit: Member);
1307 }
1308
1309 CM.finish();
1310}
1311
1312static void EmitConditionalArrayDtorCall(const CXXDestructorDecl *DD,
1313 CodeGenFunction &CGF,
1314 llvm::Value *ShouldDeleteCondition) {
1315 Address ThisPtr = CGF.LoadCXXThisAddress();
1316 llvm::BasicBlock *ScalarBB = CGF.createBasicBlock(name: "dtor.scalar");
1317 llvm::BasicBlock *callDeleteBB =
1318 CGF.createBasicBlock(name: "dtor.call_delete_after_array_destroy");
1319 llvm::BasicBlock *VectorBB = CGF.createBasicBlock(name: "dtor.vector");
1320 auto *CondTy = cast<llvm::IntegerType>(Val: ShouldDeleteCondition->getType());
1321 llvm::Value *CheckTheBitForArrayDestroy = CGF.Builder.CreateAnd(
1322 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 2));
1323 llvm::Value *ShouldDestroyArray =
1324 CGF.Builder.CreateIsNull(Arg: CheckTheBitForArrayDestroy);
1325 CGF.Builder.CreateCondBr(Cond: ShouldDestroyArray, True: ScalarBB, False: VectorBB);
1326
1327 CGF.EmitBlock(BB: VectorBB);
1328
1329 llvm::Value *numElements = nullptr;
1330 llvm::Value *allocatedPtr = nullptr;
1331 CharUnits cookieSize;
1332 QualType EltTy = DD->getThisType()->getPointeeType();
1333 CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr: ThisPtr, ElementType: EltTy, NumElements&: numElements,
1334 AllocPtr&: allocatedPtr, CookieSize&: cookieSize);
1335
1336 // Destroy the elements.
1337 QualType::DestructionKind dtorKind = EltTy.isDestructedType();
1338
1339 assert(dtorKind);
1340 assert(numElements && "no element count for a type with a destructor!");
1341
1342 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: EltTy);
1343 CharUnits elementAlign =
1344 ThisPtr.getAlignment().alignmentOfArrayElement(elementSize);
1345
1346 llvm::Value *arrayBegin = ThisPtr.emitRawPointer(CGF);
1347 llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
1348 Ty: ThisPtr.getElementType(), Ptr: arrayBegin, IdxList: numElements, Name: "delete.end");
1349
1350 // We already checked that the array is not 0-length before entering vector
1351 // deleting dtor.
1352 CGF.emitArrayDestroy(begin: arrayBegin, end: arrayEnd, elementType: EltTy, elementAlign,
1353 destroyer: CGF.getDestroyer(destructionKind: dtorKind),
1354 /*checkZeroLength*/ false, useEHCleanup: CGF.needsEHCleanup(kind: dtorKind));
1355
1356 llvm::BasicBlock *VectorBBCont = CGF.createBasicBlock(name: "dtor.vector.cont");
1357 CGF.EmitBlock(BB: VectorBBCont);
1358
1359 llvm::Value *CheckTheBitForDeleteCall = CGF.Builder.CreateAnd(
1360 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 1));
1361
1362 llvm::Value *ShouldCallDelete =
1363 CGF.Builder.CreateIsNull(Arg: CheckTheBitForDeleteCall);
1364 CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: CGF.ReturnBlock.getBlock(),
1365 False: callDeleteBB);
1366 CGF.EmitBlock(BB: callDeleteBB);
1367 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1368 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1369 if (Dtor->getArrayOperatorDelete()) {
1370 if (!Dtor->getGlobalArrayOperatorDelete()) {
1371 CGF.EmitDeleteCall(DeleteFD: Dtor->getArrayOperatorDelete(), Ptr: allocatedPtr,
1372 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1373 NumElements: numElements, CookieSize: cookieSize);
1374 } else {
1375 // If global operator[] is set, the class had its own operator delete[].
1376 // In that case, check the 4th bit. If it is set, we need to call
1377 // ::delete[].
1378 llvm::Value *CheckTheBitForGlobDeleteCall = CGF.Builder.CreateAnd(
1379 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1380
1381 llvm::Value *ShouldCallGlobDelete =
1382 CGF.Builder.CreateIsNull(Arg: CheckTheBitForGlobDeleteCall);
1383 llvm::BasicBlock *GlobDelete =
1384 CGF.createBasicBlock(name: "dtor.call_glob_delete_after_array_destroy");
1385 llvm::BasicBlock *ClassDelete =
1386 CGF.createBasicBlock(name: "dtor.call_class_delete_after_array_destroy");
1387 CGF.Builder.CreateCondBr(Cond: ShouldCallGlobDelete, True: ClassDelete, False: GlobDelete);
1388 CGF.EmitBlock(BB: ClassDelete);
1389 CGF.EmitDeleteCall(DeleteFD: Dtor->getArrayOperatorDelete(), Ptr: allocatedPtr,
1390 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1391 NumElements: numElements, CookieSize: cookieSize);
1392 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1393
1394 CGF.EmitBlock(BB: GlobDelete);
1395 // Use __global_delete wrapper instead of directly calling
1396 // ::operator delete to match MSVC's behavior. See the doc comment on
1397 // getOrCreateMSVCGlobalDeleteWrapper for details.
1398 llvm::Constant *GlobalDeleteWrapper =
1399 CGF.CGM.getOrCreateMSVCGlobalDeleteWrapper(
1400 GlobOD: Dtor->getGlobalArrayOperatorDelete());
1401 // For dllexport classes, emit forwarding bodies since the dtor is
1402 // exported and another TU may not provide the forwarding body.
1403 if (Dtor->hasAttr<DLLExportAttr>())
1404 CGF.CGM.noteDirectGlobalDelete();
1405 CGF.EmitDeleteCall(DeleteFD: Dtor->getGlobalArrayOperatorDelete(), Ptr: allocatedPtr,
1406 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1407 NumElements: numElements, CookieSize: cookieSize, CalleeOverride: GlobalDeleteWrapper);
1408 }
1409 } else {
1410 // No operators delete[] were found, so emit a trap.
1411 CGF.EmitTrapCallAndMakeUnreachable();
1412 }
1413
1414 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1415 CGF.EmitBlock(BB: ScalarBB);
1416}
1417
1418/// EmitDestructorBody - Emits the body of the current destructor.
1419void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1420 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CurGD.getDecl());
1421 CXXDtorType DtorType = CurGD.getDtorType();
1422
1423 // For an abstract class, non-base destructors are never used (and can't
1424 // be emitted in general, because vbase dtors may not have been validated
1425 // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1426 // in fact emit references to them from other compilations, so emit them
1427 // as functions containing a trap instruction.
1428 if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1429 EmitTrapCallAndMakeUnreachable();
1430 return;
1431 }
1432
1433 Stmt *Body = Dtor->getBody();
1434 if (Body) {
1435 incrementProfileCounter(S: Body);
1436 maybeCreateMCDCCondBitmap();
1437 }
1438
1439 // The call to operator delete in a deleting destructor happens
1440 // outside of the function-try-block, which means it's always
1441 // possible to delegate the destructor body to the complete
1442 // destructor. Do so.
1443 if (DtorType == Dtor_Deleting || DtorType == Dtor_VectorDeleting) {
1444 if (CXXStructorImplicitParamValue && DtorType == Dtor_VectorDeleting)
1445 EmitConditionalArrayDtorCall(DD: Dtor, CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue);
1446 RunCleanupsScope DtorEpilogue(*this);
1447 EnterDtorCleanups(Dtor, Type: Dtor_Deleting);
1448 if (HaveInsertPoint()) {
1449 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1450 EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1451 /*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1452 }
1453 return;
1454 }
1455
1456 // If the body is a function-try-block, enter the try before
1457 // anything else.
1458 bool isTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
1459 if (isTryBody)
1460 EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1461 EmitAsanPrologueOrEpilogue(Prologue: false);
1462
1463 // Enter the epilogue cleanups.
1464 RunCleanupsScope DtorEpilogue(*this);
1465
1466 // If this is the complete variant, just invoke the base variant;
1467 // the epilogue will destruct the virtual bases. But we can't do
1468 // this optimization if the body is a function-try-block, because
1469 // we'd introduce *two* handler blocks. In the Microsoft ABI, we
1470 // always delegate because we might not have a definition in this TU.
1471 switch (DtorType) {
1472 case Dtor_Unified:
1473 llvm_unreachable("not expecting a unified dtor");
1474 case Dtor_Comdat:
1475 llvm_unreachable("not expecting a COMDAT");
1476 case Dtor_Deleting:
1477 llvm_unreachable("already handled deleting case");
1478 case Dtor_VectorDeleting:
1479 llvm_unreachable("already handled vector deleting case");
1480
1481 case Dtor_Complete:
1482 assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1483 "can't emit a dtor without a body for non-Microsoft ABIs");
1484
1485 // Enter the cleanup scopes for virtual bases.
1486 EnterDtorCleanups(Dtor, Type: Dtor_Complete);
1487
1488 if (!isTryBody) {
1489 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1490 EmitCXXDestructorCall(D: Dtor, Type: Dtor_Base, /*ForVirtualBase=*/false,
1491 /*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1492 break;
1493 }
1494
1495 // Fallthrough: act like we're in the base variant.
1496 [[fallthrough]];
1497
1498 case Dtor_Base:
1499 assert(Body);
1500
1501 // Enter the cleanup scopes for fields and non-virtual bases.
1502 EnterDtorCleanups(Dtor, Type: Dtor_Base);
1503
1504 // Initialize the vtable pointers before entering the body.
1505 if (!CodeGenUtils::canSkipVTablePointerInitialization(Ctx&: getContext(), Dtor)) {
1506 // Insert the llvm.launder.invariant.group intrinsic before initializing
1507 // the vptrs to cancel any previous assumptions we might have made.
1508 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1509 CGM.getCodeGenOpts().OptimizationLevel > 0)
1510 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1511 InitializeVTablePointers(ClassDecl: Dtor->getParent());
1512 }
1513
1514 if (isTryBody)
1515 EmitStmt(S: cast<CXXTryStmt>(Val: Body)->getTryBlock());
1516 else if (Body)
1517 EmitStmt(S: Body);
1518 else {
1519 assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1520 // nothing to do besides what's in the epilogue
1521 }
1522 // -fapple-kext must inline any call to this dtor into
1523 // the caller's body.
1524 if (getLangOpts().AppleKext)
1525 CurFn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1526
1527 break;
1528 }
1529
1530 // Jump out through the epilogue cleanups.
1531 DtorEpilogue.ForceCleanup();
1532
1533 // Exit the try if applicable.
1534 if (isTryBody)
1535 ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1536}
1537
1538void CodeGenFunction::emitImplicitAssignmentOperatorBody(
1539 FunctionArgList &Args) {
1540 const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(Val: CurGD.getDecl());
1541 const Stmt *RootS = AssignOp->getBody();
1542 assert(isa<CompoundStmt>(RootS) &&
1543 "Body of an implicit assignment operator should be compound stmt.");
1544 const CompoundStmt *RootCS = cast<CompoundStmt>(Val: RootS);
1545
1546 LexicalScope Scope(*this, RootCS->getSourceRange());
1547
1548 incrementProfileCounter(S: RootCS);
1549 maybeCreateMCDCCondBitmap();
1550 AssignmentMemcpyizer AM(*this, AssignOp, Args);
1551 for (auto *I : RootCS->body())
1552 AM.emitAssignment(S: I);
1553
1554 AM.finish();
1555}
1556
1557namespace {
1558llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1559 const CXXDestructorDecl *DD) {
1560 if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1561 return CGF.EmitScalarExpr(E: ThisArg);
1562 return CGF.LoadCXXThis();
1563}
1564
1565/// Call the operator delete associated with the current destructor.
1566struct CallDtorDelete final : EHScopeStack::Cleanup {
1567 CallDtorDelete() {}
1568
1569 void Emit(CodeGenFunction &CGF, Flags flags) override {
1570 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1571 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1572 CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1573 Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1574 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl));
1575 }
1576};
1577
1578// This function implements generation of scalar deleting destructor body for
1579// the case when the destructor also accepts an implicit flag. Right now only
1580// Microsoft ABI requires deleting destructors to accept implicit flags.
1581// The flag indicates whether an operator delete should be called and whether
1582// it should be a class-specific operator delete or a global one.
1583void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1584 llvm::Value *ShouldDeleteCondition,
1585 bool ReturnAfterDelete) {
1586 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1587 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1588 const FunctionDecl *OD = Dtor->getOperatorDelete();
1589 assert(OD->isDestroyingOperatorDelete() == ReturnAfterDelete &&
1590 "unexpected value for ReturnAfterDelete");
1591 auto *CondTy = cast<llvm::IntegerType>(Val: ShouldDeleteCondition->getType());
1592 // MSVC calls global operator delete inside of the dtor body, but clang
1593 // aligned with this behavior only after a particular version. This is not
1594 // ABI-compatible with previous versions.
1595 ASTContext &Context = CGF.getContext();
1596 bool CallGlobDelete = Context.getTargetInfo().callGlobalDeleteInDeletingDtor(
1597 Context.getLangOpts());
1598 if (CallGlobDelete && OD->isDestroyingOperatorDelete()) {
1599 llvm::BasicBlock *CallDtor = CGF.createBasicBlock(name: "dtor.call_dtor");
1600 llvm::BasicBlock *DontCallDtor = CGF.createBasicBlock(name: "dtor.entry_cont");
1601 // Third bit set signals that global operator delete is called. That means
1602 // despite class having destroying operator delete which is responsible
1603 // for calling dtor, we need to call dtor because global operator delete
1604 // won't do that.
1605 llvm::Value *Check3rdBit = CGF.Builder.CreateAnd(
1606 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1607 llvm::Value *ShouldCallDtor = CGF.Builder.CreateIsNull(Arg: Check3rdBit);
1608 CGF.Builder.CreateCondBr(Cond: ShouldCallDtor, True: DontCallDtor, False: CallDtor);
1609 CGF.EmitBlock(BB: CallDtor);
1610 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1611 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1612 /*Delegating=*/false, This: CGF.LoadCXXThisAddress(),
1613 ThisTy);
1614 CGF.Builder.CreateBr(Dest: DontCallDtor);
1615 CGF.EmitBlock(BB: DontCallDtor);
1616 }
1617 llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock(name: "dtor.call_delete");
1618 llvm::BasicBlock *continueBB = CGF.createBasicBlock(name: "dtor.continue");
1619 // First bit set signals that operator delete must be called.
1620 llvm::Value *Check1stBit = CGF.Builder.CreateAnd(
1621 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 1));
1622 llvm::Value *ShouldCallDelete = CGF.Builder.CreateIsNull(Arg: Check1stBit);
1623 CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: continueBB, False: callDeleteBB);
1624
1625 CGF.EmitBlock(BB: callDeleteBB);
1626 auto EmitDeleteAndGoToEnd = [&](const FunctionDecl *DeleteOp,
1627 llvm::Constant *CalleeOverride = nullptr) {
1628 CGF.EmitDeleteCall(DeleteFD: DeleteOp, Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1629 DeleteTy: Context.getCanonicalTagType(TD: ClassDecl),
1630 /*NumElements=*/nullptr, /*CookieSize=*/CharUnits(),
1631 CalleeOverride);
1632 if (ReturnAfterDelete)
1633 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1634 else
1635 CGF.Builder.CreateBr(Dest: continueBB);
1636 };
1637 // If Sema only found a global operator delete previously, the dtor can
1638 // always call it. Otherwise we need to check the third bit and call the
1639 // appropriate operator delete, i.e. global or class-specific.
1640 if (const FunctionDecl *GlobOD = Dtor->getOperatorGlobalDelete();
1641 isa<CXXMethodDecl>(Val: OD) && GlobOD && CallGlobDelete) {
1642 // Third bit set signals that global operator delete is called, i.e.
1643 // ::delete appears on the callsite.
1644 llvm::Value *CheckTheBitForGlobDeleteCall = CGF.Builder.CreateAnd(
1645 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1646 llvm::Value *ShouldCallGlobDelete =
1647 CGF.Builder.CreateIsNull(Arg: CheckTheBitForGlobDeleteCall);
1648 llvm::BasicBlock *GlobDelete =
1649 CGF.createBasicBlock(name: "dtor.call_glob_delete");
1650 llvm::BasicBlock *ClassDelete =
1651 CGF.createBasicBlock(name: "dtor.call_class_delete");
1652 CGF.Builder.CreateCondBr(Cond: ShouldCallGlobDelete, True: ClassDelete, False: GlobDelete);
1653 CGF.EmitBlock(BB: GlobDelete);
1654
1655 // Use __global_delete wrapper instead of directly calling
1656 // ::operator delete to match MSVC's behavior. See the doc comment on
1657 // getOrCreateMSVCGlobalDeleteWrapper for details.
1658 llvm::Constant *GlobalDeleteWrapper =
1659 CGF.CGM.getOrCreateMSVCGlobalDeleteWrapper(GlobOD);
1660 // For dllexport classes, emit forwarding bodies since the dtor is
1661 // exported and another TU may not provide the forwarding body.
1662 if (Dtor->hasAttr<DLLExportAttr>())
1663 CGF.CGM.noteDirectGlobalDelete();
1664 EmitDeleteAndGoToEnd(GlobOD, GlobalDeleteWrapper);
1665 CGF.EmitBlock(BB: ClassDelete);
1666 }
1667 EmitDeleteAndGoToEnd(OD);
1668 CGF.EmitBlock(BB: continueBB);
1669}
1670
1671struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1672 llvm::Value *ShouldDeleteCondition;
1673
1674public:
1675 CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1676 : ShouldDeleteCondition(ShouldDeleteCondition) {
1677 assert(ShouldDeleteCondition != nullptr);
1678 }
1679
1680 void Emit(CodeGenFunction &CGF, Flags flags) override {
1681 EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1682 /*ReturnAfterDelete*/ false);
1683 }
1684};
1685
1686class DestroyField final : public EHScopeStack::Cleanup {
1687 const FieldDecl *field;
1688 CodeGenFunction::Destroyer *destroyer;
1689 bool useEHCleanupForArray;
1690
1691public:
1692 DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1693 bool useEHCleanupForArray)
1694 : field(field), destroyer(destroyer),
1695 useEHCleanupForArray(useEHCleanupForArray) {}
1696
1697 void Emit(CodeGenFunction &CGF, Flags flags) override {
1698 // Find the address of the field.
1699 Address thisValue = CGF.LoadCXXThisAddress();
1700 CanQualType RecordTy =
1701 CGF.getContext().getCanonicalTagType(TD: field->getParent());
1702 LValue ThisLV = CGF.MakeAddrLValue(Addr: thisValue, T: RecordTy);
1703 LValue LV = CGF.EmitLValueForField(Base: ThisLV, Field: field);
1704 assert(LV.isSimple());
1705
1706 CGF.emitDestroy(addr: LV.getAddress(), type: field->getType(), destroyer,
1707 useEHCleanupForArray: flags.isForNormalCleanup() && useEHCleanupForArray);
1708 }
1709};
1710
1711class DeclAsInlineDebugLocation {
1712 CGDebugInfo *DI;
1713 llvm::DILocation *InlinedAt;
1714 std::optional<ApplyDebugLocation> Location;
1715
1716public:
1717 DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)
1718 : DI(CGF.getDebugInfo()) {
1719 if (!DI)
1720 return;
1721 InlinedAt = DI->getInlinedAt();
1722 DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());
1723 Location.emplace(args&: CGF, args: Decl.getLocation());
1724 }
1725
1726 ~DeclAsInlineDebugLocation() {
1727 if (!DI)
1728 return;
1729 Location.reset();
1730 DI->setInlinedAt(InlinedAt);
1731 }
1732};
1733
1734static void EmitSanitizerDtorCallback(
1735 CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,
1736 std::optional<CharUnits::QuantityType> PoisonSize = {}) {
1737 CodeGenFunction::SanitizerScope SanScope(&CGF);
1738 // Pass in void pointer and size of region as arguments to runtime
1739 // function
1740 SmallVector<llvm::Value *, 2> Args = {Ptr};
1741 SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};
1742
1743 if (PoisonSize.has_value()) {
1744 Args.emplace_back(Args: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: *PoisonSize));
1745 ArgTypes.emplace_back(Args&: CGF.SizeTy);
1746 }
1747
1748 llvm::FunctionType *FnType =
1749 llvm::FunctionType::get(Result: CGF.VoidTy, Params: ArgTypes, isVarArg: false);
1750 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FnType, Name);
1751
1752 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1753}
1754
1755static void
1756EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1757 CharUnits::QuantityType PoisonSize) {
1758 EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_fields", Ptr,
1759 PoisonSize);
1760}
1761
1762/// Poison base class with a trivial destructor.
1763struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1764 const CXXRecordDecl *BaseClass;
1765 bool BaseIsVirtual;
1766 SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1767 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1768
1769 void Emit(CodeGenFunction &CGF, Flags flags) override {
1770 const CXXRecordDecl *DerivedClass =
1771 cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
1772
1773 Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1774 This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
1775
1776 const ASTRecordLayout &BaseLayout =
1777 CGF.getContext().getASTRecordLayout(D: BaseClass);
1778 CharUnits BaseSize = BaseLayout.getSize();
1779
1780 if (!BaseSize.isPositive())
1781 return;
1782
1783 // Use the base class declaration location as inline DebugLocation. All
1784 // fields of the class are destroyed.
1785 DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
1786 EmitSanitizerDtorFieldsCallback(CGF, Ptr: Addr.emitRawPointer(CGF),
1787 PoisonSize: BaseSize.getQuantity());
1788
1789 // Prevent the current stack frame from disappearing from the stack trace.
1790 CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1791 }
1792};
1793
1794class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1795 const CXXDestructorDecl *Dtor;
1796 unsigned StartIndex;
1797 unsigned EndIndex;
1798
1799public:
1800 SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1801 unsigned EndIndex)
1802 : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1803
1804 // Generate function call for handling object poisoning.
1805 // Disables tail call elimination, to prevent the current stack frame
1806 // from disappearing from the stack trace.
1807 void Emit(CodeGenFunction &CGF, Flags flags) override {
1808 const ASTContext &Context = CGF.getContext();
1809 const ASTRecordLayout &Layout =
1810 Context.getASTRecordLayout(D: Dtor->getParent());
1811
1812 // It's a first trivial field so it should be at the begining of a char,
1813 // still round up start offset just in case.
1814 CharUnits PoisonStart = Context.toCharUnitsFromBits(
1815 BitSize: Layout.getFieldOffset(FieldNo: StartIndex) + Context.getCharWidth() - 1);
1816 llvm::ConstantInt *OffsetSizePtr =
1817 llvm::ConstantInt::get(Ty: CGF.SizeTy, V: PoisonStart.getQuantity());
1818
1819 llvm::Value *OffsetPtr =
1820 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: CGF.LoadCXXThis(), IdxList: OffsetSizePtr);
1821
1822 CharUnits PoisonEnd;
1823 if (EndIndex >= Layout.getFieldCount()) {
1824 PoisonEnd = Layout.getNonVirtualSize();
1825 } else {
1826 PoisonEnd = Context.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: EndIndex));
1827 }
1828 CharUnits PoisonSize = PoisonEnd - PoisonStart;
1829 if (!PoisonSize.isPositive())
1830 return;
1831
1832 // Use the top field declaration location as inline DebugLocation.
1833 DeclAsInlineDebugLocation InlineHere(
1834 CGF, **std::next(x: Dtor->getParent()->field_begin(), n: StartIndex));
1835 EmitSanitizerDtorFieldsCallback(CGF, Ptr: OffsetPtr, PoisonSize: PoisonSize.getQuantity());
1836
1837 // Prevent the current stack frame from disappearing from the stack trace.
1838 CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1839 }
1840};
1841
1842class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1843 const CXXDestructorDecl *Dtor;
1844
1845public:
1846 SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1847
1848 // Generate function call for handling vtable pointer poisoning.
1849 void Emit(CodeGenFunction &CGF, Flags flags) override {
1850 assert(Dtor->getParent()->isDynamicClass());
1851 (void)Dtor;
1852 // Poison vtable and vtable ptr if they exist for this class.
1853 llvm::Value *VTablePtr = CGF.LoadCXXThis();
1854
1855 // Pass in void pointer and size of region as arguments to runtime
1856 // function
1857 EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_vptr", Ptr: VTablePtr);
1858 }
1859};
1860
1861class SanitizeDtorCleanupBuilder {
1862 ASTContext &Context;
1863 EHScopeStack &EHStack;
1864 const CXXDestructorDecl *DD;
1865 std::optional<unsigned> StartIndex;
1866
1867public:
1868 SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1869 const CXXDestructorDecl *DD)
1870 : Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}
1871 void PushCleanupForField(const FieldDecl *Field) {
1872 if (CodeGenUtils::isEmptyFieldForLayout(Ctx: Context, FD: Field))
1873 return;
1874 unsigned FieldIndex = Field->getFieldIndex();
1875 if (CodeGenUtils::fieldHasTrivialDestructorBody(Ctx&: Context, Field)) {
1876 if (!StartIndex)
1877 StartIndex = FieldIndex;
1878 } else if (StartIndex) {
1879 EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1880 A: *StartIndex, A: FieldIndex);
1881 StartIndex = std::nullopt;
1882 }
1883 }
1884 void End() {
1885 if (StartIndex)
1886 EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1887 A: *StartIndex, A: -1);
1888 }
1889};
1890} // end anonymous namespace
1891
1892/// Emit all code that comes at the end of class's
1893/// destructor. This is to call destructors on members and base classes
1894/// in reverse order of their construction.
1895///
1896/// For a deleting destructor, this also handles the case where a destroying
1897/// operator delete completely overrides the definition.
1898void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1899 CXXDtorType DtorType) {
1900 assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1901 "Should not emit dtor epilogue for non-exported trivial dtor!");
1902
1903 // The deleting-destructor phase just needs to call the appropriate
1904 // operator delete that Sema picked up.
1905 if (DtorType == Dtor_Deleting) {
1906 assert(DD->getOperatorDelete() &&
1907 "operator delete missing - EnterDtorCleanups");
1908 if (CXXStructorImplicitParamValue) {
1909 // If there is an implicit param to the deleting dtor, it's a boolean
1910 // telling whether this is a deleting destructor.
1911 if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1912 EmitConditionalDtorDeleteCall(CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue,
1913 /*ReturnAfterDelete*/ true);
1914 else
1915 EHStack.pushCleanup<CallDtorDeleteConditional>(
1916 Kind: NormalAndEHCleanup, A: CXXStructorImplicitParamValue);
1917 } else {
1918 if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1919 const CXXRecordDecl *ClassDecl = DD->getParent();
1920 EmitDeleteCall(DeleteFD: DD->getOperatorDelete(),
1921 Ptr: LoadThisForDtorDelete(CGF&: *this, DD),
1922 DeleteTy: getContext().getCanonicalTagType(TD: ClassDecl));
1923 EmitBranchThroughCleanup(Dest: ReturnBlock);
1924 } else {
1925 EHStack.pushCleanup<CallDtorDelete>(Kind: NormalAndEHCleanup);
1926 }
1927 }
1928 return;
1929 }
1930
1931 const CXXRecordDecl *ClassDecl = DD->getParent();
1932
1933 // Unions have no bases and do not call field destructors.
1934 if (ClassDecl->isUnion())
1935 return;
1936
1937 // The complete-destructor phase just destructs all the virtual bases.
1938 if (DtorType == Dtor_Complete) {
1939 // Poison the vtable pointer such that access after the base
1940 // and member destructors are invoked is invalid.
1941 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1942 SanOpts.has(K: SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1943 ClassDecl->isPolymorphic())
1944 EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1945
1946 // We push them in the forward order so that they'll be popped in
1947 // the reverse order.
1948 for (const auto &Base : ClassDecl->vbases()) {
1949 auto *BaseClassDecl = Base.getType()->castAsCXXRecordDecl();
1950 if (BaseClassDecl->hasTrivialDestructor()) {
1951 // Under SanitizeMemoryUseAfterDtor, poison the trivial base class
1952 // memory. For non-trival base classes the same is done in the class
1953 // destructor.
1954 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1955 SanOpts.has(K: SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1956 EHStack.pushCleanup<SanitizeDtorTrivialBase>(Kind: NormalAndEHCleanup,
1957 A: BaseClassDecl,
1958 /*BaseIsVirtual*/ A: true);
1959 } else {
1960 EHStack.pushCleanup<CallBaseDtor>(Kind: NormalAndEHCleanup, A: BaseClassDecl,
1961 /*BaseIsVirtual*/ A: true);
1962 }
1963 }
1964
1965 return;
1966 }
1967
1968 assert(DtorType == Dtor_Base);
1969 // Poison the vtable pointer if it has no virtual bases, but inherits
1970 // virtual functions.
1971 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1972 SanOpts.has(K: SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1973 ClassDecl->isPolymorphic())
1974 EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1975
1976 // Destroy non-virtual bases.
1977 for (const auto &Base : ClassDecl->bases()) {
1978 // Ignore virtual bases.
1979 if (Base.isVirtual())
1980 continue;
1981
1982 CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1983
1984 if (BaseClassDecl->hasTrivialDestructor()) {
1985 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1986 SanOpts.has(K: SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1987 EHStack.pushCleanup<SanitizeDtorTrivialBase>(Kind: NormalAndEHCleanup,
1988 A: BaseClassDecl,
1989 /*BaseIsVirtual*/ A: false);
1990 } else {
1991 EHStack.pushCleanup<CallBaseDtor>(Kind: NormalAndEHCleanup, A: BaseClassDecl,
1992 /*BaseIsVirtual*/ A: false);
1993 }
1994 }
1995
1996 // Poison fields such that access after their destructors are
1997 // invoked, and before the base class destructor runs, is invalid.
1998 bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1999 SanOpts.has(K: SanitizerKind::Memory);
2000 SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
2001
2002 // Destroy direct fields.
2003 for (const auto *Field : ClassDecl->fields()) {
2004 if (SanitizeFields)
2005 SanitizeBuilder.PushCleanupForField(Field);
2006
2007 QualType type = Field->getType();
2008 QualType::DestructionKind dtorKind = type.isDestructedType();
2009 if (!dtorKind)
2010 continue;
2011
2012 // Anonymous union members do not have their destructors called.
2013 const RecordType *RT = type->getAsUnionType();
2014 if (RT && RT->getDecl()->isAnonymousStructOrUnion())
2015 continue;
2016
2017 CleanupKind cleanupKind = getCleanupKind(kind: dtorKind);
2018 EHStack.pushCleanup<DestroyField>(
2019 Kind: cleanupKind, A: Field, A: getDestroyer(destructionKind: dtorKind), A: cleanupKind & EHCleanup);
2020 }
2021
2022 if (SanitizeFields)
2023 SanitizeBuilder.End();
2024}
2025
2026/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
2027/// constructor for each of several members of an array.
2028///
2029/// \param ctor the constructor to call for each element
2030/// \param arrayType the type of the array to initialize
2031/// \param arrayBegin an arrayType*
2032/// \param zeroInitialize true if each element should be
2033/// zero-initialized before it is constructed
2034void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2035 const ArrayType *arrayType,
2036 Address arrayBegin,
2037 const CXXConstructExpr *E,
2038 bool NewPointerIsChecked,
2039 bool zeroInitialize) {
2040 QualType elementType;
2041 llvm::Value *numElements =
2042 emitArrayLength(arrayType, baseType&: elementType, addr&: arrayBegin);
2043
2044 EmitCXXAggrConstructorCall(D: ctor, NumElements: numElements, ArrayPtr: arrayBegin, E,
2045 NewPointerIsChecked, ZeroInitialization: zeroInitialize);
2046}
2047
2048/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
2049/// constructor for each of several members of an array.
2050///
2051/// \param ctor the constructor to call for each element
2052/// \param numElements the number of elements in the array;
2053/// may be zero
2054/// \param arrayBase a T*, where T is the type constructed by ctor
2055/// \param zeroInitialize true if each element should be
2056/// zero-initialized before it is constructed
2057void CodeGenFunction::EmitCXXAggrConstructorCall(
2058 const CXXConstructorDecl *ctor, llvm::Value *numElements, Address arrayBase,
2059 const CXXConstructExpr *E, bool NewPointerIsChecked, bool zeroInitialize) {
2060 // It's legal for numElements to be zero. This can happen both
2061 // dynamically, because x can be zero in 'new A[x]', and statically,
2062 // because of GCC extensions that permit zero-length arrays. There
2063 // are probably legitimate places where we could assume that this
2064 // doesn't happen, but it's not clear that it's worth it.
2065 llvm::CondBrInst *zeroCheckBranch = nullptr;
2066
2067 // Optimize for a constant count.
2068 llvm::ConstantInt *constantCount = dyn_cast<llvm::ConstantInt>(Val: numElements);
2069 if (constantCount) {
2070 // Just skip out if the constant count is zero.
2071 if (constantCount->isZero())
2072 return;
2073
2074 // Otherwise, emit the check.
2075 } else {
2076 llvm::BasicBlock *loopBB = createBasicBlock(name: "new.ctorloop");
2077 llvm::Value *iszero = Builder.CreateIsNull(Arg: numElements, Name: "isempty");
2078 zeroCheckBranch = Builder.CreateCondBr(Cond: iszero, True: loopBB, False: loopBB);
2079 EmitBlock(BB: loopBB);
2080 }
2081
2082 // Find the end of the array.
2083 llvm::Type *elementType = arrayBase.getElementType();
2084 llvm::Value *arrayBegin = arrayBase.emitRawPointer(CGF&: *this);
2085 llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
2086 Ty: elementType, Ptr: arrayBegin, IdxList: numElements, Name: "arrayctor.end");
2087
2088 // Enter the loop, setting up a phi for the current location to initialize.
2089 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2090 llvm::BasicBlock *loopBB = createBasicBlock(name: "arrayctor.loop");
2091 EmitBlock(BB: loopBB);
2092 llvm::PHINode *cur =
2093 Builder.CreatePHI(Ty: arrayBegin->getType(), NumReservedValues: 2, Name: "arrayctor.cur");
2094 cur->addIncoming(V: arrayBegin, BB: entryBB);
2095
2096 // Inside the loop body, emit the constructor call on the array element.
2097 if (CGM.shouldEmitConvergenceTokens())
2098 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: loopBB));
2099
2100 // The alignment of the base, adjusted by the size of a single element,
2101 // provides a conservative estimate of the alignment of every element.
2102 // (This assumes we never start tracking offsetted alignments.)
2103 //
2104 // Note that these are complete objects and so we don't need to
2105 // use the non-virtual size or alignment.
2106 CanQualType type = getContext().getCanonicalTagType(TD: ctor->getParent());
2107 CharUnits eltAlignment = arrayBase.getAlignment().alignmentOfArrayElement(
2108 elementSize: getContext().getTypeSizeInChars(T: type));
2109 Address curAddr = Address(cur, elementType, eltAlignment);
2110
2111 // Zero initialize the storage, if requested.
2112 if (zeroInitialize)
2113 EmitNullInitialization(DestPtr: curAddr, Ty: type);
2114
2115 // C++ [class.temporary]p4:
2116 // There are two contexts in which temporaries are destroyed at a different
2117 // point than the end of the full-expression. The first context is when a
2118 // default constructor is called to initialize an element of an array.
2119 // If the constructor has one or more default arguments, the destruction of
2120 // every temporary created in a default argument expression is sequenced
2121 // before the construction of the next array element, if any.
2122
2123 {
2124 RunCleanupsScope Scope(*this);
2125
2126 // Evaluate the constructor and its arguments in a regular
2127 // partial-destroy cleanup.
2128 if (getLangOpts().Exceptions &&
2129 !ctor->getParent()->hasTrivialDestructor()) {
2130 Destroyer *destroyer = destroyCXXObject;
2131 pushRegularPartialArrayCleanup(arrayBegin, arrayEnd: cur, elementType: type, elementAlignment: eltAlignment,
2132 destroyer: *destroyer);
2133 }
2134 auto currAVS = AggValueSlot::forAddr(
2135 addr: curAddr, quals: type.getQualifiers(), isDestructed: AggValueSlot::IsDestructed,
2136 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2137 mayOverlap: AggValueSlot::DoesNotOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2138 isChecked: NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2139 : AggValueSlot::IsNotSanitizerChecked);
2140 EmitCXXConstructorCall(D: ctor, Type: Ctor_Complete, /*ForVirtualBase=*/false,
2141 /*Delegating=*/false, ThisAVS: currAVS, E);
2142 }
2143
2144 // Go to the next element.
2145 llvm::Value *next = Builder.CreateInBoundsGEP(
2146 Ty: elementType, Ptr: cur, IdxList: llvm::ConstantInt::get(Ty: SizeTy, V: 1), Name: "arrayctor.next");
2147 cur->addIncoming(V: next, BB: Builder.GetInsertBlock());
2148
2149 // Check whether that's the end of the loop.
2150 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: arrayEnd, Name: "arrayctor.done");
2151 llvm::BasicBlock *contBB = createBasicBlock(name: "arrayctor.cont");
2152 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2153
2154 // Patch the earlier check to skip over the loop.
2155 if (zeroCheckBranch)
2156 zeroCheckBranch->setSuccessor(idx: 0, NewSucc: contBB);
2157
2158 if (CGM.shouldEmitConvergenceTokens())
2159 ConvergenceTokenStack.pop_back();
2160
2161 EmitBlock(BB: contBB);
2162}
2163
2164void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF, Address addr,
2165 QualType type) {
2166 const CXXDestructorDecl *dtor = type->castAsCXXRecordDecl()->getDestructor();
2167 assert(!dtor->isTrivial());
2168 CGF.EmitCXXDestructorCall(D: dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
2169 /*Delegating=*/false, This: addr, ThisTy: type);
2170}
2171
2172void CodeGenFunction::EmitCXXConstructorCall(
2173 const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2174 bool Delegating, AggValueSlot ThisAVS, const CXXConstructExpr *E) {
2175 CallArgList Args;
2176 Address This = ThisAVS.getAddress();
2177 LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2178 LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
2179 llvm::Value *ThisPtr =
2180 getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType()->getPointeeType());
2181
2182 if (SlotAS != ThisAS) {
2183 unsigned TargetThisAS = getContext().getTargetAddressSpace(AS: ThisAS);
2184 llvm::Type *NewType =
2185 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetThisAS);
2186 ThisPtr = performAddrSpaceCast(Src: ThisPtr, DestTy: NewType);
2187 }
2188
2189 // Push the this ptr.
2190 Args.add(rvalue: RValue::get(V: ThisPtr), type: D->getThisType());
2191
2192 // If this is a trivial constructor, emit a memcpy now before we lose
2193 // the alignment information on the argument.
2194 // FIXME: It would be better to preserve alignment information into CallArg.
2195 if (D->isMemcpyEquivalentSpecialMember(Ctx: getContext())) {
2196 assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2197
2198 const Expr *Arg = E->getArg(Arg: 0);
2199 LValue Src = EmitCheckedLValue(E: Arg, TCK: TCK_Load);
2200 CanQualType DestTy = getContext().getCanonicalTagType(TD: D->getParent());
2201 LValue Dest = MakeAddrLValue(Addr: This, T: DestTy);
2202 EmitAggregateCopyCtor(Dest, Src, MayOverlap: ThisAVS.mayOverlap());
2203 return;
2204 }
2205
2206 // Add the rest of the user-supplied arguments.
2207 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2208 EvaluationOrder Order = E->isListInitialization()
2209 ? EvaluationOrder::ForceLeftToRight
2210 : EvaluationOrder::Default;
2211 EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments(), AC: E->getConstructor(),
2212 /*ParamsToSkip*/ 0, Order);
2213
2214 EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2215 Overlap: ThisAVS.mayOverlap(), Loc: E->getExprLoc(),
2216 NewPointerIsChecked: ThisAVS.isSanitizerChecked());
2217}
2218
2219static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2220 const CXXConstructorDecl *Ctor,
2221 CXXCtorType Type, CallArgList &Args) {
2222 // We can't forward a variadic call.
2223 if (Ctor->isVariadic())
2224 return false;
2225
2226 if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2227 // If the parameters are callee-cleanup, it's not safe to forward.
2228 for (auto *P : Ctor->parameters())
2229 if (P->needsDestruction(Ctx: CGF.getContext()))
2230 return false;
2231
2232 // Likewise if they're inalloca.
2233 const CGFunctionInfo &Info = CGF.CGM.getTypes().arrangeCXXConstructorCall(
2234 Args, D: Ctor, CtorKind: Type, ExtraPrefixArgs: 0, ExtraSuffixArgs: 0, ABIInfoFD: CGF.getCurrentFunctionDecl());
2235 if (Info.usesInAlloca())
2236 return false;
2237 }
2238
2239 // Anything else should be OK.
2240 return true;
2241}
2242
2243void CodeGenFunction::EmitCXXConstructorCall(
2244 const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2245 bool Delegating, Address This, CallArgList &Args,
2246 AggValueSlot::Overlap_t Overlap, SourceLocation Loc,
2247 bool NewPointerIsChecked, llvm::CallBase **CallOrInvoke) {
2248 const CXXRecordDecl *ClassDecl = D->getParent();
2249
2250 if (!NewPointerIsChecked)
2251 EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall, Loc, Addr: This,
2252 Type: getContext().getCanonicalTagType(TD: ClassDecl),
2253 Alignment: CharUnits::Zero());
2254
2255 if (D->isTrivial() && D->isDefaultConstructor()) {
2256 assert(Args.size() == 1 && "trivial default ctor with args");
2257 return;
2258 }
2259
2260 // If this is a trivial constructor, just emit what's needed. If this is a
2261 // union copy constructor, we must emit a memcpy, because the AST does not
2262 // model that copy.
2263 if (D->isMemcpyEquivalentSpecialMember(Ctx: getContext())) {
2264 assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2265 QualType SrcTy = D->getParamDecl(i: 0)->getType().getNonReferenceType();
2266 Address Src = makeNaturalAddressForPointer(
2267 Ptr: Args[1].getRValue(CGF&: *this).getScalarVal(), T: SrcTy);
2268 LValue SrcLVal = MakeAddrLValue(Addr: Src, T: SrcTy);
2269 CanQualType DestTy = getContext().getCanonicalTagType(TD: ClassDecl);
2270 LValue DestLVal = MakeAddrLValue(Addr: This, T: DestTy);
2271 EmitAggregateCopyCtor(Dest: DestLVal, Src: SrcLVal, MayOverlap: Overlap);
2272 return;
2273 }
2274
2275 bool PassPrototypeArgs = true;
2276 // Check whether we can actually emit the constructor before trying to do so.
2277 if (auto Inherited = D->getInheritedConstructor()) {
2278 PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2279 if (PassPrototypeArgs && !canEmitDelegateCallArgs(CGF&: *this, Ctor: D, Type, Args)) {
2280 EmitInlinedInheritingCXXConstructorCall(Ctor: D, CtorType: Type, ForVirtualBase,
2281 Delegating, Args);
2282 return;
2283 }
2284 }
2285
2286 // Insert any ABI-specific implicit constructor arguments.
2287 CGCXXABI::AddedStructorArgCounts ExtraArgs =
2288 CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D, Type, ForVirtualBase,
2289 Delegating, Args);
2290
2291 // Emit the call.
2292 llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GD: GlobalDecl(D, Type));
2293 const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2294 Args, D, CtorKind: Type, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix,
2295 ABIInfoFD: getCurrentFunctionDecl(), PassProtoArgs: PassPrototypeArgs);
2296 CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(D, Type));
2297 EmitCall(CallInfo: Info, Callee, ReturnValue: ReturnValueSlot(), Args, CallOrInvoke, IsMustTail: false, Loc);
2298
2299 // Generate vtable assumptions if we're constructing a complete object
2300 // with a vtable. We don't do this for base subobjects for two reasons:
2301 // first, it's incorrect for classes with virtual bases, and second, we're
2302 // about to overwrite the vptrs anyway.
2303 // We also have to make sure if we can refer to vtable:
2304 // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2305 // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2306 // sure that definition of vtable is not hidden,
2307 // then we are always safe to refer to it.
2308 // FIXME: It looks like InstCombine is very inefficient on dealing with
2309 // assumes. Make assumption loads require -fstrict-vtable-pointers
2310 // temporarily.
2311 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2312 ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2313 CGM.getCXXABI().canSpeculativelyEmitVTable(RD: ClassDecl) &&
2314 CGM.getCodeGenOpts().StrictVTablePointers)
2315 EmitVTableAssumptionLoads(ClassDecl, This);
2316}
2317
2318void CodeGenFunction::EmitInheritedCXXConstructorCall(
2319 const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2320 bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2321 CallArgList Args;
2322 CallArg ThisArg(RValue::get(V: getAsNaturalPointerTo(
2323 Addr: This, PointeeType: D->getThisType()->getPointeeType())),
2324 D->getThisType());
2325
2326 // Forward the parameters.
2327 if (InheritedFromVBase &&
2328 CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2329 // Nothing to do; this construction is not responsible for constructing
2330 // the base class containing the inherited constructor.
2331 // FIXME: Can we just pass undef's for the remaining arguments if we don't
2332 // have constructor variants?
2333 Args.push_back(Elt: ThisArg);
2334 } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2335 // The inheriting constructor was inlined; just inject its arguments.
2336 assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2337 "wrong number of parameters for inherited constructor call");
2338 Args = CXXInheritedCtorInitExprArgs;
2339 Args[0] = ThisArg;
2340 } else {
2341 // The inheriting constructor was not inlined. Emit delegating arguments.
2342 Args.push_back(Elt: ThisArg);
2343 const auto *OuterCtor = cast<CXXConstructorDecl>(Val: CurCodeDecl);
2344 assert(OuterCtor->getNumParams() == D->getNumParams());
2345 assert(!OuterCtor->isVariadic() && "should have been inlined");
2346
2347 for (const auto *Param : OuterCtor->parameters()) {
2348 assert(getContext().hasSameUnqualifiedType(
2349 OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2350 Param->getType()));
2351 EmitDelegateCallArg(args&: Args, param: Param, loc: E->getLocation());
2352
2353 // Forward __attribute__(pass_object_size).
2354 if (Param->hasAttr<PassObjectSizeAttr>()) {
2355 auto *POSParam = SizeArguments[Param];
2356 assert(POSParam && "missing pass_object_size value for forwarding");
2357 EmitDelegateCallArg(args&: Args, param: POSParam, loc: E->getLocation());
2358 }
2359 }
2360 }
2361
2362 EmitCXXConstructorCall(D, Type: Ctor_Base, ForVirtualBase, /*Delegating*/ false,
2363 This, Args, Overlap: AggValueSlot::MayOverlap, Loc: E->getLocation(),
2364 /*NewPointerIsChecked*/ true);
2365}
2366
2367void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2368 const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2369 bool Delegating, CallArgList &Args) {
2370 GlobalDecl GD(Ctor, CtorType);
2371 InlinedInheritingConstructorScope Scope(*this, GD);
2372 ApplyInlineDebugLocation DebugScope(*this, GD);
2373 RunCleanupsScope RunCleanups(*this);
2374
2375 // Save the arguments to be passed to the inherited constructor.
2376 CXXInheritedCtorInitExprArgs = Args;
2377
2378 FunctionArgList Params;
2379 QualType RetType = BuildFunctionArgList(GD: CurGD, Args&: Params);
2380 FnRetTy = RetType;
2381
2382 // Insert any ABI-specific implicit constructor arguments.
2383 CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D: Ctor, Type: CtorType,
2384 ForVirtualBase, Delegating, Args);
2385
2386 // Emit a simplified prolog. We only need to emit the implicit params.
2387 assert(Args.size() >= Params.size() && "too few arguments for call");
2388 for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2389 if (I < Params.size() && isa<ImplicitParamDecl>(Val: Params[I])) {
2390 const RValue &RV = Args[I].getRValue(CGF&: *this);
2391 assert(!RV.isComplex() && "complex indirect params not supported");
2392 ParamValue Val = RV.isScalar()
2393 ? ParamValue::forDirect(value: RV.getScalarVal())
2394 : ParamValue::forIndirect(addr: RV.getAggregateAddress());
2395 EmitParmDecl(D: *Params[I], Arg: Val, ArgNo: I + 1);
2396 }
2397 }
2398
2399 // Create a return value slot if the ABI implementation wants one.
2400 // FIXME: This is dumb, we should ask the ABI not to try to set the return
2401 // value instead.
2402 if (!RetType->isVoidType())
2403 ReturnValue = CreateIRTempWithoutCast(T: RetType, Name: "retval.inhctor");
2404
2405 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
2406 CXXThisValue = CXXABIThisValue;
2407
2408 // Directly emit the constructor initializers.
2409 EmitCtorPrologue(CD: Ctor, CtorType, Args&: Params);
2410}
2411
2412void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2413 llvm::Value *VTableGlobal =
2414 CGM.getCXXABI().getVTableAddressPoint(Base: Vptr.Base, VTableClass: Vptr.VTableClass);
2415 if (!VTableGlobal)
2416 return;
2417
2418 // We can just use the base offset in the complete class.
2419 CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2420
2421 if (!NonVirtualOffset.isZero())
2422 This =
2423 ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: This, nonVirtualOffset: NonVirtualOffset, virtualOffset: nullptr,
2424 derivedClass: Vptr.VTableClass, nearestVBase: Vptr.NearestVBase);
2425
2426 llvm::Value *VPtrValue =
2427 GetVTablePtr(This, VTableTy: VTableGlobal->getType(), VTableClass: Vptr.VTableClass);
2428 llvm::Value *Cmp =
2429 Builder.CreateICmpEQ(LHS: VPtrValue, RHS: VTableGlobal, Name: "cmp.vtables");
2430 Builder.CreateAssumption(Cond: Cmp);
2431}
2432
2433void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2434 Address This) {
2435 if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: ClassDecl))
2436 for (const VPtr &Vptr : getVTablePointers(VTableClass: ClassDecl))
2437 EmitVTableAssumptionLoad(Vptr, This);
2438}
2439
2440void CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(
2441 const CXXConstructorDecl *D, Address This, Address Src,
2442 const CXXConstructExpr *E) {
2443 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2444
2445 CallArgList Args;
2446
2447 // Push the this ptr.
2448 Args.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType())),
2449 type: D->getThisType());
2450
2451 // Push the src ptr.
2452 QualType QT = *(FPT->param_type_begin());
2453 llvm::Type *t = CGM.getTypes().ConvertType(T: QT);
2454 llvm::Value *Val = getAsNaturalPointerTo(Addr: Src, PointeeType: D->getThisType());
2455 llvm::Value *SrcVal = Builder.CreateBitCast(V: Val, DestTy: t);
2456 Args.add(rvalue: RValue::get(V: SrcVal), type: QT);
2457
2458 // Skip over first argument (Src).
2459 EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(RangeOrContainer: E->arguments(), N: 1), AC: E->getConstructor(),
2460 /*ParamsToSkip*/ 1);
2461
2462 EmitCXXConstructorCall(D, Type: Ctor_Complete, /*ForVirtualBase*/ false,
2463 /*Delegating*/ false, This, Args,
2464 Overlap: AggValueSlot::MayOverlap, Loc: E->getExprLoc(),
2465 /*NewPointerIsChecked*/ false);
2466}
2467
2468void CodeGenFunction::EmitDelegateCXXConstructorCall(
2469 const CXXConstructorDecl *Ctor, CXXCtorType CtorType,
2470 const FunctionArgList &Args, SourceLocation Loc) {
2471 CallArgList DelegateArgs;
2472
2473 FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2474 assert(I != E && "no parameters to constructor");
2475
2476 // this
2477 Address This = LoadCXXThisAddress();
2478 DelegateArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(
2479 Addr: This, PointeeType: (*I)->getType()->getPointeeType())),
2480 type: (*I)->getType());
2481 ++I;
2482
2483 // FIXME: The location of the VTT parameter in the parameter list is
2484 // specific to the Itanium ABI and shouldn't be hardcoded here.
2485 if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
2486 assert(I != E && "cannot skip vtt parameter, already done with args");
2487 assert((*I)->getType()->isPointerType() &&
2488 "skipping parameter not of vtt type");
2489 ++I;
2490 }
2491
2492 // Explicit arguments.
2493 for (; I != E; ++I) {
2494 const VarDecl *param = *I;
2495 // FIXME: per-argument source location
2496 EmitDelegateCallArg(args&: DelegateArgs, param, loc: Loc);
2497 }
2498
2499 EmitCXXConstructorCall(D: Ctor, Type: CtorType, /*ForVirtualBase=*/false,
2500 /*Delegating=*/true, This, Args&: DelegateArgs,
2501 Overlap: AggValueSlot::MayOverlap, Loc,
2502 /*NewPointerIsChecked=*/true);
2503}
2504
2505namespace {
2506struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2507 const CXXDestructorDecl *Dtor;
2508 Address Addr;
2509 CXXDtorType Type;
2510
2511 CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2512 CXXDtorType Type)
2513 : Dtor(D), Addr(Addr), Type(Type) {}
2514
2515 void Emit(CodeGenFunction &CGF, Flags flags) override {
2516 // We are calling the destructor from within the constructor.
2517 // Therefore, "this" should have the expected type.
2518 QualType ThisTy = Dtor->getFunctionObjectParameterType();
2519 CGF.EmitCXXDestructorCall(D: Dtor, Type, /*ForVirtualBase=*/false,
2520 /*Delegating=*/true, This: Addr, ThisTy);
2521 }
2522};
2523} // end anonymous namespace
2524
2525void CodeGenFunction::EmitDelegatingCXXConstructorCall(
2526 const CXXConstructorDecl *Ctor, const FunctionArgList &Args) {
2527 assert(Ctor->isDelegatingConstructor());
2528
2529 Address ThisPtr = LoadCXXThisAddress();
2530
2531 AggValueSlot AggSlot = AggValueSlot::forAddr(
2532 addr: ThisPtr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
2533 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2534 mayOverlap: AggValueSlot::MayOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2535 // Checks are made by the code that calls constructor.
2536 isChecked: AggValueSlot::IsSanitizerChecked);
2537
2538 EmitAggExpr(E: Ctor->init_begin()[0]->getInit(), AS: AggSlot);
2539
2540 const CXXRecordDecl *ClassDecl = Ctor->getParent();
2541 if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2542 CXXDtorType Type =
2543 CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2544
2545 EHStack.pushCleanup<CallDelegatingCtorDtor>(
2546 Kind: EHCleanup, A: ClassDecl->getDestructor(), A: ThisPtr, A: Type);
2547 }
2548}
2549
2550void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2551 CXXDtorType Type,
2552 bool ForVirtualBase,
2553 bool Delegating, Address This,
2554 QualType ThisTy) {
2555 CGM.getCXXABI().EmitDestructorCall(CGF&: *this, DD, Type, ForVirtualBase,
2556 Delegating, This, ThisTy);
2557}
2558
2559namespace {
2560struct CallLocalDtor final : EHScopeStack::Cleanup {
2561 const CXXDestructorDecl *Dtor;
2562 Address Addr;
2563 QualType Ty;
2564
2565 CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2566 : Dtor(D), Addr(Addr), Ty(Ty) {}
2567
2568 void Emit(CodeGenFunction &CGF, Flags flags) override {
2569 CGF.EmitCXXDestructorCall(DD: Dtor, Type: Dtor_Complete,
2570 /*ForVirtualBase=*/false,
2571 /*Delegating=*/false, This: Addr, ThisTy: Ty);
2572 }
2573};
2574} // end anonymous namespace
2575
2576void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2577 QualType T, Address Addr) {
2578 EHStack.pushCleanup<CallLocalDtor>(Kind: NormalAndEHCleanup, A: D, A: Addr, A: T);
2579}
2580
2581void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2582 CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2583 if (!ClassDecl)
2584 return;
2585 if (ClassDecl->hasTrivialDestructor())
2586 return;
2587
2588 const CXXDestructorDecl *D = ClassDecl->getDestructor();
2589 assert(D && D->isUsed() && "destructor not marked as used!");
2590 PushDestructorCleanup(D, T, Addr);
2591}
2592
2593void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2594 // Compute the address point.
2595 llvm::Value *VTableAddressPoint =
2596 CGM.getCXXABI().getVTableAddressPointInStructor(
2597 CGF&: *this, RD: Vptr.VTableClass, Base: Vptr.Base, NearestVBase: Vptr.NearestVBase);
2598
2599 if (!VTableAddressPoint)
2600 return;
2601
2602 // Compute where to store the address point.
2603 llvm::Value *VirtualOffset = nullptr;
2604 CharUnits NonVirtualOffset = CharUnits::Zero();
2605
2606 if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(CGF&: *this, Vptr)) {
2607 // We need to use the virtual base offset offset because the virtual base
2608 // might have a different offset in the most derived class.
2609
2610 VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2611 CGF&: *this, This: LoadCXXThisAddress(), ClassDecl: Vptr.VTableClass, BaseClassDecl: Vptr.NearestVBase);
2612 NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2613 } else {
2614 // We can just use the base offset in the complete class.
2615 NonVirtualOffset = Vptr.Base.getBaseOffset();
2616 }
2617
2618 // Apply the offsets.
2619 Address VTableField = LoadCXXThisAddress();
2620 if (!NonVirtualOffset.isZero() || VirtualOffset)
2621 VTableField = ApplyNonVirtualAndVirtualOffset(
2622 CGF&: *this, addr: VTableField, nonVirtualOffset: NonVirtualOffset, virtualOffset: VirtualOffset, derivedClass: Vptr.VTableClass,
2623 nearestVBase: Vptr.NearestVBase);
2624
2625 // Finally, store the address point. Use the same LLVM types as the field to
2626 // support optimization.
2627 unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2628 llvm::Type *PtrTy = llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: GlobalsAS);
2629 // vtable field is derived from `this` pointer, therefore they should be in
2630 // the same addr space. Note that this might not be LLVM address space 0.
2631 VTableField = VTableField.withElementType(ElemTy: PtrTy);
2632
2633 if (auto AuthenticationInfo = CGM.getVTablePointerAuthInfo(
2634 Context: this, Record: Vptr.Base.getBase(), StorageAddress: VTableField.emitRawPointer(CGF&: *this)))
2635 VTableAddressPoint =
2636 EmitPointerAuthSign(Info: *AuthenticationInfo, Pointer: VTableAddressPoint);
2637
2638 llvm::StoreInst *Store = Builder.CreateStore(Val: VTableAddressPoint, Addr: VTableField);
2639 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: PtrTy);
2640 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2641 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2642 CGM.getCodeGenOpts().StrictVTablePointers)
2643 CGM.DecorateInstructionWithInvariantGroup(I: Store, RD: Vptr.VTableClass);
2644}
2645
2646CodeGenFunction::VPtrsVector
2647CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2648 CodeGenFunction::VPtrsVector VPtrsResult;
2649 VisitedVirtualBasesSetTy VBases;
2650 getVTablePointers(Base: BaseSubobject(VTableClass, CharUnits::Zero()),
2651 /*NearestVBase=*/nullptr,
2652 /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2653 /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2654 vptrs&: VPtrsResult);
2655 return VPtrsResult;
2656}
2657
2658void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2659 const CXXRecordDecl *NearestVBase,
2660 CharUnits OffsetFromNearestVBase,
2661 bool BaseIsNonVirtualPrimaryBase,
2662 const CXXRecordDecl *VTableClass,
2663 VisitedVirtualBasesSetTy &VBases,
2664 VPtrsVector &Vptrs) {
2665 // If this base is a non-virtual primary base the address point has already
2666 // been set.
2667 if (!BaseIsNonVirtualPrimaryBase) {
2668 // Initialize the vtable pointer for this base.
2669 VPtr Vptr = {.Base: Base, .NearestVBase: NearestVBase, .OffsetFromNearestVBase: OffsetFromNearestVBase, .VTableClass: VTableClass};
2670 Vptrs.push_back(Elt: Vptr);
2671 }
2672
2673 const CXXRecordDecl *RD = Base.getBase();
2674
2675 // Traverse bases.
2676 for (const auto &I : RD->bases()) {
2677 auto *BaseDecl = I.getType()->castAsCXXRecordDecl();
2678 // Ignore classes without a vtable.
2679 if (!BaseDecl->isDynamicClass())
2680 continue;
2681
2682 CharUnits BaseOffset;
2683 CharUnits BaseOffsetFromNearestVBase;
2684 bool BaseDeclIsNonVirtualPrimaryBase;
2685
2686 if (I.isVirtual()) {
2687 // Check if we've visited this virtual base before.
2688 if (!VBases.insert(Ptr: BaseDecl).second)
2689 continue;
2690
2691 const ASTRecordLayout &Layout =
2692 getContext().getASTRecordLayout(D: VTableClass);
2693
2694 BaseOffset = Layout.getVBaseClassOffset(VBase: BaseDecl);
2695 BaseOffsetFromNearestVBase = CharUnits::Zero();
2696 BaseDeclIsNonVirtualPrimaryBase = false;
2697 } else {
2698 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2699
2700 BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(Base: BaseDecl);
2701 BaseOffsetFromNearestVBase =
2702 OffsetFromNearestVBase + Layout.getBaseClassOffset(Base: BaseDecl);
2703 BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2704 }
2705
2706 getVTablePointers(
2707 Base: BaseSubobject(BaseDecl, BaseOffset),
2708 NearestVBase: I.isVirtual() ? BaseDecl : NearestVBase, OffsetFromNearestVBase: BaseOffsetFromNearestVBase,
2709 BaseIsNonVirtualPrimaryBase: BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2710 }
2711}
2712
2713void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2714 // Ignore classes without a vtable.
2715 if (!RD->isDynamicClass())
2716 return;
2717
2718 // Initialize the vtable pointers for this class and all of its bases.
2719 if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: RD))
2720 for (const VPtr &Vptr : getVTablePointers(VTableClass: RD))
2721 InitializeVTablePointer(Vptr);
2722
2723 if (RD->getNumVBases())
2724 CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(CGF&: *this, RD);
2725}
2726
2727llvm::Value *CodeGenFunction::GetVTablePtr(Address This, llvm::Type *VTableTy,
2728 const CXXRecordDecl *RD,
2729 VTableAuthMode AuthMode) {
2730 Address VTablePtrSrc = This.withElementType(ElemTy: VTableTy);
2731 llvm::Instruction *VTable = Builder.CreateLoad(Addr: VTablePtrSrc, Name: "vtable");
2732 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: VTableTy);
2733 CGM.DecorateInstructionWithTBAA(Inst: VTable, TBAAInfo);
2734
2735 if (auto AuthenticationInfo =
2736 CGM.getVTablePointerAuthInfo(Context: this, Record: RD, StorageAddress: This.emitRawPointer(CGF&: *this))) {
2737 if (AuthMode != VTableAuthMode::UnsafeUbsanStrip) {
2738 VTable = cast<llvm::Instruction>(
2739 Val: EmitPointerAuthAuth(Info: *AuthenticationInfo, Pointer: VTable));
2740 if (AuthMode == VTableAuthMode::MustTrap) {
2741 // This is clearly suboptimal but until we have an ability
2742 // to rely on the authentication intrinsic trapping and force
2743 // an authentication to occur we don't really have a choice.
2744 VTable =
2745 cast<llvm::Instruction>(Val: Builder.CreateBitCast(V: VTable, DestTy: Int8PtrTy));
2746 Builder.CreateLoad(Addr: RawAddress(VTable, Int8Ty, CGM.getPointerAlign()),
2747 /* IsVolatile */ true);
2748 }
2749 } else {
2750 VTable = cast<llvm::Instruction>(Val: EmitPointerAuthAuth(
2751 Info: CGPointerAuthInfo(0, PointerAuthenticationMode::Strip, false, false,
2752 nullptr),
2753 Pointer: VTable));
2754 }
2755 }
2756
2757 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2758 CGM.getCodeGenOpts().StrictVTablePointers)
2759 CGM.DecorateInstructionWithInvariantGroup(I: VTable, RD);
2760
2761 return VTable;
2762}
2763
2764// If a class has a single non-virtual base and does not introduce or override
2765// virtual member functions or fields, it will have the same layout as its base.
2766// This function returns the least derived such class.
2767//
2768// Casting an instance of a base class to such a derived class is technically
2769// undefined behavior, but it is a relatively common hack for introducing member
2770// functions on class instances with specific properties (e.g. llvm::Operator)
2771// that works under most compilers and should not have security implications, so
2772// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2773static const CXXRecordDecl *
2774LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2775 if (!RD->field_empty())
2776 return RD;
2777
2778 if (RD->getNumVBases() != 0)
2779 return RD;
2780
2781 if (RD->getNumBases() != 1)
2782 return RD;
2783
2784 for (const CXXMethodDecl *MD : RD->methods()) {
2785 if (MD->isVirtual()) {
2786 // Virtual member functions are only ok if they are implicit destructors
2787 // because the implicit destructor will have the same semantics as the
2788 // base class's destructor if no fields are added.
2789 if (isa<CXXDestructorDecl>(Val: MD) && MD->isImplicit())
2790 continue;
2791 return RD;
2792 }
2793 }
2794
2795 return LeastDerivedClassWithSameLayout(
2796 RD: RD->bases_begin()->getType()->getAsCXXRecordDecl());
2797}
2798
2799void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2800 llvm::Value *VTable,
2801 SourceLocation Loc) {
2802 if (SanOpts.has(K: SanitizerKind::CFIVCall))
2803 EmitVTablePtrCheckForCall(RD, VTable, TCK: CodeGenFunction::CFITCK_VCall, Loc);
2804 // Emit the intrinsics of (type_test and assume) for the features of WPD and
2805 // speculative devirtualization. For WPD, emit the intrinsics only for the
2806 // case of non_public LTO visibility.
2807 // TODO: refactor this condition and similar ones into a function (e.g.,
2808 // ShouldEmitDevirtualizationMD) to encapsulate the details of the different
2809 // types of devirtualization.
2810 else if ((CGM.getCodeGenOpts().WholeProgramVTables &&
2811 !CGM.AlwaysHasLTOVisibilityPublic(RD)) ||
2812 CGM.getCodeGenOpts().DevirtualizeSpeculatively) {
2813 CanQualType Ty = CGM.getContext().getCanonicalTagType(TD: RD);
2814 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T: Ty);
2815 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2816
2817 // If we already know that the call has hidden LTO visibility, emit
2818 // @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD
2819 // will convert to @llvm.type.test() if we assert at link time that we have
2820 // whole program visibility.
2821 llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
2822 ? llvm::Intrinsic::type_test
2823 : llvm::Intrinsic::public_type_test;
2824 llvm::Value *TypeTest =
2825 Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VTable, TypeId});
2826 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::assume), Args: TypeTest);
2827 }
2828}
2829
2830/// Converts the CFITypeCheckKind into SanitizerKind::SanitizerOrdinal and
2831/// llvm::SanitizerStatKind.
2832static std::pair<SanitizerKind::SanitizerOrdinal, llvm::SanitizerStatKind>
2833SanitizerInfoFromCFICheckKind(CodeGenFunction::CFITypeCheckKind TCK) {
2834 switch (TCK) {
2835 case CodeGenFunction::CFITCK_VCall:
2836 return std::make_pair(x: SanitizerKind::SO_CFIVCall, y: llvm::SanStat_CFI_VCall);
2837 case CodeGenFunction::CFITCK_NVCall:
2838 return std::make_pair(x: SanitizerKind::SO_CFINVCall,
2839 y: llvm::SanStat_CFI_NVCall);
2840 case CodeGenFunction::CFITCK_DerivedCast:
2841 return std::make_pair(x: SanitizerKind::SO_CFIDerivedCast,
2842 y: llvm::SanStat_CFI_DerivedCast);
2843 case CodeGenFunction::CFITCK_UnrelatedCast:
2844 return std::make_pair(x: SanitizerKind::SO_CFIUnrelatedCast,
2845 y: llvm::SanStat_CFI_UnrelatedCast);
2846 case CodeGenFunction::CFITCK_ICall:
2847 case CodeGenFunction::CFITCK_NVMFCall:
2848 case CodeGenFunction::CFITCK_VMFCall:
2849 llvm_unreachable("unexpected sanitizer kind");
2850 }
2851 llvm_unreachable("Unknown CFITypeCheckKind enum");
2852}
2853
2854void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2855 llvm::Value *VTable,
2856 CFITypeCheckKind TCK,
2857 SourceLocation Loc) {
2858 if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2859 RD = LeastDerivedClassWithSameLayout(RD);
2860
2861 auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2862 SanitizerDebugLocation SanScope(this, {Ordinal},
2863 SanitizerHandler::CFICheckFail);
2864
2865 EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2866}
2867
2868void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2869 bool MayBeNull,
2870 CFITypeCheckKind TCK,
2871 SourceLocation Loc) {
2872 if (!getLangOpts().CPlusPlus)
2873 return;
2874
2875 const auto *ClassDecl = T->getAsCXXRecordDecl();
2876 if (!ClassDecl)
2877 return;
2878
2879 if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2880 return;
2881
2882 if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2883 ClassDecl = LeastDerivedClassWithSameLayout(RD: ClassDecl);
2884
2885 auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2886 SanitizerDebugLocation SanScope(this, {Ordinal},
2887 SanitizerHandler::CFICheckFail);
2888
2889 llvm::BasicBlock *ContBlock = nullptr;
2890
2891 if (MayBeNull) {
2892 llvm::Value *DerivedNotNull =
2893 Builder.CreateIsNotNull(Arg: Derived.emitRawPointer(CGF&: *this), Name: "cast.nonnull");
2894
2895 llvm::BasicBlock *CheckBlock = createBasicBlock(name: "cast.check");
2896 ContBlock = createBasicBlock(name: "cast.cont");
2897
2898 Builder.CreateCondBr(Cond: DerivedNotNull, True: CheckBlock, False: ContBlock);
2899
2900 EmitBlock(BB: CheckBlock);
2901 }
2902
2903 llvm::Value *VTable;
2904 std::tie(args&: VTable, args&: ClassDecl) =
2905 CGM.getCXXABI().LoadVTablePtr(CGF&: *this, This: Derived, RD: ClassDecl);
2906
2907 EmitVTablePtrCheck(RD: ClassDecl, VTable, TCK, Loc);
2908
2909 if (MayBeNull) {
2910 Builder.CreateBr(Dest: ContBlock);
2911 EmitBlock(BB: ContBlock);
2912 }
2913}
2914
2915void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2916 llvm::Value *VTable,
2917 CFITypeCheckKind TCK,
2918 SourceLocation Loc) {
2919 assert(IsSanitizerScope);
2920
2921 if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2922 !CGM.HasHiddenLTOVisibility(RD))
2923 return;
2924
2925 auto [M, _] = SanitizerInfoFromCFICheckKind(TCK);
2926 if (getContext().getNoSanitizeList().containsType(
2927 Mask: SanitizerMask::bitPosToMask(Pos: M), MangledTypeName: RD->getQualifiedNameAsString()))
2928 return;
2929
2930 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
2931 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
2932 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
2933 llvm::Value *TypeTest = Builder.CreateCall(
2934 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, TypeId});
2935
2936 EmitVTableCheck(TCK, T, MD, VTable, TypeTest, Loc,
2937 UseCrossDsoSlowPath: CGM.getCodeGenOpts().SanitizeCfiCrossDso);
2938}
2939
2940void CodeGenFunction::EmitVTableCheck(CFITypeCheckKind TCK, CanQualType T,
2941 llvm::Metadata *MD, llvm::Value *VTable,
2942 llvm::Value *TypeTest, SourceLocation Loc,
2943 bool UseCrossDsoSlowPath) {
2944 assert(IsSanitizerScope);
2945 auto [M, SSK] = SanitizerInfoFromCFICheckKind(TCK);
2946 EmitSanitizerStatReport(SSK);
2947
2948 llvm::ConstantInt *CrossDsoTypeId =
2949 UseCrossDsoSlowPath ? CGM.CreateCrossDsoCfiTypeId(MD) : nullptr;
2950 if (!CrossDsoTypeId && (CGM.getCodeGenOpts().SanitizeTrap.has(O: M) ||
2951 CGM.getCodeGenOpts().SanitizeMinimalRuntime)) {
2952 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail, StaticArgs: {},
2953 DynamicArgs: {});
2954 return;
2955 }
2956
2957 if (CGM.getCodeGenOpts().SanitizeTrap.has(O: M)) {
2958 EmitCfiSlowPathCheck(Ordinal: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: {});
2959 return;
2960 }
2961
2962 llvm::Constant *StaticData[] = {
2963 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK),
2964 EmitCheckSourceLocation(Loc),
2965 EmitCheckTypeDescriptor(T),
2966 };
2967
2968 if (CrossDsoTypeId) {
2969 EmitCfiSlowPathCheck(Ordinal: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: StaticData);
2970 return;
2971 }
2972
2973 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2974 Context&: CGM.getLLVMContext(),
2975 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
2976 llvm::Value *ValidVtable = Builder.CreateCall(
2977 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, AllVtables});
2978 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail,
2979 StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
2980}
2981
2982static std::pair<const CXXRecordDecl *, bool>
2983getCFIVCallTarget(const CodeGenFunction &CGF, const CXXRecordDecl *RD) {
2984 if (!CGF.SanOpts.has(K: SanitizerKind::CFIVCall))
2985 return {RD, false};
2986 if (!CGF.SanOpts.has(K: SanitizerKind::CFICastStrict))
2987 RD = LeastDerivedClassWithSameLayout(RD);
2988 bool ShouldEmitCFICheck = !CGF.getContext().getNoSanitizeList().containsType(
2989 Mask: SanitizerKind::CFIVCall, MangledTypeName: RD->getQualifiedNameAsString());
2990 return {RD, ShouldEmitCFICheck};
2991}
2992
2993bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2994 if (!CGM.getCodeGenOpts().WholeProgramVTables)
2995 return false;
2996
2997 auto [TargetRD, ShouldEmitCFICheck] = getCFIVCallTarget(CGF: *this, RD);
2998 if (!CGM.HasHiddenLTOVisibility(RD: TargetRD))
2999 return false;
3000
3001 // GlobalDCE's virtual function elimination only treats vtable slots as live
3002 // if they are loaded via llvm.type.checked.load, so it must always be used.
3003 return CGM.getCodeGenOpts().VirtualFunctionElimination || ShouldEmitCFICheck;
3004}
3005
3006llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
3007 const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
3008 uint64_t VTableByteOffset, SourceLocation Loc) {
3009 auto [TargetRD, ShouldEmitCFICheck] = getCFIVCallTarget(CGF: *this, RD);
3010
3011 CanQualType T = CGM.getContext().getCanonicalTagType(TD: TargetRD);
3012 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
3013 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
3014 auto CheckedLoadIntrinsic = CGM.getLangOpts().RelativeCXXABIVTables
3015 ? llvm::Intrinsic::type_checked_load_relative
3016 : llvm::Intrinsic::type_checked_load;
3017
3018 std::optional<SanitizerDebugLocation> SanScope;
3019 if (ShouldEmitCFICheck) {
3020 SanScope.emplace(args: this, args: SanitizerKind::SO_CFIVCall,
3021 args: SanitizerHandler::CFICheckFail);
3022 }
3023 llvm::Value *CheckedLoad = Builder.CreateCall(
3024 Callee: CGM.getIntrinsic(IID: CheckedLoadIntrinsic),
3025 Args: {VTable, llvm::ConstantInt::get(Ty: Int32Ty, V: VTableByteOffset), TypeId});
3026 if (ShouldEmitCFICheck) {
3027 llvm::Value *CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
3028 bool UseCrossDsoSlowPath =
3029 CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
3030 !CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIVCall);
3031 EmitVTableCheck(TCK: CFITCK_VCall, T, MD, VTable, TypeTest: CheckResult, Loc,
3032 UseCrossDsoSlowPath);
3033 }
3034 return Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0),
3035 DestTy: VTableTy);
3036}
3037
3038void CodeGenFunction::EmitForwardingCallToLambda(
3039 const CXXMethodDecl *callOperator, CallArgList &callArgs,
3040 const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {
3041 // Get the address of the call operator.
3042 if (!calleeFnInfo)
3043 calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: callOperator);
3044
3045 if (!calleePtr)
3046 calleePtr =
3047 CGM.GetAddrOfFunction(GD: GlobalDecl(callOperator),
3048 Ty: CGM.getTypes().GetFunctionType(Info: *calleeFnInfo));
3049
3050 // Prepare the return slot.
3051 const FunctionProtoType *FPT =
3052 callOperator->getType()->castAs<FunctionProtoType>();
3053 QualType resultType = FPT->getReturnType();
3054 ReturnValueSlot returnSlot;
3055 if (!resultType->isVoidType() &&
3056 calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
3057 !hasScalarEvaluationKind(T: calleeFnInfo->getReturnType()))
3058 returnSlot =
3059 ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
3060 /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
3061
3062 // We don't need to separately arrange the call arguments because
3063 // the call can't be variadic anyway --- it's impossible to forward
3064 // variadic arguments.
3065
3066 // Now emit our call.
3067 auto callee = CGCallee::forDirect(functionPtr: calleePtr, abstractInfo: GlobalDecl(callOperator));
3068 RValue RV = EmitCall(CallInfo: *calleeFnInfo, Callee: callee, ReturnValue: returnSlot, Args: callArgs);
3069
3070 // If necessary, copy the returned value into the slot.
3071 if (!resultType->isVoidType() && returnSlot.isNull()) {
3072 if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
3073 RV = RValue::get(V: EmitARCRetainAutoreleasedReturnValue(value: RV.getScalarVal()));
3074 }
3075 EmitReturnOfRValue(RV, Ty: resultType);
3076 } else
3077 EmitBranchThroughCleanup(Dest: ReturnBlock);
3078}
3079
3080void CodeGenFunction::EmitLambdaBlockInvokeBody() {
3081 const BlockDecl *BD = BlockInfo->getBlockDecl();
3082 const VarDecl *variable = BD->capture_begin()->getVariable();
3083 const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
3084 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3085
3086 if (CallOp->isVariadic()) {
3087 // FIXME: Making this work correctly is nasty because it requires either
3088 // cloning the body of the call operator or making the call operator
3089 // forward.
3090 CGM.ErrorUnsupported(D: CurCodeDecl, Type: "lambda conversion to variadic function");
3091 return;
3092 }
3093
3094 // Start building arguments for forwarding call
3095 CallArgList CallArgs;
3096
3097 CanQualType ThisType =
3098 getContext().getPointerType(T: getContext().getCanonicalTagType(TD: Lambda));
3099 Address ThisPtr = GetAddrOfBlockDecl(var: variable);
3100 CallArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: ThisPtr, PointeeType: ThisType)), type: ThisType);
3101
3102 // Add the rest of the parameters.
3103 for (auto *param : BD->parameters())
3104 EmitDelegateCallArg(args&: CallArgs, param, loc: param->getBeginLoc());
3105
3106 assert(!Lambda->isGenericLambda() &&
3107 "generic lambda interconversion to block not implemented");
3108 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3109}
3110
3111void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
3112 if (MD->isVariadic()) {
3113 // FIXME: Making this work correctly is nasty because it requires either
3114 // cloning the body of the call operator or making the call operator
3115 // forward.
3116 CGM.ErrorUnsupported(D: MD, Type: "lambda conversion to variadic function");
3117 return;
3118 }
3119
3120 const CXXRecordDecl *Lambda = MD->getParent();
3121
3122 // Start building arguments for forwarding call
3123 CallArgList CallArgs;
3124
3125 CanQualType LambdaType = getContext().getCanonicalTagType(TD: Lambda);
3126 CanQualType ThisType = getContext().getPointerType(T: LambdaType);
3127 Address ThisPtr = CreateMemTempWithoutCast(T: LambdaType, Name: "unused.capture");
3128 CallArgs.add(rvalue: RValue::get(V: ThisPtr.emitRawPointer(CGF&: *this)), type: ThisType);
3129
3130 EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3131}
3132
3133void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
3134 CallArgList &CallArgs) {
3135 // Add the rest of the forwarded parameters.
3136 for (auto *Param : MD->parameters())
3137 EmitDelegateCallArg(args&: CallArgs, param: Param, loc: Param->getBeginLoc());
3138
3139 const CXXRecordDecl *Lambda = MD->getParent();
3140 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3141 // For a generic lambda, find the corresponding call operator specialization
3142 // to which the call to the static-invoker shall be forwarded.
3143 if (Lambda->isGenericLambda()) {
3144 assert(MD->isFunctionTemplateSpecialization());
3145 const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
3146 FunctionTemplateDecl *CallOpTemplate =
3147 CallOp->getDescribedFunctionTemplate();
3148 llvm::FoldingSetInsertToken InsertToken;
3149 FunctionDecl *CorrespondingCallOpSpecialization =
3150 CallOpTemplate->findSpecialization(Args: TAL->asArray(), InsertToken);
3151 assert(CorrespondingCallOpSpecialization);
3152 CallOp = cast<CXXMethodDecl>(Val: CorrespondingCallOpSpecialization);
3153 }
3154
3155 // Special lambda forwarding when there are inalloca parameters.
3156 if (hasInAllocaArg(MD)) {
3157 const CGFunctionInfo *ImplFnInfo = nullptr;
3158 llvm::Function *ImplFn = nullptr;
3159 EmitLambdaInAllocaImplFn(CallOp, ImplFnInfo: &ImplFnInfo, ImplFn: &ImplFn);
3160
3161 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs, calleeFnInfo: ImplFnInfo, calleePtr: ImplFn);
3162 return;
3163 }
3164
3165 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3166}
3167
3168void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
3169 if (MD->isVariadic()) {
3170 // FIXME: Making this work correctly is nasty because it requires either
3171 // cloning the body of the call operator or making the call operator
3172 // forward.
3173 CGM.ErrorUnsupported(D: MD, Type: "lambda conversion to variadic function");
3174 return;
3175 }
3176
3177 // Forward %this argument.
3178 CallArgList CallArgs;
3179 CanQualType LambdaType = getContext().getCanonicalTagType(TD: MD->getParent());
3180 CanQualType ThisType = getContext().getPointerType(T: LambdaType);
3181 llvm::Value *ThisArg = CurFn->getArg(i: 0);
3182 CallArgs.add(rvalue: RValue::get(V: ThisArg), type: ThisType);
3183
3184 EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3185}
3186
3187void CodeGenFunction::EmitLambdaInAllocaImplFn(
3188 const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,
3189 llvm::Function **ImplFn) {
3190 const CGFunctionInfo &FnInfo =
3191 CGM.getTypes().arrangeCXXMethodDeclaration(MD: CallOp);
3192 llvm::Function *CallOpFn =
3193 cast<llvm::Function>(Val: CGM.GetAddrOfFunction(GD: GlobalDecl(CallOp)));
3194
3195 // Emit function containing the original call op body. __invoke will delegate
3196 // to this function.
3197 SmallVector<CanQualType, 4> ArgTypes;
3198 for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)
3199 ArgTypes.push_back(Elt: I->type);
3200 *ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(
3201 returnType: FnInfo.getReturnType(), opts: FnInfoOpts::IsDelegateCall, argTypes: ArgTypes,
3202 info: FnInfo.getExtInfo(), paramInfos: {}, args: FnInfo.getRequiredArgs(), ABIInfoFD: CallOp);
3203
3204 // Create mangled name as if this was a method named __impl. If for some
3205 // reason the name doesn't look as expected then just tack __impl to the
3206 // front.
3207 // TODO: Use the name mangler to produce the right name instead of using
3208 // string replacement.
3209 StringRef CallOpName = CallOpFn->getName();
3210 std::string ImplName;
3211 if (size_t Pos = CallOpName.find_first_of(Chars: "<lambda"))
3212 ImplName = ("?__impl@" + CallOpName.drop_front(N: Pos)).str();
3213 else
3214 ImplName = ("__impl" + CallOpName).str();
3215
3216 llvm::Function *Fn = CallOpFn->getParent()->getFunction(Name: ImplName);
3217 if (!Fn) {
3218 Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: **ImplFnInfo),
3219 Linkage: llvm::GlobalValue::InternalLinkage, N: ImplName,
3220 M&: CGM.getModule());
3221 CGM.SetInternalFunctionAttributes(GD: CallOp, F: Fn, FI: **ImplFnInfo);
3222
3223 const GlobalDecl &GD = GlobalDecl(CallOp);
3224 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
3225 CodeGenFunction(CGM).GenerateCode(GD, Fn, FnInfo: **ImplFnInfo);
3226 CGM.SetLLVMFunctionAttributesForDefinition(D, F: Fn);
3227 }
3228 *ImplFn = Fn;
3229}
3230