1//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with C++ code generation of classes
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGBlocks.h"
15#include "CGCXXABI.h"
16#include "CGDebugInfo.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "TargetInfo.h"
20#include "clang/AST/Attr.h"
21#include "clang/AST/CXXInheritance.h"
22#include "clang/AST/CharUnits.h"
23#include "clang/AST/DeclTemplate.h"
24#include "clang/AST/EvaluatedExprVisitor.h"
25#include "clang/AST/RecordLayout.h"
26#include "clang/AST/StmtCXX.h"
27#include "clang/Basic/CodeGenOptions.h"
28#include "clang/CodeGen/CGFunctionInfo.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/Metadata.h"
31#include "llvm/Support/SaveAndRestore.h"
32#include "llvm/Transforms/Utils/ModuleUtils.h"
33#include "llvm/Transforms/Utils/SanitizerStats.h"
34#include <optional>
35
36using namespace clang;
37using namespace CodeGen;
38
39/// Return the best known alignment for an unknown pointer to a
40/// particular class.
41CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
42 if (!RD->hasDefinition())
43 return CharUnits::One(); // Hopefully won't be used anywhere.
44
45 auto &layout = getContext().getASTRecordLayout(D: RD);
46
47 // If the class is final, then we know that the pointer points to an
48 // object of that type and can use the full alignment.
49 if (RD->isEffectivelyFinal())
50 return layout.getAlignment();
51
52 // Otherwise, we have to assume it could be a subclass.
53 return layout.getNonVirtualAlignment();
54}
55
56/// Return the smallest possible amount of storage that might be allocated
57/// starting from the beginning of an object of a particular class.
58///
59/// This may be smaller than sizeof(RD) if RD has virtual base classes.
60CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
61 if (!RD->hasDefinition())
62 return CharUnits::One();
63
64 auto &layout = getContext().getASTRecordLayout(D: RD);
65
66 // If the class is final, then we know that the pointer points to an
67 // object of that type and can use the full alignment.
68 if (RD->isEffectivelyFinal())
69 return layout.getSize();
70
71 // Otherwise, we have to assume it could be a subclass.
72 return std::max(a: layout.getNonVirtualSize(), b: CharUnits::One());
73}
74
75/// Return the best known alignment for a pointer to a virtual base,
76/// given the alignment of a pointer to the derived class.
77CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
78 const CXXRecordDecl *derivedClass,
79 const CXXRecordDecl *vbaseClass) {
80 // The basic idea here is that an underaligned derived pointer might
81 // indicate an underaligned base pointer.
82
83 assert(vbaseClass->isCompleteDefinition());
84 auto &baseLayout = getContext().getASTRecordLayout(D: vbaseClass);
85 CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
86
87 return getDynamicOffsetAlignment(ActualAlign: actualDerivedAlign, Class: derivedClass,
88 ExpectedTargetAlign: expectedVBaseAlign);
89}
90
91CharUnits
92CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
93 const CXXRecordDecl *baseDecl,
94 CharUnits expectedTargetAlign) {
95 // If the base is an incomplete type (which is, alas, possible with
96 // member pointers), be pessimistic.
97 if (!baseDecl->isCompleteDefinition())
98 return std::min(a: actualBaseAlign, b: expectedTargetAlign);
99
100 auto &baseLayout = getContext().getASTRecordLayout(D: baseDecl);
101 CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
102
103 // If the class is properly aligned, assume the target offset is, too.
104 //
105 // This actually isn't necessarily the right thing to do --- if the
106 // class is a complete object, but it's only properly aligned for a
107 // base subobject, then the alignments of things relative to it are
108 // probably off as well. (Note that this requires the alignment of
109 // the target to be greater than the NV alignment of the derived
110 // class.)
111 //
112 // However, our approach to this kind of under-alignment can only
113 // ever be best effort; after all, we're never going to propagate
114 // alignments through variables or parameters. Note, in particular,
115 // that constructing a polymorphic type in an address that's less
116 // than pointer-aligned will generally trap in the constructor,
117 // unless we someday add some sort of attribute to change the
118 // assumed alignment of 'this'. So our goal here is pretty much
119 // just to allow the user to explicitly say that a pointer is
120 // under-aligned and then safely access its fields and vtables.
121 if (actualBaseAlign >= expectedBaseAlign) {
122 return expectedTargetAlign;
123 }
124
125 // Otherwise, we might be offset by an arbitrary multiple of the
126 // actual alignment. The correct adjustment is to take the min of
127 // the two alignments.
128 return std::min(a: actualBaseAlign, b: expectedTargetAlign);
129}
130
131Address CodeGenFunction::LoadCXXThisAddress() {
132 assert(CurFuncDecl && "loading 'this' without a func declaration?");
133 auto *MD = cast<CXXMethodDecl>(Val: CurFuncDecl);
134
135 // Lazily compute CXXThisAlignment.
136 if (CXXThisAlignment.isZero()) {
137 // Just use the best known alignment for the parent.
138 // TODO: if we're currently emitting a complete-object ctor/dtor,
139 // we can always use the complete-object alignment.
140 CXXThisAlignment = CGM.getClassPointerAlignment(RD: MD->getParent());
141 }
142
143 return makeNaturalAddressForPointer(
144 Ptr: LoadCXXThis(), T: MD->getFunctionObjectParameterType(), Alignment: CXXThisAlignment,
145 ForPointeeType: false, BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
146}
147
148/// Emit the address of a field using a member data pointer.
149///
150/// \param E Only used for emergency diagnostics
151Address CodeGenFunction::EmitCXXMemberDataPointerAddress(
152 const Expr *E, Address base, llvm::Value *memberPtr,
153 const MemberPointerType *memberPtrType, bool IsInBounds,
154 LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
155 // Ask the ABI to compute the actual address.
156 llvm::Value *ptr = CGM.getCXXABI().EmitMemberDataPointerAddress(
157 CGF&: *this, E, Base: base, MemPtr: memberPtr, MPT: memberPtrType, IsInBounds);
158
159 QualType memberType = memberPtrType->getPointeeType();
160 CharUnits memberAlign =
161 CGM.getNaturalTypeAlignment(T: memberType, BaseInfo, TBAAInfo);
162 memberAlign = CGM.getDynamicOffsetAlignment(
163 actualBaseAlign: base.getAlignment(), baseDecl: memberPtrType->getMostRecentCXXRecordDecl(),
164 expectedTargetAlign: memberAlign);
165 return Address(ptr, ConvertTypeForMem(T: memberPtrType->getPointeeType()),
166 memberAlign);
167}
168
169CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
170 const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
171 CastExpr::path_const_iterator End) {
172 CharUnits Offset = CharUnits::Zero();
173
174 const ASTContext &Context = getContext();
175 const CXXRecordDecl *RD = DerivedClass;
176
177 for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
178 const CXXBaseSpecifier *Base = *I;
179 assert(!Base->isVirtual() && "Should not see virtual bases here!");
180
181 // Get the layout.
182 const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
183
184 const auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
185 // Add the offset.
186 Offset += Layout.getBaseClassOffset(Base: BaseDecl);
187
188 RD = BaseDecl;
189 }
190
191 return Offset;
192}
193
194llvm::Constant *CodeGenModule::GetNonVirtualBaseClassOffset(
195 const CXXRecordDecl *ClassDecl, CastExpr::path_const_iterator PathBegin,
196 CastExpr::path_const_iterator PathEnd) {
197 assert(PathBegin != PathEnd && "Base path should not be empty!");
198
199 CharUnits Offset =
200 computeNonVirtualBaseClassOffset(DerivedClass: ClassDecl, Start: PathBegin, End: PathEnd);
201 if (Offset.isZero())
202 return nullptr;
203
204 llvm::Type *PtrDiffTy =
205 getTypes().ConvertType(T: getContext().getPointerDiffType());
206
207 return llvm::ConstantInt::get(Ty: PtrDiffTy, V: Offset.getQuantity());
208}
209
210/// Gets the address of a direct base class within a complete object.
211/// This should only be used for (1) non-virtual bases or (2) virtual bases
212/// when the type is known to be complete (e.g. in complete destructors).
213///
214/// The object pointed to by 'This' is assumed to be non-null.
215Address CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(
216 Address This, const CXXRecordDecl *Derived, const CXXRecordDecl *Base,
217 bool BaseIsVirtual) {
218 // 'this' must be a pointer (in some address space) to Derived.
219 assert(This.getElementType() == ConvertType(Derived));
220
221 // Compute the offset of the virtual base.
222 CharUnits Offset;
223 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: Derived);
224 if (BaseIsVirtual)
225 Offset = Layout.getVBaseClassOffset(VBase: Base);
226 else
227 Offset = Layout.getBaseClassOffset(Base);
228
229 // Shift and cast down to the base type.
230 // TODO: for complete types, this should be possible with a GEP.
231 Address V = This;
232 if (!Offset.isZero()) {
233 V = V.withElementType(ElemTy: Int8Ty);
234 V = Builder.CreateConstInBoundsByteGEP(Addr: V, Offset);
235 }
236 return V.withElementType(ElemTy: ConvertType(T: Base));
237}
238
239static Address ApplyNonVirtualAndVirtualOffset(
240 CodeGenFunction &CGF, Address addr, CharUnits nonVirtualOffset,
241 llvm::Value *virtualOffset, const CXXRecordDecl *derivedClass,
242 const CXXRecordDecl *nearestVBase) {
243 // Assert that we have something to do.
244 assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
245
246 // Compute the offset from the static and dynamic components.
247 llvm::Value *baseOffset;
248 if (!nonVirtualOffset.isZero()) {
249 llvm::Type *OffsetType =
250 (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
251 CGF.CGM.getLangOpts().RelativeCXXABIVTables)
252 ? CGF.Int32Ty
253 : CGF.PtrDiffTy;
254 baseOffset =
255 llvm::ConstantInt::get(Ty: OffsetType, V: nonVirtualOffset.getQuantity());
256 if (virtualOffset) {
257 baseOffset = CGF.Builder.CreateAdd(LHS: virtualOffset, RHS: baseOffset);
258 }
259 } else {
260 baseOffset = virtualOffset;
261 }
262
263 // Apply the base offset.
264 llvm::Value *ptr = addr.emitRawPointer(CGF);
265 ptr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ptr, IdxList: baseOffset, Name: "add.ptr");
266
267 // If we have a virtual component, the alignment of the result will
268 // be relative only to the known alignment of that vbase.
269 CharUnits alignment;
270 if (virtualOffset) {
271 assert(nearestVBase && "virtual offset without vbase?");
272 alignment = CGF.CGM.getVBaseAlignment(actualDerivedAlign: addr.getAlignment(), derivedClass,
273 vbaseClass: nearestVBase);
274 } else {
275 alignment = addr.getAlignment();
276 }
277 alignment = alignment.alignmentAtOffset(offset: nonVirtualOffset);
278
279 return Address(ptr, CGF.Int8Ty, alignment);
280}
281
282Address CodeGenFunction::GetAddressOfBaseClass(
283 Address Value, const CXXRecordDecl *Derived,
284 CastExpr::path_const_iterator PathBegin,
285 CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
286 SourceLocation Loc) {
287 assert(PathBegin != PathEnd && "Base path should not be empty!");
288
289 CastExpr::path_const_iterator Start = PathBegin;
290 const CXXRecordDecl *VBase = nullptr;
291
292 // Sema has done some convenient canonicalization here: if the
293 // access path involved any virtual steps, the conversion path will
294 // *start* with a step down to the correct virtual base subobject,
295 // and hence will not require any further steps.
296 if ((*Start)->isVirtual()) {
297 VBase = (*Start)->getType()->castAsCXXRecordDecl();
298 ++Start;
299 }
300
301 // Compute the static offset of the ultimate destination within its
302 // allocating subobject (the virtual base, if there is one, or else
303 // the "complete" object that we see).
304 CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
305 DerivedClass: VBase ? VBase : Derived, Start, End: PathEnd);
306
307 // If there's a virtual step, we can sometimes "devirtualize" it.
308 // For now, that's limited to when the derived type is final.
309 // TODO: "devirtualize" this for accesses to known-complete objects.
310 if (VBase && Derived->hasAttr<FinalAttr>()) {
311 const ASTRecordLayout &layout = getContext().getASTRecordLayout(D: Derived);
312 CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
313 NonVirtualOffset += vBaseOffset;
314 VBase = nullptr; // we no longer have a virtual step
315 }
316
317 // Get the base pointer type.
318 llvm::Type *BaseValueTy = ConvertType(T: (PathEnd[-1])->getType());
319 llvm::Type *PtrTy = llvm::PointerType::get(
320 C&: CGM.getLLVMContext(), AddressSpace: Value.getType()->getPointerAddressSpace());
321
322 CanQualType DerivedTy = getContext().getCanonicalTagType(TD: Derived);
323 CharUnits DerivedAlign = CGM.getClassPointerAlignment(RD: Derived);
324
325 // If the static offset is zero and we don't have a virtual step,
326 // just do a bitcast; null checks are unnecessary.
327 if (NonVirtualOffset.isZero() && !VBase) {
328 if (sanitizePerformTypeCheck()) {
329 SanitizerSet SkippedChecks;
330 SkippedChecks.set(K: SanitizerKind::Null, Value: !NullCheckValue);
331 EmitTypeCheck(TCK: TCK_Upcast, Loc, V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy,
332 Alignment: DerivedAlign, SkippedChecks);
333 }
334 return Value.withElementType(ElemTy: BaseValueTy);
335 }
336
337 llvm::BasicBlock *origBB = nullptr;
338 llvm::BasicBlock *endBB = nullptr;
339
340 // Skip over the offset (and the vtable load) if we're supposed to
341 // null-check the pointer.
342 if (NullCheckValue) {
343 origBB = Builder.GetInsertBlock();
344 llvm::BasicBlock *notNullBB = createBasicBlock(name: "cast.notnull");
345 endBB = createBasicBlock(name: "cast.end");
346
347 llvm::Value *isNull = Builder.CreateIsNull(Addr: Value);
348 Builder.CreateCondBr(Cond: isNull, True: endBB, False: notNullBB);
349 EmitBlock(BB: notNullBB);
350 }
351
352 if (sanitizePerformTypeCheck()) {
353 SanitizerSet SkippedChecks;
354 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
355 EmitTypeCheck(TCK: VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
356 V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy, Alignment: DerivedAlign,
357 SkippedChecks);
358 }
359
360 // Compute the virtual offset.
361 llvm::Value *VirtualOffset = nullptr;
362 if (VBase) {
363 VirtualOffset =
364 CGM.getCXXABI().GetVirtualBaseClassOffset(CGF&: *this, This: Value, ClassDecl: Derived, BaseClassDecl: VBase);
365 }
366
367 // Apply both offsets.
368 Value = ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: Value, nonVirtualOffset: NonVirtualOffset,
369 virtualOffset: VirtualOffset, derivedClass: Derived, nearestVBase: VBase);
370
371 // Cast to the destination type.
372 Value = Value.withElementType(ElemTy: BaseValueTy);
373
374 // Build a phi if we needed a null check.
375 if (NullCheckValue) {
376 llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
377 Builder.CreateBr(Dest: endBB);
378 EmitBlock(BB: endBB);
379
380 llvm::PHINode *PHI = Builder.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "cast.result");
381 PHI->addIncoming(V: Value.emitRawPointer(CGF&: *this), BB: notNullBB);
382 PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: PtrTy), BB: origBB);
383 Value = Value.withPointer(NewPointer: PHI, IsKnownNonNull: NotKnownNonNull);
384 }
385
386 return Value;
387}
388
389Address CodeGenFunction::GetAddressOfDerivedClass(
390 Address BaseAddr, const CXXRecordDecl *Derived,
391 CastExpr::path_const_iterator PathBegin,
392 CastExpr::path_const_iterator PathEnd, bool NullCheckValue) {
393 assert(PathBegin != PathEnd && "Base path should not be empty!");
394
395 CanQualType DerivedTy = getContext().getCanonicalTagType(TD: Derived);
396 llvm::Type *DerivedValueTy = ConvertType(T: DerivedTy);
397
398 llvm::Value *NonVirtualOffset =
399 CGM.GetNonVirtualBaseClassOffset(ClassDecl: Derived, PathBegin, PathEnd);
400
401 if (!NonVirtualOffset) {
402 // No offset, we can just cast back.
403 return BaseAddr.withElementType(ElemTy: DerivedValueTy);
404 }
405
406 llvm::BasicBlock *CastNull = nullptr;
407 llvm::BasicBlock *CastNotNull = nullptr;
408 llvm::BasicBlock *CastEnd = nullptr;
409
410 if (NullCheckValue) {
411 CastNull = createBasicBlock(name: "cast.null");
412 CastNotNull = createBasicBlock(name: "cast.notnull");
413 CastEnd = createBasicBlock(name: "cast.end");
414
415 llvm::Value *IsNull = Builder.CreateIsNull(Addr: BaseAddr);
416 Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
417 EmitBlock(BB: CastNotNull);
418 }
419
420 // Apply the offset.
421 Address Addr = BaseAddr.withElementType(ElemTy: Int8Ty);
422 Addr = Builder.CreateInBoundsGEP(
423 Addr, IdxList: Builder.CreateNeg(V: NonVirtualOffset), ElementType: Int8Ty,
424 Align: CGM.getClassPointerAlignment(RD: Derived), Name: "sub.ptr");
425
426 // Just cast.
427 Addr = Addr.withElementType(ElemTy: DerivedValueTy);
428
429 // Produce a PHI if we had a null-check.
430 if (NullCheckValue) {
431 Builder.CreateBr(Dest: CastEnd);
432 EmitBlock(BB: CastNull);
433 Builder.CreateBr(Dest: CastEnd);
434 EmitBlock(BB: CastEnd);
435
436 llvm::Value *Value = Addr.emitRawPointer(CGF&: *this);
437 llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
438 PHI->addIncoming(V: Value, BB: CastNotNull);
439 PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: Value->getType()), BB: CastNull);
440 return Address(PHI, Addr.getElementType(),
441 CGM.getClassPointerAlignment(RD: Derived));
442 }
443
444 return Addr;
445}
446
447llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
448 bool ForVirtualBase,
449 bool Delegating) {
450 if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
451 // This constructor/destructor does not need a VTT parameter.
452 return nullptr;
453 }
454
455 const CXXRecordDecl *RD = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParent();
456 const CXXRecordDecl *Base = cast<CXXMethodDecl>(Val: GD.getDecl())->getParent();
457
458 uint64_t SubVTTIndex;
459
460 if (Delegating) {
461 // If this is a delegating constructor call, just load the VTT.
462 return LoadCXXVTT();
463 } else if (RD == Base) {
464 // If the record matches the base, this is the complete ctor/dtor
465 // variant calling the base variant in a class with virtual bases.
466 assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
467 "doing no-op VTT offset in base dtor/ctor?");
468 assert(!ForVirtualBase && "Can't have same class as virtual base!");
469 SubVTTIndex = 0;
470 } else {
471 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
472 CharUnits BaseOffset = ForVirtualBase ? Layout.getVBaseClassOffset(VBase: Base)
473 : Layout.getBaseClassOffset(Base);
474
475 SubVTTIndex =
476 CGM.getVTables().getSubVTTIndex(RD, Base: BaseSubobject(Base, BaseOffset));
477 assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
478 }
479
480 llvm::Value *VTT;
481 if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
482 // A VTT parameter was passed to the constructor, use it.
483 VTT = LoadCXXVTT();
484 } else {
485 // We're the complete constructor, so get the VTT by name.
486 VTT = CGM.getVTables().GetAddrOfVTT(RD);
487 }
488 return Builder.CreateConstInBoundsGEP1_64(Ty: CGM.GlobalsInt8PtrTy, Ptr: VTT,
489 Idx0: SubVTTIndex);
490}
491
492namespace {
493/// Call the destructor for a direct base class.
494struct CallBaseDtor final : EHScopeStack::Cleanup {
495 const CXXRecordDecl *BaseClass;
496 bool BaseIsVirtual;
497 CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
498 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
499
500 void Emit(CodeGenFunction &CGF, Flags flags) override {
501 const CXXRecordDecl *DerivedClass =
502 cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
503
504 const CXXDestructorDecl *D = BaseClass->getDestructor();
505 // We are already inside a destructor, so presumably the object being
506 // destroyed should have the expected type.
507 QualType ThisTy = D->getFunctionObjectParameterType();
508 Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
509 This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
510 CGF.EmitCXXDestructorCall(D, Type: Dtor_Base, ForVirtualBase: BaseIsVirtual,
511 /*Delegating=*/false, This: Addr, ThisTy);
512 }
513};
514
515/// A visitor which checks whether an initializer uses 'this' in a
516/// way which requires the vtable to be properly set.
517struct DynamicThisUseChecker
518 : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
519 typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
520
521 bool UsesThis;
522
523 DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
524
525 // Black-list all explicit and implicit references to 'this'.
526 //
527 // Do we need to worry about external references to 'this' derived
528 // from arbitrary code? If so, then anything which runs arbitrary
529 // external code might potentially access the vtable.
530 void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
531};
532} // end anonymous namespace
533
534static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
535 DynamicThisUseChecker Checker(C);
536 Checker.Visit(S: Init);
537 return Checker.UsesThis;
538}
539
540static void EmitBaseInitializer(CodeGenFunction &CGF,
541 const CXXRecordDecl *ClassDecl,
542 CXXCtorInitializer *BaseInit) {
543 assert(BaseInit->isBaseInitializer() && "Must have base initializer!");
544
545 Address ThisPtr = CGF.LoadCXXThisAddress();
546
547 const auto *BaseClassDecl = BaseInit->getBaseClass()->castAsCXXRecordDecl();
548
549 bool isBaseVirtual = BaseInit->isBaseVirtual();
550
551 // If the initializer for the base (other than the constructor
552 // itself) accesses 'this' in any way, we need to initialize the
553 // vtables.
554 if (BaseInitializerUsesThis(C&: CGF.getContext(), Init: BaseInit->getInit()))
555 CGF.InitializeVTablePointers(ClassDecl);
556
557 // We can pretend to be a complete class because it only matters for
558 // virtual bases, and we only do virtual bases for complete ctors.
559 Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
560 This: ThisPtr, Derived: ClassDecl, Base: BaseClassDecl, BaseIsVirtual: isBaseVirtual);
561 AggValueSlot AggSlot = AggValueSlot::forAddr(
562 addr: V, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
563 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
564 mayOverlap: CGF.getOverlapForBaseInit(RD: ClassDecl, BaseRD: BaseClassDecl, IsVirtual: isBaseVirtual));
565
566 CGF.EmitAggExpr(E: BaseInit->getInit(), AS: AggSlot);
567
568 if (CGF.CGM.getLangOpts().Exceptions &&
569 !BaseClassDecl->hasTrivialDestructor())
570 CGF.EHStack.pushCleanup<CallBaseDtor>(Kind: EHCleanup, A: BaseClassDecl,
571 A: isBaseVirtual);
572}
573
574static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
575 CXXCtorInitializer *MemberInit,
576 LValue &LHS) {
577 FieldDecl *Field = MemberInit->getAnyMember();
578 if (MemberInit->isIndirectMemberInitializer()) {
579 // If we are initializing an anonymous union field, drill down to the field.
580 IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
581 for (const auto *I : IndirectField->chain())
582 LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field: cast<FieldDecl>(Val: I));
583 } else {
584 LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field);
585 }
586}
587
588static void EmitMemberInitializer(CodeGenFunction &CGF,
589 const CXXRecordDecl *ClassDecl,
590 CXXCtorInitializer *MemberInit,
591 const CXXConstructorDecl *Constructor,
592 FunctionArgList &Args) {
593 ApplyAtomGroup Grp(CGF.getDebugInfo());
594 ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
595 assert(MemberInit->isAnyMemberInitializer() &&
596 "Must have member initializer!");
597 assert(MemberInit->getInit() && "Must have initializer!");
598
599 // non-static data member initializers.
600 FieldDecl *Field = MemberInit->getAnyMember();
601 QualType FieldType = Field->getType();
602
603 llvm::Value *ThisPtr = CGF.LoadCXXThis();
604 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
605 LValue LHS;
606
607 // If a base constructor is being emitted, create an LValue that has the
608 // non-virtual alignment.
609 if (CGF.CurGD.getCtorType() == Ctor_Base)
610 LHS = CGF.MakeNaturalAlignPointeeAddrLValue(V: ThisPtr, T: RecordTy);
611 else
612 LHS = CGF.MakeNaturalAlignAddrLValue(V: ThisPtr, T: RecordTy);
613
614 EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
615
616 // Special case: if we are in a copy or move constructor, and we are copying
617 // an array of PODs or classes with trivial copy constructors, ignore the
618 // AST and perform the copy we know is equivalent.
619 // FIXME: This is hacky at best... if we had a bit more explicit information
620 // in the AST, we could generalize it more easily.
621 const ConstantArrayType *Array =
622 CGF.getContext().getAsConstantArrayType(T: FieldType);
623 if (Array && Constructor->isDefaulted() &&
624 Constructor->isCopyOrMoveConstructor()) {
625 QualType BaseElementTy = CGF.getContext().getBaseElementType(VAT: Array);
626 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
627 if (BaseElementTy.isPODType(Context: CGF.getContext()) ||
628 (CE && CE->getConstructor()->isMemcpyEquivalentSpecialMember(
629 Ctx: CGF.getContext()))) {
630 unsigned SrcArgIndex =
631 CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
632 llvm::Value *SrcPtr =
633 CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: Args[SrcArgIndex]));
634 LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
635 LValue Src = CGF.EmitLValueForFieldInitialization(Base: ThisRHSLV, Field);
636
637 // Copy the aggregate.
638 CGF.EmitAggregateCopy(Dest: LHS, Src, EltTy: FieldType,
639 MayOverlap: CGF.getOverlapForFieldInit(FD: Field),
640 isVolatile: LHS.isVolatileQualified());
641 // Ensure that we destroy the objects if an exception is thrown later in
642 // the constructor.
643 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
644 if (CGF.needsEHCleanup(kind: dtorKind))
645 CGF.pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
646 return;
647 }
648 }
649
650 CGF.EmitInitializerForField(Field, LHS, Init: MemberInit->getInit());
651}
652
653void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
654 Expr *Init) {
655 QualType FieldType = Field->getType();
656 switch (getEvaluationKind(T: FieldType)) {
657 case TEK_Scalar:
658 if (LHS.isSimple()) {
659 EmitExprAsInit(init: Init, D: Field, lvalue: LHS, capturedByInit: false);
660 } else {
661 RValue RHS = RValue::get(V: EmitScalarExpr(E: Init));
662 EmitStoreThroughLValue(Src: RHS, Dst: LHS);
663 }
664 break;
665 case TEK_Complex:
666 EmitComplexExprIntoLValue(E: Init, dest: LHS, /*isInit*/ true);
667 break;
668 case TEK_Aggregate: {
669 AggValueSlot Slot = AggValueSlot::forLValue(
670 LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
671 isAliased: AggValueSlot::IsNotAliased, mayOverlap: getOverlapForFieldInit(FD: Field),
672 isZeroed: AggValueSlot::IsNotZeroed,
673 // Checks are made by the code that calls constructor.
674 isChecked: AggValueSlot::IsSanitizerChecked);
675 EmitAggExpr(E: Init, AS: Slot);
676 break;
677 }
678 }
679
680 // Ensure that we destroy this object if an exception is thrown
681 // later in the constructor.
682 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
683 if (needsEHCleanup(kind: dtorKind))
684 pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
685}
686
687/// Checks whether the given constructor is a valid subject for the
688/// complete-to-base constructor delegation optimization, i.e.
689/// emitting the complete constructor as a simple call to the base
690/// constructor.
691bool CodeGenFunction::IsConstructorDelegationValid(
692 const CXXConstructorDecl *Ctor) {
693
694 // Currently we disable the optimization for classes with virtual
695 // bases because (1) the addresses of parameter variables need to be
696 // consistent across all initializers but (2) the delegate function
697 // call necessarily creates a second copy of the parameter variable.
698 //
699 // The limiting example (purely theoretical AFAIK):
700 // struct A { A(int &c) { c++; } };
701 // struct B : virtual A {
702 // B(int count) : A(count) { printf("%d\n", count); }
703 // };
704 // ...although even this example could in principle be emitted as a
705 // delegation since the address of the parameter doesn't escape.
706 if (Ctor->getParent()->getNumVBases()) {
707 // TODO: white-list trivial vbase initializers. This case wouldn't
708 // be subject to the restrictions below.
709
710 // TODO: white-list cases where:
711 // - there are no non-reference parameters to the constructor
712 // - the initializers don't access any non-reference parameters
713 // - the initializers don't take the address of non-reference
714 // parameters
715 // - etc.
716 // If we ever add any of the above cases, remember that:
717 // - function-try-blocks will always exclude this optimization
718 // - we need to perform the constructor prologue and cleanup in
719 // EmitConstructorBody.
720
721 return false;
722 }
723
724 // We also disable the optimization for variadic functions because
725 // it's impossible to "re-pass" varargs.
726 if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
727 return false;
728
729 // FIXME: Decide if we can do a delegation of a delegating constructor.
730 if (Ctor->isDelegatingConstructor())
731 return false;
732
733 return true;
734}
735
736// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
737// to poison the extra field paddings inserted under
738// -fsanitize-address-field-padding=1|2.
739void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
740 ASTContext &Context = getContext();
741 const CXXRecordDecl *ClassDecl =
742 Prologue ? cast<CXXConstructorDecl>(Val: CurGD.getDecl())->getParent()
743 : cast<CXXDestructorDecl>(Val: CurGD.getDecl())->getParent();
744 if (!ClassDecl->mayInsertExtraPadding())
745 return;
746
747 struct SizeAndOffset {
748 uint64_t Size;
749 uint64_t Offset;
750 };
751
752 unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
753 const ASTRecordLayout &Info = Context.getASTRecordLayout(D: ClassDecl);
754
755 // Populate sizes and offsets of fields.
756 SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
757 for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
758 SSV[i].Offset =
759 Context.toCharUnitsFromBits(BitSize: Info.getFieldOffset(FieldNo: i)).getQuantity();
760
761 size_t NumFields = 0;
762 for (const auto *Field : ClassDecl->fields()) {
763 const FieldDecl *D = Field;
764 auto FieldInfo = Context.getTypeInfoInChars(T: D->getType());
765 CharUnits FieldSize = FieldInfo.Width;
766 assert(NumFields < SSV.size());
767 SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
768 NumFields++;
769 }
770 assert(NumFields == SSV.size());
771 if (SSV.size() <= 1)
772 return;
773
774 // We will insert calls to __asan_* run-time functions.
775 // LLVM AddressSanitizer pass may decide to inline them later.
776 llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
777 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, isVarArg: false);
778 llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
779 Ty: FTy, Name: Prologue ? "__asan_poison_intra_object_redzone"
780 : "__asan_unpoison_intra_object_redzone");
781
782 llvm::Value *ThisPtr = LoadCXXThis();
783 ThisPtr = Builder.CreatePtrToInt(V: ThisPtr, DestTy: IntPtrTy);
784 uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
785 // For each field check if it has sufficient padding,
786 // if so (un)poison it with a call.
787 for (size_t i = 0; i < SSV.size(); i++) {
788 uint64_t AsanAlignment = 8;
789 uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
790 uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
791 uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
792 if (PoisonSize < AsanAlignment || !SSV[i].Size ||
793 (NextField % AsanAlignment) != 0)
794 continue;
795 Builder.CreateCall(
796 Callee: F, Args: {Builder.CreateAdd(LHS: ThisPtr, RHS: Builder.getIntN(N: PtrSize, C: EndOffset)),
797 Builder.getIntN(N: PtrSize, C: PoisonSize)});
798 }
799}
800
801/// EmitConstructorBody - Emits the body of the current constructor.
802void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
803 EmitAsanPrologueOrEpilogue(Prologue: true);
804 const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(Val: CurGD.getDecl());
805 CXXCtorType CtorType = CurGD.getCtorType();
806
807 assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
808 CtorType == Ctor_Complete) &&
809 "can only generate complete ctor for this ABI");
810
811 // Before we go any further, try the complete->base constructor
812 // delegation optimization.
813 if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
814 CGM.getTarget().getCXXABI().hasConstructorVariants()) {
815 EmitDelegateCXXConstructorCall(Ctor, CtorType: Ctor_Base, Args, Loc: Ctor->getEndLoc());
816 return;
817 }
818
819 const FunctionDecl *Definition = nullptr;
820 Stmt *Body = Ctor->getBody(Definition);
821 assert(Definition == Ctor && "emitting wrong constructor body");
822
823 // Enter the function-try-block before the constructor prologue if
824 // applicable.
825 bool IsTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
826 if (IsTryBody)
827 EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
828
829 incrementProfileCounter(S: Body);
830 maybeCreateMCDCCondBitmap();
831
832 RunCleanupsScope RunCleanups(*this);
833
834 // TODO: in restricted cases, we can emit the vbase initializers of
835 // a complete ctor and then delegate to the base ctor.
836
837 // Emit the constructor prologue, i.e. the base and member
838 // initializers.
839 EmitCtorPrologue(CD: Ctor, Type: CtorType, Args);
840
841 // Emit the body of the statement.
842 if (IsTryBody)
843 EmitStmt(S: cast<CXXTryStmt>(Val: Body)->getTryBlock());
844 else if (Body)
845 EmitStmt(S: Body);
846
847 // Emit any cleanup blocks associated with the member or base
848 // initializers, which includes (along the exceptional path) the
849 // destructors for those members and bases that were fully
850 // constructed.
851 RunCleanups.ForceCleanup();
852
853 if (IsTryBody)
854 ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
855}
856
857namespace {
858/// RAII object to indicate that codegen is copying the value representation
859/// instead of the object representation. Useful when copying a struct or
860/// class which has uninitialized members and we're only performing
861/// lvalue-to-rvalue conversion on the object but not its members.
862class CopyingValueRepresentation {
863public:
864 explicit CopyingValueRepresentation(CodeGenFunction &CGF)
865 : CGF(CGF), OldSanOpts(CGF.SanOpts) {
866 CGF.SanOpts.set(K: SanitizerKind::Bool, Value: false);
867 CGF.SanOpts.set(K: SanitizerKind::Enum, Value: false);
868 }
869 ~CopyingValueRepresentation() { CGF.SanOpts = OldSanOpts; }
870
871private:
872 CodeGenFunction &CGF;
873 SanitizerSet OldSanOpts;
874};
875} // end anonymous namespace
876
877namespace {
878class FieldMemcpyizer {
879public:
880 FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
881 const VarDecl *SrcRec)
882 : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
883 RecLayout(CGF.getContext().getASTRecordLayout(D: ClassDecl)),
884 FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
885 LastFieldOffset(0), LastAddedFieldIndex(0) {}
886
887 bool isMemcpyableField(FieldDecl *F) const {
888 // Never memcpy fields when we are adding poisoned paddings.
889 if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
890 return false;
891 Qualifiers Qual = F->getType().getQualifiers();
892 if (Qual.hasVolatile() || Qual.hasObjCLifetime())
893 return false;
894 if (PointerAuthQualifier Q = F->getType().getPointerAuth();
895 Q && Q.isAddressDiscriminated())
896 return false;
897 // Non-trivially-copyable fields with pointer field protection need to be
898 // copied one by one.
899 if (!CGF.getContext().arePFPFieldsTriviallyCopyable(RD: ClassDecl) &&
900 CGF.getContext().isPFPField(Field: F))
901 return false;
902 return true;
903 }
904
905 void addMemcpyableField(FieldDecl *F) {
906 if (isEmptyFieldForLayout(Context: CGF.getContext(), FD: F))
907 return;
908 if (!FirstField)
909 addInitialField(F);
910 else
911 addNextField(F);
912 }
913
914 CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
915 ASTContext &Ctx = CGF.getContext();
916 unsigned LastFieldSize =
917 LastField->isBitField()
918 ? LastField->getBitWidthValue()
919 : Ctx.toBits(
920 CharSize: Ctx.getTypeInfoDataSizeInChars(T: LastField->getType()).Width);
921 uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
922 FirstByteOffset + Ctx.getCharWidth() - 1;
923 CharUnits MemcpySize = Ctx.toCharUnitsFromBits(BitSize: MemcpySizeBits);
924 return MemcpySize;
925 }
926
927 void emitMemcpy() {
928 // Give the subclass a chance to bail out if it feels the memcpy isn't
929 // worth it (e.g. Hasn't aggregated enough data).
930 if (!FirstField) {
931 return;
932 }
933
934 uint64_t FirstByteOffset;
935 if (FirstField->isBitField()) {
936 const CGRecordLayout &RL =
937 CGF.getTypes().getCGRecordLayout(FirstField->getParent());
938 const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FD: FirstField);
939 // FirstFieldOffset is not appropriate for bitfields,
940 // we need to use the storage offset instead.
941 FirstByteOffset = CGF.getContext().toBits(CharSize: BFInfo.StorageOffset);
942 } else {
943 FirstByteOffset = FirstFieldOffset;
944 }
945
946 CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
947 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
948 Address ThisPtr = CGF.LoadCXXThisAddress();
949 LValue DestLV = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
950 LValue Dest = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: FirstField);
951 llvm::Value *SrcPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: SrcRec));
952 LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
953 LValue Src = CGF.EmitLValueForFieldInitialization(Base: SrcLV, Field: FirstField);
954
955 emitMemcpyIR(DestPtr: Dest.isBitField() ? Dest.getBitFieldAddress()
956 : Dest.getAddress(),
957 SrcPtr: Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),
958 Size: MemcpySize);
959 reset();
960 }
961
962 void reset() { FirstField = nullptr; }
963
964protected:
965 CodeGenFunction &CGF;
966 const CXXRecordDecl *ClassDecl;
967
968private:
969 void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
970 DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
971 SrcPtr = SrcPtr.withElementType(ElemTy: CGF.Int8Ty);
972 auto *I = CGF.Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: Size.getQuantity());
973 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: nullptr);
974 }
975
976 void addInitialField(FieldDecl *F) {
977 FirstField = F;
978 LastField = F;
979 FirstFieldOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
980 LastFieldOffset = FirstFieldOffset;
981 LastAddedFieldIndex = F->getFieldIndex();
982 }
983
984 void addNextField(FieldDecl *F) {
985 // For the most part, the following invariant will hold:
986 // F->getFieldIndex() == LastAddedFieldIndex + 1
987 // The one exception is that Sema won't add a copy-initializer for an
988 // unnamed bitfield, which will show up here as a gap in the sequence.
989 assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
990 "Cannot aggregate fields out of order.");
991 LastAddedFieldIndex = F->getFieldIndex();
992
993 // The 'first' and 'last' fields are chosen by offset, rather than field
994 // index. This allows the code to support bitfields, as well as regular
995 // fields.
996 uint64_t FOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
997 if (FOffset < FirstFieldOffset) {
998 FirstField = F;
999 FirstFieldOffset = FOffset;
1000 } else if (FOffset >= LastFieldOffset) {
1001 LastField = F;
1002 LastFieldOffset = FOffset;
1003 }
1004 }
1005
1006 const VarDecl *SrcRec;
1007 const ASTRecordLayout &RecLayout;
1008 FieldDecl *FirstField;
1009 FieldDecl *LastField;
1010 uint64_t FirstFieldOffset, LastFieldOffset;
1011 unsigned LastAddedFieldIndex;
1012};
1013
1014class ConstructorMemcpyizer : public FieldMemcpyizer {
1015private:
1016 /// Get source argument for copy constructor. Returns null if not a copy
1017 /// constructor.
1018 static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1019 const CXXConstructorDecl *CD,
1020 FunctionArgList &Args) {
1021 if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1022 return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1023 return nullptr;
1024 }
1025
1026 // Returns true if a CXXCtorInitializer represents a member initialization
1027 // that can be rolled into a memcpy.
1028 bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1029 if (!MemcpyableCtor)
1030 return false;
1031 FieldDecl *Field = MemberInit->getMember();
1032 assert(Field && "No field for member init.");
1033 QualType FieldType = Field->getType();
1034 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
1035
1036 // Bail out on non-memcpyable, not-trivially-copyable members.
1037 if (!(CE && CE->getConstructor()->isMemcpyEquivalentSpecialMember(
1038 Ctx: CGF.getContext())) &&
1039 !(FieldType.isTriviallyCopyableType(Context: CGF.getContext()) ||
1040 FieldType->isReferenceType()))
1041 return false;
1042
1043 // Bail out on volatile fields.
1044 if (!isMemcpyableField(F: Field))
1045 return false;
1046
1047 // Otherwise we're good.
1048 return true;
1049 }
1050
1051public:
1052 ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1053 FunctionArgList &Args)
1054 : FieldMemcpyizer(CGF, CD->getParent(),
1055 getTrivialCopySource(CGF, CD, Args)),
1056 ConstructorDecl(CD),
1057 MemcpyableCtor(CD->isDefaulted() && CD->isCopyOrMoveConstructor() &&
1058 CGF.getLangOpts().getGC() == LangOptions::NonGC),
1059 Args(Args) {}
1060
1061 void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1062 if (isMemberInitMemcpyable(MemberInit)) {
1063 AggregatedInits.push_back(Elt: MemberInit);
1064 addMemcpyableField(F: MemberInit->getMember());
1065 } else {
1066 emitAggregatedInits();
1067 EmitMemberInitializer(CGF, ClassDecl: ConstructorDecl->getParent(), MemberInit,
1068 Constructor: ConstructorDecl, Args);
1069 }
1070 }
1071
1072 void emitAggregatedInits() {
1073 if (AggregatedInits.size() <= 1) {
1074 // This memcpy is too small to be worthwhile. Fall back on default
1075 // codegen.
1076 if (!AggregatedInits.empty()) {
1077 CopyingValueRepresentation CVR(CGF);
1078 EmitMemberInitializer(CGF, ClassDecl: ConstructorDecl->getParent(),
1079 MemberInit: AggregatedInits[0], Constructor: ConstructorDecl, Args);
1080 AggregatedInits.clear();
1081 }
1082 reset();
1083 return;
1084 }
1085
1086 pushEHDestructors();
1087 ApplyAtomGroup Grp(CGF.getDebugInfo());
1088 emitMemcpy();
1089 AggregatedInits.clear();
1090 }
1091
1092 void pushEHDestructors() {
1093 Address ThisPtr = CGF.LoadCXXThisAddress();
1094 CanQualType RecordTy = CGF.getContext().getCanonicalTagType(TD: ClassDecl);
1095 LValue LHS = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
1096
1097 for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1098 CXXCtorInitializer *MemberInit = AggregatedInits[i];
1099 QualType FieldType = MemberInit->getAnyMember()->getType();
1100 QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1101 if (!CGF.needsEHCleanup(kind: dtorKind))
1102 continue;
1103 LValue FieldLHS = LHS;
1104 EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS&: FieldLHS);
1105 CGF.pushEHDestroy(dtorKind, addr: FieldLHS.getAddress(), type: FieldType);
1106 }
1107 }
1108
1109 void finish() { emitAggregatedInits(); }
1110
1111private:
1112 const CXXConstructorDecl *ConstructorDecl;
1113 bool MemcpyableCtor;
1114 FunctionArgList &Args;
1115 SmallVector<CXXCtorInitializer *, 16> AggregatedInits;
1116};
1117
1118class AssignmentMemcpyizer : public FieldMemcpyizer {
1119private:
1120 // Returns the memcpyable field copied by the given statement, if one
1121 // exists. Otherwise returns null.
1122 FieldDecl *getMemcpyableField(Stmt *S) {
1123 if (!AssignmentsMemcpyable)
1124 return nullptr;
1125 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: S)) {
1126 // Recognise trivial assignments.
1127 if (BO->getOpcode() != BO_Assign)
1128 return nullptr;
1129 MemberExpr *ME = dyn_cast<MemberExpr>(Val: BO->getLHS());
1130 if (!ME)
1131 return nullptr;
1132 FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1133 if (!Field || !isMemcpyableField(F: Field))
1134 return nullptr;
1135 Stmt *RHS = BO->getRHS();
1136 if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(Val: RHS))
1137 RHS = EC->getSubExpr();
1138 if (!RHS)
1139 return nullptr;
1140 if (MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: RHS)) {
1141 if (ME2->getMemberDecl() == Field)
1142 return Field;
1143 }
1144 return nullptr;
1145 } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(Val: S)) {
1146 CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MCE->getCalleeDecl());
1147 if (!(MD && MD->isMemcpyEquivalentSpecialMember(Ctx: CGF.getContext())))
1148 return nullptr;
1149 MemberExpr *IOA = dyn_cast<MemberExpr>(Val: MCE->getImplicitObjectArgument());
1150 if (!IOA)
1151 return nullptr;
1152 FieldDecl *Field = dyn_cast<FieldDecl>(Val: IOA->getMemberDecl());
1153 if (!Field || !isMemcpyableField(F: Field))
1154 return nullptr;
1155 MemberExpr *Arg0 = dyn_cast<MemberExpr>(Val: MCE->getArg(Arg: 0));
1156 if (!Arg0 || Field != dyn_cast<FieldDecl>(Val: Arg0->getMemberDecl()))
1157 return nullptr;
1158 return Field;
1159 } else if (CallExpr *CE = dyn_cast<CallExpr>(Val: S)) {
1160 FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: CE->getCalleeDecl());
1161 if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1162 return nullptr;
1163 Expr *DstPtr = CE->getArg(Arg: 0);
1164 if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(Val: DstPtr))
1165 DstPtr = DC->getSubExpr();
1166 UnaryOperator *DUO = dyn_cast<UnaryOperator>(Val: DstPtr);
1167 if (!DUO || DUO->getOpcode() != UO_AddrOf)
1168 return nullptr;
1169 MemberExpr *ME = dyn_cast<MemberExpr>(Val: DUO->getSubExpr());
1170 if (!ME)
1171 return nullptr;
1172 FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1173 if (!Field || !isMemcpyableField(F: Field))
1174 return nullptr;
1175 Expr *SrcPtr = CE->getArg(Arg: 1);
1176 if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(Val: SrcPtr))
1177 SrcPtr = SC->getSubExpr();
1178 UnaryOperator *SUO = dyn_cast<UnaryOperator>(Val: SrcPtr);
1179 if (!SUO || SUO->getOpcode() != UO_AddrOf)
1180 return nullptr;
1181 MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: SUO->getSubExpr());
1182 if (!ME2 || Field != dyn_cast<FieldDecl>(Val: ME2->getMemberDecl()))
1183 return nullptr;
1184 return Field;
1185 }
1186
1187 return nullptr;
1188 }
1189
1190 bool AssignmentsMemcpyable;
1191 SmallVector<Stmt *, 16> AggregatedStmts;
1192
1193public:
1194 AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1195 FunctionArgList &Args)
1196 : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1197 AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1198 assert(Args.size() == 2);
1199 }
1200
1201 void emitAssignment(Stmt *S) {
1202 FieldDecl *F = getMemcpyableField(S);
1203 if (F) {
1204 addMemcpyableField(F);
1205 AggregatedStmts.push_back(Elt: S);
1206 } else {
1207 emitAggregatedStmts();
1208 CGF.EmitStmt(S);
1209 }
1210 }
1211
1212 void emitAggregatedStmts() {
1213 if (AggregatedStmts.size() <= 1) {
1214 if (!AggregatedStmts.empty()) {
1215 CopyingValueRepresentation CVR(CGF);
1216 CGF.EmitStmt(S: AggregatedStmts[0]);
1217 }
1218 reset();
1219 }
1220
1221 ApplyAtomGroup Grp(CGF.getDebugInfo());
1222 emitMemcpy();
1223 AggregatedStmts.clear();
1224 }
1225
1226 void finish() { emitAggregatedStmts(); }
1227};
1228
1229} // end anonymous namespace
1230
1231static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1232 const Type *BaseType = BaseInit->getBaseClass();
1233 return BaseType->castAsCXXRecordDecl()->isDynamicClass();
1234}
1235
1236/// EmitCtorPrologue - This routine generates necessary code to initialize
1237/// base classes and non-static data members belonging to this constructor.
1238void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1239 CXXCtorType CtorType,
1240 FunctionArgList &Args) {
1241 if (CD->isDelegatingConstructor())
1242 return EmitDelegatingCXXConstructorCall(Ctor: CD, Args);
1243
1244 const CXXRecordDecl *ClassDecl = CD->getParent();
1245
1246 // Virtual base initializers aren't needed if:
1247 // - This is a base ctor variant
1248 // - There are no vbases
1249 // - The class is abstract, so a complete object of it cannot be constructed
1250 //
1251 // The check for an abstract class is necessary because sema may not have
1252 // marked virtual base destructors referenced.
1253 bool ConstructVBases = CtorType != Ctor_Base &&
1254 ClassDecl->getNumVBases() != 0 &&
1255 !ClassDecl->isAbstract();
1256
1257 // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1258 // constructor of a class with virtual bases takes an additional parameter to
1259 // conditionally construct the virtual bases. Emit that check here.
1260 llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1261 if (ConstructVBases &&
1262 !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1263 BaseCtorContinueBB =
1264 CGM.getCXXABI().EmitCtorCompleteObjectHandler(CGF&: *this, RD: ClassDecl);
1265 assert(BaseCtorContinueBB);
1266 }
1267
1268 // Create three separate ranges for the different types of initializers.
1269 auto AllInits = CD->inits();
1270
1271 // Find the boundaries between the three groups.
1272 auto VirtualBaseEnd = std::find_if(
1273 first: AllInits.begin(), last: AllInits.end(), pred: [](const CXXCtorInitializer *Init) {
1274 return !(Init->isBaseInitializer() && Init->isBaseVirtual());
1275 });
1276
1277 auto NonVirtualBaseEnd = std::find_if(first: VirtualBaseEnd, last: AllInits.end(),
1278 pred: [](const CXXCtorInitializer *Init) {
1279 return !Init->isBaseInitializer();
1280 });
1281
1282 // Create the three ranges.
1283 auto VirtualBaseInits = llvm::make_range(x: AllInits.begin(), y: VirtualBaseEnd);
1284 auto NonVirtualBaseInits =
1285 llvm::make_range(x: VirtualBaseEnd, y: NonVirtualBaseEnd);
1286 auto MemberInits = llvm::make_range(x: NonVirtualBaseEnd, y: AllInits.end());
1287
1288 // Process virtual base initializers, if necessary.
1289 if (ConstructVBases) {
1290 for (CXXCtorInitializer *Initializer : VirtualBaseInits) {
1291 SaveAndRestore ThisRAII(CXXThisValue);
1292 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1293 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1294 isInitializerOfDynamicClass(BaseInit: Initializer))
1295 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1296 EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: Initializer);
1297 }
1298 }
1299
1300 if (BaseCtorContinueBB) {
1301 // Complete object handler should continue to the remaining initializers.
1302 Builder.CreateBr(Dest: BaseCtorContinueBB);
1303 EmitBlock(BB: BaseCtorContinueBB);
1304 }
1305
1306 // Then, non-virtual base initializers.
1307 for (CXXCtorInitializer *Initializer : NonVirtualBaseInits) {
1308 assert(!Initializer->isBaseVirtual());
1309 SaveAndRestore ThisRAII(CXXThisValue);
1310 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1311 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1312 isInitializerOfDynamicClass(BaseInit: Initializer))
1313 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1314 EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: Initializer);
1315 }
1316
1317 InitializeVTablePointers(ClassDecl);
1318
1319 // And finally, initialize class members.
1320 FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1321 ConstructorMemcpyizer CM(*this, CD, Args);
1322 for (CXXCtorInitializer *Member : MemberInits) {
1323 assert(!Member->isBaseInitializer());
1324 assert(Member->isAnyMemberInitializer() &&
1325 "Delegating initializer on non-delegating constructor");
1326 CM.addMemberInitializer(MemberInit: Member);
1327 }
1328
1329 CM.finish();
1330}
1331
1332static bool FieldHasTrivialDestructorBody(ASTContext &Context,
1333 const FieldDecl *Field);
1334
1335static bool
1336HasTrivialDestructorBody(ASTContext &Context,
1337 const CXXRecordDecl *BaseClassDecl,
1338 const CXXRecordDecl *MostDerivedClassDecl) {
1339 // If the destructor is trivial we don't have to check anything else.
1340 if (BaseClassDecl->hasTrivialDestructor())
1341 return true;
1342
1343 if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1344 return false;
1345
1346 // Check fields.
1347 for (const auto *Field : BaseClassDecl->fields())
1348 if (!FieldHasTrivialDestructorBody(Context, Field))
1349 return false;
1350
1351 // Check non-virtual bases.
1352 for (const auto &I : BaseClassDecl->bases()) {
1353 if (I.isVirtual())
1354 continue;
1355
1356 const auto *NonVirtualBase = I.getType()->castAsCXXRecordDecl();
1357 if (!HasTrivialDestructorBody(Context, BaseClassDecl: NonVirtualBase,
1358 MostDerivedClassDecl))
1359 return false;
1360 }
1361
1362 if (BaseClassDecl == MostDerivedClassDecl) {
1363 // Check virtual bases.
1364 for (const auto &I : BaseClassDecl->vbases()) {
1365 const auto *VirtualBase = I.getType()->castAsCXXRecordDecl();
1366 if (!HasTrivialDestructorBody(Context, BaseClassDecl: VirtualBase, MostDerivedClassDecl))
1367 return false;
1368 }
1369 }
1370
1371 return true;
1372}
1373
1374static bool FieldHasTrivialDestructorBody(ASTContext &Context,
1375 const FieldDecl *Field) {
1376 QualType FieldBaseElementType = Context.getBaseElementType(QT: Field->getType());
1377
1378 auto *FieldClassDecl = FieldBaseElementType->getAsCXXRecordDecl();
1379 if (!FieldClassDecl)
1380 return true;
1381
1382 // The destructor for an implicit anonymous union member is never invoked.
1383 if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1384 return true;
1385
1386 return HasTrivialDestructorBody(Context, BaseClassDecl: FieldClassDecl, MostDerivedClassDecl: FieldClassDecl);
1387}
1388
1389/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1390/// any vtable pointers before calling this destructor.
1391static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1392 const CXXDestructorDecl *Dtor) {
1393 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1394 if (!ClassDecl->isDynamicClass())
1395 return true;
1396
1397 // For a final class, the vtable pointer is known to already point to the
1398 // class's vtable.
1399 if (ClassDecl->isEffectivelyFinal())
1400 return true;
1401
1402 if (!Dtor->hasTrivialBody())
1403 return false;
1404
1405 // Check the fields.
1406 for (const auto *Field : ClassDecl->fields())
1407 if (!FieldHasTrivialDestructorBody(Context&: CGF.getContext(), Field))
1408 return false;
1409
1410 return true;
1411}
1412
1413static void EmitConditionalArrayDtorCall(const CXXDestructorDecl *DD,
1414 CodeGenFunction &CGF,
1415 llvm::Value *ShouldDeleteCondition) {
1416 Address ThisPtr = CGF.LoadCXXThisAddress();
1417 llvm::BasicBlock *ScalarBB = CGF.createBasicBlock(name: "dtor.scalar");
1418 llvm::BasicBlock *callDeleteBB =
1419 CGF.createBasicBlock(name: "dtor.call_delete_after_array_destroy");
1420 llvm::BasicBlock *VectorBB = CGF.createBasicBlock(name: "dtor.vector");
1421 auto *CondTy = cast<llvm::IntegerType>(Val: ShouldDeleteCondition->getType());
1422 llvm::Value *CheckTheBitForArrayDestroy = CGF.Builder.CreateAnd(
1423 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 2));
1424 llvm::Value *ShouldDestroyArray =
1425 CGF.Builder.CreateIsNull(Arg: CheckTheBitForArrayDestroy);
1426 CGF.Builder.CreateCondBr(Cond: ShouldDestroyArray, True: ScalarBB, False: VectorBB);
1427
1428 CGF.EmitBlock(BB: VectorBB);
1429
1430 llvm::Value *numElements = nullptr;
1431 llvm::Value *allocatedPtr = nullptr;
1432 CharUnits cookieSize;
1433 QualType EltTy = DD->getThisType()->getPointeeType();
1434 CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr: ThisPtr, ElementType: EltTy, NumElements&: numElements,
1435 AllocPtr&: allocatedPtr, CookieSize&: cookieSize);
1436
1437 // Destroy the elements.
1438 QualType::DestructionKind dtorKind = EltTy.isDestructedType();
1439
1440 assert(dtorKind);
1441 assert(numElements && "no element count for a type with a destructor!");
1442
1443 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: EltTy);
1444 CharUnits elementAlign =
1445 ThisPtr.getAlignment().alignmentOfArrayElement(elementSize);
1446
1447 llvm::Value *arrayBegin = ThisPtr.emitRawPointer(CGF);
1448 llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
1449 Ty: ThisPtr.getElementType(), Ptr: arrayBegin, IdxList: numElements, Name: "delete.end");
1450
1451 // We already checked that the array is not 0-length before entering vector
1452 // deleting dtor.
1453 CGF.emitArrayDestroy(begin: arrayBegin, end: arrayEnd, elementType: EltTy, elementAlign,
1454 destroyer: CGF.getDestroyer(destructionKind: dtorKind),
1455 /*checkZeroLength*/ false, useEHCleanup: CGF.needsEHCleanup(kind: dtorKind));
1456
1457 llvm::BasicBlock *VectorBBCont = CGF.createBasicBlock(name: "dtor.vector.cont");
1458 CGF.EmitBlock(BB: VectorBBCont);
1459
1460 llvm::Value *CheckTheBitForDeleteCall = CGF.Builder.CreateAnd(
1461 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 1));
1462
1463 llvm::Value *ShouldCallDelete =
1464 CGF.Builder.CreateIsNull(Arg: CheckTheBitForDeleteCall);
1465 CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: CGF.ReturnBlock.getBlock(),
1466 False: callDeleteBB);
1467 CGF.EmitBlock(BB: callDeleteBB);
1468 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1469 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1470 if (Dtor->getArrayOperatorDelete()) {
1471 if (!Dtor->getGlobalArrayOperatorDelete()) {
1472 CGF.EmitDeleteCall(DeleteFD: Dtor->getArrayOperatorDelete(), Ptr: allocatedPtr,
1473 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1474 NumElements: numElements, CookieSize: cookieSize);
1475 } else {
1476 // If global operator[] is set, the class had its own operator delete[].
1477 // In that case, check the 4th bit. If it is set, we need to call
1478 // ::delete[].
1479 llvm::Value *CheckTheBitForGlobDeleteCall = CGF.Builder.CreateAnd(
1480 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1481
1482 llvm::Value *ShouldCallGlobDelete =
1483 CGF.Builder.CreateIsNull(Arg: CheckTheBitForGlobDeleteCall);
1484 llvm::BasicBlock *GlobDelete =
1485 CGF.createBasicBlock(name: "dtor.call_glob_delete_after_array_destroy");
1486 llvm::BasicBlock *ClassDelete =
1487 CGF.createBasicBlock(name: "dtor.call_class_delete_after_array_destroy");
1488 CGF.Builder.CreateCondBr(Cond: ShouldCallGlobDelete, True: ClassDelete, False: GlobDelete);
1489 CGF.EmitBlock(BB: ClassDelete);
1490 CGF.EmitDeleteCall(DeleteFD: Dtor->getArrayOperatorDelete(), Ptr: allocatedPtr,
1491 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1492 NumElements: numElements, CookieSize: cookieSize);
1493 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1494
1495 CGF.EmitBlock(BB: GlobDelete);
1496 // Use __global_delete wrapper instead of directly calling
1497 // ::operator delete to match MSVC's behavior. See the doc comment on
1498 // getOrCreateMSVCGlobalDeleteWrapper for details.
1499 llvm::Constant *GlobalDeleteWrapper =
1500 CGF.CGM.getOrCreateMSVCGlobalDeleteWrapper(
1501 GlobOD: Dtor->getGlobalArrayOperatorDelete());
1502 // For dllexport classes, emit forwarding bodies since the dtor is
1503 // exported and another TU may not provide the forwarding body.
1504 if (Dtor->hasAttr<DLLExportAttr>())
1505 CGF.CGM.noteDirectGlobalDelete();
1506 CGF.EmitDeleteCall(DeleteFD: Dtor->getGlobalArrayOperatorDelete(), Ptr: allocatedPtr,
1507 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl),
1508 NumElements: numElements, CookieSize: cookieSize, CalleeOverride: GlobalDeleteWrapper);
1509 }
1510 } else {
1511 // No operators delete[] were found, so emit a trap.
1512 llvm::CallInst *TrapCall = CGF.EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1513 TrapCall->setDoesNotReturn();
1514 TrapCall->setDoesNotThrow();
1515 CGF.Builder.CreateUnreachable();
1516 CGF.Builder.ClearInsertionPoint();
1517 }
1518
1519 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1520 CGF.EmitBlock(BB: ScalarBB);
1521}
1522
1523/// EmitDestructorBody - Emits the body of the current destructor.
1524void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1525 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CurGD.getDecl());
1526 CXXDtorType DtorType = CurGD.getDtorType();
1527
1528 // For an abstract class, non-base destructors are never used (and can't
1529 // be emitted in general, because vbase dtors may not have been validated
1530 // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1531 // in fact emit references to them from other compilations, so emit them
1532 // as functions containing a trap instruction.
1533 if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1534 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1535 TrapCall->setDoesNotReturn();
1536 TrapCall->setDoesNotThrow();
1537 Builder.CreateUnreachable();
1538 Builder.ClearInsertionPoint();
1539 return;
1540 }
1541
1542 Stmt *Body = Dtor->getBody();
1543 if (Body) {
1544 incrementProfileCounter(S: Body);
1545 maybeCreateMCDCCondBitmap();
1546 }
1547
1548 // The call to operator delete in a deleting destructor happens
1549 // outside of the function-try-block, which means it's always
1550 // possible to delegate the destructor body to the complete
1551 // destructor. Do so.
1552 if (DtorType == Dtor_Deleting || DtorType == Dtor_VectorDeleting) {
1553 if (CXXStructorImplicitParamValue && DtorType == Dtor_VectorDeleting)
1554 EmitConditionalArrayDtorCall(DD: Dtor, CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue);
1555 RunCleanupsScope DtorEpilogue(*this);
1556 EnterDtorCleanups(Dtor, Type: Dtor_Deleting);
1557 if (HaveInsertPoint()) {
1558 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1559 EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1560 /*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1561 }
1562 return;
1563 }
1564
1565 // If the body is a function-try-block, enter the try before
1566 // anything else.
1567 bool isTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
1568 if (isTryBody)
1569 EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1570 EmitAsanPrologueOrEpilogue(Prologue: false);
1571
1572 // Enter the epilogue cleanups.
1573 RunCleanupsScope DtorEpilogue(*this);
1574
1575 // If this is the complete variant, just invoke the base variant;
1576 // the epilogue will destruct the virtual bases. But we can't do
1577 // this optimization if the body is a function-try-block, because
1578 // we'd introduce *two* handler blocks. In the Microsoft ABI, we
1579 // always delegate because we might not have a definition in this TU.
1580 switch (DtorType) {
1581 case Dtor_Unified:
1582 llvm_unreachable("not expecting a unified dtor");
1583 case Dtor_Comdat:
1584 llvm_unreachable("not expecting a COMDAT");
1585 case Dtor_Deleting:
1586 llvm_unreachable("already handled deleting case");
1587 case Dtor_VectorDeleting:
1588 llvm_unreachable("already handled vector deleting case");
1589
1590 case Dtor_Complete:
1591 assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1592 "can't emit a dtor without a body for non-Microsoft ABIs");
1593
1594 // Enter the cleanup scopes for virtual bases.
1595 EnterDtorCleanups(Dtor, Type: Dtor_Complete);
1596
1597 if (!isTryBody) {
1598 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1599 EmitCXXDestructorCall(D: Dtor, Type: Dtor_Base, /*ForVirtualBase=*/false,
1600 /*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1601 break;
1602 }
1603
1604 // Fallthrough: act like we're in the base variant.
1605 [[fallthrough]];
1606
1607 case Dtor_Base:
1608 assert(Body);
1609
1610 // Enter the cleanup scopes for fields and non-virtual bases.
1611 EnterDtorCleanups(Dtor, Type: Dtor_Base);
1612
1613 // Initialize the vtable pointers before entering the body.
1614 if (!CanSkipVTablePointerInitialization(CGF&: *this, Dtor)) {
1615 // Insert the llvm.launder.invariant.group intrinsic before initializing
1616 // the vptrs to cancel any previous assumptions we might have made.
1617 if (CGM.getCodeGenOpts().StrictVTablePointers &&
1618 CGM.getCodeGenOpts().OptimizationLevel > 0)
1619 CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1620 InitializeVTablePointers(ClassDecl: Dtor->getParent());
1621 }
1622
1623 if (isTryBody)
1624 EmitStmt(S: cast<CXXTryStmt>(Val: Body)->getTryBlock());
1625 else if (Body)
1626 EmitStmt(S: Body);
1627 else {
1628 assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1629 // nothing to do besides what's in the epilogue
1630 }
1631 // -fapple-kext must inline any call to this dtor into
1632 // the caller's body.
1633 if (getLangOpts().AppleKext)
1634 CurFn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1635
1636 break;
1637 }
1638
1639 // Jump out through the epilogue cleanups.
1640 DtorEpilogue.ForceCleanup();
1641
1642 // Exit the try if applicable.
1643 if (isTryBody)
1644 ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1645}
1646
1647void CodeGenFunction::emitImplicitAssignmentOperatorBody(
1648 FunctionArgList &Args) {
1649 const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(Val: CurGD.getDecl());
1650 const Stmt *RootS = AssignOp->getBody();
1651 assert(isa<CompoundStmt>(RootS) &&
1652 "Body of an implicit assignment operator should be compound stmt.");
1653 const CompoundStmt *RootCS = cast<CompoundStmt>(Val: RootS);
1654
1655 LexicalScope Scope(*this, RootCS->getSourceRange());
1656
1657 incrementProfileCounter(S: RootCS);
1658 maybeCreateMCDCCondBitmap();
1659 AssignmentMemcpyizer AM(*this, AssignOp, Args);
1660 for (auto *I : RootCS->body())
1661 AM.emitAssignment(S: I);
1662
1663 AM.finish();
1664}
1665
1666namespace {
1667llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1668 const CXXDestructorDecl *DD) {
1669 if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1670 return CGF.EmitScalarExpr(E: ThisArg);
1671 return CGF.LoadCXXThis();
1672}
1673
1674/// Call the operator delete associated with the current destructor.
1675struct CallDtorDelete final : EHScopeStack::Cleanup {
1676 CallDtorDelete() {}
1677
1678 void Emit(CodeGenFunction &CGF, Flags flags) override {
1679 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1680 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1681 CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1682 Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1683 DeleteTy: CGF.getContext().getCanonicalTagType(TD: ClassDecl));
1684 }
1685};
1686
1687// This function implements generation of scalar deleting destructor body for
1688// the case when the destructor also accepts an implicit flag. Right now only
1689// Microsoft ABI requires deleting destructors to accept implicit flags.
1690// The flag indicates whether an operator delete should be called and whether
1691// it should be a class-specific operator delete or a global one.
1692void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1693 llvm::Value *ShouldDeleteCondition,
1694 bool ReturnAfterDelete) {
1695 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1696 const CXXRecordDecl *ClassDecl = Dtor->getParent();
1697 const FunctionDecl *OD = Dtor->getOperatorDelete();
1698 assert(OD->isDestroyingOperatorDelete() == ReturnAfterDelete &&
1699 "unexpected value for ReturnAfterDelete");
1700 auto *CondTy = cast<llvm::IntegerType>(Val: ShouldDeleteCondition->getType());
1701 // MSVC calls global operator delete inside of the dtor body, but clang
1702 // aligned with this behavior only after a particular version. This is not
1703 // ABI-compatible with previous versions.
1704 ASTContext &Context = CGF.getContext();
1705 bool CallGlobDelete = Context.getTargetInfo().callGlobalDeleteInDeletingDtor(
1706 Context.getLangOpts());
1707 if (CallGlobDelete && OD->isDestroyingOperatorDelete()) {
1708 llvm::BasicBlock *CallDtor = CGF.createBasicBlock(name: "dtor.call_dtor");
1709 llvm::BasicBlock *DontCallDtor = CGF.createBasicBlock(name: "dtor.entry_cont");
1710 // Third bit set signals that global operator delete is called. That means
1711 // despite class having destroying operator delete which is responsible
1712 // for calling dtor, we need to call dtor because global operator delete
1713 // won't do that.
1714 llvm::Value *Check3rdBit = CGF.Builder.CreateAnd(
1715 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1716 llvm::Value *ShouldCallDtor = CGF.Builder.CreateIsNull(Arg: Check3rdBit);
1717 CGF.Builder.CreateCondBr(Cond: ShouldCallDtor, True: DontCallDtor, False: CallDtor);
1718 CGF.EmitBlock(BB: CallDtor);
1719 QualType ThisTy = Dtor->getFunctionObjectParameterType();
1720 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1721 /*Delegating=*/false, This: CGF.LoadCXXThisAddress(),
1722 ThisTy);
1723 CGF.Builder.CreateBr(Dest: DontCallDtor);
1724 CGF.EmitBlock(BB: DontCallDtor);
1725 }
1726 llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock(name: "dtor.call_delete");
1727 llvm::BasicBlock *continueBB = CGF.createBasicBlock(name: "dtor.continue");
1728 // First bit set signals that operator delete must be called.
1729 llvm::Value *Check1stBit = CGF.Builder.CreateAnd(
1730 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 1));
1731 llvm::Value *ShouldCallDelete = CGF.Builder.CreateIsNull(Arg: Check1stBit);
1732 CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: continueBB, False: callDeleteBB);
1733
1734 CGF.EmitBlock(BB: callDeleteBB);
1735 auto EmitDeleteAndGoToEnd = [&](const FunctionDecl *DeleteOp,
1736 llvm::Constant *CalleeOverride = nullptr) {
1737 CGF.EmitDeleteCall(DeleteFD: DeleteOp, Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1738 DeleteTy: Context.getCanonicalTagType(TD: ClassDecl),
1739 /*NumElements=*/nullptr, /*CookieSize=*/CharUnits(),
1740 CalleeOverride);
1741 if (ReturnAfterDelete)
1742 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1743 else
1744 CGF.Builder.CreateBr(Dest: continueBB);
1745 };
1746 // If Sema only found a global operator delete previously, the dtor can
1747 // always call it. Otherwise we need to check the third bit and call the
1748 // appropriate operator delete, i.e. global or class-specific.
1749 if (const FunctionDecl *GlobOD = Dtor->getOperatorGlobalDelete();
1750 isa<CXXMethodDecl>(Val: OD) && GlobOD && CallGlobDelete) {
1751 // Third bit set signals that global operator delete is called, i.e.
1752 // ::delete appears on the callsite.
1753 llvm::Value *CheckTheBitForGlobDeleteCall = CGF.Builder.CreateAnd(
1754 LHS: ShouldDeleteCondition, RHS: llvm::ConstantInt::get(Ty: CondTy, V: 4));
1755 llvm::Value *ShouldCallGlobDelete =
1756 CGF.Builder.CreateIsNull(Arg: CheckTheBitForGlobDeleteCall);
1757 llvm::BasicBlock *GlobDelete =
1758 CGF.createBasicBlock(name: "dtor.call_glob_delete");
1759 llvm::BasicBlock *ClassDelete =
1760 CGF.createBasicBlock(name: "dtor.call_class_delete");
1761 CGF.Builder.CreateCondBr(Cond: ShouldCallGlobDelete, True: ClassDelete, False: GlobDelete);
1762 CGF.EmitBlock(BB: GlobDelete);
1763
1764 // Use __global_delete wrapper instead of directly calling
1765 // ::operator delete to match MSVC's behavior. See the doc comment on
1766 // getOrCreateMSVCGlobalDeleteWrapper for details.
1767 llvm::Constant *GlobalDeleteWrapper =
1768 CGF.CGM.getOrCreateMSVCGlobalDeleteWrapper(GlobOD);
1769 // For dllexport classes, emit forwarding bodies since the dtor is
1770 // exported and another TU may not provide the forwarding body.
1771 if (Dtor->hasAttr<DLLExportAttr>())
1772 CGF.CGM.noteDirectGlobalDelete();
1773 EmitDeleteAndGoToEnd(GlobOD, GlobalDeleteWrapper);
1774 CGF.EmitBlock(BB: ClassDelete);
1775 }
1776 EmitDeleteAndGoToEnd(OD);
1777 CGF.EmitBlock(BB: continueBB);
1778}
1779
1780struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1781 llvm::Value *ShouldDeleteCondition;
1782
1783public:
1784 CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1785 : ShouldDeleteCondition(ShouldDeleteCondition) {
1786 assert(ShouldDeleteCondition != nullptr);
1787 }
1788
1789 void Emit(CodeGenFunction &CGF, Flags flags) override {
1790 EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1791 /*ReturnAfterDelete*/ false);
1792 }
1793};
1794
1795class DestroyField final : public EHScopeStack::Cleanup {
1796 const FieldDecl *field;
1797 CodeGenFunction::Destroyer *destroyer;
1798 bool useEHCleanupForArray;
1799
1800public:
1801 DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1802 bool useEHCleanupForArray)
1803 : field(field), destroyer(destroyer),
1804 useEHCleanupForArray(useEHCleanupForArray) {}
1805
1806 void Emit(CodeGenFunction &CGF, Flags flags) override {
1807 // Find the address of the field.
1808 Address thisValue = CGF.LoadCXXThisAddress();
1809 CanQualType RecordTy =
1810 CGF.getContext().getCanonicalTagType(TD: field->getParent());
1811 LValue ThisLV = CGF.MakeAddrLValue(Addr: thisValue, T: RecordTy);
1812 LValue LV = CGF.EmitLValueForField(Base: ThisLV, Field: field);
1813 assert(LV.isSimple());
1814
1815 CGF.emitDestroy(addr: LV.getAddress(), type: field->getType(), destroyer,
1816 useEHCleanupForArray: flags.isForNormalCleanup() && useEHCleanupForArray);
1817 }
1818};
1819
1820class DeclAsInlineDebugLocation {
1821 CGDebugInfo *DI;
1822 llvm::DILocation *InlinedAt;
1823 std::optional<ApplyDebugLocation> Location;
1824
1825public:
1826 DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)
1827 : DI(CGF.getDebugInfo()) {
1828 if (!DI)
1829 return;
1830 InlinedAt = DI->getInlinedAt();
1831 DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());
1832 Location.emplace(args&: CGF, args: Decl.getLocation());
1833 }
1834
1835 ~DeclAsInlineDebugLocation() {
1836 if (!DI)
1837 return;
1838 Location.reset();
1839 DI->setInlinedAt(InlinedAt);
1840 }
1841};
1842
1843static void EmitSanitizerDtorCallback(
1844 CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,
1845 std::optional<CharUnits::QuantityType> PoisonSize = {}) {
1846 CodeGenFunction::SanitizerScope SanScope(&CGF);
1847 // Pass in void pointer and size of region as arguments to runtime
1848 // function
1849 SmallVector<llvm::Value *, 2> Args = {Ptr};
1850 SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};
1851
1852 if (PoisonSize.has_value()) {
1853 Args.emplace_back(Args: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: *PoisonSize));
1854 ArgTypes.emplace_back(Args&: CGF.SizeTy);
1855 }
1856
1857 llvm::FunctionType *FnType =
1858 llvm::FunctionType::get(Result: CGF.VoidTy, Params: ArgTypes, isVarArg: false);
1859 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FnType, Name);
1860
1861 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1862}
1863
1864static void
1865EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1866 CharUnits::QuantityType PoisonSize) {
1867 EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_fields", Ptr,
1868 PoisonSize);
1869}
1870
1871/// Poison base class with a trivial destructor.
1872struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1873 const CXXRecordDecl *BaseClass;
1874 bool BaseIsVirtual;
1875 SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1876 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1877
1878 void Emit(CodeGenFunction &CGF, Flags flags) override {
1879 const CXXRecordDecl *DerivedClass =
1880 cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
1881
1882 Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1883 This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
1884
1885 const ASTRecordLayout &BaseLayout =
1886 CGF.getContext().getASTRecordLayout(D: BaseClass);
1887 CharUnits BaseSize = BaseLayout.getSize();
1888
1889 if (!BaseSize.isPositive())
1890 return;
1891
1892 // Use the base class declaration location as inline DebugLocation. All
1893 // fields of the class are destroyed.
1894 DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
1895 EmitSanitizerDtorFieldsCallback(CGF, Ptr: Addr.emitRawPointer(CGF),
1896 PoisonSize: BaseSize.getQuantity());
1897
1898 // Prevent the current stack frame from disappearing from the stack trace.
1899 CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1900 }
1901};
1902
1903class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1904 const CXXDestructorDecl *Dtor;
1905 unsigned StartIndex;
1906 unsigned EndIndex;
1907
1908public:
1909 SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1910 unsigned EndIndex)
1911 : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1912
1913 // Generate function call for handling object poisoning.
1914 // Disables tail call elimination, to prevent the current stack frame
1915 // from disappearing from the stack trace.
1916 void Emit(CodeGenFunction &CGF, Flags flags) override {
1917 const ASTContext &Context = CGF.getContext();
1918 const ASTRecordLayout &Layout =
1919 Context.getASTRecordLayout(D: Dtor->getParent());
1920
1921 // It's a first trivial field so it should be at the begining of a char,
1922 // still round up start offset just in case.
1923 CharUnits PoisonStart = Context.toCharUnitsFromBits(
1924 BitSize: Layout.getFieldOffset(FieldNo: StartIndex) + Context.getCharWidth() - 1);
1925 llvm::ConstantInt *OffsetSizePtr =
1926 llvm::ConstantInt::get(Ty: CGF.SizeTy, V: PoisonStart.getQuantity());
1927
1928 llvm::Value *OffsetPtr =
1929 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: CGF.LoadCXXThis(), IdxList: OffsetSizePtr);
1930
1931 CharUnits PoisonEnd;
1932 if (EndIndex >= Layout.getFieldCount()) {
1933 PoisonEnd = Layout.getNonVirtualSize();
1934 } else {
1935 PoisonEnd = Context.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: EndIndex));
1936 }
1937 CharUnits PoisonSize = PoisonEnd - PoisonStart;
1938 if (!PoisonSize.isPositive())
1939 return;
1940
1941 // Use the top field declaration location as inline DebugLocation.
1942 DeclAsInlineDebugLocation InlineHere(
1943 CGF, **std::next(x: Dtor->getParent()->field_begin(), n: StartIndex));
1944 EmitSanitizerDtorFieldsCallback(CGF, Ptr: OffsetPtr, PoisonSize: PoisonSize.getQuantity());
1945
1946 // Prevent the current stack frame from disappearing from the stack trace.
1947 CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1948 }
1949};
1950
1951class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1952 const CXXDestructorDecl *Dtor;
1953
1954public:
1955 SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1956
1957 // Generate function call for handling vtable pointer poisoning.
1958 void Emit(CodeGenFunction &CGF, Flags flags) override {
1959 assert(Dtor->getParent()->isDynamicClass());
1960 (void)Dtor;
1961 // Poison vtable and vtable ptr if they exist for this class.
1962 llvm::Value *VTablePtr = CGF.LoadCXXThis();
1963
1964 // Pass in void pointer and size of region as arguments to runtime
1965 // function
1966 EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_vptr", Ptr: VTablePtr);
1967 }
1968};
1969
1970class SanitizeDtorCleanupBuilder {
1971 ASTContext &Context;
1972 EHScopeStack &EHStack;
1973 const CXXDestructorDecl *DD;
1974 std::optional<unsigned> StartIndex;
1975
1976public:
1977 SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1978 const CXXDestructorDecl *DD)
1979 : Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}
1980 void PushCleanupForField(const FieldDecl *Field) {
1981 if (isEmptyFieldForLayout(Context, FD: Field))
1982 return;
1983 unsigned FieldIndex = Field->getFieldIndex();
1984 if (FieldHasTrivialDestructorBody(Context, Field)) {
1985 if (!StartIndex)
1986 StartIndex = FieldIndex;
1987 } else if (StartIndex) {
1988 EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1989 A: *StartIndex, A: FieldIndex);
1990 StartIndex = std::nullopt;
1991 }
1992 }
1993 void End() {
1994 if (StartIndex)
1995 EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1996 A: *StartIndex, A: -1);
1997 }
1998};
1999} // end anonymous namespace
2000
2001/// Emit all code that comes at the end of class's
2002/// destructor. This is to call destructors on members and base classes
2003/// in reverse order of their construction.
2004///
2005/// For a deleting destructor, this also handles the case where a destroying
2006/// operator delete completely overrides the definition.
2007void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
2008 CXXDtorType DtorType) {
2009 assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
2010 "Should not emit dtor epilogue for non-exported trivial dtor!");
2011
2012 // The deleting-destructor phase just needs to call the appropriate
2013 // operator delete that Sema picked up.
2014 if (DtorType == Dtor_Deleting) {
2015 assert(DD->getOperatorDelete() &&
2016 "operator delete missing - EnterDtorCleanups");
2017 if (CXXStructorImplicitParamValue) {
2018 // If there is an implicit param to the deleting dtor, it's a boolean
2019 // telling whether this is a deleting destructor.
2020 if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
2021 EmitConditionalDtorDeleteCall(CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue,
2022 /*ReturnAfterDelete*/ true);
2023 else
2024 EHStack.pushCleanup<CallDtorDeleteConditional>(
2025 Kind: NormalAndEHCleanup, A: CXXStructorImplicitParamValue);
2026 } else {
2027 if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
2028 const CXXRecordDecl *ClassDecl = DD->getParent();
2029 EmitDeleteCall(DeleteFD: DD->getOperatorDelete(),
2030 Ptr: LoadThisForDtorDelete(CGF&: *this, DD),
2031 DeleteTy: getContext().getCanonicalTagType(TD: ClassDecl));
2032 EmitBranchThroughCleanup(Dest: ReturnBlock);
2033 } else {
2034 EHStack.pushCleanup<CallDtorDelete>(Kind: NormalAndEHCleanup);
2035 }
2036 }
2037 return;
2038 }
2039
2040 const CXXRecordDecl *ClassDecl = DD->getParent();
2041
2042 // Unions have no bases and do not call field destructors.
2043 if (ClassDecl->isUnion())
2044 return;
2045
2046 // The complete-destructor phase just destructs all the virtual bases.
2047 if (DtorType == Dtor_Complete) {
2048 // Poison the vtable pointer such that access after the base
2049 // and member destructors are invoked is invalid.
2050 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
2051 SanOpts.has(K: SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
2052 ClassDecl->isPolymorphic())
2053 EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
2054
2055 // We push them in the forward order so that they'll be popped in
2056 // the reverse order.
2057 for (const auto &Base : ClassDecl->vbases()) {
2058 auto *BaseClassDecl = Base.getType()->castAsCXXRecordDecl();
2059 if (BaseClassDecl->hasTrivialDestructor()) {
2060 // Under SanitizeMemoryUseAfterDtor, poison the trivial base class
2061 // memory. For non-trival base classes the same is done in the class
2062 // destructor.
2063 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
2064 SanOpts.has(K: SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
2065 EHStack.pushCleanup<SanitizeDtorTrivialBase>(Kind: NormalAndEHCleanup,
2066 A: BaseClassDecl,
2067 /*BaseIsVirtual*/ A: true);
2068 } else {
2069 EHStack.pushCleanup<CallBaseDtor>(Kind: NormalAndEHCleanup, A: BaseClassDecl,
2070 /*BaseIsVirtual*/ A: true);
2071 }
2072 }
2073
2074 return;
2075 }
2076
2077 assert(DtorType == Dtor_Base);
2078 // Poison the vtable pointer if it has no virtual bases, but inherits
2079 // virtual functions.
2080 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
2081 SanOpts.has(K: SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
2082 ClassDecl->isPolymorphic())
2083 EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
2084
2085 // Destroy non-virtual bases.
2086 for (const auto &Base : ClassDecl->bases()) {
2087 // Ignore virtual bases.
2088 if (Base.isVirtual())
2089 continue;
2090
2091 CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
2092
2093 if (BaseClassDecl->hasTrivialDestructor()) {
2094 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
2095 SanOpts.has(K: SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
2096 EHStack.pushCleanup<SanitizeDtorTrivialBase>(Kind: NormalAndEHCleanup,
2097 A: BaseClassDecl,
2098 /*BaseIsVirtual*/ A: false);
2099 } else {
2100 EHStack.pushCleanup<CallBaseDtor>(Kind: NormalAndEHCleanup, A: BaseClassDecl,
2101 /*BaseIsVirtual*/ A: false);
2102 }
2103 }
2104
2105 // Poison fields such that access after their destructors are
2106 // invoked, and before the base class destructor runs, is invalid.
2107 bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
2108 SanOpts.has(K: SanitizerKind::Memory);
2109 SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
2110
2111 // Destroy direct fields.
2112 for (const auto *Field : ClassDecl->fields()) {
2113 if (SanitizeFields)
2114 SanitizeBuilder.PushCleanupForField(Field);
2115
2116 QualType type = Field->getType();
2117 QualType::DestructionKind dtorKind = type.isDestructedType();
2118 if (!dtorKind)
2119 continue;
2120
2121 // Anonymous union members do not have their destructors called.
2122 const RecordType *RT = type->getAsUnionType();
2123 if (RT && RT->getDecl()->isAnonymousStructOrUnion())
2124 continue;
2125
2126 CleanupKind cleanupKind = getCleanupKind(kind: dtorKind);
2127 EHStack.pushCleanup<DestroyField>(
2128 Kind: cleanupKind, A: Field, A: getDestroyer(destructionKind: dtorKind), A: cleanupKind & EHCleanup);
2129 }
2130
2131 if (SanitizeFields)
2132 SanitizeBuilder.End();
2133}
2134
2135/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
2136/// constructor for each of several members of an array.
2137///
2138/// \param ctor the constructor to call for each element
2139/// \param arrayType the type of the array to initialize
2140/// \param arrayBegin an arrayType*
2141/// \param zeroInitialize true if each element should be
2142/// zero-initialized before it is constructed
2143void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2144 const ArrayType *arrayType,
2145 Address arrayBegin,
2146 const CXXConstructExpr *E,
2147 bool NewPointerIsChecked,
2148 bool zeroInitialize) {
2149 QualType elementType;
2150 llvm::Value *numElements =
2151 emitArrayLength(arrayType, baseType&: elementType, addr&: arrayBegin);
2152
2153 EmitCXXAggrConstructorCall(D: ctor, NumElements: numElements, ArrayPtr: arrayBegin, E,
2154 NewPointerIsChecked, ZeroInitialization: zeroInitialize);
2155}
2156
2157/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
2158/// constructor for each of several members of an array.
2159///
2160/// \param ctor the constructor to call for each element
2161/// \param numElements the number of elements in the array;
2162/// may be zero
2163/// \param arrayBase a T*, where T is the type constructed by ctor
2164/// \param zeroInitialize true if each element should be
2165/// zero-initialized before it is constructed
2166void CodeGenFunction::EmitCXXAggrConstructorCall(
2167 const CXXConstructorDecl *ctor, llvm::Value *numElements, Address arrayBase,
2168 const CXXConstructExpr *E, bool NewPointerIsChecked, bool zeroInitialize) {
2169 // It's legal for numElements to be zero. This can happen both
2170 // dynamically, because x can be zero in 'new A[x]', and statically,
2171 // because of GCC extensions that permit zero-length arrays. There
2172 // are probably legitimate places where we could assume that this
2173 // doesn't happen, but it's not clear that it's worth it.
2174 llvm::CondBrInst *zeroCheckBranch = nullptr;
2175
2176 // Optimize for a constant count.
2177 llvm::ConstantInt *constantCount = dyn_cast<llvm::ConstantInt>(Val: numElements);
2178 if (constantCount) {
2179 // Just skip out if the constant count is zero.
2180 if (constantCount->isZero())
2181 return;
2182
2183 // Otherwise, emit the check.
2184 } else {
2185 llvm::BasicBlock *loopBB = createBasicBlock(name: "new.ctorloop");
2186 llvm::Value *iszero = Builder.CreateIsNull(Arg: numElements, Name: "isempty");
2187 zeroCheckBranch = Builder.CreateCondBr(Cond: iszero, True: loopBB, False: loopBB);
2188 EmitBlock(BB: loopBB);
2189 }
2190
2191 // Find the end of the array.
2192 llvm::Type *elementType = arrayBase.getElementType();
2193 llvm::Value *arrayBegin = arrayBase.emitRawPointer(CGF&: *this);
2194 llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
2195 Ty: elementType, Ptr: arrayBegin, IdxList: numElements, Name: "arrayctor.end");
2196
2197 // Enter the loop, setting up a phi for the current location to initialize.
2198 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2199 llvm::BasicBlock *loopBB = createBasicBlock(name: "arrayctor.loop");
2200 EmitBlock(BB: loopBB);
2201 llvm::PHINode *cur =
2202 Builder.CreatePHI(Ty: arrayBegin->getType(), NumReservedValues: 2, Name: "arrayctor.cur");
2203 cur->addIncoming(V: arrayBegin, BB: entryBB);
2204
2205 // Inside the loop body, emit the constructor call on the array element.
2206 if (CGM.shouldEmitConvergenceTokens())
2207 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: loopBB));
2208
2209 // The alignment of the base, adjusted by the size of a single element,
2210 // provides a conservative estimate of the alignment of every element.
2211 // (This assumes we never start tracking offsetted alignments.)
2212 //
2213 // Note that these are complete objects and so we don't need to
2214 // use the non-virtual size or alignment.
2215 CanQualType type = getContext().getCanonicalTagType(TD: ctor->getParent());
2216 CharUnits eltAlignment = arrayBase.getAlignment().alignmentOfArrayElement(
2217 elementSize: getContext().getTypeSizeInChars(T: type));
2218 Address curAddr = Address(cur, elementType, eltAlignment);
2219
2220 // Zero initialize the storage, if requested.
2221 if (zeroInitialize)
2222 EmitNullInitialization(DestPtr: curAddr, Ty: type);
2223
2224 // C++ [class.temporary]p4:
2225 // There are two contexts in which temporaries are destroyed at a different
2226 // point than the end of the full-expression. The first context is when a
2227 // default constructor is called to initialize an element of an array.
2228 // If the constructor has one or more default arguments, the destruction of
2229 // every temporary created in a default argument expression is sequenced
2230 // before the construction of the next array element, if any.
2231
2232 {
2233 RunCleanupsScope Scope(*this);
2234
2235 // Evaluate the constructor and its arguments in a regular
2236 // partial-destroy cleanup.
2237 if (getLangOpts().Exceptions &&
2238 !ctor->getParent()->hasTrivialDestructor()) {
2239 Destroyer *destroyer = destroyCXXObject;
2240 pushRegularPartialArrayCleanup(arrayBegin, arrayEnd: cur, elementType: type, elementAlignment: eltAlignment,
2241 destroyer: *destroyer);
2242 }
2243 auto currAVS = AggValueSlot::forAddr(
2244 addr: curAddr, quals: type.getQualifiers(), isDestructed: AggValueSlot::IsDestructed,
2245 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2246 mayOverlap: AggValueSlot::DoesNotOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2247 isChecked: NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2248 : AggValueSlot::IsNotSanitizerChecked);
2249 EmitCXXConstructorCall(D: ctor, Type: Ctor_Complete, /*ForVirtualBase=*/false,
2250 /*Delegating=*/false, ThisAVS: currAVS, E);
2251 }
2252
2253 // Go to the next element.
2254 llvm::Value *next = Builder.CreateInBoundsGEP(
2255 Ty: elementType, Ptr: cur, IdxList: llvm::ConstantInt::get(Ty: SizeTy, V: 1), Name: "arrayctor.next");
2256 cur->addIncoming(V: next, BB: Builder.GetInsertBlock());
2257
2258 // Check whether that's the end of the loop.
2259 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: arrayEnd, Name: "arrayctor.done");
2260 llvm::BasicBlock *contBB = createBasicBlock(name: "arrayctor.cont");
2261 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2262
2263 // Patch the earlier check to skip over the loop.
2264 if (zeroCheckBranch)
2265 zeroCheckBranch->setSuccessor(idx: 0, NewSucc: contBB);
2266
2267 if (CGM.shouldEmitConvergenceTokens())
2268 ConvergenceTokenStack.pop_back();
2269
2270 EmitBlock(BB: contBB);
2271}
2272
2273void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF, Address addr,
2274 QualType type) {
2275 const CXXDestructorDecl *dtor = type->castAsCXXRecordDecl()->getDestructor();
2276 assert(!dtor->isTrivial());
2277 CGF.EmitCXXDestructorCall(D: dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
2278 /*Delegating=*/false, This: addr, ThisTy: type);
2279}
2280
2281void CodeGenFunction::EmitCXXConstructorCall(
2282 const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2283 bool Delegating, AggValueSlot ThisAVS, const CXXConstructExpr *E) {
2284 CallArgList Args;
2285 Address This = ThisAVS.getAddress();
2286 LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2287 LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
2288 llvm::Value *ThisPtr =
2289 getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType()->getPointeeType());
2290
2291 if (SlotAS != ThisAS) {
2292 unsigned TargetThisAS = getContext().getTargetAddressSpace(AS: ThisAS);
2293 llvm::Type *NewType =
2294 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetThisAS);
2295 ThisPtr = performAddrSpaceCast(Src: ThisPtr, DestTy: NewType);
2296 }
2297
2298 // Push the this ptr.
2299 Args.add(rvalue: RValue::get(V: ThisPtr), type: D->getThisType());
2300
2301 // If this is a trivial constructor, emit a memcpy now before we lose
2302 // the alignment information on the argument.
2303 // FIXME: It would be better to preserve alignment information into CallArg.
2304 if (D->isMemcpyEquivalentSpecialMember(Ctx: getContext())) {
2305 assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2306
2307 const Expr *Arg = E->getArg(Arg: 0);
2308 LValue Src = EmitCheckedLValue(E: Arg, TCK: TCK_Load);
2309 CanQualType DestTy = getContext().getCanonicalTagType(TD: D->getParent());
2310 LValue Dest = MakeAddrLValue(Addr: This, T: DestTy);
2311 EmitAggregateCopyCtor(Dest, Src, MayOverlap: ThisAVS.mayOverlap());
2312 return;
2313 }
2314
2315 // Add the rest of the user-supplied arguments.
2316 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2317 EvaluationOrder Order = E->isListInitialization()
2318 ? EvaluationOrder::ForceLeftToRight
2319 : EvaluationOrder::Default;
2320 EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments(), AC: E->getConstructor(),
2321 /*ParamsToSkip*/ 0, Order);
2322
2323 EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2324 Overlap: ThisAVS.mayOverlap(), Loc: E->getExprLoc(),
2325 NewPointerIsChecked: ThisAVS.isSanitizerChecked());
2326}
2327
2328static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2329 const CXXConstructorDecl *Ctor,
2330 CXXCtorType Type, CallArgList &Args) {
2331 // We can't forward a variadic call.
2332 if (Ctor->isVariadic())
2333 return false;
2334
2335 if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2336 // If the parameters are callee-cleanup, it's not safe to forward.
2337 for (auto *P : Ctor->parameters())
2338 if (P->needsDestruction(Ctx: CGF.getContext()))
2339 return false;
2340
2341 // Likewise if they're inalloca.
2342 const CGFunctionInfo &Info = CGF.CGM.getTypes().arrangeCXXConstructorCall(
2343 Args, D: Ctor, CtorKind: Type, ExtraPrefixArgs: 0, ExtraSuffixArgs: 0, ABIInfoFD: CGF.getCurrentFunctionDecl());
2344 if (Info.usesInAlloca())
2345 return false;
2346 }
2347
2348 // Anything else should be OK.
2349 return true;
2350}
2351
2352void CodeGenFunction::EmitCXXConstructorCall(
2353 const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2354 bool Delegating, Address This, CallArgList &Args,
2355 AggValueSlot::Overlap_t Overlap, SourceLocation Loc,
2356 bool NewPointerIsChecked, llvm::CallBase **CallOrInvoke) {
2357 const CXXRecordDecl *ClassDecl = D->getParent();
2358
2359 if (!NewPointerIsChecked)
2360 EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall, Loc, Addr: This,
2361 Type: getContext().getCanonicalTagType(TD: ClassDecl),
2362 Alignment: CharUnits::Zero());
2363
2364 if (D->isTrivial() && D->isDefaultConstructor()) {
2365 assert(Args.size() == 1 && "trivial default ctor with args");
2366 return;
2367 }
2368
2369 // If this is a trivial constructor, just emit what's needed. If this is a
2370 // union copy constructor, we must emit a memcpy, because the AST does not
2371 // model that copy.
2372 if (D->isMemcpyEquivalentSpecialMember(Ctx: getContext())) {
2373 assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2374 QualType SrcTy = D->getParamDecl(i: 0)->getType().getNonReferenceType();
2375 Address Src = makeNaturalAddressForPointer(
2376 Ptr: Args[1].getRValue(CGF&: *this).getScalarVal(), T: SrcTy);
2377 LValue SrcLVal = MakeAddrLValue(Addr: Src, T: SrcTy);
2378 CanQualType DestTy = getContext().getCanonicalTagType(TD: ClassDecl);
2379 LValue DestLVal = MakeAddrLValue(Addr: This, T: DestTy);
2380 EmitAggregateCopyCtor(Dest: DestLVal, Src: SrcLVal, MayOverlap: Overlap);
2381 return;
2382 }
2383
2384 bool PassPrototypeArgs = true;
2385 // Check whether we can actually emit the constructor before trying to do so.
2386 if (auto Inherited = D->getInheritedConstructor()) {
2387 PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2388 if (PassPrototypeArgs && !canEmitDelegateCallArgs(CGF&: *this, Ctor: D, Type, Args)) {
2389 EmitInlinedInheritingCXXConstructorCall(Ctor: D, CtorType: Type, ForVirtualBase,
2390 Delegating, Args);
2391 return;
2392 }
2393 }
2394
2395 // Insert any ABI-specific implicit constructor arguments.
2396 CGCXXABI::AddedStructorArgCounts ExtraArgs =
2397 CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D, Type, ForVirtualBase,
2398 Delegating, Args);
2399
2400 // Emit the call.
2401 llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GD: GlobalDecl(D, Type));
2402 const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2403 Args, D, CtorKind: Type, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix,
2404 ABIInfoFD: getCurrentFunctionDecl(), PassProtoArgs: PassPrototypeArgs);
2405 CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(D, Type));
2406 EmitCall(CallInfo: Info, Callee, ReturnValue: ReturnValueSlot(), Args, CallOrInvoke, IsMustTail: false, Loc);
2407
2408 // Generate vtable assumptions if we're constructing a complete object
2409 // with a vtable. We don't do this for base subobjects for two reasons:
2410 // first, it's incorrect for classes with virtual bases, and second, we're
2411 // about to overwrite the vptrs anyway.
2412 // We also have to make sure if we can refer to vtable:
2413 // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2414 // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2415 // sure that definition of vtable is not hidden,
2416 // then we are always safe to refer to it.
2417 // FIXME: It looks like InstCombine is very inefficient on dealing with
2418 // assumes. Make assumption loads require -fstrict-vtable-pointers
2419 // temporarily.
2420 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2421 ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2422 CGM.getCXXABI().canSpeculativelyEmitVTable(RD: ClassDecl) &&
2423 CGM.getCodeGenOpts().StrictVTablePointers)
2424 EmitVTableAssumptionLoads(ClassDecl, This);
2425}
2426
2427void CodeGenFunction::EmitInheritedCXXConstructorCall(
2428 const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2429 bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2430 CallArgList Args;
2431 CallArg ThisArg(RValue::get(V: getAsNaturalPointerTo(
2432 Addr: This, PointeeType: D->getThisType()->getPointeeType())),
2433 D->getThisType());
2434
2435 // Forward the parameters.
2436 if (InheritedFromVBase &&
2437 CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2438 // Nothing to do; this construction is not responsible for constructing
2439 // the base class containing the inherited constructor.
2440 // FIXME: Can we just pass undef's for the remaining arguments if we don't
2441 // have constructor variants?
2442 Args.push_back(Elt: ThisArg);
2443 } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2444 // The inheriting constructor was inlined; just inject its arguments.
2445 assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2446 "wrong number of parameters for inherited constructor call");
2447 Args = CXXInheritedCtorInitExprArgs;
2448 Args[0] = ThisArg;
2449 } else {
2450 // The inheriting constructor was not inlined. Emit delegating arguments.
2451 Args.push_back(Elt: ThisArg);
2452 const auto *OuterCtor = cast<CXXConstructorDecl>(Val: CurCodeDecl);
2453 assert(OuterCtor->getNumParams() == D->getNumParams());
2454 assert(!OuterCtor->isVariadic() && "should have been inlined");
2455
2456 for (const auto *Param : OuterCtor->parameters()) {
2457 assert(getContext().hasSameUnqualifiedType(
2458 OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2459 Param->getType()));
2460 EmitDelegateCallArg(args&: Args, param: Param, loc: E->getLocation());
2461
2462 // Forward __attribute__(pass_object_size).
2463 if (Param->hasAttr<PassObjectSizeAttr>()) {
2464 auto *POSParam = SizeArguments[Param];
2465 assert(POSParam && "missing pass_object_size value for forwarding");
2466 EmitDelegateCallArg(args&: Args, param: POSParam, loc: E->getLocation());
2467 }
2468 }
2469 }
2470
2471 EmitCXXConstructorCall(D, Type: Ctor_Base, ForVirtualBase, /*Delegating*/ false,
2472 This, Args, Overlap: AggValueSlot::MayOverlap, Loc: E->getLocation(),
2473 /*NewPointerIsChecked*/ true);
2474}
2475
2476void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2477 const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2478 bool Delegating, CallArgList &Args) {
2479 GlobalDecl GD(Ctor, CtorType);
2480 InlinedInheritingConstructorScope Scope(*this, GD);
2481 ApplyInlineDebugLocation DebugScope(*this, GD);
2482 RunCleanupsScope RunCleanups(*this);
2483
2484 // Save the arguments to be passed to the inherited constructor.
2485 CXXInheritedCtorInitExprArgs = Args;
2486
2487 FunctionArgList Params;
2488 QualType RetType = BuildFunctionArgList(GD: CurGD, Args&: Params);
2489 FnRetTy = RetType;
2490
2491 // Insert any ABI-specific implicit constructor arguments.
2492 CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D: Ctor, Type: CtorType,
2493 ForVirtualBase, Delegating, Args);
2494
2495 // Emit a simplified prolog. We only need to emit the implicit params.
2496 assert(Args.size() >= Params.size() && "too few arguments for call");
2497 for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2498 if (I < Params.size() && isa<ImplicitParamDecl>(Val: Params[I])) {
2499 const RValue &RV = Args[I].getRValue(CGF&: *this);
2500 assert(!RV.isComplex() && "complex indirect params not supported");
2501 ParamValue Val = RV.isScalar()
2502 ? ParamValue::forDirect(value: RV.getScalarVal())
2503 : ParamValue::forIndirect(addr: RV.getAggregateAddress());
2504 EmitParmDecl(D: *Params[I], Arg: Val, ArgNo: I + 1);
2505 }
2506 }
2507
2508 // Create a return value slot if the ABI implementation wants one.
2509 // FIXME: This is dumb, we should ask the ABI not to try to set the return
2510 // value instead.
2511 if (!RetType->isVoidType())
2512 ReturnValue = CreateIRTempWithoutCast(T: RetType, Name: "retval.inhctor");
2513
2514 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
2515 CXXThisValue = CXXABIThisValue;
2516
2517 // Directly emit the constructor initializers.
2518 EmitCtorPrologue(CD: Ctor, CtorType, Args&: Params);
2519}
2520
2521void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2522 llvm::Value *VTableGlobal =
2523 CGM.getCXXABI().getVTableAddressPoint(Base: Vptr.Base, VTableClass: Vptr.VTableClass);
2524 if (!VTableGlobal)
2525 return;
2526
2527 // We can just use the base offset in the complete class.
2528 CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2529
2530 if (!NonVirtualOffset.isZero())
2531 This =
2532 ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: This, nonVirtualOffset: NonVirtualOffset, virtualOffset: nullptr,
2533 derivedClass: Vptr.VTableClass, nearestVBase: Vptr.NearestVBase);
2534
2535 llvm::Value *VPtrValue =
2536 GetVTablePtr(This, VTableTy: VTableGlobal->getType(), VTableClass: Vptr.VTableClass);
2537 llvm::Value *Cmp =
2538 Builder.CreateICmpEQ(LHS: VPtrValue, RHS: VTableGlobal, Name: "cmp.vtables");
2539 Builder.CreateAssumption(Cond: Cmp);
2540}
2541
2542void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2543 Address This) {
2544 if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: ClassDecl))
2545 for (const VPtr &Vptr : getVTablePointers(VTableClass: ClassDecl))
2546 EmitVTableAssumptionLoad(Vptr, This);
2547}
2548
2549void CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(
2550 const CXXConstructorDecl *D, Address This, Address Src,
2551 const CXXConstructExpr *E) {
2552 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2553
2554 CallArgList Args;
2555
2556 // Push the this ptr.
2557 Args.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType())),
2558 type: D->getThisType());
2559
2560 // Push the src ptr.
2561 QualType QT = *(FPT->param_type_begin());
2562 llvm::Type *t = CGM.getTypes().ConvertType(T: QT);
2563 llvm::Value *Val = getAsNaturalPointerTo(Addr: Src, PointeeType: D->getThisType());
2564 llvm::Value *SrcVal = Builder.CreateBitCast(V: Val, DestTy: t);
2565 Args.add(rvalue: RValue::get(V: SrcVal), type: QT);
2566
2567 // Skip over first argument (Src).
2568 EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(RangeOrContainer: E->arguments(), N: 1), AC: E->getConstructor(),
2569 /*ParamsToSkip*/ 1);
2570
2571 EmitCXXConstructorCall(D, Type: Ctor_Complete, /*ForVirtualBase*/ false,
2572 /*Delegating*/ false, This, Args,
2573 Overlap: AggValueSlot::MayOverlap, Loc: E->getExprLoc(),
2574 /*NewPointerIsChecked*/ false);
2575}
2576
2577void CodeGenFunction::EmitDelegateCXXConstructorCall(
2578 const CXXConstructorDecl *Ctor, CXXCtorType CtorType,
2579 const FunctionArgList &Args, SourceLocation Loc) {
2580 CallArgList DelegateArgs;
2581
2582 FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2583 assert(I != E && "no parameters to constructor");
2584
2585 // this
2586 Address This = LoadCXXThisAddress();
2587 DelegateArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(
2588 Addr: This, PointeeType: (*I)->getType()->getPointeeType())),
2589 type: (*I)->getType());
2590 ++I;
2591
2592 // FIXME: The location of the VTT parameter in the parameter list is
2593 // specific to the Itanium ABI and shouldn't be hardcoded here.
2594 if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
2595 assert(I != E && "cannot skip vtt parameter, already done with args");
2596 assert((*I)->getType()->isPointerType() &&
2597 "skipping parameter not of vtt type");
2598 ++I;
2599 }
2600
2601 // Explicit arguments.
2602 for (; I != E; ++I) {
2603 const VarDecl *param = *I;
2604 // FIXME: per-argument source location
2605 EmitDelegateCallArg(args&: DelegateArgs, param, loc: Loc);
2606 }
2607
2608 EmitCXXConstructorCall(D: Ctor, Type: CtorType, /*ForVirtualBase=*/false,
2609 /*Delegating=*/true, This, Args&: DelegateArgs,
2610 Overlap: AggValueSlot::MayOverlap, Loc,
2611 /*NewPointerIsChecked=*/true);
2612}
2613
2614namespace {
2615struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2616 const CXXDestructorDecl *Dtor;
2617 Address Addr;
2618 CXXDtorType Type;
2619
2620 CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2621 CXXDtorType Type)
2622 : Dtor(D), Addr(Addr), Type(Type) {}
2623
2624 void Emit(CodeGenFunction &CGF, Flags flags) override {
2625 // We are calling the destructor from within the constructor.
2626 // Therefore, "this" should have the expected type.
2627 QualType ThisTy = Dtor->getFunctionObjectParameterType();
2628 CGF.EmitCXXDestructorCall(D: Dtor, Type, /*ForVirtualBase=*/false,
2629 /*Delegating=*/true, This: Addr, ThisTy);
2630 }
2631};
2632} // end anonymous namespace
2633
2634void CodeGenFunction::EmitDelegatingCXXConstructorCall(
2635 const CXXConstructorDecl *Ctor, const FunctionArgList &Args) {
2636 assert(Ctor->isDelegatingConstructor());
2637
2638 Address ThisPtr = LoadCXXThisAddress();
2639
2640 AggValueSlot AggSlot = AggValueSlot::forAddr(
2641 addr: ThisPtr, quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed,
2642 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2643 mayOverlap: AggValueSlot::MayOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2644 // Checks are made by the code that calls constructor.
2645 isChecked: AggValueSlot::IsSanitizerChecked);
2646
2647 EmitAggExpr(E: Ctor->init_begin()[0]->getInit(), AS: AggSlot);
2648
2649 const CXXRecordDecl *ClassDecl = Ctor->getParent();
2650 if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2651 CXXDtorType Type =
2652 CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2653
2654 EHStack.pushCleanup<CallDelegatingCtorDtor>(
2655 Kind: EHCleanup, A: ClassDecl->getDestructor(), A: ThisPtr, A: Type);
2656 }
2657}
2658
2659void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2660 CXXDtorType Type,
2661 bool ForVirtualBase,
2662 bool Delegating, Address This,
2663 QualType ThisTy) {
2664 CGM.getCXXABI().EmitDestructorCall(CGF&: *this, DD, Type, ForVirtualBase,
2665 Delegating, This, ThisTy);
2666}
2667
2668namespace {
2669struct CallLocalDtor final : EHScopeStack::Cleanup {
2670 const CXXDestructorDecl *Dtor;
2671 Address Addr;
2672 QualType Ty;
2673
2674 CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2675 : Dtor(D), Addr(Addr), Ty(Ty) {}
2676
2677 void Emit(CodeGenFunction &CGF, Flags flags) override {
2678 CGF.EmitCXXDestructorCall(DD: Dtor, Type: Dtor_Complete,
2679 /*ForVirtualBase=*/false,
2680 /*Delegating=*/false, This: Addr, ThisTy: Ty);
2681 }
2682};
2683} // end anonymous namespace
2684
2685void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2686 QualType T, Address Addr) {
2687 EHStack.pushCleanup<CallLocalDtor>(Kind: NormalAndEHCleanup, A: D, A: Addr, A: T);
2688}
2689
2690void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2691 CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2692 if (!ClassDecl)
2693 return;
2694 if (ClassDecl->hasTrivialDestructor())
2695 return;
2696
2697 const CXXDestructorDecl *D = ClassDecl->getDestructor();
2698 assert(D && D->isUsed() && "destructor not marked as used!");
2699 PushDestructorCleanup(D, T, Addr);
2700}
2701
2702void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2703 // Compute the address point.
2704 llvm::Value *VTableAddressPoint =
2705 CGM.getCXXABI().getVTableAddressPointInStructor(
2706 CGF&: *this, RD: Vptr.VTableClass, Base: Vptr.Base, NearestVBase: Vptr.NearestVBase);
2707
2708 if (!VTableAddressPoint)
2709 return;
2710
2711 // Compute where to store the address point.
2712 llvm::Value *VirtualOffset = nullptr;
2713 CharUnits NonVirtualOffset = CharUnits::Zero();
2714
2715 if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(CGF&: *this, Vptr)) {
2716 // We need to use the virtual base offset offset because the virtual base
2717 // might have a different offset in the most derived class.
2718
2719 VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2720 CGF&: *this, This: LoadCXXThisAddress(), ClassDecl: Vptr.VTableClass, BaseClassDecl: Vptr.NearestVBase);
2721 NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2722 } else {
2723 // We can just use the base offset in the complete class.
2724 NonVirtualOffset = Vptr.Base.getBaseOffset();
2725 }
2726
2727 // Apply the offsets.
2728 Address VTableField = LoadCXXThisAddress();
2729 if (!NonVirtualOffset.isZero() || VirtualOffset)
2730 VTableField = ApplyNonVirtualAndVirtualOffset(
2731 CGF&: *this, addr: VTableField, nonVirtualOffset: NonVirtualOffset, virtualOffset: VirtualOffset, derivedClass: Vptr.VTableClass,
2732 nearestVBase: Vptr.NearestVBase);
2733
2734 // Finally, store the address point. Use the same LLVM types as the field to
2735 // support optimization.
2736 unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2737 llvm::Type *PtrTy = llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: GlobalsAS);
2738 // vtable field is derived from `this` pointer, therefore they should be in
2739 // the same addr space. Note that this might not be LLVM address space 0.
2740 VTableField = VTableField.withElementType(ElemTy: PtrTy);
2741
2742 if (auto AuthenticationInfo = CGM.getVTablePointerAuthInfo(
2743 Context: this, Record: Vptr.Base.getBase(), StorageAddress: VTableField.emitRawPointer(CGF&: *this)))
2744 VTableAddressPoint =
2745 EmitPointerAuthSign(Info: *AuthenticationInfo, Pointer: VTableAddressPoint);
2746
2747 llvm::StoreInst *Store = Builder.CreateStore(Val: VTableAddressPoint, Addr: VTableField);
2748 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: PtrTy);
2749 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2750 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2751 CGM.getCodeGenOpts().StrictVTablePointers)
2752 CGM.DecorateInstructionWithInvariantGroup(I: Store, RD: Vptr.VTableClass);
2753}
2754
2755CodeGenFunction::VPtrsVector
2756CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2757 CodeGenFunction::VPtrsVector VPtrsResult;
2758 VisitedVirtualBasesSetTy VBases;
2759 getVTablePointers(Base: BaseSubobject(VTableClass, CharUnits::Zero()),
2760 /*NearestVBase=*/nullptr,
2761 /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2762 /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2763 vptrs&: VPtrsResult);
2764 return VPtrsResult;
2765}
2766
2767void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2768 const CXXRecordDecl *NearestVBase,
2769 CharUnits OffsetFromNearestVBase,
2770 bool BaseIsNonVirtualPrimaryBase,
2771 const CXXRecordDecl *VTableClass,
2772 VisitedVirtualBasesSetTy &VBases,
2773 VPtrsVector &Vptrs) {
2774 // If this base is a non-virtual primary base the address point has already
2775 // been set.
2776 if (!BaseIsNonVirtualPrimaryBase) {
2777 // Initialize the vtable pointer for this base.
2778 VPtr Vptr = {.Base: Base, .NearestVBase: NearestVBase, .OffsetFromNearestVBase: OffsetFromNearestVBase, .VTableClass: VTableClass};
2779 Vptrs.push_back(Elt: Vptr);
2780 }
2781
2782 const CXXRecordDecl *RD = Base.getBase();
2783
2784 // Traverse bases.
2785 for (const auto &I : RD->bases()) {
2786 auto *BaseDecl = I.getType()->castAsCXXRecordDecl();
2787 // Ignore classes without a vtable.
2788 if (!BaseDecl->isDynamicClass())
2789 continue;
2790
2791 CharUnits BaseOffset;
2792 CharUnits BaseOffsetFromNearestVBase;
2793 bool BaseDeclIsNonVirtualPrimaryBase;
2794
2795 if (I.isVirtual()) {
2796 // Check if we've visited this virtual base before.
2797 if (!VBases.insert(Ptr: BaseDecl).second)
2798 continue;
2799
2800 const ASTRecordLayout &Layout =
2801 getContext().getASTRecordLayout(D: VTableClass);
2802
2803 BaseOffset = Layout.getVBaseClassOffset(VBase: BaseDecl);
2804 BaseOffsetFromNearestVBase = CharUnits::Zero();
2805 BaseDeclIsNonVirtualPrimaryBase = false;
2806 } else {
2807 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2808
2809 BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(Base: BaseDecl);
2810 BaseOffsetFromNearestVBase =
2811 OffsetFromNearestVBase + Layout.getBaseClassOffset(Base: BaseDecl);
2812 BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2813 }
2814
2815 getVTablePointers(
2816 Base: BaseSubobject(BaseDecl, BaseOffset),
2817 NearestVBase: I.isVirtual() ? BaseDecl : NearestVBase, OffsetFromNearestVBase: BaseOffsetFromNearestVBase,
2818 BaseIsNonVirtualPrimaryBase: BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2819 }
2820}
2821
2822void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2823 // Ignore classes without a vtable.
2824 if (!RD->isDynamicClass())
2825 return;
2826
2827 // Initialize the vtable pointers for this class and all of its bases.
2828 if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: RD))
2829 for (const VPtr &Vptr : getVTablePointers(VTableClass: RD))
2830 InitializeVTablePointer(Vptr);
2831
2832 if (RD->getNumVBases())
2833 CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(CGF&: *this, RD);
2834}
2835
2836llvm::Value *CodeGenFunction::GetVTablePtr(Address This, llvm::Type *VTableTy,
2837 const CXXRecordDecl *RD,
2838 VTableAuthMode AuthMode) {
2839 Address VTablePtrSrc = This.withElementType(ElemTy: VTableTy);
2840 llvm::Instruction *VTable = Builder.CreateLoad(Addr: VTablePtrSrc, Name: "vtable");
2841 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: VTableTy);
2842 CGM.DecorateInstructionWithTBAA(Inst: VTable, TBAAInfo);
2843
2844 if (auto AuthenticationInfo =
2845 CGM.getVTablePointerAuthInfo(Context: this, Record: RD, StorageAddress: This.emitRawPointer(CGF&: *this))) {
2846 if (AuthMode != VTableAuthMode::UnsafeUbsanStrip) {
2847 VTable = cast<llvm::Instruction>(
2848 Val: EmitPointerAuthAuth(Info: *AuthenticationInfo, Pointer: VTable));
2849 if (AuthMode == VTableAuthMode::MustTrap) {
2850 // This is clearly suboptimal but until we have an ability
2851 // to rely on the authentication intrinsic trapping and force
2852 // an authentication to occur we don't really have a choice.
2853 VTable =
2854 cast<llvm::Instruction>(Val: Builder.CreateBitCast(V: VTable, DestTy: Int8PtrTy));
2855 Builder.CreateLoad(Addr: RawAddress(VTable, Int8Ty, CGM.getPointerAlign()),
2856 /* IsVolatile */ true);
2857 }
2858 } else {
2859 VTable = cast<llvm::Instruction>(Val: EmitPointerAuthAuth(
2860 Info: CGPointerAuthInfo(0, PointerAuthenticationMode::Strip, false, false,
2861 nullptr),
2862 Pointer: VTable));
2863 }
2864 }
2865
2866 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2867 CGM.getCodeGenOpts().StrictVTablePointers)
2868 CGM.DecorateInstructionWithInvariantGroup(I: VTable, RD);
2869
2870 return VTable;
2871}
2872
2873// If a class has a single non-virtual base and does not introduce or override
2874// virtual member functions or fields, it will have the same layout as its base.
2875// This function returns the least derived such class.
2876//
2877// Casting an instance of a base class to such a derived class is technically
2878// undefined behavior, but it is a relatively common hack for introducing member
2879// functions on class instances with specific properties (e.g. llvm::Operator)
2880// that works under most compilers and should not have security implications, so
2881// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2882static const CXXRecordDecl *
2883LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2884 if (!RD->field_empty())
2885 return RD;
2886
2887 if (RD->getNumVBases() != 0)
2888 return RD;
2889
2890 if (RD->getNumBases() != 1)
2891 return RD;
2892
2893 for (const CXXMethodDecl *MD : RD->methods()) {
2894 if (MD->isVirtual()) {
2895 // Virtual member functions are only ok if they are implicit destructors
2896 // because the implicit destructor will have the same semantics as the
2897 // base class's destructor if no fields are added.
2898 if (isa<CXXDestructorDecl>(Val: MD) && MD->isImplicit())
2899 continue;
2900 return RD;
2901 }
2902 }
2903
2904 return LeastDerivedClassWithSameLayout(
2905 RD: RD->bases_begin()->getType()->getAsCXXRecordDecl());
2906}
2907
2908void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2909 llvm::Value *VTable,
2910 SourceLocation Loc) {
2911 if (SanOpts.has(K: SanitizerKind::CFIVCall))
2912 EmitVTablePtrCheckForCall(RD, VTable, TCK: CodeGenFunction::CFITCK_VCall, Loc);
2913 // Emit the intrinsics of (type_test and assume) for the features of WPD and
2914 // speculative devirtualization. For WPD, emit the intrinsics only for the
2915 // case of non_public LTO visibility.
2916 // TODO: refactor this condition and similar ones into a function (e.g.,
2917 // ShouldEmitDevirtualizationMD) to encapsulate the details of the different
2918 // types of devirtualization.
2919 else if ((CGM.getCodeGenOpts().WholeProgramVTables &&
2920 !CGM.AlwaysHasLTOVisibilityPublic(RD)) ||
2921 CGM.getCodeGenOpts().DevirtualizeSpeculatively) {
2922 CanQualType Ty = CGM.getContext().getCanonicalTagType(TD: RD);
2923 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T: Ty);
2924 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2925
2926 // If we already know that the call has hidden LTO visibility, emit
2927 // @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD
2928 // will convert to @llvm.type.test() if we assert at link time that we have
2929 // whole program visibility.
2930 llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
2931 ? llvm::Intrinsic::type_test
2932 : llvm::Intrinsic::public_type_test;
2933 llvm::Value *TypeTest =
2934 Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VTable, TypeId});
2935 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::assume), Args: TypeTest);
2936 }
2937}
2938
2939/// Converts the CFITypeCheckKind into SanitizerKind::SanitizerOrdinal and
2940/// llvm::SanitizerStatKind.
2941static std::pair<SanitizerKind::SanitizerOrdinal, llvm::SanitizerStatKind>
2942SanitizerInfoFromCFICheckKind(CodeGenFunction::CFITypeCheckKind TCK) {
2943 switch (TCK) {
2944 case CodeGenFunction::CFITCK_VCall:
2945 return std::make_pair(x: SanitizerKind::SO_CFIVCall, y: llvm::SanStat_CFI_VCall);
2946 case CodeGenFunction::CFITCK_NVCall:
2947 return std::make_pair(x: SanitizerKind::SO_CFINVCall,
2948 y: llvm::SanStat_CFI_NVCall);
2949 case CodeGenFunction::CFITCK_DerivedCast:
2950 return std::make_pair(x: SanitizerKind::SO_CFIDerivedCast,
2951 y: llvm::SanStat_CFI_DerivedCast);
2952 case CodeGenFunction::CFITCK_UnrelatedCast:
2953 return std::make_pair(x: SanitizerKind::SO_CFIUnrelatedCast,
2954 y: llvm::SanStat_CFI_UnrelatedCast);
2955 case CodeGenFunction::CFITCK_ICall:
2956 case CodeGenFunction::CFITCK_NVMFCall:
2957 case CodeGenFunction::CFITCK_VMFCall:
2958 llvm_unreachable("unexpected sanitizer kind");
2959 }
2960 llvm_unreachable("Unknown CFITypeCheckKind enum");
2961}
2962
2963void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2964 llvm::Value *VTable,
2965 CFITypeCheckKind TCK,
2966 SourceLocation Loc) {
2967 if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2968 RD = LeastDerivedClassWithSameLayout(RD);
2969
2970 auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2971 SanitizerDebugLocation SanScope(this, {Ordinal},
2972 SanitizerHandler::CFICheckFail);
2973
2974 EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2975}
2976
2977void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2978 bool MayBeNull,
2979 CFITypeCheckKind TCK,
2980 SourceLocation Loc) {
2981 if (!getLangOpts().CPlusPlus)
2982 return;
2983
2984 const auto *ClassDecl = T->getAsCXXRecordDecl();
2985 if (!ClassDecl)
2986 return;
2987
2988 if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2989 return;
2990
2991 if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2992 ClassDecl = LeastDerivedClassWithSameLayout(RD: ClassDecl);
2993
2994 auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2995 SanitizerDebugLocation SanScope(this, {Ordinal},
2996 SanitizerHandler::CFICheckFail);
2997
2998 llvm::BasicBlock *ContBlock = nullptr;
2999
3000 if (MayBeNull) {
3001 llvm::Value *DerivedNotNull =
3002 Builder.CreateIsNotNull(Arg: Derived.emitRawPointer(CGF&: *this), Name: "cast.nonnull");
3003
3004 llvm::BasicBlock *CheckBlock = createBasicBlock(name: "cast.check");
3005 ContBlock = createBasicBlock(name: "cast.cont");
3006
3007 Builder.CreateCondBr(Cond: DerivedNotNull, True: CheckBlock, False: ContBlock);
3008
3009 EmitBlock(BB: CheckBlock);
3010 }
3011
3012 llvm::Value *VTable;
3013 std::tie(args&: VTable, args&: ClassDecl) =
3014 CGM.getCXXABI().LoadVTablePtr(CGF&: *this, This: Derived, RD: ClassDecl);
3015
3016 EmitVTablePtrCheck(RD: ClassDecl, VTable, TCK, Loc);
3017
3018 if (MayBeNull) {
3019 Builder.CreateBr(Dest: ContBlock);
3020 EmitBlock(BB: ContBlock);
3021 }
3022}
3023
3024void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
3025 llvm::Value *VTable,
3026 CFITypeCheckKind TCK,
3027 SourceLocation Loc) {
3028 assert(IsSanitizerScope);
3029
3030 if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
3031 !CGM.HasHiddenLTOVisibility(RD))
3032 return;
3033
3034 auto [M, SSK] = SanitizerInfoFromCFICheckKind(TCK);
3035
3036 std::string TypeName = RD->getQualifiedNameAsString();
3037 if (getContext().getNoSanitizeList().containsType(
3038 Mask: SanitizerMask::bitPosToMask(Pos: M), MangledTypeName: TypeName))
3039 return;
3040
3041 EmitSanitizerStatReport(SSK);
3042
3043 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
3044 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
3045 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
3046
3047 llvm::Value *TypeTest = Builder.CreateCall(
3048 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, TypeId});
3049
3050 llvm::Constant *StaticData[] = {
3051 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK),
3052 EmitCheckSourceLocation(Loc),
3053 EmitCheckTypeDescriptor(T),
3054 };
3055
3056 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
3057 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
3058 EmitCfiSlowPathCheck(Ordinal: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: StaticData);
3059 return;
3060 }
3061
3062 if (CGM.getCodeGenOpts().SanitizeTrap.has(O: M)) {
3063 bool NoMerge = !CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: M);
3064 EmitTrapCheck(Checked: TypeTest, CheckHandlerID: SanitizerHandler::CFICheckFail, NoMerge);
3065 return;
3066 }
3067
3068 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3069 Context&: CGM.getLLVMContext(),
3070 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
3071 llvm::Value *ValidVtable = Builder.CreateCall(
3072 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, AllVtables});
3073 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail,
3074 StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
3075}
3076
3077bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
3078 if ((!CGM.getCodeGenOpts().WholeProgramVTables ||
3079 !CGM.HasHiddenLTOVisibility(RD)) &&
3080 !CGM.getCodeGenOpts().DevirtualizeSpeculatively)
3081 return false;
3082
3083 if (CGM.getCodeGenOpts().VirtualFunctionElimination)
3084 return true;
3085
3086 if (!SanOpts.has(K: SanitizerKind::CFIVCall) ||
3087 !CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIVCall))
3088 return false;
3089
3090 std::string TypeName = RD->getQualifiedNameAsString();
3091 return !getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
3092 MangledTypeName: TypeName);
3093}
3094
3095llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
3096 const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
3097 uint64_t VTableByteOffset) {
3098 auto CheckOrdinal = SanitizerKind::SO_CFIVCall;
3099 auto CheckHandler = SanitizerHandler::CFICheckFail;
3100 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3101
3102 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_VCall);
3103
3104 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
3105 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
3106 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
3107
3108 auto CheckedLoadIntrinsic = CGM.getLangOpts().RelativeCXXABIVTables
3109 ? llvm::Intrinsic::type_checked_load_relative
3110 : llvm::Intrinsic::type_checked_load;
3111 llvm::Value *CheckedLoad = Builder.CreateCall(
3112 Callee: CGM.getIntrinsic(IID: CheckedLoadIntrinsic),
3113 Args: {VTable, llvm::ConstantInt::get(Ty: Int32Ty, V: VTableByteOffset), TypeId});
3114
3115 llvm::Value *CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
3116
3117 std::string TypeName = RD->getQualifiedNameAsString();
3118 if (SanOpts.has(K: SanitizerKind::CFIVCall) &&
3119 !getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
3120 MangledTypeName: TypeName)) {
3121 EmitCheck(Checked: std::make_pair(x&: CheckResult, y&: CheckOrdinal), Check: CheckHandler, StaticArgs: {}, DynamicArgs: {});
3122 }
3123
3124 return Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0),
3125 DestTy: VTableTy);
3126}
3127
3128void CodeGenFunction::EmitForwardingCallToLambda(
3129 const CXXMethodDecl *callOperator, CallArgList &callArgs,
3130 const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {
3131 // Get the address of the call operator.
3132 if (!calleeFnInfo)
3133 calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: callOperator);
3134
3135 if (!calleePtr)
3136 calleePtr =
3137 CGM.GetAddrOfFunction(GD: GlobalDecl(callOperator),
3138 Ty: CGM.getTypes().GetFunctionType(Info: *calleeFnInfo));
3139
3140 // Prepare the return slot.
3141 const FunctionProtoType *FPT =
3142 callOperator->getType()->castAs<FunctionProtoType>();
3143 QualType resultType = FPT->getReturnType();
3144 ReturnValueSlot returnSlot;
3145 if (!resultType->isVoidType() &&
3146 calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
3147 !hasScalarEvaluationKind(T: calleeFnInfo->getReturnType()))
3148 returnSlot =
3149 ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
3150 /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
3151
3152 // We don't need to separately arrange the call arguments because
3153 // the call can't be variadic anyway --- it's impossible to forward
3154 // variadic arguments.
3155
3156 // Now emit our call.
3157 auto callee = CGCallee::forDirect(functionPtr: calleePtr, abstractInfo: GlobalDecl(callOperator));
3158 RValue RV = EmitCall(CallInfo: *calleeFnInfo, Callee: callee, ReturnValue: returnSlot, Args: callArgs);
3159
3160 // If necessary, copy the returned value into the slot.
3161 if (!resultType->isVoidType() && returnSlot.isNull()) {
3162 if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
3163 RV = RValue::get(V: EmitARCRetainAutoreleasedReturnValue(value: RV.getScalarVal()));
3164 }
3165 EmitReturnOfRValue(RV, Ty: resultType);
3166 } else
3167 EmitBranchThroughCleanup(Dest: ReturnBlock);
3168}
3169
3170void CodeGenFunction::EmitLambdaBlockInvokeBody() {
3171 const BlockDecl *BD = BlockInfo->getBlockDecl();
3172 const VarDecl *variable = BD->capture_begin()->getVariable();
3173 const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
3174 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3175
3176 if (CallOp->isVariadic()) {
3177 // FIXME: Making this work correctly is nasty because it requires either
3178 // cloning the body of the call operator or making the call operator
3179 // forward.
3180 CGM.ErrorUnsupported(D: CurCodeDecl, Type: "lambda conversion to variadic function");
3181 return;
3182 }
3183
3184 // Start building arguments for forwarding call
3185 CallArgList CallArgs;
3186
3187 CanQualType ThisType =
3188 getContext().getPointerType(T: getContext().getCanonicalTagType(TD: Lambda));
3189 Address ThisPtr = GetAddrOfBlockDecl(var: variable);
3190 CallArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: ThisPtr, PointeeType: ThisType)), type: ThisType);
3191
3192 // Add the rest of the parameters.
3193 for (auto *param : BD->parameters())
3194 EmitDelegateCallArg(args&: CallArgs, param, loc: param->getBeginLoc());
3195
3196 assert(!Lambda->isGenericLambda() &&
3197 "generic lambda interconversion to block not implemented");
3198 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3199}
3200
3201void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
3202 if (MD->isVariadic()) {
3203 // FIXME: Making this work correctly is nasty because it requires either
3204 // cloning the body of the call operator or making the call operator
3205 // forward.
3206 CGM.ErrorUnsupported(D: MD, Type: "lambda conversion to variadic function");
3207 return;
3208 }
3209
3210 const CXXRecordDecl *Lambda = MD->getParent();
3211
3212 // Start building arguments for forwarding call
3213 CallArgList CallArgs;
3214
3215 CanQualType LambdaType = getContext().getCanonicalTagType(TD: Lambda);
3216 CanQualType ThisType = getContext().getPointerType(T: LambdaType);
3217 Address ThisPtr = CreateMemTempWithoutCast(T: LambdaType, Name: "unused.capture");
3218 CallArgs.add(rvalue: RValue::get(V: ThisPtr.emitRawPointer(CGF&: *this)), type: ThisType);
3219
3220 EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3221}
3222
3223void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
3224 CallArgList &CallArgs) {
3225 // Add the rest of the forwarded parameters.
3226 for (auto *Param : MD->parameters())
3227 EmitDelegateCallArg(args&: CallArgs, param: Param, loc: Param->getBeginLoc());
3228
3229 const CXXRecordDecl *Lambda = MD->getParent();
3230 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3231 // For a generic lambda, find the corresponding call operator specialization
3232 // to which the call to the static-invoker shall be forwarded.
3233 if (Lambda->isGenericLambda()) {
3234 assert(MD->isFunctionTemplateSpecialization());
3235 const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
3236 FunctionTemplateDecl *CallOpTemplate =
3237 CallOp->getDescribedFunctionTemplate();
3238 void *InsertPos = nullptr;
3239 FunctionDecl *CorrespondingCallOpSpecialization =
3240 CallOpTemplate->findSpecialization(Args: TAL->asArray(), InsertPos);
3241 assert(CorrespondingCallOpSpecialization);
3242 CallOp = cast<CXXMethodDecl>(Val: CorrespondingCallOpSpecialization);
3243 }
3244
3245 // Special lambda forwarding when there are inalloca parameters.
3246 if (hasInAllocaArg(MD)) {
3247 const CGFunctionInfo *ImplFnInfo = nullptr;
3248 llvm::Function *ImplFn = nullptr;
3249 EmitLambdaInAllocaImplFn(CallOp, ImplFnInfo: &ImplFnInfo, ImplFn: &ImplFn);
3250
3251 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs, calleeFnInfo: ImplFnInfo, calleePtr: ImplFn);
3252 return;
3253 }
3254
3255 EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3256}
3257
3258void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
3259 if (MD->isVariadic()) {
3260 // FIXME: Making this work correctly is nasty because it requires either
3261 // cloning the body of the call operator or making the call operator
3262 // forward.
3263 CGM.ErrorUnsupported(D: MD, Type: "lambda conversion to variadic function");
3264 return;
3265 }
3266
3267 // Forward %this argument.
3268 CallArgList CallArgs;
3269 CanQualType LambdaType = getContext().getCanonicalTagType(TD: MD->getParent());
3270 CanQualType ThisType = getContext().getPointerType(T: LambdaType);
3271 llvm::Value *ThisArg = CurFn->getArg(i: 0);
3272 CallArgs.add(rvalue: RValue::get(V: ThisArg), type: ThisType);
3273
3274 EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3275}
3276
3277void CodeGenFunction::EmitLambdaInAllocaImplFn(
3278 const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,
3279 llvm::Function **ImplFn) {
3280 const CGFunctionInfo &FnInfo =
3281 CGM.getTypes().arrangeCXXMethodDeclaration(MD: CallOp);
3282 llvm::Function *CallOpFn =
3283 cast<llvm::Function>(Val: CGM.GetAddrOfFunction(GD: GlobalDecl(CallOp)));
3284
3285 // Emit function containing the original call op body. __invoke will delegate
3286 // to this function.
3287 SmallVector<CanQualType, 4> ArgTypes;
3288 for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)
3289 ArgTypes.push_back(Elt: I->type);
3290 *ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(
3291 returnType: FnInfo.getReturnType(), opts: FnInfoOpts::IsDelegateCall, argTypes: ArgTypes,
3292 info: FnInfo.getExtInfo(), paramInfos: {}, args: FnInfo.getRequiredArgs(), ABIInfoFD: CallOp);
3293
3294 // Create mangled name as if this was a method named __impl. If for some
3295 // reason the name doesn't look as expected then just tack __impl to the
3296 // front.
3297 // TODO: Use the name mangler to produce the right name instead of using
3298 // string replacement.
3299 StringRef CallOpName = CallOpFn->getName();
3300 std::string ImplName;
3301 if (size_t Pos = CallOpName.find_first_of(Chars: "<lambda"))
3302 ImplName = ("?__impl@" + CallOpName.drop_front(N: Pos)).str();
3303 else
3304 ImplName = ("__impl" + CallOpName).str();
3305
3306 llvm::Function *Fn = CallOpFn->getParent()->getFunction(Name: ImplName);
3307 if (!Fn) {
3308 Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: **ImplFnInfo),
3309 Linkage: llvm::GlobalValue::InternalLinkage, N: ImplName,
3310 M&: CGM.getModule());
3311 CGM.SetInternalFunctionAttributes(GD: CallOp, F: Fn, FI: **ImplFnInfo);
3312
3313 const GlobalDecl &GD = GlobalDecl(CallOp);
3314 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
3315 CodeGenFunction(CGM).GenerateCode(GD, Fn, FnInfo: **ImplFnInfo);
3316 CGM.SetLLVMFunctionAttributesForDefinition(D, F: Fn);
3317 }
3318 *ImplFn = Fn;
3319}
3320