1//===------- ItaniumCXXABI.cpp - Emit LLVM Code from ASTs for a Module ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides C++ code generation targeting the Itanium C++ ABI. The class
10// in this file generates structures that follow the Itanium C++ ABI, which is
11// documented at:
12// https://itanium-cxx-abi.github.io/cxx-abi/abi.html
13// https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
14//
15// It also supports the closely-related ARM ABI, documented at:
16// https://developer.arm.com/documentation/ihi0041/g/
17//
18//===----------------------------------------------------------------------===//
19
20#include "CGCXXABI.h"
21#include "CGCleanup.h"
22#include "CGDebugInfo.h"
23#include "CGRecordLayout.h"
24#include "CGVTables.h"
25#include "CodeGenFunction.h"
26#include "CodeGenModule.h"
27#include "TargetInfo.h"
28#include "clang/AST/Attr.h"
29#include "clang/AST/Mangle.h"
30#include "clang/AST/StmtCXX.h"
31#include "clang/AST/Type.h"
32#include "clang/CodeGen/ConstantInitBuilder.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/Value.h"
38#include "llvm/Support/ConvertEBCDIC.h"
39#include "llvm/Support/ScopedPrinter.h"
40
41#include <optional>
42
43using namespace clang;
44using namespace CodeGen;
45
46namespace {
47class ItaniumCXXABI : public CodeGen::CGCXXABI {
48 /// VTables - All the vtables which have been defined.
49 llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
50
51 /// All the thread wrapper functions that have been used.
52 llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
53 ThreadWrappers;
54
55protected:
56 bool UseARMMethodPtrABI;
57 bool UseARMGuardVarABI;
58 bool Use32BitVTableOffsetABI;
59
60 ItaniumMangleContext &getMangleContext() {
61 return cast<ItaniumMangleContext>(Val&: CodeGen::CGCXXABI::getMangleContext());
62 }
63
64public:
65 ItaniumCXXABI(CodeGen::CodeGenModule &CGM,
66 bool UseARMMethodPtrABI = false,
67 bool UseARMGuardVarABI = false) :
68 CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI),
69 UseARMGuardVarABI(UseARMGuardVarABI),
70 Use32BitVTableOffsetABI(false) { }
71
72 bool classifyReturnType(CGFunctionInfo &FI) const override;
73
74 RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override {
75 // If C++ prohibits us from making a copy, pass by address.
76 if (!RD->canPassInRegisters())
77 return RAA_Indirect;
78 return RAA_Default;
79 }
80
81 bool isThisCompleteObject(GlobalDecl GD) const override {
82 // The Itanium ABI has separate complete-object vs. base-object
83 // variants of both constructors and destructors.
84 if (isa<CXXDestructorDecl>(Val: GD.getDecl())) {
85 switch (GD.getDtorType()) {
86 case Dtor_Complete:
87 case Dtor_Deleting:
88 return true;
89
90 case Dtor_Base:
91 return false;
92
93 case Dtor_Comdat:
94 llvm_unreachable("emitting dtor comdat as function?");
95 case Dtor_Unified:
96 llvm_unreachable("emitting unified dtor as function?");
97 case Dtor_VectorDeleting:
98 llvm_unreachable("unexpected dtor kind for this ABI");
99 }
100 llvm_unreachable("bad dtor kind");
101 }
102 if (isa<CXXConstructorDecl>(Val: GD.getDecl())) {
103 switch (GD.getCtorType()) {
104 case Ctor_Complete:
105 return true;
106
107 case Ctor_Base:
108 return false;
109
110 case Ctor_CopyingClosure:
111 case Ctor_DefaultClosure:
112 llvm_unreachable("closure ctors in Itanium ABI?");
113
114 case Ctor_Comdat:
115 llvm_unreachable("emitting ctor comdat as function?");
116
117 case Ctor_Unified:
118 llvm_unreachable("emitting unified ctor as function?");
119 }
120 llvm_unreachable("bad dtor kind");
121 }
122
123 // No other kinds.
124 return false;
125 }
126
127 bool isZeroInitializable(const MemberPointerType *MPT) override;
128
129 llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
130
131 CGCallee
132 EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
133 const Expr *E,
134 Address This,
135 llvm::Value *&ThisPtrForCall,
136 llvm::Value *MemFnPtr,
137 const MemberPointerType *MPT) override;
138
139 llvm::Value *EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
140 Address Base, llvm::Value *MemPtr,
141 const MemberPointerType *MPT,
142 bool IsInBounds) override;
143
144 llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
145 const CastExpr *E,
146 llvm::Value *Src) override;
147 llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
148 llvm::Constant *Src) override;
149
150 llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
151
152 llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
153 llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
154 CharUnits offset) override;
155 llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
156 llvm::Constant *BuildMemberPointer(const CXXMethodDecl *MD,
157 CharUnits ThisAdjustment);
158
159 llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
160 llvm::Value *L, llvm::Value *R,
161 const MemberPointerType *MPT,
162 bool Inequality) override;
163
164 llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
165 llvm::Value *Addr,
166 const MemberPointerType *MPT) override;
167
168 void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
169 Address Ptr, QualType ElementType,
170 const CXXDestructorDecl *Dtor) override;
171
172 void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
173 void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
174
175 void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
176
177 llvm::CallInst *
178 emitTerminateForUnexpectedException(CodeGenFunction &CGF,
179 llvm::Value *Exn) override;
180
181 void EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD);
182 llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
183 CatchTypeInfo
184 getAddrOfCXXCatchHandlerType(QualType Ty,
185 QualType CatchHandlerType) override {
186 return CatchTypeInfo{.RTTI: getAddrOfRTTIDescriptor(Ty), .Flags: 0};
187 }
188
189 bool shouldTypeidBeNullChecked(QualType SrcRecordTy) override;
190 void EmitBadTypeidCall(CodeGenFunction &CGF) override;
191 llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
192 Address ThisPtr,
193 llvm::Type *StdTypeInfoPtrTy) override;
194
195 bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
196 QualType SrcRecordTy) override;
197
198 /// Determine whether we know that all instances of type RecordTy will have
199 /// the same vtable pointer values, that is distinct from all other vtable
200 /// pointers. While this is required by the Itanium ABI, it doesn't happen in
201 /// practice in some cases due to language extensions.
202 bool hasUniqueVTablePointer(QualType RecordTy) {
203 const CXXRecordDecl *RD = RecordTy->getAsCXXRecordDecl();
204
205 // The exact dynamic_cast optimization relies on the vtable having a unique
206 // address. -fno-assume-unique-vtables disables it, and under -fapple-kext
207 // multiple definitions of the same vtable may be emitted.
208 if (CGM.getCodeGenOpts().DisableExactDynamicCast ||
209 getContext().getLangOpts().AppleKext)
210 return false;
211
212 // If the type_info* would be null, the vtable might be merged with that of
213 // another type.
214 if (!CGM.shouldEmitRTTI())
215 return false;
216
217 // If there's only one definition of the vtable in the program, it has a
218 // unique address.
219 if (!llvm::GlobalValue::isWeakForLinker(Linkage: CGM.getVTableLinkage(RD)))
220 return true;
221
222 // Even if there are multiple definitions of the vtable, they are required
223 // by the ABI to use the same symbol name, so should be merged at load
224 // time. However, if the class has hidden visibility, there can be
225 // different versions of the class in different modules, and the ABI
226 // library might treat them as being the same.
227 if (CGM.GetLLVMVisibility(V: RD->getVisibility()) !=
228 llvm::GlobalValue::DefaultVisibility)
229 return false;
230
231 // A vague-linkage (weak) vtable on a target whose ABI may duplicate it can
232 // be emitted with a distinct address in more than one image, so its address
233 // cannot be assumed unique.
234 return !CGM.mayVTableBeDuplicated(Linkage: CGM.getVTableLinkage(RD));
235 }
236
237 bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
238 return hasUniqueVTablePointer(RecordTy: DestRecordTy);
239 }
240
241 std::optional<ExactDynamicCastInfo>
242 getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
243 QualType DestRecordTy) override;
244
245 llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
246 QualType SrcRecordTy, QualType DestTy,
247 QualType DestRecordTy,
248 llvm::BasicBlock *CastEnd) override;
249
250 llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address ThisAddr,
251 QualType SrcRecordTy, QualType DestTy,
252 QualType DestRecordTy,
253 const ExactDynamicCastInfo &CastInfo,
254 llvm::BasicBlock *CastSuccess,
255 llvm::BasicBlock *CastFail) override;
256
257 llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
258 QualType SrcRecordTy) override;
259
260 bool EmitBadCastCall(CodeGenFunction &CGF) override;
261
262 llvm::Value *
263 GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
264 const CXXRecordDecl *ClassDecl,
265 const CXXRecordDecl *BaseClassDecl) override;
266
267 void EmitCXXConstructors(const CXXConstructorDecl *D) override;
268
269 AddedStructorArgCounts
270 buildStructorSignature(GlobalDecl GD,
271 SmallVectorImpl<CanQualType> &ArgTys) override;
272
273 bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
274 CXXDtorType DT) const override {
275 // Itanium does not emit any destructor variant as an inline thunk.
276 // Delegating may occur as an optimization, but all variants are either
277 // emitted with external linkage or as linkonce if they are inline and used.
278 return false;
279 }
280
281 void EmitCXXDestructors(const CXXDestructorDecl *D) override;
282
283 void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
284 FunctionArgList &Params) override;
285
286 void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
287
288 AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
289 const CXXConstructorDecl *D,
290 CXXCtorType Type,
291 bool ForVirtualBase,
292 bool Delegating) override;
293
294 llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
295 const CXXDestructorDecl *DD,
296 CXXDtorType Type,
297 bool ForVirtualBase,
298 bool Delegating) override;
299
300 void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
301 CXXDtorType Type, bool ForVirtualBase,
302 bool Delegating, Address This,
303 QualType ThisTy) override;
304
305 void emitVTableDefinitions(CodeGenVTables &CGVT,
306 const CXXRecordDecl *RD) override;
307
308 bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
309 CodeGenFunction::VPtr Vptr) override;
310
311 bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
312 return true;
313 }
314
315 llvm::Constant *
316 getVTableAddressPoint(BaseSubobject Base,
317 const CXXRecordDecl *VTableClass) override;
318
319 llvm::Value *getVTableAddressPointInStructor(
320 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
321 BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
322
323 llvm::Value *getVTableAddressPointInStructorWithVTT(
324 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
325 BaseSubobject Base, const CXXRecordDecl *NearestVBase);
326
327 llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
328 CharUnits VPtrOffset) override;
329
330 CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
331 Address This, llvm::Type *Ty,
332 SourceLocation Loc) override;
333
334 llvm::Value *
335 EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor,
336 CXXDtorType DtorType, Address This,
337 DeleteOrMemberCallExpr E,
338 llvm::CallBase **CallOrInvoke) override;
339
340 void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
341
342 bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override;
343 bool canSpeculativelyEmitVTableAsBaseClass(const CXXRecordDecl *RD) const;
344
345 void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
346 bool ReturnAdjustment) override {
347 // Allow inlining of thunks by emitting them with available_externally
348 // linkage together with vtables when needed.
349 if (ForVTable && !Thunk->hasLocalLinkage())
350 Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
351 CGM.setGVProperties(GV: Thunk, GD);
352 }
353
354 bool exportThunk() override { return true; }
355
356 llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
357 const CXXRecordDecl *UnadjustedThisClass,
358 const ThunkInfo &TI) override;
359
360 llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
361 const CXXRecordDecl *UnadjustedRetClass,
362 const ReturnAdjustment &RA) override;
363
364 size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
365 FunctionArgList &Args) const override {
366 assert(!Args.empty() && "expected the arglist to not be empty!");
367 return Args.size() - 1;
368 }
369
370 StringRef GetPureVirtualCallName() override { return "__cxa_pure_virtual"; }
371 StringRef GetDeletedVirtualCallName() override
372 { return "__cxa_deleted_virtual"; }
373
374 CharUnits getArrayCookieSizeImpl(QualType elementType) override;
375 Address InitializeArrayCookie(CodeGenFunction &CGF,
376 Address NewPtr,
377 llvm::Value *NumElements,
378 const CXXNewExpr *expr,
379 QualType ElementType) override;
380 llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
381 Address allocPtr,
382 CharUnits cookieSize) override;
383
384 void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
385 llvm::GlobalVariable *DeclPtr,
386 bool PerformInit) override;
387 void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
388 llvm::FunctionCallee dtor,
389 llvm::Constant *addr) override;
390
391 llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD,
392 llvm::Value *Val);
393 void EmitThreadLocalInitFuncs(
394 CodeGenModule &CGM,
395 ArrayRef<const VarDecl *> CXXThreadLocals,
396 ArrayRef<llvm::Function *> CXXThreadLocalInits,
397 ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
398
399 bool usesThreadWrapperFunction(const VarDecl *VD) const override {
400 return !isEmittedWithConstantInitializer(VD) ||
401 mayNeedDestruction(VD);
402 }
403 LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
404 QualType LValType) override;
405
406 bool NeedsVTTParameter(GlobalDecl GD) override;
407
408 llvm::Constant *
409 getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD);
410
411 /**************************** RTTI Uniqueness ******************************/
412
413protected:
414 /// Returns true if the ABI requires RTTI type_info objects to be unique
415 /// across a program.
416 virtual bool shouldRTTIBeUnique() const { return true; }
417
418public:
419 /// What sort of unique-RTTI behavior should we use?
420 enum RTTIUniquenessKind {
421 /// We are guaranteeing, or need to guarantee, that the RTTI string
422 /// is unique.
423 RUK_Unique,
424
425 /// We are not guaranteeing uniqueness for the RTTI string, so we
426 /// can demote to hidden visibility but must use string comparisons.
427 RUK_NonUniqueHidden,
428
429 /// We are not guaranteeing uniqueness for the RTTI string, so we
430 /// have to use string comparisons, but we also have to emit it with
431 /// non-hidden visibility.
432 RUK_NonUniqueVisible
433 };
434
435 /// Return the required visibility status for the given type and linkage in
436 /// the current ABI.
437 RTTIUniquenessKind
438 classifyRTTIUniqueness(QualType CanTy,
439 llvm::GlobalValue::LinkageTypes Linkage) const;
440 friend class ItaniumRTTIBuilder;
441
442 void emitCXXStructor(GlobalDecl GD) override;
443
444 std::pair<llvm::Value *, const CXXRecordDecl *>
445 LoadVTablePtr(CodeGenFunction &CGF, Address This,
446 const CXXRecordDecl *RD) override;
447
448 private:
449 llvm::Constant *
450 getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD);
451
452 bool hasAnyUnusedVirtualInlineFunction(const CXXRecordDecl *RD) const {
453 const auto &VtableLayout =
454 CGM.getItaniumVTableContext().getVTableLayout(RD);
455
456 for (const auto &VtableComponent : VtableLayout.vtable_components()) {
457 // Skip empty slot.
458 if (!VtableComponent.isUsedFunctionPointerKind())
459 continue;
460
461 const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
462 const FunctionDecl *FD = Method->getDefinition();
463 const bool IsInlined =
464 Method->getCanonicalDecl()->isInlined() || (FD && FD->isInlined());
465 if (!IsInlined)
466 continue;
467
468 StringRef Name = CGM.getMangledName(
469 GD: VtableComponent.getGlobalDecl(/*HasVectorDeletingDtors=*/false));
470 auto *Entry = CGM.GetGlobalValue(Ref: Name);
471 // This checks if virtual inline function has already been emitted.
472 // Note that it is possible that this inline function would be emitted
473 // after trying to emit vtable speculatively. Because of this we do
474 // an extra pass after emitting all deferred vtables to find and emit
475 // these vtables opportunistically.
476 if (!Entry || Entry->isDeclaration())
477 return true;
478 }
479 return false;
480 }
481
482 bool isVTableHidden(const CXXRecordDecl *RD) const {
483 const auto &VtableLayout =
484 CGM.getItaniumVTableContext().getVTableLayout(RD);
485
486 for (const auto &VtableComponent : VtableLayout.vtable_components()) {
487 if (VtableComponent.isRTTIKind()) {
488 const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl();
489 if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility)
490 return true;
491 } else if (VtableComponent.isUsedFunctionPointerKind()) {
492 const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
493 if (Method->getVisibility() == Visibility::HiddenVisibility &&
494 !Method->isDefined())
495 return true;
496 }
497 }
498 return false;
499 }
500};
501
502class ARMCXXABI : public ItaniumCXXABI {
503public:
504 ARMCXXABI(CodeGen::CodeGenModule &CGM) :
505 ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
506 /*UseARMGuardVarABI=*/true) {}
507
508 bool constructorsAndDestructorsReturnThis() const override { return true; }
509
510 void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV,
511 QualType ResTy) override;
512
513 CharUnits getArrayCookieSizeImpl(QualType elementType) override;
514 Address InitializeArrayCookie(CodeGenFunction &CGF,
515 Address NewPtr,
516 llvm::Value *NumElements,
517 const CXXNewExpr *expr,
518 QualType ElementType) override;
519 llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
520 CharUnits cookieSize) override;
521};
522
523class AppleARM64CXXABI : public ARMCXXABI {
524public:
525 AppleARM64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
526 Use32BitVTableOffsetABI = true;
527 }
528
529 // ARM64 libraries are prepared for non-unique RTTI.
530 bool shouldRTTIBeUnique() const override { return false; }
531};
532
533class FuchsiaCXXABI final : public ItaniumCXXABI {
534public:
535 explicit FuchsiaCXXABI(CodeGen::CodeGenModule &CGM)
536 : ItaniumCXXABI(CGM) {}
537
538private:
539 bool constructorsAndDestructorsReturnThis() const override { return true; }
540};
541
542class WebAssemblyCXXABI final : public ItaniumCXXABI {
543public:
544 explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM)
545 : ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
546 /*UseARMGuardVarABI=*/true) {}
547 void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
548 llvm::CallInst *
549 emitTerminateForUnexpectedException(CodeGenFunction &CGF,
550 llvm::Value *Exn) override;
551
552private:
553 bool constructorsAndDestructorsReturnThis() const override { return true; }
554 bool canCallMismatchedFunctionType() const override { return false; }
555};
556
557class XLCXXABI final : public ItaniumCXXABI {
558public:
559 explicit XLCXXABI(CodeGen::CodeGenModule &CGM)
560 : ItaniumCXXABI(CGM) {}
561
562 void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
563 llvm::FunctionCallee dtor,
564 llvm::Constant *addr) override;
565
566 bool useSinitAndSterm() const override { return true; }
567
568private:
569 void emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
570 llvm::Constant *addr);
571};
572}
573
574CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
575 switch (CGM.getContext().getCXXABIKind()) {
576 // For IR-generation purposes, there's no significant difference
577 // between the ARM and iOS ABIs.
578 case TargetCXXABI::GenericARM:
579 case TargetCXXABI::iOS:
580 case TargetCXXABI::WatchOS:
581 return new ARMCXXABI(CGM);
582
583 case TargetCXXABI::AppleARM64:
584 return new AppleARM64CXXABI(CGM);
585
586 case TargetCXXABI::Fuchsia:
587 return new FuchsiaCXXABI(CGM);
588
589 // Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
590 // include the other 32-bit ARM oddities: constructor/destructor return values
591 // and array cookies.
592 case TargetCXXABI::GenericAArch64:
593 return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
594 /*UseARMGuardVarABI=*/true);
595
596 case TargetCXXABI::GenericMIPS:
597 return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
598
599 case TargetCXXABI::WebAssembly:
600 return new WebAssemblyCXXABI(CGM);
601
602 case TargetCXXABI::XL:
603 return new XLCXXABI(CGM);
604
605 case TargetCXXABI::GenericItanium:
606 return new ItaniumCXXABI(CGM);
607
608 case TargetCXXABI::Microsoft:
609 llvm_unreachable("Microsoft ABI is not Itanium-based");
610 }
611 llvm_unreachable("bad ABI kind");
612}
613
614llvm::Type *
615ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
616 if (MPT->isMemberDataPointer())
617 return CGM.PtrDiffTy;
618 return llvm::StructType::get(elt1: CGM.PtrDiffTy, elts: CGM.PtrDiffTy);
619}
620
621/// In the Itanium and ARM ABIs, method pointers have the form:
622/// struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
623///
624/// In the Itanium ABI:
625/// - method pointers are virtual if (memptr.ptr & 1) is nonzero
626/// - the this-adjustment is (memptr.adj)
627/// - the virtual offset is (memptr.ptr - 1)
628///
629/// In the ARM ABI:
630/// - method pointers are virtual if (memptr.adj & 1) is nonzero
631/// - the this-adjustment is (memptr.adj >> 1)
632/// - the virtual offset is (memptr.ptr)
633/// ARM uses 'adj' for the virtual flag because Thumb functions
634/// may be only single-byte aligned.
635///
636/// If the member is virtual, the adjusted 'this' pointer points
637/// to a vtable pointer from which the virtual offset is applied.
638///
639/// If the member is non-virtual, memptr.ptr is the address of
640/// the function to call.
641CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
642 CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
643 llvm::Value *&ThisPtrForCall,
644 llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
645 CGBuilderTy &Builder = CGF.Builder;
646
647 const FunctionProtoType *FPT =
648 MPT->getPointeeType()->castAs<FunctionProtoType>();
649 auto *RD = MPT->getMostRecentCXXRecordDecl();
650
651 llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 1);
652
653 llvm::BasicBlock *FnVirtual = CGF.createBasicBlock(name: "memptr.virtual");
654 llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock(name: "memptr.nonvirtual");
655 llvm::BasicBlock *FnEnd = CGF.createBasicBlock(name: "memptr.end");
656
657 // Extract memptr.adj, which is in the second field.
658 llvm::Value *RawAdj = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 1, Name: "memptr.adj");
659
660 // Compute the true adjustment.
661 llvm::Value *Adj = RawAdj;
662 if (UseARMMethodPtrABI)
663 Adj = Builder.CreateAShr(LHS: Adj, RHS: ptrdiff_1, Name: "memptr.adj.shifted");
664
665 // Apply the adjustment and cast back to the original struct type
666 // for consistency.
667 llvm::Value *This = ThisAddr.emitRawPointer(CGF);
668 This = Builder.CreateInBoundsGEP(Ty: Builder.getInt8Ty(), Ptr: This, IdxList: Adj);
669 ThisPtrForCall = This;
670
671 // Load the function pointer.
672 llvm::Value *FnAsInt = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 0, Name: "memptr.ptr");
673
674 // If the LSB in the function pointer is 1, the function pointer points to
675 // a virtual function.
676 llvm::Value *IsVirtual;
677 if (UseARMMethodPtrABI)
678 IsVirtual = Builder.CreateAnd(LHS: RawAdj, RHS: ptrdiff_1);
679 else
680 IsVirtual = Builder.CreateAnd(LHS: FnAsInt, RHS: ptrdiff_1);
681 IsVirtual = Builder.CreateIsNotNull(Arg: IsVirtual, Name: "memptr.isvirtual");
682 Builder.CreateCondBr(Cond: IsVirtual, True: FnVirtual, False: FnNonVirtual);
683
684 // In the virtual path, the adjustment left 'This' pointing to the
685 // vtable of the correct base subobject. The "function pointer" is an
686 // offset within the vtable (+1 for the virtual flag on non-ARM).
687 CGF.EmitBlock(BB: FnVirtual);
688
689 // Cast the adjusted this to a pointer to vtable pointer and load.
690 llvm::Type *VTableTy = CGF.CGM.GlobalsInt8PtrTy;
691 CharUnits VTablePtrAlign =
692 CGF.CGM.getDynamicOffsetAlignment(ActualAlign: ThisAddr.getAlignment(), Class: RD,
693 ExpectedTargetAlign: CGF.getPointerAlign());
694 llvm::Value *VTable = CGF.GetVTablePtr(
695 This: Address(This, ThisAddr.getElementType(), VTablePtrAlign), VTableTy, VTableClass: RD);
696
697 // Apply the offset.
698 // On ARM64, to reserve extra space in virtual member function pointers,
699 // we only pay attention to the low 32 bits of the offset.
700 llvm::Value *VTableOffset = FnAsInt;
701 if (!UseARMMethodPtrABI)
702 VTableOffset = Builder.CreateSub(LHS: VTableOffset, RHS: ptrdiff_1);
703 if (Use32BitVTableOffsetABI) {
704 VTableOffset = Builder.CreateTrunc(V: VTableOffset, DestTy: CGF.Int32Ty);
705 VTableOffset = Builder.CreateZExt(V: VTableOffset, DestTy: CGM.PtrDiffTy);
706 }
707
708 // Check the address of the function pointer if CFI on member function
709 // pointers is enabled.
710 llvm::Constant *CheckSourceLocation;
711 llvm::Constant *CheckTypeDesc;
712 bool ShouldEmitCFICheck = CGF.SanOpts.has(K: SanitizerKind::CFIMFCall) &&
713 CGM.HasHiddenLTOVisibility(RD);
714
715 if (ShouldEmitCFICheck) {
716 if (const auto *BinOp = dyn_cast<BinaryOperator>(Val: E)) {
717 if (BinOp->isPtrMemOp() &&
718 BinOp->getRHS()
719 ->getType()
720 ->hasPointeeToCFIUncheckedCalleeFunctionType())
721 ShouldEmitCFICheck = false;
722 }
723 }
724
725 bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
726 CGM.HasHiddenLTOVisibility(RD);
727 // TODO: Update this name not to be restricted to WPD only
728 // as we now emit the vtable info info for speculative devirtualization as
729 // well.
730 bool ShouldEmitWPDInfo =
731 (CGM.getCodeGenOpts().WholeProgramVTables &&
732 // Don't insert type tests if we are forcing public visibility.
733 !CGM.AlwaysHasLTOVisibilityPublic(RD)) ||
734 CGM.getCodeGenOpts().DevirtualizeSpeculatively;
735 llvm::Value *VirtualFn = nullptr;
736
737 {
738 auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
739 auto CheckHandler = SanitizerHandler::CFICheckFail;
740 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
741
742 llvm::Value *TypeId = nullptr;
743 llvm::Value *CheckResult = nullptr;
744
745 if (ShouldEmitCFICheck || ShouldEmitVFEInfo || ShouldEmitWPDInfo) {
746 // If doing CFI, VFE or WPD, we will need the metadata node to check
747 // against.
748 llvm::Metadata *MD =
749 CGM.CreateMetadataIdentifierForVirtualMemPtrType(T: QualType(MPT, 0));
750 TypeId = llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
751 }
752
753 if (ShouldEmitVFEInfo) {
754 llvm::Value *VFPAddr =
755 Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
756
757 // If doing VFE, load from the vtable with a type.checked.load intrinsic
758 // call. Note that we use the GEP to calculate the address to load from
759 // and pass 0 as the offset to the intrinsic. This is because every
760 // vtable slot of the correct type is marked with matching metadata, and
761 // we know that the load must be from one of these slots.
762 llvm::Value *CheckedLoad = Builder.CreateCall(
763 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_checked_load),
764 Args: {VFPAddr, llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0), TypeId});
765 CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
766 VirtualFn = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0);
767 } else {
768 // When not doing VFE, emit a normal load, as it allows more
769 // optimisations than type.checked.load.
770 if (ShouldEmitCFICheck || ShouldEmitWPDInfo) {
771 llvm::Value *VFPAddr =
772 Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
773 llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
774 ? llvm::Intrinsic::type_test
775 : llvm::Intrinsic::public_type_test;
776
777 CheckResult =
778 Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VFPAddr, TypeId});
779 }
780
781 if (CGM.getLangOpts().RelativeCXXABIVTables) {
782 VirtualFn = CGF.Builder.CreateCall(
783 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative,
784 Tys: {VTableOffset->getType()}),
785 Args: {VTable, VTableOffset});
786 } else {
787 llvm::Value *VFPAddr =
788 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
789 VirtualFn = CGF.Builder.CreateAlignedLoad(Ty: CGF.DefaultPtrTy, Addr: VFPAddr,
790 Align: CGF.getPointerAlign(),
791 Name: "memptr.virtualfn");
792 }
793 }
794 assert(VirtualFn && "Virtual fuction pointer not created!");
795 assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || !ShouldEmitWPDInfo ||
796 CheckResult) &&
797 "Check result required but not created!");
798
799 if (ShouldEmitCFICheck) {
800 // If doing CFI, emit the check.
801 CheckSourceLocation = CGF.EmitCheckSourceLocation(Loc: E->getBeginLoc());
802 CheckTypeDesc = CGF.EmitCheckTypeDescriptor(T: QualType(MPT, 0));
803 llvm::Constant *StaticData[] = {
804 llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_VMFCall),
805 CheckSourceLocation,
806 CheckTypeDesc,
807 };
808
809 if (CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIMFCall)) {
810 CGF.EmitTrapCheck(Checked: CheckResult, CheckHandlerID: CheckHandler);
811 } else {
812 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
813 Context&: CGM.getLLVMContext(),
814 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
815 llvm::Value *ValidVtable = Builder.CreateCall(
816 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, AllVtables});
817 CGF.EmitCheck(Checked: std::make_pair(x&: CheckResult, y&: CheckOrdinal), Check: CheckHandler,
818 StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
819 }
820
821 FnVirtual = Builder.GetInsertBlock();
822 }
823 } // End of sanitizer scope
824
825 CGF.EmitBranch(Block: FnEnd);
826
827 // In the non-virtual path, the function pointer is actually a
828 // function pointer.
829 CGF.EmitBlock(BB: FnNonVirtual);
830 llvm::Value *NonVirtualFn =
831 Builder.CreateIntToPtr(V: FnAsInt, DestTy: CGF.DefaultPtrTy, Name: "memptr.nonvirtualfn");
832
833 // Check the function pointer if CFI on member function pointers is enabled.
834 if (ShouldEmitCFICheck) {
835 CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
836 if (RD->hasDefinition()) {
837 auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
838 auto CheckHandler = SanitizerHandler::CFICheckFail;
839 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
840
841 llvm::Constant *StaticData[] = {
842 llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_NVMFCall),
843 CheckSourceLocation,
844 CheckTypeDesc,
845 };
846
847 llvm::Value *Bit = Builder.getFalse();
848 for (const CXXRecordDecl *Base : CGM.getMostBaseClasses(RD)) {
849 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(
850 T: getContext().getMemberPointerType(T: MPT->getPointeeType(),
851 /*Qualifier=*/std::nullopt,
852 Cls: Base->getCanonicalDecl()));
853 llvm::Value *TypeId =
854 llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
855
856 llvm::Value *TypeTest =
857 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
858 Args: {NonVirtualFn, TypeId});
859 Bit = Builder.CreateOr(LHS: Bit, RHS: TypeTest);
860 }
861
862 CGF.EmitCheck(Checked: std::make_pair(x&: Bit, y&: CheckOrdinal), Check: CheckHandler, StaticArgs: StaticData,
863 DynamicArgs: {NonVirtualFn, llvm::UndefValue::get(T: CGF.IntPtrTy)});
864
865 FnNonVirtual = Builder.GetInsertBlock();
866 }
867 }
868
869 // We're done.
870 CGF.EmitBlock(BB: FnEnd);
871 llvm::PHINode *CalleePtr = Builder.CreatePHI(Ty: CGF.DefaultPtrTy, NumReservedValues: 2);
872 CalleePtr->addIncoming(V: VirtualFn, BB: FnVirtual);
873 CalleePtr->addIncoming(V: NonVirtualFn, BB: FnNonVirtual);
874
875 CGPointerAuthInfo PointerAuth;
876
877 if (const auto &Schema =
878 CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers) {
879 llvm::PHINode *DiscriminatorPHI = Builder.CreatePHI(Ty: CGF.IntPtrTy, NumReservedValues: 2);
880 DiscriminatorPHI->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
881 BB: FnVirtual);
882 const auto &AuthInfo =
883 CGM.getMemberFunctionPointerAuthInfo(FT: QualType(MPT, 0));
884 assert(Schema.getKey() == AuthInfo.getKey() &&
885 "Keys for virtual and non-virtual member functions must match");
886 auto *NonVirtualDiscriminator = AuthInfo.getDiscriminator();
887 DiscriminatorPHI->addIncoming(V: NonVirtualDiscriminator, BB: FnNonVirtual);
888 PointerAuth = CGPointerAuthInfo(
889 Schema.getKey(), Schema.getAuthenticationMode(), Schema.isIsaPointer(),
890 Schema.authenticatesNullValues(), DiscriminatorPHI);
891 }
892
893 CGCallee Callee(FPT, CalleePtr, PointerAuth);
894 return Callee;
895}
896
897/// Compute an l-value by applying the given pointer-to-member to a
898/// base object.
899llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
900 CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
901 const MemberPointerType *MPT, bool IsInBounds) {
902 assert(MemPtr->getType() == CGM.PtrDiffTy);
903
904 CGBuilderTy &Builder = CGF.Builder;
905
906 // Apply the offset.
907 llvm::Value *BaseAddr = Base.emitRawPointer(CGF);
908 return Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: BaseAddr, IdxList: MemPtr, Name: "memptr.offset",
909 NW: IsInBounds ? llvm::GEPNoWrapFlags::inBounds()
910 : llvm::GEPNoWrapFlags::none());
911}
912
913// See if it's possible to return a constant signed pointer.
914static llvm::Constant *pointerAuthResignConstant(
915 llvm::Value *Ptr, const CGPointerAuthInfo &CurAuthInfo,
916 const CGPointerAuthInfo &NewAuthInfo, CodeGenModule &CGM) {
917 const auto *CPA = dyn_cast<llvm::ConstantPtrAuth>(Val: Ptr);
918
919 if (!CPA)
920 return nullptr;
921
922 assert(CPA->getKey()->getZExtValue() == CurAuthInfo.getKey() &&
923 CPA->getAddrDiscriminator()->isNullValue() &&
924 CPA->getDiscriminator() == CurAuthInfo.getDiscriminator() &&
925 "unexpected key or discriminators");
926
927 return CGM.getConstantSignedPointer(
928 Pointer: CPA->getPointer(), Key: NewAuthInfo.getKey(), StorageAddress: nullptr,
929 OtherDiscriminator: cast<llvm::ConstantInt>(Val: NewAuthInfo.getDiscriminator()));
930}
931
932/// Perform a bitcast, derived-to-base, or base-to-derived member pointer
933/// conversion.
934///
935/// Bitcast conversions are always a no-op under Itanium.
936///
937/// Obligatory offset/adjustment diagram:
938/// <-- offset --> <-- adjustment -->
939/// |--------------------------|----------------------|--------------------|
940/// ^Derived address point ^Base address point ^Member address point
941///
942/// So when converting a base member pointer to a derived member pointer,
943/// we add the offset to the adjustment because the address point has
944/// decreased; and conversely, when converting a derived MP to a base MP
945/// we subtract the offset from the adjustment because the address point
946/// has increased.
947///
948/// The standard forbids (at compile time) conversion to and from
949/// virtual bases, which is why we don't have to consider them here.
950///
951/// The standard forbids (at run time) casting a derived MP to a base
952/// MP when the derived MP does not point to a member of the base.
953/// This is why -1 is a reasonable choice for null data member
954/// pointers.
955llvm::Value *
956ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
957 const CastExpr *E,
958 llvm::Value *src) {
959 // Use constant emission if we can.
960 if (isa<llvm::Constant>(Val: src))
961 return EmitMemberPointerConversion(E, Src: cast<llvm::Constant>(Val: src));
962
963 assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
964 E->getCastKind() == CK_BaseToDerivedMemberPointer ||
965 E->getCastKind() == CK_ReinterpretMemberPointer);
966
967 CGBuilderTy &Builder = CGF.Builder;
968 QualType DstType = E->getType();
969
970 if (DstType->isMemberFunctionPointerType()) {
971 if (const auto &NewAuthInfo =
972 CGM.getMemberFunctionPointerAuthInfo(FT: DstType)) {
973 QualType SrcType = E->getSubExpr()->getType();
974 assert(SrcType->isMemberFunctionPointerType());
975 const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: SrcType);
976 llvm::Value *MemFnPtr = Builder.CreateExtractValue(Agg: src, Idxs: 0, Name: "memptr.ptr");
977 llvm::Type *OrigTy = MemFnPtr->getType();
978
979 llvm::BasicBlock *StartBB = Builder.GetInsertBlock();
980 llvm::BasicBlock *ResignBB = CGF.createBasicBlock(name: "resign");
981 llvm::BasicBlock *MergeBB = CGF.createBasicBlock(name: "merge");
982
983 // Check whether we have a virtual offset or a pointer to a function.
984 assert(UseARMMethodPtrABI && "ARM ABI expected");
985 llvm::Value *Adj = Builder.CreateExtractValue(Agg: src, Idxs: 1, Name: "memptr.adj");
986 llvm::Constant *Ptrdiff_1 = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 1);
987 llvm::Value *AndVal = Builder.CreateAnd(LHS: Adj, RHS: Ptrdiff_1);
988 llvm::Value *IsVirtualOffset =
989 Builder.CreateIsNotNull(Arg: AndVal, Name: "is.virtual.offset");
990 Builder.CreateCondBr(Cond: IsVirtualOffset, True: MergeBB, False: ResignBB);
991
992 CGF.EmitBlock(BB: ResignBB);
993 llvm::Type *PtrTy = llvm::PointerType::getUnqual(C&: CGM.getLLVMContext());
994 MemFnPtr = Builder.CreateIntToPtr(V: MemFnPtr, DestTy: PtrTy);
995 MemFnPtr =
996 CGF.emitPointerAuthResign(Pointer: MemFnPtr, PointerType: SrcType, CurAuthInfo, NewAuthInfo,
997 IsKnownNonNull: isa<llvm::Constant>(Val: src));
998 MemFnPtr = Builder.CreatePtrToInt(V: MemFnPtr, DestTy: OrigTy);
999 llvm::Value *ResignedVal = Builder.CreateInsertValue(Agg: src, Val: MemFnPtr, Idxs: 0);
1000 ResignBB = Builder.GetInsertBlock();
1001
1002 CGF.EmitBlock(BB: MergeBB);
1003 llvm::PHINode *NewSrc = Builder.CreatePHI(Ty: src->getType(), NumReservedValues: 2);
1004 NewSrc->addIncoming(V: src, BB: StartBB);
1005 NewSrc->addIncoming(V: ResignedVal, BB: ResignBB);
1006 src = NewSrc;
1007 }
1008 }
1009
1010 // Under Itanium, reinterprets don't require any additional processing.
1011 if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
1012
1013 llvm::Constant *adj = getMemberPointerAdjustment(E);
1014 if (!adj) return src;
1015
1016 bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
1017
1018 const MemberPointerType *destTy =
1019 E->getType()->castAs<MemberPointerType>();
1020
1021 // For member data pointers, this is just a matter of adding the
1022 // offset if the source is non-null.
1023 if (destTy->isMemberDataPointer()) {
1024 llvm::Value *dst;
1025 if (isDerivedToBase)
1026 dst = Builder.CreateNSWSub(LHS: src, RHS: adj, Name: "adj");
1027 else
1028 dst = Builder.CreateNSWAdd(LHS: src, RHS: adj, Name: "adj");
1029
1030 // Null check.
1031 llvm::Value *null = llvm::Constant::getAllOnesValue(Ty: src->getType());
1032 llvm::Value *isNull = Builder.CreateICmpEQ(LHS: src, RHS: null, Name: "memptr.isnull");
1033 return Builder.CreateSelect(C: isNull, True: src, False: dst);
1034 }
1035
1036 // The this-adjustment is left-shifted by 1 on ARM.
1037 if (UseARMMethodPtrABI) {
1038 uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
1039 offset <<= 1;
1040 adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
1041 }
1042
1043 llvm::Value *srcAdj = Builder.CreateExtractValue(Agg: src, Idxs: 1, Name: "src.adj");
1044 llvm::Value *dstAdj;
1045 if (isDerivedToBase)
1046 dstAdj = Builder.CreateNSWSub(LHS: srcAdj, RHS: adj, Name: "adj");
1047 else
1048 dstAdj = Builder.CreateNSWAdd(LHS: srcAdj, RHS: adj, Name: "adj");
1049
1050 return Builder.CreateInsertValue(Agg: src, Val: dstAdj, Idxs: 1);
1051}
1052
1053static llvm::Constant *
1054pointerAuthResignMemberFunctionPointer(llvm::Constant *Src, QualType DestType,
1055 QualType SrcType, CodeGenModule &CGM) {
1056 assert(DestType->isMemberFunctionPointerType() &&
1057 SrcType->isMemberFunctionPointerType() &&
1058 "member function pointers expected");
1059 if (DestType == SrcType)
1060 return Src;
1061
1062 const auto &NewAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: DestType);
1063 const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: SrcType);
1064
1065 if (!NewAuthInfo && !CurAuthInfo)
1066 return Src;
1067
1068 llvm::Constant *MemFnPtr = Src->getAggregateElement(Elt: 0u);
1069 if (MemFnPtr->getNumOperands() == 0) {
1070 // src must be a pair of null pointers.
1071 assert(isa<llvm::ConstantInt>(MemFnPtr) && "constant int expected");
1072 return Src;
1073 }
1074
1075 llvm::Constant *ConstPtr = pointerAuthResignConstant(
1076 Ptr: cast<llvm::User>(Val: MemFnPtr)->getOperand(i: 0), CurAuthInfo, NewAuthInfo, CGM);
1077 ConstPtr = llvm::ConstantExpr::getPtrToInt(C: ConstPtr, Ty: MemFnPtr->getType());
1078 return ConstantFoldInsertValueInstruction(Agg: Src, Val: ConstPtr, Idxs: 0);
1079}
1080
1081llvm::Constant *
1082ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
1083 llvm::Constant *src) {
1084 assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
1085 E->getCastKind() == CK_BaseToDerivedMemberPointer ||
1086 E->getCastKind() == CK_ReinterpretMemberPointer);
1087
1088 QualType DstType = E->getType();
1089
1090 if (DstType->isMemberFunctionPointerType())
1091 src = pointerAuthResignMemberFunctionPointer(
1092 Src: src, DestType: DstType, SrcType: E->getSubExpr()->getType(), CGM);
1093
1094 // Under Itanium, reinterprets don't require any additional processing.
1095 if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
1096
1097 // If the adjustment is trivial, we don't need to do anything.
1098 llvm::Constant *adj = getMemberPointerAdjustment(E);
1099 if (!adj) return src;
1100
1101 bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
1102
1103 const MemberPointerType *destTy =
1104 E->getType()->castAs<MemberPointerType>();
1105
1106 // For member data pointers, this is just a matter of adding the
1107 // offset if the source is non-null.
1108 if (destTy->isMemberDataPointer()) {
1109 // null maps to null.
1110 if (src->isAllOnesValue()) return src;
1111
1112 if (isDerivedToBase)
1113 return llvm::ConstantExpr::getNSWSub(C1: src, C2: adj);
1114 else
1115 return llvm::ConstantExpr::getNSWAdd(C1: src, C2: adj);
1116 }
1117
1118 // The this-adjustment is left-shifted by 1 on ARM.
1119 if (UseARMMethodPtrABI) {
1120 uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
1121 offset <<= 1;
1122 adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
1123 }
1124
1125 llvm::Constant *srcAdj = src->getAggregateElement(Elt: 1);
1126 llvm::Constant *dstAdj;
1127 if (isDerivedToBase)
1128 dstAdj = llvm::ConstantExpr::getNSWSub(C1: srcAdj, C2: adj);
1129 else
1130 dstAdj = llvm::ConstantExpr::getNSWAdd(C1: srcAdj, C2: adj);
1131
1132 llvm::Constant *res = ConstantFoldInsertValueInstruction(Agg: src, Val: dstAdj, Idxs: 1);
1133 assert(res != nullptr && "Folding must succeed");
1134 return res;
1135}
1136
1137llvm::Constant *
1138ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
1139 // Itanium C++ ABI 2.3:
1140 // A NULL pointer is represented as -1.
1141 if (MPT->isMemberDataPointer())
1142 return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: -1ULL, /*isSigned=*/IsSigned: true);
1143
1144 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 0);
1145 llvm::Constant *Values[2] = { Zero, Zero };
1146 return llvm::ConstantStruct::getAnon(V: Values);
1147}
1148
1149llvm::Constant *
1150ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
1151 CharUnits offset) {
1152 // Itanium C++ ABI 2.3:
1153 // A pointer to data member is an offset from the base address of
1154 // the class object containing it, represented as a ptrdiff_t
1155 return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: offset.getQuantity());
1156}
1157
1158llvm::Constant *
1159ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
1160 return BuildMemberPointer(MD, ThisAdjustment: CharUnits::Zero());
1161}
1162
1163llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD,
1164 CharUnits ThisAdjustment) {
1165 assert(MD->isInstance() && "Member function must not be static!");
1166
1167 CodeGenTypes &Types = CGM.getTypes();
1168
1169 // Get the function pointer (or index if this is a virtual function).
1170 llvm::Constant *MemPtr[2];
1171 if (MD->isVirtual()) {
1172 uint64_t Index = CGM.getItaniumVTableContext().getMethodVTableIndex(GD: MD);
1173 uint64_t VTableOffset;
1174 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1175 // Multiply by 4-byte relative offsets.
1176 VTableOffset = Index * 4;
1177 } else {
1178 const ASTContext &Context = getContext();
1179 CharUnits PointerWidth = Context.toCharUnitsFromBits(
1180 BitSize: Context.getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default));
1181 VTableOffset = Index * PointerWidth.getQuantity();
1182 }
1183
1184 if (UseARMMethodPtrABI) {
1185 // ARM C++ ABI 3.2.1:
1186 // This ABI specifies that adj contains twice the this
1187 // adjustment, plus 1 if the member function is virtual. The
1188 // least significant bit of adj then makes exactly the same
1189 // discrimination as the least significant bit of ptr does for
1190 // Itanium.
1191
1192 // We cannot use the Itanium ABI's representation for virtual member
1193 // function pointers under pointer authentication because it would
1194 // require us to store both the virtual offset and the constant
1195 // discriminator in the pointer, which would be immediately vulnerable
1196 // to attack. Instead we introduce a thunk that does the virtual dispatch
1197 // and store it as if it were a non-virtual member function. This means
1198 // that virtual function pointers may not compare equal anymore, but
1199 // fortunately they aren't required to by the standard, and we do make
1200 // a best-effort attempt to re-use the thunk.
1201 //
1202 // To support interoperation with code in which pointer authentication
1203 // is disabled, derefencing a member function pointer must still handle
1204 // the virtual case, but it can use a discriminator which should never
1205 // be valid.
1206 const auto &Schema =
1207 CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers;
1208 if (Schema)
1209 MemPtr[0] = llvm::ConstantExpr::getPtrToInt(
1210 C: getSignedVirtualMemberFunctionPointer(MD), Ty: CGM.PtrDiffTy);
1211 else
1212 MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset);
1213 // Don't set the LSB of adj to 1 if pointer authentication for member
1214 // function pointers is enabled.
1215 MemPtr[1] = llvm::ConstantInt::get(
1216 Ty: CGM.PtrDiffTy, V: 2 * ThisAdjustment.getQuantity() + !Schema);
1217 } else {
1218 // Itanium C++ ABI 2.3:
1219 // For a virtual function, [the pointer field] is 1 plus the
1220 // virtual table offset (in bytes) of the function,
1221 // represented as a ptrdiff_t.
1222 MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset + 1);
1223 MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1224 V: ThisAdjustment.getQuantity());
1225 }
1226 } else {
1227 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1228 llvm::Type *Ty;
1229 // Check whether the function has a computable LLVM signature.
1230 if (Types.isFuncTypeConvertible(FT: FPT)) {
1231 // The function has a computable LLVM signature; use the correct type.
1232 Ty = Types.GetFunctionType(Info: Types.arrangeCXXMethodDeclaration(MD));
1233 } else {
1234 // Use an arbitrary non-function type to tell GetAddrOfFunction that the
1235 // function type is incomplete.
1236 Ty = CGM.PtrDiffTy;
1237 }
1238 llvm::Constant *addr = CGM.getMemberFunctionPointer(FD: MD, Ty);
1239
1240 MemPtr[0] = llvm::ConstantExpr::getPtrToInt(C: addr, Ty: CGM.PtrDiffTy);
1241 MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1242 V: (UseARMMethodPtrABI ? 2 : 1) *
1243 ThisAdjustment.getQuantity());
1244 }
1245
1246 return llvm::ConstantStruct::getAnon(V: MemPtr);
1247}
1248
1249llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
1250 QualType MPType) {
1251 const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
1252 const ValueDecl *MPD = MP.getMemberPointerDecl();
1253 if (!MPD)
1254 return EmitNullMemberPointer(MPT);
1255
1256 CharUnits ThisAdjustment = getContext().getMemberPointerPathAdjustment(MP);
1257
1258 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MPD)) {
1259 llvm::Constant *Src = BuildMemberPointer(MD, ThisAdjustment);
1260 QualType SrcType = getContext().getMemberPointerType(
1261 T: MD->getType(), /*Qualifier=*/std::nullopt, Cls: MD->getParent());
1262 return pointerAuthResignMemberFunctionPointer(Src, DestType: MPType, SrcType, CGM);
1263 }
1264
1265 getContext().recordMemberDataPointerEvaluation(VD: MPD);
1266 CharUnits FieldOffset =
1267 getContext().toCharUnitsFromBits(BitSize: getContext().getFieldOffset(FD: MPD));
1268 return EmitMemberDataPointer(MPT, offset: ThisAdjustment + FieldOffset);
1269}
1270
1271/// The comparison algorithm is pretty easy: the member pointers are
1272/// the same if they're either bitwise identical *or* both null.
1273///
1274/// ARM is different here only because null-ness is more complicated.
1275llvm::Value *
1276ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
1277 llvm::Value *L,
1278 llvm::Value *R,
1279 const MemberPointerType *MPT,
1280 bool Inequality) {
1281 CGBuilderTy &Builder = CGF.Builder;
1282
1283 llvm::ICmpInst::Predicate Eq;
1284 llvm::Instruction::BinaryOps And, Or;
1285 if (Inequality) {
1286 Eq = llvm::ICmpInst::ICMP_NE;
1287 And = llvm::Instruction::Or;
1288 Or = llvm::Instruction::And;
1289 } else {
1290 Eq = llvm::ICmpInst::ICMP_EQ;
1291 And = llvm::Instruction::And;
1292 Or = llvm::Instruction::Or;
1293 }
1294
1295 // Member data pointers are easy because there's a unique null
1296 // value, so it just comes down to bitwise equality.
1297 if (MPT->isMemberDataPointer())
1298 return Builder.CreateICmp(P: Eq, LHS: L, RHS: R);
1299
1300 // For member function pointers, the tautologies are more complex.
1301 // The Itanium tautology is:
1302 // (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
1303 // The ARM tautology is:
1304 // (L == R) <==> (L.ptr == R.ptr &&
1305 // (L.adj == R.adj ||
1306 // (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
1307 // The inequality tautologies have exactly the same structure, except
1308 // applying De Morgan's laws.
1309
1310 llvm::Value *LPtr = Builder.CreateExtractValue(Agg: L, Idxs: 0, Name: "lhs.memptr.ptr");
1311 llvm::Value *RPtr = Builder.CreateExtractValue(Agg: R, Idxs: 0, Name: "rhs.memptr.ptr");
1312
1313 // This condition tests whether L.ptr == R.ptr. This must always be
1314 // true for equality to hold.
1315 llvm::Value *PtrEq = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: RPtr, Name: "cmp.ptr");
1316
1317 // This condition, together with the assumption that L.ptr == R.ptr,
1318 // tests whether the pointers are both null. ARM imposes an extra
1319 // condition.
1320 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: LPtr->getType());
1321 llvm::Value *EqZero = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: Zero, Name: "cmp.ptr.null");
1322
1323 // This condition tests whether L.adj == R.adj. If this isn't
1324 // true, the pointers are unequal unless they're both null.
1325 llvm::Value *LAdj = Builder.CreateExtractValue(Agg: L, Idxs: 1, Name: "lhs.memptr.adj");
1326 llvm::Value *RAdj = Builder.CreateExtractValue(Agg: R, Idxs: 1, Name: "rhs.memptr.adj");
1327 llvm::Value *AdjEq = Builder.CreateICmp(P: Eq, LHS: LAdj, RHS: RAdj, Name: "cmp.adj");
1328
1329 // Null member function pointers on ARM clear the low bit of Adj,
1330 // so the zero condition has to check that neither low bit is set.
1331 if (UseARMMethodPtrABI) {
1332 llvm::Value *One = llvm::ConstantInt::get(Ty: LPtr->getType(), V: 1);
1333
1334 // Compute (l.adj | r.adj) & 1 and test it against zero.
1335 llvm::Value *OrAdj = Builder.CreateOr(LHS: LAdj, RHS: RAdj, Name: "or.adj");
1336 llvm::Value *OrAdjAnd1 = Builder.CreateAnd(LHS: OrAdj, RHS: One);
1337 llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(P: Eq, LHS: OrAdjAnd1, RHS: Zero,
1338 Name: "cmp.or.adj");
1339 EqZero = Builder.CreateBinOp(Opc: And, LHS: EqZero, RHS: OrAdjAnd1EqZero);
1340 }
1341
1342 // Tie together all our conditions.
1343 llvm::Value *Result = Builder.CreateBinOp(Opc: Or, LHS: EqZero, RHS: AdjEq);
1344 Result = Builder.CreateBinOp(Opc: And, LHS: PtrEq, RHS: Result,
1345 Name: Inequality ? "memptr.ne" : "memptr.eq");
1346 return Result;
1347}
1348
1349llvm::Value *
1350ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
1351 llvm::Value *MemPtr,
1352 const MemberPointerType *MPT) {
1353 CGBuilderTy &Builder = CGF.Builder;
1354
1355 /// For member data pointers, this is just a check against -1.
1356 if (MPT->isMemberDataPointer()) {
1357 assert(MemPtr->getType() == CGM.PtrDiffTy);
1358 llvm::Value *NegativeOne =
1359 llvm::Constant::getAllOnesValue(Ty: MemPtr->getType());
1360 return Builder.CreateICmpNE(LHS: MemPtr, RHS: NegativeOne, Name: "memptr.tobool");
1361 }
1362
1363 // In Itanium, a member function pointer is not null if 'ptr' is not null.
1364 llvm::Value *Ptr = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 0, Name: "memptr.ptr");
1365
1366 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 0);
1367 llvm::Value *Result = Builder.CreateICmpNE(LHS: Ptr, RHS: Zero, Name: "memptr.tobool");
1368
1369 // On ARM, a member function pointer is also non-null if the low bit of 'adj'
1370 // (the virtual bit) is set.
1371 if (UseARMMethodPtrABI) {
1372 llvm::Constant *One = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 1);
1373 llvm::Value *Adj = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 1, Name: "memptr.adj");
1374 llvm::Value *VirtualBit = Builder.CreateAnd(LHS: Adj, RHS: One, Name: "memptr.virtualbit");
1375 llvm::Value *IsVirtual = Builder.CreateICmpNE(LHS: VirtualBit, RHS: Zero,
1376 Name: "memptr.isvirtual");
1377 Result = Builder.CreateOr(LHS: Result, RHS: IsVirtual);
1378 }
1379
1380 return Result;
1381}
1382
1383bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
1384 const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
1385 if (!RD)
1386 return false;
1387
1388 // If C++ prohibits us from making a copy, return by address using the target
1389 // hook getSRetAddrSpace to decide the AS.
1390 if (!RD->canPassInRegisters()) {
1391 auto Align = CGM.getContext().getTypeAlignInChars(T: FI.getReturnType());
1392 LangAS SRetAS = CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
1393 unsigned AS = CGM.getContext().getTargetAddressSpace(AS: SRetAS);
1394 FI.getReturnInfo() =
1395 ABIArgInfo::getIndirect(Alignment: Align, /*AddrSpace=*/AS, /*ByVal=*/false);
1396 return true;
1397 }
1398 return false;
1399}
1400
1401/// The Itanium ABI requires non-zero initialization only for data
1402/// member pointers, for which '0' is a valid offset.
1403bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
1404 return MPT->isMemberFunctionPointer();
1405}
1406
1407/// The Itanium ABI always places an offset to the complete object
1408/// at entry -2 in the vtable.
1409void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
1410 const CXXDeleteExpr *DE,
1411 Address Ptr,
1412 QualType ElementType,
1413 const CXXDestructorDecl *Dtor) {
1414 bool UseGlobalDelete = DE->isGlobalDelete();
1415 if (UseGlobalDelete) {
1416 // Derive the complete-object pointer, which is what we need
1417 // to pass to the deallocation function.
1418
1419 // Grab the vtable pointer as an intptr_t*.
1420 auto *ClassDecl = ElementType->castAsCXXRecordDecl();
1421 llvm::Value *VTable = CGF.GetVTablePtr(This: Ptr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1422
1423 // Track back to entry -2 and pull out the offset there.
1424 llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
1425 Ty: CGF.IntPtrTy, Ptr: VTable, Idx0: -2, Name: "complete-offset.ptr");
1426 llvm::Value *Offset = CGF.Builder.CreateAlignedLoad(Ty: CGF.IntPtrTy, Addr: OffsetPtr,
1427 Align: CGF.getPointerAlign());
1428
1429 // Apply the offset.
1430 llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
1431 CompletePtr =
1432 CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: CompletePtr, IdxList: Offset);
1433
1434 // If we're supposed to call the global delete, make sure we do so
1435 // even if the destructor throws.
1436 CGF.pushCallObjectDeleteCleanup(OperatorDelete: DE->getOperatorDelete(), CompletePtr,
1437 ElementType);
1438 }
1439
1440 // FIXME: Provide a source location here even though there's no
1441 // CXXMemberCallExpr for dtor call.
1442 CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
1443 EmitVirtualDestructorCall(CGF, Dtor, DtorType, This: Ptr, E: DE,
1444 /*CallOrInvoke=*/nullptr);
1445
1446 if (UseGlobalDelete)
1447 CGF.PopCleanupBlock();
1448}
1449
1450void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
1451 // void __cxa_rethrow();
1452
1453 llvm::FunctionType *FTy =
1454 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
1455
1456 llvm::FunctionCallee Fn = CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_rethrow");
1457
1458 if (isNoReturn)
1459 CGF.EmitNoreturnRuntimeCallOrInvoke(callee: Fn, args: {});
1460 else
1461 CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1462}
1463
1464static llvm::FunctionCallee getAllocateExceptionFn(CodeGenModule &CGM) {
1465 // void *__cxa_allocate_exception(size_t thrown_size);
1466
1467 llvm::FunctionType *FTy =
1468 llvm::FunctionType::get(Result: CGM.Int8PtrTy, Params: CGM.SizeTy, /*isVarArg=*/false);
1469
1470 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_allocate_exception");
1471}
1472
1473static llvm::FunctionCallee getThrowFn(CodeGenModule &CGM) {
1474 // void __cxa_throw(void *thrown_exception, std::type_info *tinfo,
1475 // void (*dest) (void *));
1476
1477 llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.GlobalsInt8PtrTy, CGM.Int8PtrTy };
1478 llvm::FunctionType *FTy =
1479 llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, /*isVarArg=*/false);
1480
1481 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_throw");
1482}
1483
1484void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
1485 QualType ThrowType = E->getSubExpr()->getType();
1486 // Now allocate the exception object.
1487 llvm::Type *SizeTy = CGF.ConvertType(T: getContext().getSizeType());
1488 uint64_t TypeSize = getContext().getTypeSizeInChars(T: ThrowType).getQuantity();
1489
1490 llvm::FunctionCallee AllocExceptionFn = getAllocateExceptionFn(CGM);
1491 llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
1492 callee: AllocExceptionFn, args: llvm::ConstantInt::get(Ty: SizeTy, V: TypeSize), name: "exception");
1493
1494 CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment();
1495 CGF.EmitAnyExprToExn(
1496 E: E->getSubExpr(), Addr: Address(ExceptionPtr, CGM.Int8Ty, ExnAlign));
1497
1498 // Now throw the exception.
1499 llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(Ty: ThrowType,
1500 /*ForEH=*/true);
1501
1502 // The address of the destructor. If the exception type has a
1503 // trivial destructor (or isn't a record), we just pass null.
1504 llvm::Constant *Dtor = nullptr;
1505 if (const auto *Record = ThrowType->getAsCXXRecordDecl();
1506 Record && !Record->hasTrivialDestructor()) {
1507 // __cxa_throw is declared to take its destructor as void (*)(void *). We
1508 // must match that if function pointers can be authenticated with a
1509 // discriminator based on their type.
1510 const ASTContext &Ctx = getContext();
1511 QualType DtorTy = Ctx.getFunctionType(ResultTy: Ctx.VoidTy, Args: {Ctx.VoidPtrTy},
1512 EPI: FunctionProtoType::ExtProtoInfo());
1513
1514 CXXDestructorDecl *DtorD = Record->getDestructor();
1515 Dtor = CGM.getAddrOfCXXStructor(GD: GlobalDecl(DtorD, Dtor_Complete));
1516 Dtor = CGM.getFunctionPointer(Pointer: Dtor, FunctionType: DtorTy);
1517 }
1518 if (!Dtor) Dtor = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
1519
1520 llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor };
1521 CGF.EmitNoreturnRuntimeCallOrInvoke(callee: getThrowFn(CGM), args);
1522}
1523
1524static llvm::FunctionCallee getItaniumDynamicCastFn(CodeGenFunction &CGF) {
1525 // void *__dynamic_cast(const void *sub,
1526 // global_as const abi::__class_type_info *src,
1527 // global_as const abi::__class_type_info *dst,
1528 // std::ptrdiff_t src2dst_offset);
1529
1530 llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
1531 llvm::Type *GlobInt8PtrTy = CGF.GlobalsInt8PtrTy;
1532 llvm::Type *PtrDiffTy =
1533 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1534
1535 llvm::Type *Args[4] = { Int8PtrTy, GlobInt8PtrTy, GlobInt8PtrTy, PtrDiffTy };
1536
1537 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int8PtrTy, Params: Args, isVarArg: false);
1538
1539 // Mark the function as nounwind willreturn readonly.
1540 llvm::AttrBuilder FuncAttrs(CGF.getLLVMContext());
1541 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1542 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
1543 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
1544 llvm::AttributeList Attrs = llvm::AttributeList::get(
1545 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: FuncAttrs);
1546
1547 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__dynamic_cast", ExtraAttrs: Attrs);
1548}
1549
1550static llvm::FunctionCallee getBadCastFn(CodeGenFunction &CGF) {
1551 // void __cxa_bad_cast();
1552 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1553 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_cast");
1554}
1555
1556/// Compute the src2dst_offset hint as described in the
1557/// Itanium C++ ABI [2.9.7]
1558static CharUnits computeOffsetHint(ASTContext &Context,
1559 const CXXRecordDecl *Src,
1560 const CXXRecordDecl *Dst) {
1561 CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
1562 /*DetectVirtual=*/false);
1563
1564 // If Dst is not derived from Src we can skip the whole computation below and
1565 // return that Src is not a public base of Dst. Record all inheritance paths.
1566 if (!Dst->isDerivedFrom(Base: Src, Paths))
1567 return CharUnits::fromQuantity(Quantity: -2ULL);
1568
1569 unsigned NumPublicPaths = 0;
1570 CharUnits Offset;
1571
1572 // Now walk all possible inheritance paths.
1573 for (const CXXBasePath &Path : Paths) {
1574 if (Path.Access != AS_public) // Ignore non-public inheritance.
1575 continue;
1576
1577 ++NumPublicPaths;
1578
1579 for (const CXXBasePathElement &PathElement : Path) {
1580 // If the path contains a virtual base class we can't give any hint.
1581 // -1: no hint.
1582 if (PathElement.Base->isVirtual())
1583 return CharUnits::fromQuantity(Quantity: -1ULL);
1584
1585 if (NumPublicPaths > 1) // Won't use offsets, skip computation.
1586 continue;
1587
1588 // Accumulate the base class offsets.
1589 const ASTRecordLayout &L = Context.getASTRecordLayout(D: PathElement.Class);
1590 Offset += L.getBaseClassOffset(
1591 Base: PathElement.Base->getType()->getAsCXXRecordDecl());
1592 }
1593 }
1594
1595 // -2: Src is not a public base of Dst.
1596 if (NumPublicPaths == 0)
1597 return CharUnits::fromQuantity(Quantity: -2ULL);
1598
1599 // -3: Src is a multiple public base type but never a virtual base type.
1600 if (NumPublicPaths > 1)
1601 return CharUnits::fromQuantity(Quantity: -3ULL);
1602
1603 // Otherwise, the Src type is a unique public nonvirtual base type of Dst.
1604 // Return the offset of Src from the origin of Dst.
1605 return Offset;
1606}
1607
1608static llvm::FunctionCallee getBadTypeidFn(CodeGenFunction &CGF) {
1609 // void __cxa_bad_typeid();
1610 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1611
1612 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_typeid");
1613}
1614
1615bool ItaniumCXXABI::shouldTypeidBeNullChecked(QualType SrcRecordTy) {
1616 return true;
1617}
1618
1619void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
1620 llvm::FunctionCallee Fn = getBadTypeidFn(CGF);
1621 llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1622 Call->setDoesNotReturn();
1623 CGF.Builder.CreateUnreachable();
1624}
1625
1626llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
1627 QualType SrcRecordTy,
1628 Address ThisPtr,
1629 llvm::Type *StdTypeInfoPtrTy) {
1630 auto *ClassDecl = SrcRecordTy->castAsCXXRecordDecl();
1631 llvm::Value *Value = CGF.GetVTablePtr(This: ThisPtr, VTableTy: CGM.GlobalsInt8PtrTy,
1632 VTableClass: ClassDecl);
1633
1634 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1635 // Load the type info.
1636 Value = CGF.Builder.CreateCall(
1637 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative, Tys: {CGM.Int32Ty}),
1638 Args: {Value, llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: -4)});
1639 } else {
1640 // Load the type info.
1641 Value =
1642 CGF.Builder.CreateConstInBoundsGEP1_64(Ty: StdTypeInfoPtrTy, Ptr: Value, Idx0: -1ULL);
1643 }
1644 return CGF.Builder.CreateAlignedLoad(Ty: StdTypeInfoPtrTy, Addr: Value,
1645 Align: CGF.getPointerAlign());
1646}
1647
1648bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1649 QualType SrcRecordTy) {
1650 return SrcIsPtr;
1651}
1652
1653llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
1654 CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1655 QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1656 llvm::Type *PtrDiffLTy =
1657 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1658
1659 llvm::Value *SrcRTTI =
1660 CGF.CGM.GetAddrOfRTTIDescriptor(Ty: SrcRecordTy.getUnqualifiedType());
1661 llvm::Value *DestRTTI =
1662 CGF.CGM.GetAddrOfRTTIDescriptor(Ty: DestRecordTy.getUnqualifiedType());
1663
1664 // Compute the offset hint.
1665 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1666 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1667 llvm::Value *OffsetHint = llvm::ConstantInt::getSigned(
1668 Ty: PtrDiffLTy,
1669 V: computeOffsetHint(Context&: CGF.getContext(), Src: SrcDecl, Dst: DestDecl).getQuantity());
1670
1671 // Emit the call to __dynamic_cast.
1672 llvm::Value *Value = ThisAddr.emitRawPointer(CGF);
1673 if (CGM.getCodeGenOpts().PointerAuth.CXXVTablePointers) {
1674 // We perform a no-op load of the vtable pointer here to force an
1675 // authentication. In environments that do not support pointer
1676 // authentication this is a an actual no-op that will be elided. When
1677 // pointer authentication is supported and enforced on vtable pointers this
1678 // load can trap.
1679 llvm::Value *Vtable =
1680 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGM.Int8PtrTy, VTableClass: SrcDecl,
1681 AuthMode: CodeGenFunction::VTableAuthMode::MustTrap);
1682 assert(Vtable);
1683 (void)Vtable;
1684 }
1685
1686 llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
1687 Value = CGF.EmitNounwindRuntimeCall(callee: getItaniumDynamicCastFn(CGF), args);
1688
1689 /// C++ [expr.dynamic.cast]p9:
1690 /// A failed cast to reference type throws std::bad_cast
1691 if (DestTy->isReferenceType()) {
1692 llvm::BasicBlock *BadCastBlock =
1693 CGF.createBasicBlock(name: "dynamic_cast.bad_cast");
1694
1695 llvm::Value *IsNull = CGF.Builder.CreateIsNull(Arg: Value);
1696 CGF.Builder.CreateCondBr(Cond: IsNull, True: BadCastBlock, False: CastEnd);
1697
1698 CGF.EmitBlock(BB: BadCastBlock);
1699 EmitBadCastCall(CGF);
1700 }
1701
1702 return Value;
1703}
1704
1705std::optional<CGCXXABI::ExactDynamicCastInfo>
1706ItaniumCXXABI::getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
1707 QualType DestRecordTy) {
1708 assert(shouldEmitExactDynamicCast(DestRecordTy));
1709
1710 ASTContext &Context = getContext();
1711
1712 // Find all the inheritance paths.
1713 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1714 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1715 CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
1716 /*DetectVirtual=*/false);
1717 (void)DestDecl->isDerivedFrom(Base: SrcDecl, Paths);
1718
1719 // Find an offset within `DestDecl` where a `SrcDecl` instance and its vptr
1720 // might appear.
1721 std::optional<CharUnits> Offset;
1722 for (const CXXBasePath &Path : Paths) {
1723 // dynamic_cast only finds public inheritance paths.
1724 if (Path.Access != AS_public)
1725 continue;
1726
1727 CharUnits PathOffset;
1728 for (const CXXBasePathElement &PathElement : Path) {
1729 // Find the offset along this inheritance step.
1730 const CXXRecordDecl *Base =
1731 PathElement.Base->getType()->getAsCXXRecordDecl();
1732 if (PathElement.Base->isVirtual()) {
1733 // For a virtual base class, we know that the derived class is exactly
1734 // DestDecl, so we can use the vbase offset from its layout.
1735 const ASTRecordLayout &L = Context.getASTRecordLayout(D: DestDecl);
1736 PathOffset = L.getVBaseClassOffset(VBase: Base);
1737 } else {
1738 const ASTRecordLayout &L =
1739 Context.getASTRecordLayout(D: PathElement.Class);
1740 PathOffset += L.getBaseClassOffset(Base);
1741 }
1742 }
1743
1744 if (!Offset)
1745 Offset = PathOffset;
1746 else if (Offset != PathOffset) {
1747 // Base appears in at least two different places.
1748 return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/true,
1749 .Offset: CharUnits::Zero()};
1750 }
1751 }
1752 if (!Offset)
1753 return std::nullopt;
1754 return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/false, .Offset: *Offset};
1755}
1756
1757llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
1758 CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1759 QualType DestTy, QualType DestRecordTy,
1760 const ExactDynamicCastInfo &ExactCastInfo, llvm::BasicBlock *CastSuccess,
1761 llvm::BasicBlock *CastFail) {
1762 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1763 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1764 auto AuthenticateVTable = [&](Address ThisAddr, const CXXRecordDecl *Decl) {
1765 if (!CGF.getLangOpts().PointerAuthCalls)
1766 return;
1767 (void)CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: Decl,
1768 AuthMode: CodeGenFunction::VTableAuthMode::MustTrap);
1769 };
1770
1771 bool PerformPostCastAuthentication = false;
1772 llvm::Value *VTable = nullptr;
1773 if (ExactCastInfo.RequiresCastToPrimaryBase) {
1774 // Base appears in at least two different places. Find the most-derived
1775 // object and see if it's a DestDecl. Note that the most-derived object
1776 // must be at least as aligned as this base class subobject, and must
1777 // have a vptr at offset 0.
1778 llvm::Value *PrimaryBase =
1779 emitDynamicCastToVoid(CGF, Value: ThisAddr, SrcRecordTy);
1780 ThisAddr = Address(PrimaryBase, CGF.VoidPtrTy, ThisAddr.getAlignment());
1781 SrcDecl = DestDecl;
1782 // This unauthenticated load is unavoidable, so we're relying on the
1783 // authenticated load in the dynamic cast to void, and we'll manually
1784 // authenticate the resulting v-table at the end of the cast check.
1785 PerformPostCastAuthentication = CGF.getLangOpts().PointerAuthCalls;
1786 CGPointerAuthInfo StrippingAuthInfo(0, PointerAuthenticationMode::Strip,
1787 false, false, nullptr);
1788 Address VTablePtrPtr = ThisAddr.withElementType(ElemTy: CGF.VoidPtrPtrTy);
1789 VTable = CGF.Builder.CreateLoad(Addr: VTablePtrPtr, Name: "vtable");
1790 if (PerformPostCastAuthentication)
1791 VTable = CGF.EmitPointerAuthAuth(Info: StrippingAuthInfo, Pointer: VTable);
1792 } else
1793 VTable = CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: SrcDecl);
1794
1795 // Compare the vptr against the expected vptr for the destination type at
1796 // this offset.
1797 llvm::Constant *ExpectedVTable = getVTableAddressPoint(
1798 Base: BaseSubobject(SrcDecl, ExactCastInfo.Offset), VTableClass: DestDecl);
1799 llvm::Value *Success = CGF.Builder.CreateICmpEQ(LHS: VTable, RHS: ExpectedVTable);
1800 llvm::Value *AdjustedThisPtr = ThisAddr.emitRawPointer(CGF);
1801
1802 if (!ExactCastInfo.Offset.isZero()) {
1803 CharUnits::QuantityType Offset = ExactCastInfo.Offset.getQuantity();
1804 llvm::Constant *OffsetConstant =
1805 llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: -Offset);
1806 AdjustedThisPtr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.CharTy, Ptr: AdjustedThisPtr,
1807 IdxList: OffsetConstant);
1808 PerformPostCastAuthentication = CGF.getLangOpts().PointerAuthCalls;
1809 }
1810
1811 if (PerformPostCastAuthentication) {
1812 // If we've changed the object pointer we authenticate the vtable pointer
1813 // of the resulting object.
1814 llvm::BasicBlock *NonNullBlock = CGF.Builder.GetInsertBlock();
1815 llvm::BasicBlock *PostCastAuthSuccess =
1816 CGF.createBasicBlock(name: "dynamic_cast.postauth.success");
1817 llvm::BasicBlock *PostCastAuthComplete =
1818 CGF.createBasicBlock(name: "dynamic_cast.postauth.complete");
1819 CGF.Builder.CreateCondBr(Cond: Success, True: PostCastAuthSuccess,
1820 False: PostCastAuthComplete);
1821 CGF.EmitBlock(BB: PostCastAuthSuccess);
1822 Address AdjustedThisAddr =
1823 Address(AdjustedThisPtr, CGF.IntPtrTy, CGF.getPointerAlign());
1824 AuthenticateVTable(AdjustedThisAddr, DestDecl);
1825 CGF.EmitBranch(Block: PostCastAuthComplete);
1826 CGF.EmitBlock(BB: PostCastAuthComplete);
1827 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: AdjustedThisPtr->getType(), NumReservedValues: 2);
1828 PHI->addIncoming(V: AdjustedThisPtr, BB: PostCastAuthSuccess);
1829 llvm::Value *NullValue =
1830 llvm::Constant::getNullValue(Ty: AdjustedThisPtr->getType());
1831 PHI->addIncoming(V: NullValue, BB: NonNullBlock);
1832 AdjustedThisPtr = PHI;
1833 }
1834 CGF.Builder.CreateCondBr(Cond: Success, True: CastSuccess, False: CastFail);
1835 return AdjustedThisPtr;
1836}
1837
1838llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
1839 Address ThisAddr,
1840 QualType SrcRecordTy) {
1841 auto *ClassDecl = SrcRecordTy->castAsCXXRecordDecl();
1842 llvm::Value *OffsetToTop;
1843 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1844 // Get the vtable pointer.
1845 llvm::Value *VTable =
1846 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1847
1848 // Get the offset-to-top from the vtable.
1849 OffsetToTop =
1850 CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGM.Int32Ty, Ptr: VTable, Idx0: -2U);
1851 OffsetToTop = CGF.Builder.CreateAlignedLoad(
1852 Ty: CGM.Int32Ty, Addr: OffsetToTop, Align: CharUnits::fromQuantity(Quantity: 4), Name: "offset.to.top");
1853 } else {
1854 llvm::Type *PtrDiffLTy =
1855 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1856
1857 // Get the vtable pointer.
1858 llvm::Value *VTable =
1859 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1860
1861 // Get the offset-to-top from the vtable.
1862 OffsetToTop =
1863 CGF.Builder.CreateConstInBoundsGEP1_64(Ty: PtrDiffLTy, Ptr: VTable, Idx0: -2ULL);
1864 OffsetToTop = CGF.Builder.CreateAlignedLoad(
1865 Ty: PtrDiffLTy, Addr: OffsetToTop, Align: CGF.getPointerAlign(), Name: "offset.to.top");
1866 }
1867 // Finally, add the offset to the pointer.
1868 return CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ThisAddr.emitRawPointer(CGF),
1869 IdxList: OffsetToTop);
1870}
1871
1872bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1873 llvm::FunctionCallee Fn = getBadCastFn(CGF);
1874 llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1875 Call->setDoesNotReturn();
1876 CGF.Builder.CreateUnreachable();
1877 return true;
1878}
1879
1880llvm::Value *
1881ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
1882 Address This,
1883 const CXXRecordDecl *ClassDecl,
1884 const CXXRecordDecl *BaseClassDecl) {
1885 llvm::Value *VTablePtr = CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: ClassDecl);
1886 CharUnits VBaseOffsetOffset =
1887 CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD: ClassDecl,
1888 VBase: BaseClassDecl);
1889 llvm::Value *VBaseOffsetPtr =
1890 CGF.Builder.CreateConstGEP1_64(
1891 Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VBaseOffsetOffset.getQuantity(),
1892 Name: "vbase.offset.ptr");
1893
1894 llvm::Value *VBaseOffset;
1895 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1896 VBaseOffset = CGF.Builder.CreateAlignedLoad(
1897 Ty: CGF.Int32Ty, Addr: VBaseOffsetPtr, Align: CharUnits::fromQuantity(Quantity: 4),
1898 Name: "vbase.offset");
1899 } else {
1900 VBaseOffset = CGF.Builder.CreateAlignedLoad(
1901 Ty: CGM.PtrDiffTy, Addr: VBaseOffsetPtr, Align: CGF.getPointerAlign(), Name: "vbase.offset");
1902 }
1903 return VBaseOffset;
1904}
1905
1906void ItaniumCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1907 // Just make sure we're in sync with TargetCXXABI.
1908 assert(CGM.getTarget().getCXXABI().hasConstructorVariants());
1909
1910 // The constructor used for constructing this as a base class;
1911 // ignores virtual bases.
1912 CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Base));
1913
1914 // The constructor used for constructing this as a complete class;
1915 // constructs the virtual bases, then calls the base constructor.
1916 if (!D->getParent()->isAbstract()) {
1917 // We don't need to emit the complete ctor if the class is abstract.
1918 CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Complete));
1919 }
1920}
1921
1922CGCXXABI::AddedStructorArgCounts
1923ItaniumCXXABI::buildStructorSignature(GlobalDecl GD,
1924 SmallVectorImpl<CanQualType> &ArgTys) {
1925 ASTContext &Context = getContext();
1926
1927 // All parameters are already in place except VTT, which goes after 'this'.
1928 // These are Clang types, so we don't need to worry about sret yet.
1929
1930 // Check if we need to add a VTT parameter (which has type global void **).
1931 if ((isa<CXXConstructorDecl>(Val: GD.getDecl()) ? GD.getCtorType() == Ctor_Base
1932 : GD.getDtorType() == Dtor_Base) &&
1933 cast<CXXMethodDecl>(Val: GD.getDecl())->getParent()->getNumVBases() != 0) {
1934 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1935 QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1936 ArgTys.insert(I: ArgTys.begin() + 1,
1937 Elt: Context.getPointerType(T: CanQualType::CreateUnsafe(Other: Q)));
1938 return AddedStructorArgCounts::prefix(N: 1);
1939 }
1940 return AddedStructorArgCounts{};
1941}
1942
1943void ItaniumCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1944 // The destructor used for destructing this as a base class; ignores
1945 // virtual bases.
1946 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Base));
1947
1948 // The destructor used for destructing this as a most-derived class;
1949 // call the base destructor and then destructs any virtual bases.
1950 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Complete));
1951
1952 // The destructor in a virtual table is always a 'deleting'
1953 // destructor, which calls the complete destructor and then uses the
1954 // appropriate operator delete.
1955 if (D->isVirtual())
1956 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Deleting));
1957}
1958
1959void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1960 QualType &ResTy,
1961 FunctionArgList &Params) {
1962 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1963 assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1964
1965 // Check if we need a VTT parameter as well.
1966 if (NeedsVTTParameter(GD: CGF.CurGD)) {
1967 ASTContext &Context = getContext();
1968
1969 // FIXME: avoid the fake decl
1970 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1971 QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1972 QualType T = Context.getPointerType(T: Q);
1973 auto *VTTDecl = ImplicitParamDecl::Create(
1974 C&: Context, /*DC=*/nullptr, IdLoc: MD->getLocation(), Id: &Context.Idents.get(Name: "vtt"),
1975 T, ParamKind: ImplicitParamKind::CXXVTT);
1976 Params.insert(I: Params.begin() + 1, Elt: VTTDecl);
1977 getStructorImplicitParamDecl(CGF) = VTTDecl;
1978 }
1979}
1980
1981void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1982 // Naked functions have no prolog.
1983 if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
1984 return;
1985
1986 /// Initialize the 'this' slot. In the Itanium C++ ABI, no prologue
1987 /// adjustments are required, because they are all handled by thunks.
1988 setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
1989
1990 /// Initialize the 'vtt' slot if needed.
1991 if (getStructorImplicitParamDecl(CGF)) {
1992 getStructorImplicitParamValue(CGF) = CGF.Builder.CreateLoad(
1993 Addr: CGF.GetAddrOfLocalVar(VD: getStructorImplicitParamDecl(CGF)), Name: "vtt");
1994 }
1995
1996 /// If this is a function that the ABI specifies returns 'this', initialize
1997 /// the return slot to 'this' at the start of the function.
1998 ///
1999 /// Unlike the setting of return types, this is done within the ABI
2000 /// implementation instead of by clients of CGCXXABI because:
2001 /// 1) getThisValue is currently protected
2002 /// 2) in theory, an ABI could implement 'this' returns some other way;
2003 /// HasThisReturn only specifies a contract, not the implementation
2004 if (HasThisReturn(GD: CGF.CurGD))
2005 CGF.Builder.CreateStore(Val: getThisValue(CGF), Addr: CGF.ReturnValue);
2006}
2007
2008CGCXXABI::AddedStructorArgs ItaniumCXXABI::getImplicitConstructorArgs(
2009 CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
2010 bool ForVirtualBase, bool Delegating) {
2011 if (!NeedsVTTParameter(GD: GlobalDecl(D, Type)))
2012 return AddedStructorArgs{};
2013
2014 // Insert the implicit 'vtt' argument as the second argument. Make sure to
2015 // correctly reflect its address space, which can differ from generic on
2016 // some targets.
2017 llvm::Value *VTT =
2018 CGF.GetVTTParameter(GD: GlobalDecl(D, Type), ForVirtualBase, Delegating);
2019 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
2020 QualType Q = getContext().getAddrSpaceQualType(T: getContext().VoidPtrTy, AddressSpace: AS);
2021 QualType VTTTy = getContext().getPointerType(T: Q);
2022 return AddedStructorArgs::prefix(Args: {{.Value: VTT, .Type: VTTTy}});
2023}
2024
2025llvm::Value *ItaniumCXXABI::getCXXDestructorImplicitParam(
2026 CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
2027 bool ForVirtualBase, bool Delegating) {
2028 GlobalDecl GD(DD, Type);
2029 return CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
2030}
2031
2032void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
2033 const CXXDestructorDecl *DD,
2034 CXXDtorType Type, bool ForVirtualBase,
2035 bool Delegating, Address This,
2036 QualType ThisTy) {
2037 GlobalDecl GD(DD, Type);
2038 llvm::Value *VTT =
2039 getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating);
2040 QualType VTTTy = getContext().getPointerType(T: getContext().VoidPtrTy);
2041
2042 CGCallee Callee;
2043 if (getContext().getLangOpts().AppleKext &&
2044 Type != Dtor_Base && DD->isVirtual())
2045 Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, RD: DD->getParent());
2046 else
2047 Callee = CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD), abstractInfo: GD);
2048
2049 CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: CGF.getAsNaturalPointerTo(Addr: This, PointeeType: ThisTy),
2050 ThisTy, ImplicitParam: VTT, ImplicitParamTy: VTTTy, E: nullptr);
2051}
2052
2053// Check if any non-inline method has the specified attribute.
2054template <typename T>
2055static bool CXXRecordNonInlineHasAttr(const CXXRecordDecl *RD) {
2056 for (const auto *D : RD->noload_decls()) {
2057 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2058 if (FD->isInlined() || FD->doesThisDeclarationHaveABody() ||
2059 FD->isPureVirtual())
2060 continue;
2061 if (D->hasAttr<T>())
2062 return true;
2063 }
2064 }
2065
2066 return false;
2067}
2068
2069static void setVTableSelectiveDLLImportExport(CodeGenModule &CGM,
2070 llvm::GlobalVariable *VTable,
2071 const CXXRecordDecl *RD) {
2072 if (VTable->getDLLStorageClass() !=
2073 llvm::GlobalVariable::DefaultStorageClass ||
2074 RD->hasAttr<DLLImportAttr>() || RD->hasAttr<DLLExportAttr>())
2075 return;
2076
2077 if (CGM.getVTables().isVTableExternal(RD)) {
2078 if (CXXRecordNonInlineHasAttr<DLLImportAttr>(RD))
2079 VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2080 } else if (CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
2081 VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2082}
2083
2084void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
2085 const CXXRecordDecl *RD) {
2086 llvm::GlobalVariable *VTable = getAddrOfVTable(RD, VPtrOffset: CharUnits());
2087 if (VTable->hasInitializer())
2088 return;
2089
2090 ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
2091 const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
2092 llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
2093 llvm::Constant *RTTI =
2094 CGM.GetAddrOfRTTIDescriptor(Ty: CGM.getContext().getCanonicalTagType(TD: RD));
2095
2096 // Create and set the initializer.
2097 ConstantInitBuilder builder(CGM);
2098 auto components = builder.beginStruct();
2099 CGVT.createVTableInitializer(builder&: components, layout: VTLayout, rtti: RTTI,
2100 vtableHasLocalLinkage: llvm::GlobalValue::isLocalLinkage(Linkage));
2101 components.finishAndSetAsInitializer(global: VTable);
2102
2103 // Set the correct linkage.
2104 VTable->setLinkage(Linkage);
2105
2106 // On a target that may duplicate vtables, a weak vtable does not have a
2107 // unique address, so its address is insignificant and it can be marked
2108 // unnamed_addr.
2109 if (CGM.mayVTableBeDuplicated(Linkage: VTable->getLinkage()))
2110 VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2111
2112 if (CGM.supportsCOMDAT() && VTable->isWeakForLinker())
2113 VTable->setComdat(CGM.getModule().getOrInsertComdat(Name: VTable->getName()));
2114
2115 if (CGM.getTarget().hasPS4DLLImportExport())
2116 setVTableSelectiveDLLImportExport(CGM, VTable, RD);
2117
2118 // Set the right visibility.
2119 CGM.setGVProperties(GV: VTable, D: RD);
2120
2121 // If this is the magic class __cxxabiv1::__fundamental_type_info,
2122 // we will emit the typeinfo for the fundamental types. This is the
2123 // same behaviour as GCC.
2124 const DeclContext *DC = RD->getDeclContext();
2125 if (RD->getIdentifier() &&
2126 RD->getIdentifier()->isStr(Str: "__fundamental_type_info") &&
2127 isa<NamespaceDecl>(Val: DC) && cast<NamespaceDecl>(Val: DC)->getIdentifier() &&
2128 cast<NamespaceDecl>(Val: DC)->getIdentifier()->isStr(Str: "__cxxabiv1") &&
2129 DC->getParent()->isTranslationUnit())
2130 EmitFundamentalRTTIDescriptors(RD);
2131
2132 // Always emit type metadata on non-available_externally definitions, and on
2133 // available_externally definitions if we are performing whole program
2134 // devirtualization or speculative devirtualization. We need the type metadata
2135 // on all vtable definitions to ensure we associate derived classes with base
2136 // classes defined in headers but with a strong definition only in a shared
2137 // library.
2138 if (!VTable->isDeclarationForLinker() ||
2139 CGM.getCodeGenOpts().WholeProgramVTables ||
2140 CGM.getCodeGenOpts().DevirtualizeSpeculatively) {
2141 CGM.EmitVTableTypeMetadata(RD, VTable, VTLayout);
2142 // For available_externally definitions, add the vtable to
2143 // @llvm.compiler.used so that it isn't deleted before whole program
2144 // analysis.
2145 if (VTable->isDeclarationForLinker()) {
2146 assert(CGM.getCodeGenOpts().WholeProgramVTables ||
2147 CGM.getCodeGenOpts().DevirtualizeSpeculatively);
2148 CGM.addCompilerUsedGlobal(GV: VTable);
2149 }
2150 }
2151
2152 if (CGM.getLangOpts().RelativeCXXABIVTables) {
2153 CGVT.RemoveHwasanMetadata(GV: VTable);
2154 if (!VTable->isDSOLocal())
2155 CGVT.GenerateRelativeVTableAlias(VTable, AliasNameRef: VTable->getName());
2156 }
2157
2158 // Emit symbol for debugger only if requested debug info.
2159 if (CGDebugInfo *DI = CGM.getModuleDebugInfo())
2160 DI->emitVTableSymbol(VTable, RD);
2161}
2162
2163bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
2164 CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
2165 if (Vptr.NearestVBase == nullptr)
2166 return false;
2167 return NeedsVTTParameter(GD: CGF.CurGD);
2168}
2169
2170llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
2171 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
2172 const CXXRecordDecl *NearestVBase) {
2173
2174 if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
2175 NeedsVTTParameter(GD: CGF.CurGD)) {
2176 return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base,
2177 NearestVBase);
2178 }
2179 return getVTableAddressPoint(Base, VTableClass);
2180}
2181
2182llvm::Constant *
2183ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base,
2184 const CXXRecordDecl *VTableClass) {
2185 llvm::GlobalValue *VTable = getAddrOfVTable(RD: VTableClass, VPtrOffset: CharUnits());
2186
2187 // Find the appropriate vtable within the vtable group, and the address point
2188 // within that vtable.
2189 const VTableLayout &Layout =
2190 CGM.getItaniumVTableContext().getVTableLayout(RD: VTableClass);
2191 VTableLayout::AddressPointLocation AddressPoint =
2192 Layout.getAddressPoint(Base);
2193 llvm::Value *Indices[] = {
2194 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
2195 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.VTableIndex),
2196 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.AddressPointIndex),
2197 };
2198
2199 // Add inrange attribute to indicate that only the VTableIndex can be
2200 // accessed.
2201 unsigned ComponentSize =
2202 CGM.getDataLayout().getTypeAllocSize(Ty: CGM.getVTableComponentType());
2203 unsigned VTableSize =
2204 ComponentSize * Layout.getVTableSize(i: AddressPoint.VTableIndex);
2205 unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex;
2206 llvm::ConstantRange InRange(
2207 llvm::APInt(32, (int)-Offset, true),
2208 llvm::APInt(32, (int)(VTableSize - Offset), true));
2209 return llvm::ConstantExpr::getGetElementPtr(
2210 Ty: VTable->getValueType(), C: VTable, IdxList: Indices, /*InBounds=*/NW: true, InRange);
2211}
2212
2213llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
2214 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
2215 const CXXRecordDecl *NearestVBase) {
2216 assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
2217 NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT");
2218
2219 // Get the secondary vpointer index.
2220 uint64_t VirtualPointerIndex =
2221 CGM.getVTables().getSecondaryVirtualPointerIndex(RD: VTableClass, Base);
2222
2223 /// Load the VTT.
2224 llvm::Value *VTT = CGF.LoadCXXVTT();
2225 if (VirtualPointerIndex)
2226 VTT = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.GlobalsVoidPtrTy, Ptr: VTT,
2227 Idx0: VirtualPointerIndex);
2228
2229 // And load the address point from the VTT.
2230 llvm::Value *AP =
2231 CGF.Builder.CreateAlignedLoad(Ty: CGF.GlobalsVoidPtrTy, Addr: VTT,
2232 Align: CGF.getPointerAlign());
2233
2234 if (auto &Schema = CGF.CGM.getCodeGenOpts().PointerAuth.CXXVTTVTablePointers) {
2235 CGPointerAuthInfo PointerAuth = CGF.EmitPointerAuthInfo(Schema, StorageAddress: VTT,
2236 SchemaDecl: GlobalDecl(),
2237 SchemaType: QualType());
2238 AP = CGF.EmitPointerAuthAuth(Info: PointerAuth, Pointer: AP);
2239 }
2240
2241 return AP;
2242}
2243
2244llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
2245 CharUnits VPtrOffset) {
2246 assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
2247
2248 llvm::GlobalVariable *&VTable = VTables[RD];
2249 if (VTable)
2250 return VTable;
2251
2252 // Queue up this vtable for possible deferred emission.
2253 CGM.addDeferredVTable(RD);
2254
2255 SmallString<256> Name;
2256 llvm::raw_svector_ostream Out(Name);
2257 getMangleContext().mangleCXXVTable(RD, Out);
2258
2259 const VTableLayout &VTLayout =
2260 CGM.getItaniumVTableContext().getVTableLayout(RD);
2261 llvm::Type *VTableType = CGM.getVTables().getVTableType(layout: VTLayout);
2262
2263 // Use pointer to global alignment for the vtable. Otherwise we would align
2264 // them based on the size of the initializer which doesn't make sense as only
2265 // single values are read.
2266 unsigned PAlign = CGM.getVtableGlobalVarAlignment();
2267
2268 VTable = CGM.CreateOrReplaceCXXRuntimeVariable(
2269 Name, Ty: VTableType, Linkage: llvm::GlobalValue::ExternalLinkage,
2270 Alignment: getContext().toCharUnitsFromBits(BitSize: PAlign).getAsAlign());
2271 if (!CGM.shouldEmitRTTI())
2272 VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2273
2274 if (CGM.getTarget().hasPS4DLLImportExport())
2275 setVTableSelectiveDLLImportExport(CGM, VTable, RD);
2276
2277 CGM.setGVProperties(GV: VTable, D: RD);
2278 return VTable;
2279}
2280
2281CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
2282 GlobalDecl GD,
2283 Address This,
2284 llvm::Type *Ty,
2285 SourceLocation Loc) {
2286 llvm::Type *PtrTy = CGM.GlobalsInt8PtrTy;
2287 auto *MethodDecl = cast<CXXMethodDecl>(Val: GD.getDecl());
2288 llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy: PtrTy, VTableClass: MethodDecl->getParent());
2289
2290 // For the translation of virtual functions, we need to map the (potential)
2291 // host vtable to the device vtable. This is done by calling the runtime
2292 // function
2293 // __llvm_omp_indirect_call_lookup.
2294 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
2295 auto *NewPtrTy = CGM.VoidPtrTy;
2296 llvm::Type *RtlFnArgs[] = {NewPtrTy};
2297 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
2298 Ty: llvm::FunctionType::get(Result: NewPtrTy, Params: RtlFnArgs, isVarArg: false),
2299 Name: "__llvm_omp_indirect_call_lookup");
2300 auto *BackupTy = VTable->getType();
2301 // Need to convert to generic address space
2302 VTable = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: VTable, DestTy: NewPtrTy);
2303 VTable = CGF.EmitRuntimeCall(callee: DeviceRtlFn, args: {VTable});
2304 // convert to original address space
2305 VTable = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: VTable, DestTy: BackupTy);
2306 }
2307
2308 uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
2309 llvm::Value *VFunc, *VTableSlotPtr = nullptr;
2310 auto &Schema = CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers;
2311
2312 llvm::Type *ComponentTy = CGM.getVTables().getVTableComponentType();
2313 uint64_t ByteOffset =
2314 VTableIndex * CGM.getDataLayout().getTypeSizeInBits(Ty: ComponentTy) / 8;
2315
2316 if (!Schema && CGF.ShouldEmitVTableTypeCheckedLoad(RD: MethodDecl->getParent())) {
2317 VFunc = CGF.EmitVTableTypeCheckedLoad(RD: MethodDecl->getParent(), VTable,
2318 VTableTy: PtrTy, VTableByteOffset: ByteOffset);
2319 } else {
2320 CGF.EmitTypeMetadataCodeForVCall(RD: MethodDecl->getParent(), VTable, Loc);
2321
2322 llvm::Value *VFuncLoad;
2323 if (CGM.getLangOpts().RelativeCXXABIVTables) {
2324 VFuncLoad = CGF.Builder.CreateCall(
2325 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative, Tys: {CGM.Int32Ty}),
2326 Args: {VTable, llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: ByteOffset)});
2327 } else {
2328 VTableSlotPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2329 Ty: PtrTy, Ptr: VTable, Idx0: VTableIndex, Name: "vfn");
2330 VFuncLoad = CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Addr: VTableSlotPtr,
2331 Align: CGF.getPointerAlign());
2332 }
2333
2334 // Add !invariant.load md to virtual function load to indicate that
2335 // function didn't change inside vtable.
2336 // It's safe to add it without -fstrict-vtable-pointers, but it would not
2337 // help in devirtualization because it will only matter if we will have 2
2338 // the same virtual function loads from the same vtable load, which won't
2339 // happen without enabled devirtualization with -fstrict-vtable-pointers.
2340 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2341 CGM.getCodeGenOpts().StrictVTablePointers) {
2342 if (auto *VFuncLoadInstr = dyn_cast<llvm::Instruction>(Val: VFuncLoad)) {
2343 VFuncLoadInstr->setMetadata(
2344 KindID: llvm::LLVMContext::MD_invariant_load,
2345 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(),
2346 MDs: llvm::ArrayRef<llvm::Metadata *>()));
2347 }
2348 }
2349 VFunc = VFuncLoad;
2350 }
2351
2352 CGPointerAuthInfo PointerAuth;
2353 if (Schema) {
2354 assert(VTableSlotPtr && "virtual function pointer not set");
2355 GD = CGM.getItaniumVTableContext().findOriginalMethod(GD: GD.getCanonicalDecl());
2356 PointerAuth = CGF.EmitPointerAuthInfo(Schema, StorageAddress: VTableSlotPtr, SchemaDecl: GD, SchemaType: QualType());
2357 }
2358 CGCallee Callee(GD, VFunc, PointerAuth);
2359 return Callee;
2360}
2361
2362llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
2363 CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
2364 Address This, DeleteOrMemberCallExpr E, llvm::CallBase **CallOrInvoke) {
2365 auto *CE = dyn_cast<const CXXMemberCallExpr *>(Val&: E);
2366 auto *D = dyn_cast<const CXXDeleteExpr *>(Val&: E);
2367 assert((CE != nullptr) ^ (D != nullptr));
2368 assert(CE == nullptr || CE->arguments().empty());
2369 assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
2370
2371 GlobalDecl GD(Dtor, DtorType);
2372 const CGFunctionInfo *FInfo =
2373 &CGM.getTypes().arrangeCXXStructorDeclaration(GD);
2374 llvm::FunctionType *Ty = CGF.CGM.getTypes().GetFunctionType(Info: *FInfo);
2375 CGCallee Callee = CGCallee::forVirtual(CE, MD: GD, Addr: This, FTy: Ty);
2376
2377 QualType ThisTy;
2378 if (CE) {
2379 ThisTy = CE->getObjectType();
2380 } else {
2381 ThisTy = D->getDestroyedType();
2382 }
2383
2384 CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: This.emitRawPointer(CGF), ThisTy,
2385 ImplicitParam: nullptr, ImplicitParamTy: QualType(), E: nullptr, CallOrInvoke);
2386 return nullptr;
2387}
2388
2389void ItaniumCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
2390 CodeGenVTables &VTables = CGM.getVTables();
2391 llvm::GlobalVariable *VTT = VTables.GetAddrOfVTT(RD);
2392 VTables.EmitVTTDefinition(VTT, Linkage: CGM.getVTableLinkage(RD), RD);
2393}
2394
2395bool ItaniumCXXABI::canSpeculativelyEmitVTableAsBaseClass(
2396 const CXXRecordDecl *RD) const {
2397 // We don't emit available_externally vtables if we are in -fapple-kext mode
2398 // because kext mode does not permit devirtualization.
2399 if (CGM.getLangOpts().AppleKext)
2400 return false;
2401
2402 // If the vtable is hidden then it is not safe to emit an available_externally
2403 // copy of vtable.
2404 if (isVTableHidden(RD))
2405 return false;
2406
2407 if (CGM.getCodeGenOpts().ForceEmitVTables)
2408 return true;
2409
2410 // A speculative vtable can only be generated if all virtual inline functions
2411 // defined by this class are emitted. The vtable in the final program contains
2412 // for each virtual inline function not used in the current TU a function that
2413 // is equivalent to the unused function. The function in the actual vtable
2414 // does not have to be declared under the same symbol (e.g., a virtual
2415 // destructor that can be substituted with its base class's destructor). Since
2416 // inline functions are emitted lazily and this emissions does not account for
2417 // speculative emission of a vtable, we might generate a speculative vtable
2418 // with references to inline functions that are not emitted under that name.
2419 // This can lead to problems when devirtualizing a call to such a function,
2420 // that result in linking errors. Hence, if there are any unused virtual
2421 // inline function, we cannot emit the speculative vtable.
2422 // FIXME we can still emit a copy of the vtable if we
2423 // can emit definition of the inline functions.
2424 if (hasAnyUnusedVirtualInlineFunction(RD))
2425 return false;
2426
2427 // For a class with virtual bases, we must also be able to speculatively
2428 // emit the VTT, because CodeGen doesn't have separate notions of "can emit
2429 // the vtable" and "can emit the VTT". For a base subobject, this means we
2430 // need to be able to emit non-virtual base vtables.
2431 if (RD->getNumVBases()) {
2432 for (const auto &B : RD->bases()) {
2433 auto *BRD = B.getType()->getAsCXXRecordDecl();
2434 assert(BRD && "no class for base specifier");
2435 if (B.isVirtual() || !BRD->isDynamicClass())
2436 continue;
2437 if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2438 return false;
2439 }
2440 }
2441
2442 return true;
2443}
2444
2445bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const {
2446 if (!canSpeculativelyEmitVTableAsBaseClass(RD))
2447 return false;
2448
2449 if (RD->shouldEmitInExternalSource())
2450 return false;
2451
2452 // For a complete-object vtable (or more specifically, for the VTT), we need
2453 // to be able to speculatively emit the vtables of all dynamic virtual bases.
2454 for (const auto &B : RD->vbases()) {
2455 auto *BRD = B.getType()->getAsCXXRecordDecl();
2456 assert(BRD && "no class for base specifier");
2457 if (!BRD->isDynamicClass())
2458 continue;
2459 if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2460 return false;
2461 }
2462
2463 return true;
2464}
2465static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
2466 Address InitialPtr,
2467 const CXXRecordDecl *UnadjustedClass,
2468 int64_t NonVirtualAdjustment,
2469 int64_t VirtualAdjustment,
2470 bool IsReturnAdjustment) {
2471 if (!NonVirtualAdjustment && !VirtualAdjustment)
2472 return InitialPtr.emitRawPointer(CGF);
2473
2474 Address V = InitialPtr.withElementType(ElemTy: CGF.Int8Ty);
2475
2476 // In a base-to-derived cast, the non-virtual adjustment is applied first.
2477 if (NonVirtualAdjustment && !IsReturnAdjustment) {
2478 V = CGF.Builder.CreateConstInBoundsByteGEP(Addr: V,
2479 Offset: CharUnits::fromQuantity(Quantity: NonVirtualAdjustment));
2480 }
2481
2482 // Perform the virtual adjustment if we have one.
2483 llvm::Value *ResultPtr;
2484 if (VirtualAdjustment) {
2485 llvm::Value *VTablePtr =
2486 CGF.GetVTablePtr(This: V, VTableTy: CGF.Int8PtrTy, VTableClass: UnadjustedClass);
2487
2488 llvm::Value *Offset;
2489 llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2490 Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VirtualAdjustment);
2491 if (CGF.CGM.getLangOpts().RelativeCXXABIVTables) {
2492 // Load the adjustment offset from the vtable as a 32-bit int.
2493 Offset =
2494 CGF.Builder.CreateAlignedLoad(Ty: CGF.Int32Ty, Addr: OffsetPtr,
2495 Align: CharUnits::fromQuantity(Quantity: 4));
2496 } else {
2497 llvm::Type *PtrDiffTy =
2498 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
2499
2500 // Load the adjustment offset from the vtable.
2501 Offset = CGF.Builder.CreateAlignedLoad(Ty: PtrDiffTy, Addr: OffsetPtr,
2502 Align: CGF.getPointerAlign());
2503 }
2504 // Adjust our pointer.
2505 ResultPtr = CGF.Builder.CreateInBoundsGEP(Ty: V.getElementType(),
2506 Ptr: V.emitRawPointer(CGF), IdxList: Offset);
2507 } else {
2508 ResultPtr = V.emitRawPointer(CGF);
2509 }
2510
2511 // In a derived-to-base conversion, the non-virtual adjustment is
2512 // applied second.
2513 if (NonVirtualAdjustment && IsReturnAdjustment) {
2514 ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.Int8Ty, Ptr: ResultPtr,
2515 Idx0: NonVirtualAdjustment);
2516 }
2517
2518 return ResultPtr;
2519}
2520
2521llvm::Value *
2522ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This,
2523 const CXXRecordDecl *UnadjustedClass,
2524 const ThunkInfo &TI) {
2525 return performTypeAdjustment(CGF, InitialPtr: This, UnadjustedClass, NonVirtualAdjustment: TI.This.NonVirtual,
2526 VirtualAdjustment: TI.This.Virtual.Itanium.VCallOffsetOffset,
2527 /*IsReturnAdjustment=*/false);
2528}
2529
2530llvm::Value *
2531ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
2532 const CXXRecordDecl *UnadjustedClass,
2533 const ReturnAdjustment &RA) {
2534 return performTypeAdjustment(CGF, InitialPtr: Ret, UnadjustedClass, NonVirtualAdjustment: RA.NonVirtual,
2535 VirtualAdjustment: RA.Virtual.Itanium.VBaseOffsetOffset,
2536 /*IsReturnAdjustment=*/true);
2537}
2538
2539void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
2540 RValue RV, QualType ResultType) {
2541 if (!isa<CXXDestructorDecl>(Val: CGF.CurGD.getDecl()))
2542 return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
2543
2544 // Destructor thunks in the ARM ABI have indeterminate results.
2545 llvm::Type *T = CGF.ReturnValue.getElementType();
2546 RValue Undef = RValue::get(V: llvm::UndefValue::get(T));
2547 return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV: Undef, ResultType);
2548}
2549
2550/************************** Array allocation cookies **************************/
2551
2552CharUnits ItaniumCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2553 // The array cookie is a size_t; pad that up to the element alignment.
2554 // The cookie is actually right-justified in that space.
2555 return std::max(a: CharUnits::fromQuantity(Quantity: CGM.SizeSizeInBytes),
2556 b: CGM.getContext().getPreferredTypeAlignInChars(T: elementType));
2557}
2558
2559Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2560 Address NewPtr,
2561 llvm::Value *NumElements,
2562 const CXXNewExpr *expr,
2563 QualType ElementType) {
2564 assert(requiresArrayCookie(expr));
2565
2566 unsigned AS = NewPtr.getAddressSpace();
2567
2568 ASTContext &Ctx = getContext();
2569 CharUnits SizeSize = CGF.getSizeSize();
2570
2571 // The size of the cookie.
2572 CharUnits CookieSize =
2573 std::max(a: SizeSize, b: Ctx.getPreferredTypeAlignInChars(T: ElementType));
2574 assert(CookieSize == getArrayCookieSizeImpl(ElementType));
2575
2576 // Compute an offset to the cookie.
2577 Address CookiePtr = NewPtr;
2578 CharUnits CookieOffset = CookieSize - SizeSize;
2579 if (!CookieOffset.isZero())
2580 CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: CookiePtr, Offset: CookieOffset);
2581
2582 // Write the number of elements into the appropriate slot.
2583 Address NumElementsPtr = CookiePtr.withElementType(ElemTy: CGF.SizeTy);
2584 llvm::Instruction *SI = CGF.Builder.CreateStore(Val: NumElements, Addr: NumElementsPtr);
2585
2586 // Handle the array cookie specially in ASan.
2587 if (CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) && AS == 0 &&
2588 (expr->getOperatorNew()->isReplaceableGlobalAllocationFunction() ||
2589 CGM.getCodeGenOpts().SanitizeAddressPoisonCustomArrayCookie)) {
2590 // The store to the CookiePtr does not need to be instrumented.
2591 SI->setNoSanitizeMetadata();
2592 llvm::FunctionType *FTy =
2593 llvm::FunctionType::get(Result: CGM.VoidTy, Params: NumElementsPtr.getType(), isVarArg: false);
2594 llvm::FunctionCallee F =
2595 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_poison_cxx_array_cookie");
2596 CGF.Builder.CreateCall(Callee: F, Args: NumElementsPtr.emitRawPointer(CGF));
2597 }
2598
2599 // Finally, compute a pointer to the actual data buffer by skipping
2600 // over the cookie completely.
2601 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: NewPtr, Offset: CookieSize);
2602}
2603
2604llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2605 Address allocPtr,
2606 CharUnits cookieSize) {
2607 // The element size is right-justified in the cookie.
2608 Address numElementsPtr = allocPtr;
2609 CharUnits numElementsOffset = cookieSize - CGF.getSizeSize();
2610 if (!numElementsOffset.isZero())
2611 numElementsPtr =
2612 CGF.Builder.CreateConstInBoundsByteGEP(Addr: numElementsPtr, Offset: numElementsOffset);
2613
2614 unsigned AS = allocPtr.getAddressSpace();
2615 numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2616 if (!CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) || AS != 0)
2617 return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2618 // In asan mode emit a function call instead of a regular load and let the
2619 // run-time deal with it: if the shadow is properly poisoned return the
2620 // cookie, otherwise return 0 to avoid an infinite loop calling DTORs.
2621 // We can't simply ignore this load using nosanitize metadata because
2622 // the metadata may be lost.
2623 llvm::FunctionType *FTy =
2624 llvm::FunctionType::get(Result: CGF.SizeTy, Params: CGF.DefaultPtrTy, isVarArg: false);
2625 llvm::FunctionCallee F =
2626 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_load_cxx_array_cookie");
2627 return CGF.Builder.CreateCall(Callee: F, Args: numElementsPtr.emitRawPointer(CGF));
2628}
2629
2630CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2631 // ARM says that the cookie is always:
2632 // struct array_cookie {
2633 // std::size_t element_size; // element_size != 0
2634 // std::size_t element_count;
2635 // };
2636 // But the base ABI doesn't give anything an alignment greater than
2637 // 8, so we can dismiss this as typical ABI-author blindness to
2638 // actual language complexity and round up to the element alignment.
2639 return std::max(a: CharUnits::fromQuantity(Quantity: 2 * CGM.SizeSizeInBytes),
2640 b: CGM.getContext().getTypeAlignInChars(T: elementType));
2641}
2642
2643Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2644 Address newPtr,
2645 llvm::Value *numElements,
2646 const CXXNewExpr *expr,
2647 QualType elementType) {
2648 assert(requiresArrayCookie(expr));
2649
2650 // The cookie is always at the start of the buffer.
2651 Address cookie = newPtr;
2652
2653 // The first element is the element size.
2654 cookie = cookie.withElementType(ElemTy: CGF.SizeTy);
2655 llvm::Value *elementSize = llvm::ConstantInt::get(Ty: CGF.SizeTy,
2656 V: getContext().getTypeSizeInChars(T: elementType).getQuantity());
2657 CGF.Builder.CreateStore(Val: elementSize, Addr: cookie);
2658
2659 // The second element is the element count.
2660 cookie = CGF.Builder.CreateConstInBoundsGEP(Addr: cookie, Index: 1);
2661 CGF.Builder.CreateStore(Val: numElements, Addr: cookie);
2662
2663 // Finally, compute a pointer to the actual data buffer by skipping
2664 // over the cookie completely.
2665 CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
2666 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: newPtr, Offset: cookieSize);
2667}
2668
2669llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2670 Address allocPtr,
2671 CharUnits cookieSize) {
2672 // The number of elements is at offset sizeof(size_t) relative to
2673 // the allocated pointer.
2674 Address numElementsPtr
2675 = CGF.Builder.CreateConstInBoundsByteGEP(Addr: allocPtr, Offset: CGF.getSizeSize());
2676
2677 numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2678 return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2679}
2680
2681/*********************** Static local initialization **************************/
2682
2683static llvm::FunctionCallee getGuardAcquireFn(CodeGenModule &CGM,
2684 llvm::PointerType *GuardPtrTy) {
2685 // int __cxa_guard_acquire(__guard *guard_object);
2686 llvm::FunctionType *FTy =
2687 llvm::FunctionType::get(Result: CGM.getTypes().ConvertType(T: CGM.getContext().IntTy),
2688 Params: GuardPtrTy, /*isVarArg=*/false);
2689 return CGM.CreateRuntimeFunction(
2690 Ty: FTy, Name: "__cxa_guard_acquire",
2691 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2692 Index: llvm::AttributeList::FunctionIndex,
2693 Kinds: llvm::Attribute::NoUnwind));
2694}
2695
2696static llvm::FunctionCallee getGuardReleaseFn(CodeGenModule &CGM,
2697 llvm::PointerType *GuardPtrTy) {
2698 // void __cxa_guard_release(__guard *guard_object);
2699 llvm::FunctionType *FTy =
2700 llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2701 return CGM.CreateRuntimeFunction(
2702 Ty: FTy, Name: "__cxa_guard_release",
2703 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2704 Index: llvm::AttributeList::FunctionIndex,
2705 Kinds: llvm::Attribute::NoUnwind));
2706}
2707
2708static llvm::FunctionCallee getGuardAbortFn(CodeGenModule &CGM,
2709 llvm::PointerType *GuardPtrTy) {
2710 // void __cxa_guard_abort(__guard *guard_object);
2711 llvm::FunctionType *FTy =
2712 llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2713 return CGM.CreateRuntimeFunction(
2714 Ty: FTy, Name: "__cxa_guard_abort",
2715 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2716 Index: llvm::AttributeList::FunctionIndex,
2717 Kinds: llvm::Attribute::NoUnwind));
2718}
2719
2720namespace {
2721 struct CallGuardAbort final : EHScopeStack::Cleanup {
2722 llvm::GlobalVariable *Guard;
2723 CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
2724
2725 void Emit(CodeGenFunction &CGF, Flags flags) override {
2726 CGF.EmitNounwindRuntimeCall(callee: getGuardAbortFn(CGM&: CGF.CGM, GuardPtrTy: Guard->getType()),
2727 args: Guard);
2728 }
2729 };
2730}
2731
2732/// The ARM code here follows the Itanium code closely enough that we
2733/// just special-case it at particular places.
2734void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
2735 const VarDecl &D,
2736 llvm::GlobalVariable *var,
2737 bool shouldPerformInit) {
2738 CGBuilderTy &Builder = CGF.Builder;
2739
2740 // Inline variables that weren't instantiated from variable templates have
2741 // partially-ordered initialization within their translation unit.
2742 bool NonTemplateInline =
2743 D.isInline() &&
2744 !isTemplateInstantiation(Kind: D.getTemplateSpecializationKind());
2745
2746 // We only need to use thread-safe statics for local non-TLS variables and
2747 // inline variables; other global initialization is always single-threaded
2748 // or (through lazy dynamic loading in multiple threads) unsequenced.
2749 bool threadsafe = getContext().getLangOpts().ThreadsafeStatics &&
2750 (D.isLocalVarDecl() || NonTemplateInline) &&
2751 !D.getTLSKind();
2752
2753 // If we have a global variable with internal linkage and thread-safe statics
2754 // are disabled, we can just let the guard variable be of type i8.
2755 bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
2756
2757 llvm::IntegerType *guardTy;
2758 CharUnits guardAlignment;
2759 if (useInt8GuardVariable) {
2760 guardTy = CGF.Int8Ty;
2761 guardAlignment = CharUnits::One();
2762 } else {
2763 // Guard variables are 64 bits in the generic ABI and size width on ARM
2764 // (i.e. 32-bit on AArch32, 64-bit on AArch64).
2765 if (UseARMGuardVarABI) {
2766 guardTy = CGF.SizeTy;
2767 guardAlignment = CGF.getSizeAlign();
2768 } else {
2769 guardTy = CGF.Int64Ty;
2770 guardAlignment =
2771 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getABITypeAlign(Ty: guardTy));
2772 }
2773 }
2774 llvm::PointerType *guardPtrTy = llvm::PointerType::get(
2775 C&: CGF.CGM.getLLVMContext(),
2776 AddressSpace: CGF.CGM.getDataLayout().getDefaultGlobalsAddressSpace());
2777
2778 // Create the guard variable if we don't already have it (as we
2779 // might if we're double-emitting this function body).
2780 llvm::GlobalVariable *guard = CGM.getStaticLocalDeclGuardAddress(D: &D);
2781 if (!guard) {
2782 // Mangle the name for the guard.
2783 SmallString<256> guardName;
2784 {
2785 llvm::raw_svector_ostream out(guardName);
2786 getMangleContext().mangleStaticGuardVariable(D: &D, out);
2787 }
2788
2789 // Create the guard variable with a zero-initializer.
2790 // Just absorb linkage, visibility and dll storage class from the guarded
2791 // variable.
2792 guard = new llvm::GlobalVariable(CGM.getModule(), guardTy,
2793 false, var->getLinkage(),
2794 llvm::ConstantInt::get(Ty: guardTy, V: 0),
2795 guardName.str());
2796 guard->setDSOLocal(var->isDSOLocal());
2797 guard->setVisibility(var->getVisibility());
2798 guard->setDLLStorageClass(var->getDLLStorageClass());
2799 // If the variable is thread-local, so is its guard variable.
2800 guard->setThreadLocalMode(var->getThreadLocalMode());
2801 guard->setAlignment(guardAlignment.getAsAlign());
2802
2803 // The ABI says: "It is suggested that it be emitted in the same COMDAT
2804 // group as the associated data object." In practice, this doesn't work for
2805 // non-ELF and non-Wasm object formats, so only do it for ELF and Wasm.
2806 llvm::Comdat *C = var->getComdat();
2807 if (!D.isLocalVarDecl() && C &&
2808 (CGM.getTarget().getTriple().isOSBinFormatELF() ||
2809 CGM.getTarget().getTriple().isOSBinFormatWasm())) {
2810 guard->setComdat(C);
2811 } else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) {
2812 guard->setComdat(CGM.getModule().getOrInsertComdat(Name: guard->getName()));
2813 }
2814
2815 CGM.setStaticLocalDeclGuardAddress(D: &D, C: guard);
2816 }
2817
2818 Address guardAddr = Address(guard, guard->getValueType(), guardAlignment);
2819
2820 // Test whether the variable has completed initialization.
2821 //
2822 // Itanium C++ ABI 3.3.2:
2823 // The following is pseudo-code showing how these functions can be used:
2824 // if (obj_guard.first_byte == 0) {
2825 // if ( __cxa_guard_acquire (&obj_guard) ) {
2826 // try {
2827 // ... initialize the object ...;
2828 // } catch (...) {
2829 // __cxa_guard_abort (&obj_guard);
2830 // throw;
2831 // }
2832 // ... queue object destructor with __cxa_atexit() ...;
2833 // __cxa_guard_release (&obj_guard);
2834 // }
2835 // }
2836 //
2837 // If threadsafe statics are enabled, but we don't have inline atomics, just
2838 // call __cxa_guard_acquire unconditionally. The "inline" check isn't
2839 // actually inline, and the user might not expect calls to __atomic libcalls.
2840
2841 unsigned MaxInlineWidthInBits = CGF.getTarget().getMaxAtomicInlineWidth();
2842 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2843 if (!threadsafe || MaxInlineWidthInBits) {
2844 // Load the first byte of the guard variable.
2845 llvm::LoadInst *LI =
2846 Builder.CreateLoad(Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2847
2848 // Itanium ABI:
2849 // An implementation supporting thread-safety on multiprocessor
2850 // systems must also guarantee that references to the initialized
2851 // object do not occur before the load of the initialization flag.
2852 //
2853 // In LLVM, we do this by marking the load Acquire.
2854 if (threadsafe)
2855 LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
2856
2857 // For ARM, we should only check the first bit, rather than the entire byte:
2858 //
2859 // ARM C++ ABI 3.2.3.1:
2860 // To support the potential use of initialization guard variables
2861 // as semaphores that are the target of ARM SWP and LDREX/STREX
2862 // synchronizing instructions we define a static initialization
2863 // guard variable to be a 4-byte aligned, 4-byte word with the
2864 // following inline access protocol.
2865 // #define INITIALIZED 1
2866 // if ((obj_guard & INITIALIZED) != INITIALIZED) {
2867 // if (__cxa_guard_acquire(&obj_guard))
2868 // ...
2869 // }
2870 //
2871 // and similarly for ARM64:
2872 //
2873 // ARM64 C++ ABI 3.2.2:
2874 // This ABI instead only specifies the value bit 0 of the static guard
2875 // variable; all other bits are platform defined. Bit 0 shall be 0 when the
2876 // variable is not initialized and 1 when it is.
2877 llvm::Value *V =
2878 (UseARMGuardVarABI && !useInt8GuardVariable)
2879 ? Builder.CreateAnd(LHS: LI, RHS: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1))
2880 : LI;
2881 llvm::Value *NeedsInit = Builder.CreateIsNull(Arg: V, Name: "guard.uninitialized");
2882
2883 llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock(name: "init.check");
2884
2885 // Check if the first byte of the guard variable is zero.
2886 CGF.EmitCXXGuardedInitBranch(NeedsInit, InitBlock: InitCheckBlock, NoInitBlock: EndBlock,
2887 Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2888
2889 CGF.EmitBlock(BB: InitCheckBlock);
2890 }
2891
2892 // The semantics of dynamic initialization of variables with static or thread
2893 // storage duration depends on whether they are declared at block-scope. The
2894 // initialization of such variables at block-scope can be aborted with an
2895 // exception and later retried (per C++20 [stmt.dcl]p4), and recursive entry
2896 // to their initialization has undefined behavior (also per C++20
2897 // [stmt.dcl]p4). For such variables declared at non-block scope, exceptions
2898 // lead to termination (per C++20 [except.terminate]p1), and recursive
2899 // references to the variables are governed only by the lifetime rules (per
2900 // C++20 [class.cdtor]p2), which means such references are perfectly fine as
2901 // long as they avoid touching memory. As a result, block-scope variables must
2902 // not be marked as initialized until after initialization completes (unless
2903 // the mark is reverted following an exception), but non-block-scope variables
2904 // must be marked prior to initialization so that recursive accesses during
2905 // initialization do not restart initialization.
2906
2907 // Variables used when coping with thread-safe statics and exceptions.
2908 if (threadsafe) {
2909 // Call __cxa_guard_acquire.
2910 llvm::Value *V
2911 = CGF.EmitNounwindRuntimeCall(callee: getGuardAcquireFn(CGM, GuardPtrTy: guardPtrTy), args: guard);
2912
2913 llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2914
2915 Builder.CreateCondBr(Cond: Builder.CreateIsNotNull(Arg: V, Name: "tobool"),
2916 True: InitBlock, False: EndBlock);
2917
2918 // Call __cxa_guard_abort along the exceptional edge.
2919 CGF.EHStack.pushCleanup<CallGuardAbort>(Kind: EHCleanup, A: guard);
2920
2921 CGF.EmitBlock(BB: InitBlock);
2922 } else if (!D.isLocalVarDecl()) {
2923 // For non-local variables, store 1 into the first byte of the guard
2924 // variable before the object initialization begins so that references
2925 // to the variable during initialization don't restart initialization.
2926 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2927 Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2928 }
2929
2930 // Emit the initializer and add a global destructor if appropriate.
2931 CGF.EmitCXXGlobalVarDeclInit(D, GV: var, PerformInit: shouldPerformInit);
2932
2933 if (threadsafe) {
2934 // Pop the guard-abort cleanup if we pushed one.
2935 CGF.PopCleanupBlock();
2936
2937 // Call __cxa_guard_release. This cannot throw.
2938 CGF.EmitNounwindRuntimeCall(callee: getGuardReleaseFn(CGM, GuardPtrTy: guardPtrTy),
2939 args: guardAddr.emitRawPointer(CGF));
2940 } else if (D.isLocalVarDecl()) {
2941 // For local variables, store 1 into the first byte of the guard variable
2942 // after the object initialization completes so that initialization is
2943 // retried if initialization is interrupted by an exception.
2944 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2945 Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2946 }
2947
2948 CGF.EmitBlock(BB: EndBlock);
2949}
2950
2951/// Register a global destructor using __cxa_atexit.
2952static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF,
2953 llvm::FunctionCallee dtor,
2954 llvm::Constant *addr, bool TLS) {
2955 assert(!CGF.getTarget().getTriple().isOSAIX() &&
2956 "unexpected call to emitGlobalDtorWithCXAAtExit");
2957 assert((TLS || CGF.getTypes().getCodeGenOpts().CXAAtExit) &&
2958 "__cxa_atexit is disabled");
2959 const char *Name = "__cxa_atexit";
2960 if (TLS) {
2961 const llvm::Triple &T = CGF.getTarget().getTriple();
2962 Name = T.isOSDarwin() ? "_tlv_atexit" : "__cxa_thread_atexit";
2963 }
2964
2965 // We're assuming that the destructor function is something we can
2966 // reasonably call with the default CC.
2967 llvm::Type *dtorTy = CGF.DefaultPtrTy;
2968
2969 // Preserve address space of addr.
2970 auto AddrAS = addr ? addr->getType()->getPointerAddressSpace() : 0;
2971 auto AddrPtrTy = AddrAS ? llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: AddrAS)
2972 : CGF.Int8PtrTy;
2973
2974 // Create a variable that binds the atexit to this shared object.
2975 llvm::Constant *handle =
2976 CGF.CGM.CreateRuntimeVariable(Ty: CGF.Int8Ty, Name: "__dso_handle");
2977 auto *GV = cast<llvm::GlobalValue>(Val: handle->stripPointerCasts());
2978 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
2979
2980 // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
2981 llvm::Type *paramTys[] = {dtorTy, AddrPtrTy, handle->getType()};
2982 llvm::FunctionType *atexitTy =
2983 llvm::FunctionType::get(Result: CGF.IntTy, Params: paramTys, isVarArg: false);
2984
2985 // Fetch the actual function.
2986 llvm::FunctionCallee atexit = CGF.CGM.CreateRuntimeFunction(Ty: atexitTy, Name);
2987 if (llvm::Function *fn = dyn_cast<llvm::Function>(Val: atexit.getCallee()))
2988 fn->setDoesNotThrow();
2989
2990 const auto &Context = CGF.CGM.getContext();
2991 FunctionProtoType::ExtProtoInfo EPI(Context.getDefaultCallingConvention(
2992 /*IsVariadic=*/false, /*IsCXXMethod=*/false));
2993 QualType fnType =
2994 Context.getFunctionType(ResultTy: Context.VoidTy, Args: {Context.VoidPtrTy}, EPI);
2995 llvm::Value *dtorCallee = dtor.getCallee();
2996 dtorCallee =
2997 CGF.CGM.getFunctionPointer(Pointer: cast<llvm::Constant>(Val: dtorCallee), FunctionType: fnType);
2998
2999 if (dtorCallee->getType()->getPointerAddressSpace() != AddrAS)
3000 dtorCallee = CGF.performAddrSpaceCast(Src: dtorCallee, DestTy: AddrPtrTy);
3001
3002 if (!addr)
3003 // addr is null when we are trying to register a dtor annotated with
3004 // __attribute__((destructor)) in a constructor function. Using null here is
3005 // okay because this argument is just passed back to the destructor
3006 // function.
3007 addr = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
3008
3009 llvm::Value *args[] = {dtorCallee, addr, handle};
3010 CGF.EmitNounwindRuntimeCall(callee: atexit, args);
3011}
3012
3013static llvm::Function *createGlobalInitOrCleanupFn(CodeGen::CodeGenModule &CGM,
3014 StringRef FnName) {
3015 // Create a function that registers/unregisters destructors that have the same
3016 // priority.
3017 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
3018 llvm::Function *GlobalInitOrCleanupFn = CGM.CreateGlobalInitOrCleanUpFunction(
3019 ty: FTy, name: FnName, FI: CGM.getTypes().arrangeNullaryFunction(), Loc: SourceLocation());
3020
3021 return GlobalInitOrCleanupFn;
3022}
3023
3024void CodeGenModule::unregisterGlobalDtorsWithUnAtExit() {
3025 for (const auto &I : DtorsUsingAtExit) {
3026 int Priority = I.first;
3027 std::string GlobalCleanupFnName =
3028 std::string("__GLOBAL_cleanup_") + llvm::to_string(Value: Priority);
3029
3030 llvm::Function *GlobalCleanupFn =
3031 createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalCleanupFnName);
3032
3033 CodeGenFunction CGF(*this);
3034 CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalCleanupFn,
3035 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
3036 Loc: SourceLocation(), StartLoc: SourceLocation());
3037 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
3038
3039 // Get the destructor function type, void(*)(void).
3040 llvm::FunctionType *dtorFuncTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
3041
3042 // Destructor functions are run/unregistered in non-ascending
3043 // order of their priorities.
3044 const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
3045 auto itv = Dtors.rbegin();
3046 while (itv != Dtors.rend()) {
3047 llvm::Function *Dtor = *itv;
3048
3049 // We're assuming that the destructor function is something we can
3050 // reasonably call with the correct CC.
3051 llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub: Dtor);
3052 llvm::Value *NeedsDestruct =
3053 CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
3054
3055 llvm::BasicBlock *DestructCallBlock =
3056 CGF.createBasicBlock(name: "destruct.call");
3057 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(
3058 name: (itv + 1) != Dtors.rend() ? "unatexit.call" : "destruct.end");
3059 // Check if unatexit returns a value of 0. If it does, jump to
3060 // DestructCallBlock, otherwise jump to EndBlock directly.
3061 CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
3062
3063 CGF.EmitBlock(BB: DestructCallBlock);
3064
3065 // Emit the call to casted Dtor.
3066 llvm::CallInst *CI = CGF.Builder.CreateCall(FTy: dtorFuncTy, Callee: Dtor);
3067 // Make sure the call and the callee agree on calling convention.
3068 CI->setCallingConv(Dtor->getCallingConv());
3069
3070 CGF.EmitBlock(BB: EndBlock);
3071
3072 itv++;
3073 }
3074
3075 CGF.FinishFunction();
3076 AddGlobalDtor(Dtor: GlobalCleanupFn, Priority);
3077 }
3078}
3079
3080void CodeGenModule::registerGlobalDtorsWithAtExit() {
3081 for (const auto &I : DtorsUsingAtExit) {
3082 int Priority = I.first;
3083 std::string GlobalInitFnName =
3084 std::string("__GLOBAL_init_") + llvm::to_string(Value: Priority);
3085 llvm::Function *GlobalInitFn =
3086 createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalInitFnName);
3087
3088 CodeGenFunction CGF(*this);
3089 CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalInitFn,
3090 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
3091 Loc: SourceLocation(), StartLoc: SourceLocation());
3092 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
3093
3094 // Since constructor functions are run in non-descending order of their
3095 // priorities, destructors are registered in non-descending order of their
3096 // priorities, and since destructor functions are run in the reverse order
3097 // of their registration, destructor functions are run in non-ascending
3098 // order of their priorities.
3099 const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
3100 for (auto *Dtor : Dtors) {
3101 // Register the destructor function calling __cxa_atexit if it is
3102 // available. Otherwise fall back on calling atexit.
3103 if (getCodeGenOpts().CXAAtExit) {
3104 emitGlobalDtorWithCXAAtExit(CGF, dtor: Dtor, addr: nullptr, TLS: false);
3105 } else {
3106 // We're assuming that the destructor function is something we can
3107 // reasonably call with the correct CC.
3108 CGF.registerGlobalDtorWithAtExit(dtorStub: Dtor);
3109 }
3110 }
3111
3112 CGF.FinishFunction();
3113 AddGlobalCtor(Ctor: GlobalInitFn, Priority);
3114 }
3115
3116 if (getCXXABI().useSinitAndSterm())
3117 unregisterGlobalDtorsWithUnAtExit();
3118}
3119
3120/// Register a global destructor as best as we know how.
3121void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
3122 llvm::FunctionCallee dtor,
3123 llvm::Constant *addr) {
3124 if (D.isNoDestroy(CGM.getContext()))
3125 return;
3126
3127 // HLSL doesn't support atexit.
3128 if (CGM.getLangOpts().HLSL)
3129 return CGM.AddCXXDtorEntry(DtorFn: dtor, Object: addr);
3130
3131 // OpenMP offloading supports C++ constructors and destructors but we do not
3132 // always have 'atexit' available. Instead lower these to use the LLVM global
3133 // destructors which we can handle directly in the runtime. Note that this is
3134 // not strictly 1-to-1 with using `atexit` because we no longer tear down
3135 // globals in reverse order of when they were constructed.
3136 if (!CGM.getLangOpts().hasAtExit() && !D.isStaticLocal())
3137 return CGF.registerGlobalDtorWithLLVM(D, fn: dtor, addr);
3138
3139 // emitGlobalDtorWithCXAAtExit will emit a call to either __cxa_thread_atexit
3140 // or __cxa_atexit depending on whether this VarDecl is a thread-local storage
3141 // or not. CXAAtExit controls only __cxa_atexit, so use it if it is enabled.
3142 // We can always use __cxa_thread_atexit.
3143 if (CGM.getCodeGenOpts().CXAAtExit || D.getTLSKind())
3144 return emitGlobalDtorWithCXAAtExit(CGF, dtor, addr, TLS: D.getTLSKind());
3145
3146 // In Apple kexts, we want to add a global destructor entry.
3147 // FIXME: shouldn't this be guarded by some variable?
3148 if (CGM.getLangOpts().AppleKext) {
3149 // Generate a global destructor entry.
3150 return CGM.AddCXXDtorEntry(DtorFn: dtor, Object: addr);
3151 }
3152
3153 CGF.registerGlobalDtorWithAtExit(D, fn: dtor, addr);
3154}
3155
3156static bool isThreadWrapperReplaceable(const VarDecl *VD,
3157 CodeGen::CodeGenModule &CGM) {
3158 assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!");
3159 // Darwin prefers to have references to thread local variables to go through
3160 // the thread wrapper instead of directly referencing the backing variable.
3161 return VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3162 CGM.getTarget().getTriple().isOSDarwin();
3163}
3164
3165/// Get the appropriate linkage for the wrapper function. This is essentially
3166/// the weak form of the variable's linkage; every translation unit which needs
3167/// the wrapper emits a copy, and we want the linker to merge them.
3168static llvm::GlobalValue::LinkageTypes
3169getThreadLocalWrapperLinkage(const VarDecl *VD, CodeGen::CodeGenModule &CGM) {
3170 llvm::GlobalValue::LinkageTypes VarLinkage =
3171 CGM.getLLVMLinkageVarDefinition(VD);
3172
3173 // For internal linkage variables, we don't need an external or weak wrapper.
3174 if (llvm::GlobalValue::isLocalLinkage(Linkage: VarLinkage))
3175 return VarLinkage;
3176
3177 // If the thread wrapper is replaceable, give it appropriate linkage.
3178 if (isThreadWrapperReplaceable(VD, CGM))
3179 if (!llvm::GlobalVariable::isLinkOnceLinkage(Linkage: VarLinkage) &&
3180 !llvm::GlobalVariable::isWeakODRLinkage(Linkage: VarLinkage))
3181 return VarLinkage;
3182 return llvm::GlobalValue::WeakODRLinkage;
3183}
3184
3185llvm::Function *
3186ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
3187 llvm::Value *Val) {
3188 // Mangle the name for the thread_local wrapper function.
3189 SmallString<256> WrapperName;
3190 {
3191 llvm::raw_svector_ostream Out(WrapperName);
3192 getMangleContext().mangleItaniumThreadLocalWrapper(D: VD, Out);
3193 }
3194
3195 // FIXME: If VD is a definition, we should regenerate the function attributes
3196 // before returning.
3197 if (llvm::Value *V = CGM.getModule().getNamedValue(Name: WrapperName))
3198 return cast<llvm::Function>(Val: V);
3199
3200 QualType RetQT = VD->getType();
3201 if (RetQT->isReferenceType())
3202 RetQT = RetQT.getNonReferenceType();
3203
3204 const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
3205 resultType: getContext().getPointerType(T: RetQT), args: FunctionArgList());
3206
3207 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FI);
3208 llvm::Function *Wrapper =
3209 llvm::Function::Create(Ty: FnTy, Linkage: getThreadLocalWrapperLinkage(VD, CGM),
3210 N: WrapperName.str(), M: &CGM.getModule());
3211
3212 if (CGM.supportsCOMDAT() && Wrapper->isWeakForLinker())
3213 Wrapper->setComdat(CGM.getModule().getOrInsertComdat(Name: Wrapper->getName()));
3214
3215 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Wrapper, /*IsThunk=*/false);
3216
3217 // Always resolve references to the wrapper at link time.
3218 if (!Wrapper->hasLocalLinkage())
3219 if (!isThreadWrapperReplaceable(VD, CGM) ||
3220 llvm::GlobalVariable::isLinkOnceLinkage(Linkage: Wrapper->getLinkage()) ||
3221 llvm::GlobalVariable::isWeakODRLinkage(Linkage: Wrapper->getLinkage()) ||
3222 VD->getVisibility() == HiddenVisibility)
3223 Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility);
3224
3225 if (isThreadWrapperReplaceable(VD, CGM)) {
3226 Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3227 Wrapper->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3228 }
3229
3230 ThreadWrappers.push_back(Elt: {VD, Wrapper});
3231 return Wrapper;
3232}
3233
3234void ItaniumCXXABI::EmitThreadLocalInitFuncs(
3235 CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
3236 ArrayRef<llvm::Function *> CXXThreadLocalInits,
3237 ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
3238 llvm::Function *InitFunc = nullptr;
3239
3240 // Separate initializers into those with ordered (or partially-ordered)
3241 // initialization and those with unordered initialization.
3242 llvm::SmallVector<llvm::Function *, 8> OrderedInits;
3243 llvm::SmallDenseMap<const VarDecl *, llvm::Function *> UnorderedInits;
3244 for (unsigned I = 0; I != CXXThreadLocalInits.size(); ++I) {
3245 if (isTemplateInstantiation(
3246 Kind: CXXThreadLocalInitVars[I]->getTemplateSpecializationKind()))
3247 UnorderedInits[CXXThreadLocalInitVars[I]->getCanonicalDecl()] =
3248 CXXThreadLocalInits[I];
3249 else
3250 OrderedInits.push_back(Elt: CXXThreadLocalInits[I]);
3251 }
3252
3253 if (!OrderedInits.empty()) {
3254 // Generate a guarded initialization function.
3255 llvm::FunctionType *FTy =
3256 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
3257 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3258 InitFunc = CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: "__tls_init", FI,
3259 Loc: SourceLocation(),
3260 /*TLS=*/true);
3261 llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
3262 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
3263 llvm::GlobalVariable::InternalLinkage,
3264 llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 0), "__tls_guard");
3265 Guard->setThreadLocal(true);
3266 Guard->setThreadLocalMode(CGM.GetDefaultLLVMTLSModel());
3267
3268 CharUnits GuardAlign = CharUnits::One();
3269 Guard->setAlignment(GuardAlign.getAsAlign());
3270
3271 CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(
3272 Fn: InitFunc, CXXThreadLocals: OrderedInits, Guard: ConstantAddress(Guard, CGM.Int8Ty, GuardAlign));
3273 // On Darwin platforms, use CXX_FAST_TLS calling convention.
3274 if (CGM.getTarget().getTriple().isOSDarwin()) {
3275 InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3276 InitFunc->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3277 }
3278 }
3279
3280 // Create declarations for thread wrappers for all thread-local variables
3281 // with non-discardable definitions in this translation unit.
3282 for (const VarDecl *VD : CXXThreadLocals) {
3283 if (VD->hasDefinition() &&
3284 !isDiscardableGVALinkage(L: getContext().GetGVALinkageForVariable(VD))) {
3285 llvm::GlobalValue *GV = CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD));
3286 getOrCreateThreadLocalWrapper(VD, Val: GV);
3287 }
3288 }
3289
3290 // Emit all referenced thread wrappers.
3291 for (auto VDAndWrapper : ThreadWrappers) {
3292 const VarDecl *VD = VDAndWrapper.first;
3293 llvm::GlobalVariable *Var =
3294 cast<llvm::GlobalVariable>(Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD)));
3295 llvm::Function *Wrapper = VDAndWrapper.second;
3296
3297 // Some targets require that all access to thread local variables go through
3298 // the thread wrapper. This means that we cannot attempt to create a thread
3299 // wrapper or a thread helper.
3300 if (!VD->hasDefinition()) {
3301 if (isThreadWrapperReplaceable(VD, CGM)) {
3302 Wrapper->setLinkage(llvm::Function::ExternalLinkage);
3303 continue;
3304 }
3305
3306 // If this isn't a TU in which this variable is defined, the thread
3307 // wrapper is discardable.
3308 if (Wrapper->getLinkage() == llvm::Function::WeakODRLinkage)
3309 Wrapper->setLinkage(llvm::Function::LinkOnceODRLinkage);
3310 }
3311
3312 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Wrapper);
3313
3314 // Mangle the name for the thread_local initialization function.
3315 SmallString<256> InitFnName;
3316 {
3317 llvm::raw_svector_ostream Out(InitFnName);
3318 getMangleContext().mangleItaniumThreadLocalInit(D: VD, Out);
3319 }
3320
3321 llvm::FunctionType *InitFnTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
3322
3323 // If we have a definition for the variable, emit the initialization
3324 // function as an alias to the global Init function (if any). Otherwise,
3325 // produce a declaration of the initialization function.
3326 llvm::GlobalValue *Init = nullptr;
3327 bool InitIsInitFunc = false;
3328 bool HasConstantInitialization = false;
3329 if (!usesThreadWrapperFunction(VD)) {
3330 HasConstantInitialization = true;
3331 } else if (VD->hasDefinition()) {
3332 InitIsInitFunc = true;
3333 llvm::Function *InitFuncToUse = InitFunc;
3334 if (isTemplateInstantiation(Kind: VD->getTemplateSpecializationKind()))
3335 InitFuncToUse = UnorderedInits.lookup(Val: VD->getCanonicalDecl());
3336 if (InitFuncToUse)
3337 Init = llvm::GlobalAlias::create(Linkage: Var->getLinkage(), Name: InitFnName.str(),
3338 Aliasee: InitFuncToUse);
3339 } else {
3340 // Emit a weak global function referring to the initialization function.
3341 // This function will not exist if the TU defining the thread_local
3342 // variable in question does not need any dynamic initialization for
3343 // its thread_local variables.
3344 Init = llvm::Function::Create(Ty: InitFnTy,
3345 Linkage: llvm::GlobalVariable::ExternalWeakLinkage,
3346 N: InitFnName.str(), M: &CGM.getModule());
3347 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3348 CGM.SetLLVMFunctionAttributes(
3349 GD: GlobalDecl(), Info: FI, F: cast<llvm::Function>(Val: Init), /*IsThunk=*/false);
3350 }
3351
3352 if (Init) {
3353 Init->setVisibility(Var->getVisibility());
3354 // Don't mark an extern_weak function DSO local on windows.
3355 if (!CGM.getTriple().isOSWindows() || !Init->hasExternalWeakLinkage())
3356 Init->setDSOLocal(Var->isDSOLocal());
3357 }
3358
3359 llvm::LLVMContext &Context = CGM.getModule().getContext();
3360
3361 // The linker on AIX is not happy with missing weak symbols. However,
3362 // other TUs will not know whether the initialization routine exists
3363 // so create an empty, init function to satisfy the linker.
3364 // This is needed whenever a thread wrapper function is not used, and
3365 // also when the symbol is weak.
3366 if (CGM.getTriple().isOSAIX() && VD->hasDefinition() &&
3367 isEmittedWithConstantInitializer(VD, InspectInitForWeakDef: true) &&
3368 !mayNeedDestruction(VD)) {
3369 // Init should be null. If it were non-null, then the logic above would
3370 // either be defining the function to be an alias or declaring the
3371 // function with the expectation that the definition of the variable
3372 // is elsewhere.
3373 assert(Init == nullptr && "Expected Init to be null.");
3374
3375 llvm::Function *Func = llvm::Function::Create(
3376 Ty: InitFnTy, Linkage: Var->getLinkage(), N: InitFnName.str(), M: &CGM.getModule());
3377 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3378 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI,
3379 F: cast<llvm::Function>(Val: Func),
3380 /*IsThunk=*/false);
3381 // Create a function body that just returns
3382 llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Func);
3383 CGBuilderTy Builder(CGM, Entry);
3384 Builder.CreateRetVoid();
3385 }
3386
3387 llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3388 CGBuilderTy Builder(CGM, Entry);
3389 if (HasConstantInitialization) {
3390 // No dynamic initialization to invoke.
3391 } else if (InitIsInitFunc) {
3392 if (Init) {
3393 llvm::CallInst *CallVal = Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3394 if (isThreadWrapperReplaceable(VD, CGM)) {
3395 CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3396 llvm::Function *Fn =
3397 cast<llvm::Function>(Val: cast<llvm::GlobalAlias>(Val: Init)->getAliasee());
3398 Fn->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3399 }
3400 }
3401 } else if (CGM.getTriple().isOSAIX()) {
3402 // On AIX, except if constinit and also neither of class type or of
3403 // (possibly multi-dimensional) array of class type, thread_local vars
3404 // will have init routines regardless of whether they are
3405 // const-initialized. Since the routine is guaranteed to exist, we can
3406 // unconditionally call it without testing for its existance. This
3407 // avoids potentially unresolved weak symbols which the AIX linker
3408 // isn't happy with.
3409 Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3410 } else {
3411 // Don't know whether we have an init function. Call it if it exists.
3412 llvm::Value *Have = Builder.CreateIsNotNull(Arg: Init);
3413 llvm::BasicBlock *InitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3414 llvm::BasicBlock *ExitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3415 Builder.CreateCondBr(Cond: Have, True: InitBB, False: ExitBB);
3416
3417 Builder.SetInsertPoint(InitBB);
3418 Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3419 Builder.CreateBr(Dest: ExitBB);
3420
3421 Builder.SetInsertPoint(ExitBB);
3422 }
3423
3424 // For a reference, the result of the wrapper function is a pointer to
3425 // the referenced object.
3426 llvm::Value *Val = Builder.CreateThreadLocalAddress(Ptr: Var);
3427
3428 if (VD->getType()->isReferenceType()) {
3429 CharUnits Align = CGM.getContext().getDeclAlign(D: VD);
3430 Val = Builder.CreateAlignedLoad(Ty: Var->getValueType(), Addr: Val, Align);
3431 }
3432 Val = Builder.CreateAddrSpaceCast(V: Val, DestTy: Wrapper->getReturnType());
3433
3434 Builder.CreateRet(V: Val);
3435 }
3436}
3437
3438LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
3439 const VarDecl *VD,
3440 QualType LValType) {
3441 llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3442 llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val);
3443
3444 llvm::CallInst *CallVal = CGF.Builder.CreateCall(Callee: Wrapper);
3445 CallVal->setCallingConv(Wrapper->getCallingConv());
3446
3447 LValue LV;
3448 if (VD->getType()->isReferenceType())
3449 LV = CGF.MakeNaturalAlignRawAddrLValue(V: CallVal, T: LValType);
3450 else
3451 LV = CGF.MakeRawAddrLValue(V: CallVal, T: LValType,
3452 Alignment: CGF.getContext().getDeclAlign(D: VD));
3453 // FIXME: need setObjCGCLValueClass?
3454 return LV;
3455}
3456
3457/// Return whether the given global decl needs a VTT parameter, which it does
3458/// if it's a base constructor or destructor with virtual bases.
3459bool ItaniumCXXABI::NeedsVTTParameter(GlobalDecl GD) {
3460 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
3461
3462 // We don't have any virtual bases, just return early.
3463 if (!MD->getParent()->getNumVBases())
3464 return false;
3465
3466 // Check if we have a base constructor.
3467 if (isa<CXXConstructorDecl>(Val: MD) && GD.getCtorType() == Ctor_Base)
3468 return true;
3469
3470 // Check if we have a base destructor.
3471 if (isa<CXXDestructorDecl>(Val: MD) && GD.getDtorType() == Dtor_Base)
3472 return true;
3473
3474 return false;
3475}
3476
3477llvm::Constant *
3478ItaniumCXXABI::getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD) {
3479 SmallString<256> MethodName;
3480 llvm::raw_svector_ostream Out(MethodName);
3481 getMangleContext().mangleCXXName(GD: MD, Out);
3482 MethodName += "_vfpthunk_";
3483 StringRef ThunkName = MethodName.str();
3484 llvm::Function *ThunkFn;
3485 if ((ThunkFn = cast_or_null<llvm::Function>(
3486 Val: CGM.getModule().getNamedValue(Name: ThunkName))))
3487 return ThunkFn;
3488
3489 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeCXXMethodDeclaration(MD);
3490 llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
3491 llvm::GlobalValue::LinkageTypes Linkage =
3492 MD->isExternallyVisible() ? llvm::GlobalValue::LinkOnceODRLinkage
3493 : llvm::GlobalValue::InternalLinkage;
3494 ThunkFn =
3495 llvm::Function::Create(Ty: ThunkTy, Linkage, N: ThunkName, M: &CGM.getModule());
3496 if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
3497 ThunkFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3498 assert(ThunkFn->getName() == ThunkName && "name was uniqued!");
3499
3500 CGM.SetLLVMFunctionAttributes(GD: MD, Info: FnInfo, F: ThunkFn, /*IsThunk=*/true);
3501 CGM.SetLLVMFunctionAttributesForDefinition(D: MD, F: ThunkFn);
3502
3503 // Stack protection sometimes gets inserted after the musttail call.
3504 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtect);
3505 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtectStrong);
3506 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtectReq);
3507
3508 // Start codegen.
3509 CodeGenFunction CGF(CGM);
3510 CGF.CurGD = GlobalDecl(MD);
3511 CGF.CurFuncIsThunk = true;
3512
3513 // Build FunctionArgs.
3514 FunctionArgList FunctionArgs;
3515 CGF.BuildFunctionArgList(GD: CGF.CurGD, Args&: FunctionArgs);
3516
3517 CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
3518 Args: FunctionArgs, Loc: MD->getLocation(), StartLoc: SourceLocation());
3519
3520 // Emit an artificial location for this function.
3521 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
3522
3523 llvm::Value *ThisVal = loadIncomingCXXThis(CGF);
3524 setCXXABIThisValue(CGF, ThisPtr: ThisVal);
3525
3526 CallArgList CallArgs;
3527 for (const VarDecl *VD : FunctionArgs)
3528 CGF.EmitDelegateCallArg(args&: CallArgs, param: VD, loc: SourceLocation());
3529
3530 const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
3531 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FPT, /*this*/ additional: 1);
3532 const CGFunctionInfo &CallInfo =
3533 CGM.getTypes().arrangeCXXMethodCall(args: CallArgs, type: FPT, required: Required, numPrefixArgs: 0, ABIInfoFD: MD);
3534 CGCallee Callee = CGCallee::forVirtual(CE: nullptr, MD: GlobalDecl(MD),
3535 Addr: getThisAddress(CGF), FTy: ThunkTy);
3536 llvm::CallBase *CallOrInvoke;
3537 CGF.EmitCall(CallInfo, Callee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: &CallOrInvoke,
3538 /*IsMustTail=*/true, Loc: SourceLocation(), IsVirtualFunctionPointerThunk: true);
3539 auto *Call = cast<llvm::CallInst>(Val: CallOrInvoke);
3540 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
3541 if (Call->getType()->isVoidTy())
3542 CGF.Builder.CreateRetVoid();
3543 else
3544 CGF.Builder.CreateRet(V: Call);
3545
3546 // Finish the function to maintain CodeGenFunction invariants.
3547 // FIXME: Don't emit unreachable code.
3548 CGF.EmitBlock(BB: CGF.createBasicBlock());
3549 CGF.FinishFunction();
3550 return ThunkFn;
3551}
3552
3553namespace {
3554class ItaniumRTTIBuilder {
3555 CodeGenModule &CGM; // Per-module state.
3556 llvm::LLVMContext &VMContext;
3557 const ItaniumCXXABI &CXXABI; // Per-module state.
3558
3559 /// Fields - The fields of the RTTI descriptor currently being built.
3560 SmallVector<llvm::Constant *, 16> Fields;
3561
3562 /// GetAddrOfTypeName - Returns the mangled type name of the given type.
3563 llvm::GlobalVariable *
3564 GetAddrOfTypeName(QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage);
3565
3566 /// GetAddrOfExternalRTTIDescriptor - Returns the constant for the RTTI
3567 /// descriptor of the given type.
3568 llvm::Constant *GetAddrOfExternalRTTIDescriptor(QualType Ty);
3569
3570 /// BuildVTablePointer - Build the vtable pointer for the given type.
3571 void BuildVTablePointer(const Type *Ty, llvm::Constant *StorageAddress);
3572
3573 /// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
3574 /// inheritance, according to the Itanium C++ ABI, 2.9.5p6b.
3575 void BuildSIClassTypeInfo(const CXXRecordDecl *RD);
3576
3577 /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
3578 /// classes with bases that do not satisfy the abi::__si_class_type_info
3579 /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
3580 void BuildVMIClassTypeInfo(const CXXRecordDecl *RD);
3581
3582 /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, used
3583 /// for pointer types.
3584 void BuildPointerTypeInfo(QualType PointeeTy);
3585
3586 /// BuildObjCObjectTypeInfo - Build the appropriate kind of
3587 /// type_info for an object type.
3588 void BuildObjCObjectTypeInfo(const ObjCObjectType *Ty);
3589
3590 /// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
3591 /// struct, used for member pointer types.
3592 void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty);
3593
3594public:
3595 ItaniumRTTIBuilder(const ItaniumCXXABI &ABI)
3596 : CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {}
3597
3598 // Pointer type info flags.
3599 enum {
3600 /// PTI_Const - Type has const qualifier.
3601 PTI_Const = 0x1,
3602
3603 /// PTI_Volatile - Type has volatile qualifier.
3604 PTI_Volatile = 0x2,
3605
3606 /// PTI_Restrict - Type has restrict qualifier.
3607 PTI_Restrict = 0x4,
3608
3609 /// PTI_Incomplete - Type is incomplete.
3610 PTI_Incomplete = 0x8,
3611
3612 /// PTI_ContainingClassIncomplete - Containing class is incomplete.
3613 /// (in pointer to member).
3614 PTI_ContainingClassIncomplete = 0x10,
3615
3616 /// PTI_TransactionSafe - Pointee is transaction_safe function (C++ TM TS).
3617 //PTI_TransactionSafe = 0x20,
3618
3619 /// PTI_Noexcept - Pointee is noexcept function (C++1z).
3620 PTI_Noexcept = 0x40,
3621 };
3622
3623 // VMI type info flags.
3624 enum {
3625 /// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
3626 VMI_NonDiamondRepeat = 0x1,
3627
3628 /// VMI_DiamondShaped - Class is diamond shaped.
3629 VMI_DiamondShaped = 0x2
3630 };
3631
3632 // Base class type info flags.
3633 enum {
3634 /// BCTI_Virtual - Base class is virtual.
3635 BCTI_Virtual = 0x1,
3636
3637 /// BCTI_Public - Base class is public.
3638 BCTI_Public = 0x2
3639 };
3640
3641 /// BuildTypeInfo - Build the RTTI type info struct for the given type, or
3642 /// link to an existing RTTI descriptor if one already exists.
3643 llvm::Constant *BuildTypeInfo(QualType Ty);
3644
3645 /// BuildTypeInfo - Build the RTTI type info struct for the given type.
3646 llvm::Constant *BuildTypeInfo(
3647 QualType Ty,
3648 llvm::GlobalVariable::LinkageTypes Linkage,
3649 llvm::GlobalValue::VisibilityTypes Visibility,
3650 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass);
3651};
3652}
3653
3654llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName(
3655 QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) {
3656 SmallString<256> Name;
3657 llvm::raw_svector_ostream Out(Name);
3658 CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(T: Ty, Out);
3659
3660 // We know that the mangled name of the type starts at index 4 of the
3661 // mangled name of the typename, so we can just index into it in order to
3662 // get the mangled name of the type.
3663 llvm::Constant *Init;
3664 if (CGM.getTriple().isOSzOS()) {
3665 // On z/OS, typename is stored as 2 encodings: EBCDIC followed by ASCII.
3666 SmallString<256> DualEncodedName;
3667 llvm::ConverterEBCDIC::convertToEBCDIC(Source: Name.substr(Start: 4), Result&: DualEncodedName);
3668 DualEncodedName += '\0';
3669 DualEncodedName += Name.substr(Start: 4);
3670 Init = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: DualEncodedName);
3671 } else
3672 Init = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Name.substr(Start: 4));
3673
3674 auto Align = CGM.getContext().getTypeAlignInChars(T: CGM.getContext().CharTy);
3675
3676 llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
3677 Name, Ty: Init->getType(), Linkage, Alignment: Align.getAsAlign());
3678
3679 GV->setInitializer(Init);
3680
3681 return GV;
3682}
3683
3684llvm::Constant *
3685ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
3686 // Mangle the RTTI name.
3687 SmallString<256> Name;
3688 llvm::raw_svector_ostream Out(Name);
3689 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
3690
3691 // Look for an existing global.
3692 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
3693
3694 if (!GV) {
3695 // Create a new global variable.
3696 // Note for the future: If we would ever like to do deferred emission of
3697 // RTTI, check if emitting vtables opportunistically need any adjustment.
3698
3699 GV = new llvm::GlobalVariable(
3700 CGM.getModule(), CGM.GlobalsInt8PtrTy,
3701 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, Name);
3702 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
3703 CGM.setGVProperties(GV, D: RD);
3704 // Import the typeinfo symbol when all non-inline virtual methods are
3705 // imported.
3706 if (CGM.getTarget().hasPS4DLLImportExport()) {
3707 if (RD && CXXRecordNonInlineHasAttr<DLLImportAttr>(RD)) {
3708 GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
3709 CGM.setDSOLocal(GV);
3710 }
3711 }
3712 }
3713
3714 return GV;
3715}
3716
3717/// TypeInfoIsInStandardLibrary - Given a builtin type, returns whether the type
3718/// info for that type is defined in the standard library.
3719static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
3720 // Itanium C++ ABI 2.9.2:
3721 // Basic type information (e.g. for "int", "bool", etc.) will be kept in
3722 // the run-time support library. Specifically, the run-time support
3723 // library should contain type_info objects for the types X, X* and
3724 // X const*, for every X in: void, std::nullptr_t, bool, wchar_t, char,
3725 // unsigned char, signed char, short, unsigned short, int, unsigned int,
3726 // long, unsigned long, long long, unsigned long long, float, double,
3727 // long double, char16_t, char32_t, and the IEEE 754r decimal and
3728 // half-precision floating point types.
3729 //
3730 // GCC also emits RTTI for __int128.
3731 // FIXME: We do not emit RTTI information for decimal types here.
3732
3733 // Types added here must also be added to EmitFundamentalRTTIDescriptors.
3734 switch (Ty->getKind()) {
3735 case BuiltinType::Void:
3736 case BuiltinType::NullPtr:
3737 case BuiltinType::Bool:
3738 case BuiltinType::WChar_S:
3739 case BuiltinType::WChar_U:
3740 case BuiltinType::Char_U:
3741 case BuiltinType::Char_S:
3742 case BuiltinType::UChar:
3743 case BuiltinType::SChar:
3744 case BuiltinType::Short:
3745 case BuiltinType::UShort:
3746 case BuiltinType::Int:
3747 case BuiltinType::UInt:
3748 case BuiltinType::Long:
3749 case BuiltinType::ULong:
3750 case BuiltinType::LongLong:
3751 case BuiltinType::ULongLong:
3752 case BuiltinType::Half:
3753 case BuiltinType::Float:
3754 case BuiltinType::Double:
3755 case BuiltinType::LongDouble:
3756 case BuiltinType::Float16:
3757 case BuiltinType::Float128:
3758 case BuiltinType::Ibm128:
3759 case BuiltinType::Char8:
3760 case BuiltinType::Char16:
3761 case BuiltinType::Char32:
3762 case BuiltinType::Int128:
3763 case BuiltinType::UInt128:
3764 return true;
3765
3766#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
3767 case BuiltinType::Id:
3768#include "clang/Basic/OpenCLImageTypes.def"
3769#define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
3770 case BuiltinType::Id:
3771#include "clang/Basic/OpenCLExtensionTypes.def"
3772 case BuiltinType::OCLSampler:
3773 case BuiltinType::OCLEvent:
3774 case BuiltinType::OCLClkEvent:
3775 case BuiltinType::OCLQueue:
3776 case BuiltinType::OCLReserveID:
3777#define SVE_TYPE(Name, Id, SingletonId) \
3778 case BuiltinType::Id:
3779#include "clang/Basic/AArch64ACLETypes.def"
3780#define PPC_VECTOR_TYPE(Name, Id, Size) \
3781 case BuiltinType::Id:
3782#include "clang/Basic/PPCTypes.def"
3783#define RVV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3784#include "clang/Basic/RISCVVTypes.def"
3785#define WASM_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3786#include "clang/Basic/WebAssemblyReferenceTypes.def"
3787#define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align) case BuiltinType::Id:
3788#include "clang/Basic/AMDGPUTypes.def"
3789#define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3790#include "clang/Basic/HLSLIntangibleTypes.def"
3791 case BuiltinType::ShortAccum:
3792 case BuiltinType::Accum:
3793 case BuiltinType::LongAccum:
3794 case BuiltinType::UShortAccum:
3795 case BuiltinType::UAccum:
3796 case BuiltinType::ULongAccum:
3797 case BuiltinType::ShortFract:
3798 case BuiltinType::Fract:
3799 case BuiltinType::LongFract:
3800 case BuiltinType::UShortFract:
3801 case BuiltinType::UFract:
3802 case BuiltinType::ULongFract:
3803 case BuiltinType::SatShortAccum:
3804 case BuiltinType::SatAccum:
3805 case BuiltinType::SatLongAccum:
3806 case BuiltinType::SatUShortAccum:
3807 case BuiltinType::SatUAccum:
3808 case BuiltinType::SatULongAccum:
3809 case BuiltinType::SatShortFract:
3810 case BuiltinType::SatFract:
3811 case BuiltinType::SatLongFract:
3812 case BuiltinType::SatUShortFract:
3813 case BuiltinType::SatUFract:
3814 case BuiltinType::SatULongFract:
3815 case BuiltinType::BFloat16:
3816 return false;
3817
3818 case BuiltinType::Dependent:
3819#define BUILTIN_TYPE(Id, SingletonId)
3820#define PLACEHOLDER_TYPE(Id, SingletonId) \
3821 case BuiltinType::Id:
3822#include "clang/AST/BuiltinTypes.def"
3823 llvm_unreachable("asking for RRTI for a placeholder type!");
3824
3825 case BuiltinType::ObjCId:
3826 case BuiltinType::ObjCClass:
3827 case BuiltinType::ObjCSel:
3828 llvm_unreachable("FIXME: Objective-C types are unsupported!");
3829 }
3830
3831 llvm_unreachable("Invalid BuiltinType Kind!");
3832}
3833
3834static bool TypeInfoIsInStandardLibrary(const PointerType *PointerTy) {
3835 QualType PointeeTy = PointerTy->getPointeeType();
3836 const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: PointeeTy);
3837 if (!BuiltinTy)
3838 return false;
3839
3840 // Check the qualifiers.
3841 Qualifiers Quals = PointeeTy.getQualifiers();
3842 Quals.removeConst();
3843
3844 if (!Quals.empty())
3845 return false;
3846
3847 return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3848}
3849
3850/// IsStandardLibraryRTTIDescriptor - Returns whether the type
3851/// information for the given type exists in the standard library.
3852static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
3853 // Type info for builtin types is defined in the standard library.
3854 if (const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: Ty))
3855 return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3856
3857 // Type info for some pointer types to builtin types is defined in the
3858 // standard library.
3859 if (const PointerType *PointerTy = dyn_cast<PointerType>(Val&: Ty))
3860 return TypeInfoIsInStandardLibrary(PointerTy);
3861
3862 return false;
3863}
3864
3865/// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
3866/// the given type exists somewhere else, and that we should not emit the type
3867/// information in this translation unit. Assumes that it is not a
3868/// standard-library type.
3869static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
3870 QualType Ty) {
3871 ASTContext &Context = CGM.getContext();
3872
3873 // If RTTI is disabled, assume it might be disabled in the
3874 // translation unit that defines any potential key function, too.
3875 if (!Context.getLangOpts().RTTI) return false;
3876
3877 if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
3878 const CXXRecordDecl *RD =
3879 cast<CXXRecordDecl>(Val: RecordTy->getDecl())->getDefinitionOrSelf();
3880 if (!RD->hasDefinition())
3881 return false;
3882
3883 if (!RD->isDynamicClass())
3884 return false;
3885
3886 // FIXME: this may need to be reconsidered if the key function
3887 // changes.
3888 // N.B. We must always emit the RTTI data ourselves if there exists a key
3889 // function.
3890 bool IsDLLImport = RD->hasAttr<DLLImportAttr>();
3891
3892 // Don't import the RTTI but emit it locally.
3893 if (CGM.getTriple().isOSCygMing())
3894 return false;
3895
3896 if (CGM.getVTables().isVTableExternal(RD)) {
3897 if (CGM.getTarget().hasPS4DLLImportExport())
3898 return true;
3899
3900 return IsDLLImport && !CGM.getTriple().isWindowsItaniumEnvironment()
3901 ? false
3902 : true;
3903 }
3904 if (IsDLLImport)
3905 return true;
3906 }
3907
3908 return false;
3909}
3910
3911/// IsIncompleteClassType - Returns whether the given record type is incomplete.
3912static bool IsIncompleteClassType(const RecordType *RecordTy) {
3913 return !RecordTy->getDecl()->getDefinitionOrSelf()->isCompleteDefinition();
3914}
3915
3916/// ContainsIncompleteClassType - Returns whether the given type contains an
3917/// incomplete class type. This is true if
3918///
3919/// * The given type is an incomplete class type.
3920/// * The given type is a pointer type whose pointee type contains an
3921/// incomplete class type.
3922/// * The given type is a member pointer type whose class is an incomplete
3923/// class type.
3924/// * The given type is a member pointer type whoise pointee type contains an
3925/// incomplete class type.
3926/// is an indirect or direct pointer to an incomplete class type.
3927static bool ContainsIncompleteClassType(QualType Ty) {
3928 if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
3929 if (IsIncompleteClassType(RecordTy))
3930 return true;
3931 }
3932
3933 if (const PointerType *PointerTy = dyn_cast<PointerType>(Val&: Ty))
3934 return ContainsIncompleteClassType(Ty: PointerTy->getPointeeType());
3935
3936 if (const MemberPointerType *MemberPointerTy =
3937 dyn_cast<MemberPointerType>(Val&: Ty)) {
3938 // Check if the class type is incomplete.
3939 if (!MemberPointerTy->getMostRecentCXXRecordDecl()->hasDefinition())
3940 return true;
3941
3942 return ContainsIncompleteClassType(Ty: MemberPointerTy->getPointeeType());
3943 }
3944
3945 return false;
3946}
3947
3948// CanUseSingleInheritance - Return whether the given record decl has a "single,
3949// public, non-virtual base at offset zero (i.e. the derived class is dynamic
3950// iff the base is)", according to Itanium C++ ABI, 2.95p6b.
3951static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
3952 // Check the number of bases.
3953 if (RD->getNumBases() != 1)
3954 return false;
3955
3956 // Get the base.
3957 CXXRecordDecl::base_class_const_iterator Base = RD->bases_begin();
3958
3959 // Check that the base is not virtual.
3960 if (Base->isVirtual())
3961 return false;
3962
3963 // Check that the base is public.
3964 if (Base->getAccessSpecifier() != AS_public)
3965 return false;
3966
3967 // Check that the class is dynamic iff the base is.
3968 auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
3969 if (!BaseDecl->isEmpty() &&
3970 BaseDecl->isDynamicClass() != RD->isDynamicClass())
3971 return false;
3972
3973 return true;
3974}
3975
3976void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
3977 llvm::Constant *StorageAddress) {
3978 // abi::__class_type_info.
3979 static const char * const ClassTypeInfo =
3980 "_ZTVN10__cxxabiv117__class_type_infoE";
3981 // abi::__si_class_type_info.
3982 static const char * const SIClassTypeInfo =
3983 "_ZTVN10__cxxabiv120__si_class_type_infoE";
3984 // abi::__vmi_class_type_info.
3985 static const char * const VMIClassTypeInfo =
3986 "_ZTVN10__cxxabiv121__vmi_class_type_infoE";
3987
3988 const char *VTableName = nullptr;
3989
3990 switch (Ty->getTypeClass()) {
3991#define TYPE(Class, Base)
3992#define ABSTRACT_TYPE(Class, Base)
3993#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
3994#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
3995#define DEPENDENT_TYPE(Class, Base) case Type::Class:
3996#include "clang/AST/TypeNodes.inc"
3997 llvm_unreachable("Non-canonical and dependent types shouldn't get here");
3998
3999 case Type::LValueReference:
4000 case Type::RValueReference:
4001 llvm_unreachable("References shouldn't get here");
4002
4003 case Type::Auto:
4004 case Type::DeducedTemplateSpecialization:
4005 llvm_unreachable("Undeduced type shouldn't get here");
4006
4007 case Type::Pipe:
4008 llvm_unreachable("Pipe types shouldn't get here");
4009
4010 case Type::ArrayParameter:
4011 llvm_unreachable("Array Parameter types should not get here.");
4012
4013 case Type::Builtin:
4014 case Type::BitInt:
4015 case Type::OverflowBehavior:
4016 // GCC treats vector and complex types as fundamental types.
4017 case Type::Vector:
4018 case Type::ExtVector:
4019 case Type::ConstantMatrix:
4020 case Type::Complex:
4021 case Type::Atomic:
4022 // FIXME: GCC treats block pointers as fundamental types?!
4023 case Type::BlockPointer:
4024 // abi::__fundamental_type_info.
4025 VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE";
4026 break;
4027
4028 case Type::ConstantArray:
4029 case Type::IncompleteArray:
4030 case Type::VariableArray:
4031 // abi::__array_type_info.
4032 VTableName = "_ZTVN10__cxxabiv117__array_type_infoE";
4033 break;
4034
4035 case Type::FunctionNoProto:
4036 case Type::FunctionProto:
4037 // abi::__function_type_info.
4038 VTableName = "_ZTVN10__cxxabiv120__function_type_infoE";
4039 break;
4040
4041 case Type::Enum:
4042 // abi::__enum_type_info.
4043 VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE";
4044 break;
4045
4046 case Type::Record: {
4047 const auto *RD = cast<CXXRecordDecl>(Val: cast<RecordType>(Val: Ty)->getDecl())
4048 ->getDefinitionOrSelf();
4049
4050 if (!RD->hasDefinition() || !RD->getNumBases()) {
4051 VTableName = ClassTypeInfo;
4052 } else if (CanUseSingleInheritance(RD)) {
4053 VTableName = SIClassTypeInfo;
4054 } else {
4055 VTableName = VMIClassTypeInfo;
4056 }
4057
4058 break;
4059 }
4060
4061 case Type::ObjCObject:
4062 // Ignore protocol qualifiers.
4063 Ty = cast<ObjCObjectType>(Val: Ty)->getBaseType().getTypePtr();
4064
4065 // Handle id and Class.
4066 if (isa<BuiltinType>(Val: Ty)) {
4067 VTableName = ClassTypeInfo;
4068 break;
4069 }
4070
4071 assert(isa<ObjCInterfaceType>(Ty));
4072 [[fallthrough]];
4073
4074 case Type::ObjCInterface:
4075 if (cast<ObjCInterfaceType>(Val: Ty)->getDecl()->getSuperClass()) {
4076 VTableName = SIClassTypeInfo;
4077 } else {
4078 VTableName = ClassTypeInfo;
4079 }
4080 break;
4081
4082 case Type::ObjCObjectPointer:
4083 case Type::Pointer:
4084 // abi::__pointer_type_info.
4085 VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE";
4086 break;
4087
4088 case Type::MemberPointer:
4089 // abi::__pointer_to_member_type_info.
4090 VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE";
4091 break;
4092
4093 case Type::HLSLAttributedResource:
4094 case Type::HLSLInlineSpirv:
4095 llvm_unreachable("HLSL doesn't support virtual functions");
4096 }
4097
4098 llvm::Constant *VTable = nullptr;
4099
4100 // Check if the alias exists. If it doesn't, then get or create the global.
4101 if (CGM.getLangOpts().RelativeCXXABIVTables)
4102 VTable = CGM.getModule().getNamedAlias(Name: VTableName);
4103 if (!VTable) {
4104 llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.GlobalsInt8PtrTy, NumElements: 0);
4105 VTable = CGM.getModule().getOrInsertGlobal(Name: VTableName, Ty);
4106 }
4107
4108 CGM.setDSOLocal(cast<llvm::GlobalValue>(Val: VTable->stripPointerCasts()));
4109
4110 llvm::Type *PtrDiffTy =
4111 CGM.getTypes().ConvertType(T: CGM.getContext().getPointerDiffType());
4112
4113 // The vtable address point is 2.
4114 if (CGM.getLangOpts().RelativeCXXABIVTables) {
4115 // The vtable address point is 8 bytes after its start:
4116 // 4 for the offset to top + 4 for the relative offset to rtti.
4117 llvm::Constant *Eight = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 8);
4118 VTable = llvm::ConstantExpr::getInBoundsPtrAdd(Ptr: VTable, Offset: Eight);
4119 } else {
4120 llvm::Constant *Two = llvm::ConstantInt::get(Ty: PtrDiffTy, V: 2);
4121 VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(Ty: CGM.GlobalsInt8PtrTy,
4122 C: VTable, Idx: Two);
4123 }
4124
4125 if (const auto &Schema =
4126 CGM.getCodeGenOpts().PointerAuth.CXXTypeInfoVTablePointer)
4127 VTable = CGM.getConstantSignedPointer(
4128 Pointer: VTable, Schema,
4129 StorageAddress: Schema.isAddressDiscriminated() ? StorageAddress : nullptr,
4130 SchemaDecl: GlobalDecl(), SchemaType: QualType(Ty, 0));
4131
4132 Fields.push_back(Elt: VTable);
4133}
4134
4135/// Return the linkage that the type info and type info name constants
4136/// should have for the given type.
4137static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM,
4138 QualType Ty) {
4139 // Itanium C++ ABI 2.9.5p7:
4140 // In addition, it and all of the intermediate abi::__pointer_type_info
4141 // structs in the chain down to the abi::__class_type_info for the
4142 // incomplete class type must be prevented from resolving to the
4143 // corresponding type_info structs for the complete class type, possibly
4144 // by making them local static objects. Finally, a dummy class RTTI is
4145 // generated for the incomplete type that will not resolve to the final
4146 // complete class RTTI (because the latter need not exist), possibly by
4147 // making it a local static object.
4148 if (ContainsIncompleteClassType(Ty))
4149 return llvm::GlobalValue::InternalLinkage;
4150
4151 switch (Ty->getLinkage()) {
4152 case Linkage::Invalid:
4153 llvm_unreachable("Linkage hasn't been computed!");
4154
4155 case Linkage::None:
4156 case Linkage::Internal:
4157 case Linkage::UniqueExternal:
4158 return llvm::GlobalValue::InternalLinkage;
4159
4160 case Linkage::VisibleNone:
4161 case Linkage::Module:
4162 case Linkage::External:
4163 // RTTI is not enabled, which means that this type info struct is going
4164 // to be used for exception handling. Give it linkonce_odr linkage.
4165 if (!CGM.getLangOpts().RTTI)
4166 return llvm::GlobalValue::LinkOnceODRLinkage;
4167
4168 if (const RecordType *Record = dyn_cast<RecordType>(Val&: Ty)) {
4169 const auto *RD =
4170 cast<CXXRecordDecl>(Val: Record->getDecl())->getDefinitionOrSelf();
4171 if (RD->hasAttr<WeakAttr>())
4172 return llvm::GlobalValue::WeakODRLinkage;
4173 if (CGM.getTriple().isWindowsItaniumEnvironment())
4174 if (RD->hasAttr<DLLImportAttr>() &&
4175 ShouldUseExternalRTTIDescriptor(CGM, Ty))
4176 return llvm::GlobalValue::ExternalLinkage;
4177 // MinGW always uses LinkOnceODRLinkage for type info.
4178 if (RD->isDynamicClass() &&
4179 !CGM.getContext().getTargetInfo().getTriple().isOSCygMing())
4180 return CGM.getVTableLinkage(RD);
4181 }
4182
4183 return llvm::GlobalValue::LinkOnceODRLinkage;
4184 }
4185
4186 llvm_unreachable("Invalid linkage!");
4187}
4188
4189llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty) {
4190 // We want to operate on the canonical type.
4191 Ty = Ty.getCanonicalType();
4192
4193 // Check if we've already emitted an RTTI descriptor for this type.
4194 SmallString<256> Name;
4195 llvm::raw_svector_ostream Out(Name);
4196 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
4197
4198 llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
4199 if (OldGV && !OldGV->isDeclaration()) {
4200 assert(!OldGV->hasAvailableExternallyLinkage() &&
4201 "available_externally typeinfos not yet implemented");
4202
4203 return OldGV;
4204 }
4205
4206 // Check if there is already an external RTTI descriptor for this type.
4207 if (IsStandardLibraryRTTIDescriptor(Ty) ||
4208 ShouldUseExternalRTTIDescriptor(CGM, Ty))
4209 return GetAddrOfExternalRTTIDescriptor(Ty);
4210
4211 // Emit the standard library with external linkage.
4212 llvm::GlobalVariable::LinkageTypes Linkage = getTypeInfoLinkage(CGM, Ty);
4213
4214 // Give the type_info object and name the formal visibility of the
4215 // type itself.
4216 llvm::GlobalValue::VisibilityTypes llvmVisibility;
4217 if (llvm::GlobalValue::isLocalLinkage(Linkage))
4218 // If the linkage is local, only default visibility makes sense.
4219 llvmVisibility = llvm::GlobalValue::DefaultVisibility;
4220 else if (CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage) ==
4221 ItaniumCXXABI::RUK_NonUniqueHidden)
4222 llvmVisibility = llvm::GlobalValue::HiddenVisibility;
4223 else
4224 llvmVisibility = CodeGenModule::GetLLVMVisibility(V: Ty->getVisibility());
4225
4226 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
4227 llvm::GlobalValue::DefaultStorageClass;
4228 if (auto RD = Ty->getAsCXXRecordDecl()) {
4229 if ((CGM.getTriple().isWindowsItaniumEnvironment() &&
4230 RD->hasAttr<DLLExportAttr>()) ||
4231 (CGM.shouldMapVisibilityToDLLExport(D: RD) &&
4232 !llvm::GlobalValue::isLocalLinkage(Linkage) &&
4233 llvmVisibility == llvm::GlobalValue::DefaultVisibility))
4234 DLLStorageClass = llvm::GlobalValue::DLLExportStorageClass;
4235 }
4236 return BuildTypeInfo(Ty, Linkage, Visibility: llvmVisibility, DLLStorageClass);
4237}
4238
4239llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
4240 QualType Ty,
4241 llvm::GlobalVariable::LinkageTypes Linkage,
4242 llvm::GlobalValue::VisibilityTypes Visibility,
4243 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass) {
4244 SmallString<256> Name;
4245 llvm::raw_svector_ostream Out(Name);
4246 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
4247 llvm::Module &M = CGM.getModule();
4248 llvm::GlobalVariable *OldGV = M.getNamedGlobal(Name);
4249 // int8 is an arbitrary type to be replaced later with replaceInitializer.
4250 llvm::GlobalVariable *GV =
4251 new llvm::GlobalVariable(M, CGM.Int8Ty, /*isConstant=*/true, Linkage,
4252 /*Initializer=*/nullptr, Name);
4253
4254 // Add the vtable pointer.
4255 BuildVTablePointer(Ty: cast<Type>(Val&: Ty), StorageAddress: GV);
4256
4257 // And the name.
4258 llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
4259 llvm::Constant *TypeNameField;
4260
4261 // If we're supposed to demote the visibility, be sure to set a flag
4262 // to use a string comparison for type_info comparisons.
4263 ItaniumCXXABI::RTTIUniquenessKind RTTIUniqueness =
4264 CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage);
4265 if (RTTIUniqueness != ItaniumCXXABI::RUK_Unique) {
4266 // The flag is the sign bit, which on ARM64 is defined to be clear
4267 // for global pointers. This is very ARM64-specific.
4268 TypeNameField = llvm::ConstantExpr::getPtrToInt(C: TypeName, Ty: CGM.Int64Ty);
4269 llvm::Constant *flag =
4270 llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: ((uint64_t)1) << 63);
4271 TypeNameField = llvm::ConstantExpr::getAdd(C1: TypeNameField, C2: flag);
4272 TypeNameField =
4273 llvm::ConstantExpr::getIntToPtr(C: TypeNameField, Ty: CGM.GlobalsInt8PtrTy);
4274 } else {
4275 TypeNameField = TypeName;
4276 }
4277 Fields.push_back(Elt: TypeNameField);
4278
4279 switch (Ty->getTypeClass()) {
4280#define TYPE(Class, Base)
4281#define ABSTRACT_TYPE(Class, Base)
4282#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
4283#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
4284#define DEPENDENT_TYPE(Class, Base) case Type::Class:
4285#include "clang/AST/TypeNodes.inc"
4286 llvm_unreachable("Non-canonical and dependent types shouldn't get here");
4287
4288 // GCC treats vector types as fundamental types.
4289 case Type::Builtin:
4290 case Type::Vector:
4291 case Type::ExtVector:
4292 case Type::ConstantMatrix:
4293 case Type::Complex:
4294 case Type::BlockPointer:
4295 // Itanium C++ ABI 2.9.5p4:
4296 // abi::__fundamental_type_info adds no data members to std::type_info.
4297 break;
4298
4299 case Type::LValueReference:
4300 case Type::RValueReference:
4301 llvm_unreachable("References shouldn't get here");
4302
4303 case Type::Auto:
4304 case Type::DeducedTemplateSpecialization:
4305 llvm_unreachable("Undeduced type shouldn't get here");
4306
4307 case Type::Pipe:
4308 break;
4309
4310 case Type::BitInt:
4311 break;
4312
4313 case Type::ConstantArray:
4314 case Type::IncompleteArray:
4315 case Type::VariableArray:
4316 case Type::ArrayParameter:
4317 // Itanium C++ ABI 2.9.5p5:
4318 // abi::__array_type_info adds no data members to std::type_info.
4319 break;
4320
4321 case Type::FunctionNoProto:
4322 case Type::FunctionProto:
4323 // Itanium C++ ABI 2.9.5p5:
4324 // abi::__function_type_info adds no data members to std::type_info.
4325 break;
4326
4327 case Type::Enum:
4328 // Itanium C++ ABI 2.9.5p5:
4329 // abi::__enum_type_info adds no data members to std::type_info.
4330 break;
4331
4332 case Type::Record: {
4333 const auto *RD = cast<CXXRecordDecl>(Val: cast<RecordType>(Val&: Ty)->getDecl())
4334 ->getDefinitionOrSelf();
4335 if (!RD->hasDefinition() || !RD->getNumBases()) {
4336 // We don't need to emit any fields.
4337 break;
4338 }
4339
4340 if (CanUseSingleInheritance(RD))
4341 BuildSIClassTypeInfo(RD);
4342 else
4343 BuildVMIClassTypeInfo(RD);
4344
4345 break;
4346 }
4347
4348 case Type::ObjCObject:
4349 case Type::ObjCInterface:
4350 BuildObjCObjectTypeInfo(Ty: cast<ObjCObjectType>(Val&: Ty));
4351 break;
4352
4353 case Type::ObjCObjectPointer:
4354 BuildPointerTypeInfo(PointeeTy: cast<ObjCObjectPointerType>(Val&: Ty)->getPointeeType());
4355 break;
4356
4357 case Type::Pointer:
4358 BuildPointerTypeInfo(PointeeTy: cast<PointerType>(Val&: Ty)->getPointeeType());
4359 break;
4360
4361 case Type::MemberPointer:
4362 BuildPointerToMemberTypeInfo(Ty: cast<MemberPointerType>(Val&: Ty));
4363 break;
4364
4365 case Type::Atomic:
4366 // No fields, at least for the moment.
4367 break;
4368
4369 case Type::OverflowBehavior:
4370 break;
4371
4372 case Type::HLSLAttributedResource:
4373 case Type::HLSLInlineSpirv:
4374 llvm_unreachable("HLSL doesn't support RTTI");
4375 }
4376
4377 GV->replaceInitializer(InitVal: llvm::ConstantStruct::getAnon(V: Fields));
4378
4379 // Export the typeinfo in the same circumstances as the vtable is exported.
4380 auto GVDLLStorageClass = DLLStorageClass;
4381 if (CGM.getTarget().hasPS4DLLImportExport() &&
4382 GVDLLStorageClass != llvm::GlobalVariable::DLLExportStorageClass) {
4383 if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
4384 const auto *RD =
4385 cast<CXXRecordDecl>(Val: RecordTy->getDecl())->getDefinitionOrSelf();
4386 if (RD->hasAttr<DLLExportAttr>() ||
4387 CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
4388 GVDLLStorageClass = llvm::GlobalVariable::DLLExportStorageClass;
4389 }
4390 }
4391
4392 // If there's already an old global variable, replace it with the new one.
4393 if (OldGV) {
4394 GV->takeName(V: OldGV);
4395 OldGV->replaceAllUsesWith(V: GV);
4396 OldGV->eraseFromParent();
4397 }
4398
4399 if (CGM.supportsCOMDAT() && GV->isWeakForLinker())
4400 GV->setComdat(M.getOrInsertComdat(Name: GV->getName()));
4401
4402 CharUnits Align = CGM.getContext().toCharUnitsFromBits(
4403 BitSize: CGM.getTarget().getPointerAlign(AddrSpace: CGM.GetGlobalVarAddressSpace(D: nullptr)));
4404 GV->setAlignment(Align.getAsAlign());
4405
4406 // The Itanium ABI specifies that type_info objects must be globally
4407 // unique, with one exception: if the type is an incomplete class
4408 // type or a (possibly indirect) pointer to one. That exception
4409 // affects the general case of comparing type_info objects produced
4410 // by the typeid operator, which is why the comparison operators on
4411 // std::type_info generally use the type_info name pointers instead
4412 // of the object addresses. However, the language's built-in uses
4413 // of RTTI generally require class types to be complete, even when
4414 // manipulating pointers to those class types. This allows the
4415 // implementation of dynamic_cast to rely on address equality tests,
4416 // which is much faster.
4417
4418 // All of this is to say that it's important that both the type_info
4419 // object and the type_info name be uniqued when weakly emitted.
4420
4421 TypeName->setVisibility(Visibility);
4422 CGM.setDSOLocal(TypeName);
4423
4424 GV->setVisibility(Visibility);
4425 CGM.setDSOLocal(GV);
4426
4427 TypeName->setDLLStorageClass(DLLStorageClass);
4428 GV->setDLLStorageClass(GVDLLStorageClass);
4429
4430 TypeName->setPartition(CGM.getCodeGenOpts().SymbolPartition);
4431 GV->setPartition(CGM.getCodeGenOpts().SymbolPartition);
4432
4433 return GV;
4434}
4435
4436/// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info
4437/// for the given Objective-C object type.
4438void ItaniumRTTIBuilder::BuildObjCObjectTypeInfo(const ObjCObjectType *OT) {
4439 // Drop qualifiers.
4440 const Type *T = OT->getBaseType().getTypePtr();
4441 assert(isa<BuiltinType>(T) || isa<ObjCInterfaceType>(T));
4442
4443 // The builtin types are abi::__class_type_infos and don't require
4444 // extra fields.
4445 if (isa<BuiltinType>(Val: T)) return;
4446
4447 ObjCInterfaceDecl *Class = cast<ObjCInterfaceType>(Val: T)->getDecl();
4448 ObjCInterfaceDecl *Super = Class->getSuperClass();
4449
4450 // Root classes are also __class_type_info.
4451 if (!Super) return;
4452
4453 QualType SuperTy = CGM.getContext().getObjCInterfaceType(Decl: Super);
4454
4455 // Everything else is single inheritance.
4456 llvm::Constant *BaseTypeInfo =
4457 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: SuperTy);
4458 Fields.push_back(Elt: BaseTypeInfo);
4459}
4460
4461/// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
4462/// inheritance, according to the Itanium C++ ABI, 2.95p6b.
4463void ItaniumRTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
4464 // Itanium C++ ABI 2.9.5p6b:
4465 // It adds to abi::__class_type_info a single member pointing to the
4466 // type_info structure for the base type,
4467 llvm::Constant *BaseTypeInfo =
4468 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: RD->bases_begin()->getType());
4469 Fields.push_back(Elt: BaseTypeInfo);
4470}
4471
4472namespace {
4473 /// SeenBases - Contains virtual and non-virtual bases seen when traversing
4474 /// a class hierarchy.
4475 struct SeenBases {
4476 llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
4477 llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
4478 };
4479}
4480
4481/// ComputeVMIClassTypeInfoFlags - Compute the value of the flags member in
4482/// abi::__vmi_class_type_info.
4483///
4484static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
4485 SeenBases &Bases) {
4486
4487 unsigned Flags = 0;
4488
4489 auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
4490 if (Base->isVirtual()) {
4491 // Mark the virtual base as seen.
4492 if (!Bases.VirtualBases.insert(Ptr: BaseDecl).second) {
4493 // If this virtual base has been seen before, then the class is diamond
4494 // shaped.
4495 Flags |= ItaniumRTTIBuilder::VMI_DiamondShaped;
4496 } else {
4497 if (Bases.NonVirtualBases.count(Ptr: BaseDecl))
4498 Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4499 }
4500 } else {
4501 // Mark the non-virtual base as seen.
4502 if (!Bases.NonVirtualBases.insert(Ptr: BaseDecl).second) {
4503 // If this non-virtual base has been seen before, then the class has non-
4504 // diamond shaped repeated inheritance.
4505 Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4506 } else {
4507 if (Bases.VirtualBases.count(Ptr: BaseDecl))
4508 Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4509 }
4510 }
4511
4512 // Walk all bases.
4513 for (const auto &I : BaseDecl->bases())
4514 Flags |= ComputeVMIClassTypeInfoFlags(Base: &I, Bases);
4515
4516 return Flags;
4517}
4518
4519static unsigned ComputeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
4520 unsigned Flags = 0;
4521 SeenBases Bases;
4522
4523 // Walk all bases.
4524 for (const auto &I : RD->bases())
4525 Flags |= ComputeVMIClassTypeInfoFlags(Base: &I, Bases);
4526
4527 return Flags;
4528}
4529
4530/// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
4531/// classes with bases that do not satisfy the abi::__si_class_type_info
4532/// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
4533void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
4534 llvm::Type *UnsignedIntLTy =
4535 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4536
4537 // Itanium C++ ABI 2.9.5p6c:
4538 // __flags is a word with flags describing details about the class
4539 // structure, which may be referenced by using the __flags_masks
4540 // enumeration. These flags refer to both direct and indirect bases.
4541 unsigned Flags = ComputeVMIClassTypeInfoFlags(RD);
4542 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4543
4544 // Itanium C++ ABI 2.9.5p6c:
4545 // __base_count is a word with the number of direct proper base class
4546 // descriptions that follow.
4547 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: RD->getNumBases()));
4548
4549 if (!RD->getNumBases())
4550 return;
4551
4552 // Now add the base class descriptions.
4553
4554 // Itanium C++ ABI 2.9.5p6c:
4555 // __base_info[] is an array of base class descriptions -- one for every
4556 // direct proper base. Each description is of the type:
4557 //
4558 // struct abi::__base_class_type_info {
4559 // public:
4560 // const __class_type_info *__base_type;
4561 // long __offset_flags;
4562 //
4563 // enum __offset_flags_masks {
4564 // __virtual_mask = 0x1,
4565 // __public_mask = 0x2,
4566 // __offset_shift = 8
4567 // };
4568 // };
4569
4570 // If we're in mingw and 'long' isn't wide enough for a pointer, use 'long
4571 // long' instead of 'long' for __offset_flags. libstdc++abi uses long long on
4572 // LLP64 platforms.
4573 // FIXME: Consider updating libc++abi to match, and extend this logic to all
4574 // LLP64 platforms.
4575 QualType OffsetFlagsTy = CGM.getContext().LongTy;
4576 const TargetInfo &TI = CGM.getContext().getTargetInfo();
4577 if (TI.getTriple().isOSCygMing() &&
4578 TI.getPointerWidth(AddrSpace: LangAS::Default) > TI.getLongWidth())
4579 OffsetFlagsTy = CGM.getContext().LongLongTy;
4580 llvm::Type *OffsetFlagsLTy =
4581 CGM.getTypes().ConvertType(T: OffsetFlagsTy);
4582
4583 for (const auto &Base : RD->bases()) {
4584 // The __base_type member points to the RTTI for the base type.
4585 Fields.push_back(Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: Base.getType()));
4586
4587 auto *BaseDecl = Base.getType()->castAsCXXRecordDecl();
4588 int64_t OffsetFlags = 0;
4589
4590 // All but the lower 8 bits of __offset_flags are a signed offset.
4591 // For a non-virtual base, this is the offset in the object of the base
4592 // subobject. For a virtual base, this is the offset in the virtual table of
4593 // the virtual base offset for the virtual base referenced (negative).
4594 CharUnits Offset;
4595 if (Base.isVirtual())
4596 Offset =
4597 CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD, VBase: BaseDecl);
4598 else {
4599 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
4600 Offset = Layout.getBaseClassOffset(Base: BaseDecl);
4601 };
4602
4603 OffsetFlags = uint64_t(Offset.getQuantity()) << 8;
4604
4605 // The low-order byte of __offset_flags contains flags, as given by the
4606 // masks from the enumeration __offset_flags_masks.
4607 if (Base.isVirtual())
4608 OffsetFlags |= BCTI_Virtual;
4609 if (Base.getAccessSpecifier() == AS_public)
4610 OffsetFlags |= BCTI_Public;
4611
4612 Fields.push_back(Elt: llvm::ConstantInt::getSigned(Ty: OffsetFlagsLTy, V: OffsetFlags));
4613 }
4614}
4615
4616/// Compute the flags for a __pbase_type_info, and remove the corresponding
4617/// pieces from \p Type.
4618static unsigned extractPBaseFlags(ASTContext &Ctx, QualType &Type) {
4619 unsigned Flags = 0;
4620
4621 if (Type.isConstQualified())
4622 Flags |= ItaniumRTTIBuilder::PTI_Const;
4623 if (Type.isVolatileQualified())
4624 Flags |= ItaniumRTTIBuilder::PTI_Volatile;
4625 if (Type.isRestrictQualified())
4626 Flags |= ItaniumRTTIBuilder::PTI_Restrict;
4627 Type = Type.getUnqualifiedType();
4628
4629 // Itanium C++ ABI 2.9.5p7:
4630 // When the abi::__pbase_type_info is for a direct or indirect pointer to an
4631 // incomplete class type, the incomplete target type flag is set.
4632 if (ContainsIncompleteClassType(Ty: Type))
4633 Flags |= ItaniumRTTIBuilder::PTI_Incomplete;
4634
4635 if (auto *Proto = Type->getAs<FunctionProtoType>()) {
4636 if (Proto->isNothrow()) {
4637 Flags |= ItaniumRTTIBuilder::PTI_Noexcept;
4638 Type = Ctx.getFunctionTypeWithExceptionSpec(Orig: Type, ESI: EST_None);
4639 }
4640 }
4641
4642 return Flags;
4643}
4644
4645/// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct,
4646/// used for pointer types.
4647void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {
4648 // Itanium C++ ABI 2.9.5p7:
4649 // __flags is a flag word describing the cv-qualification and other
4650 // attributes of the type pointed to
4651 unsigned Flags = extractPBaseFlags(Ctx&: CGM.getContext(), Type&: PointeeTy);
4652
4653 llvm::Type *UnsignedIntLTy =
4654 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4655 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4656
4657 // Itanium C++ ABI 2.9.5p7:
4658 // __pointee is a pointer to the std::type_info derivation for the
4659 // unqualified type being pointed to.
4660 llvm::Constant *PointeeTypeInfo =
4661 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4662 Fields.push_back(Elt: PointeeTypeInfo);
4663}
4664
4665/// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
4666/// struct, used for member pointer types.
4667void
4668ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
4669 QualType PointeeTy = Ty->getPointeeType();
4670
4671 // Itanium C++ ABI 2.9.5p7:
4672 // __flags is a flag word describing the cv-qualification and other
4673 // attributes of the type pointed to.
4674 unsigned Flags = extractPBaseFlags(Ctx&: CGM.getContext(), Type&: PointeeTy);
4675
4676 const auto *RD = Ty->getMostRecentCXXRecordDecl();
4677 if (!RD->hasDefinition())
4678 Flags |= PTI_ContainingClassIncomplete;
4679
4680 llvm::Type *UnsignedIntLTy =
4681 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4682 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4683
4684 // Itanium C++ ABI 2.9.5p7:
4685 // __pointee is a pointer to the std::type_info derivation for the
4686 // unqualified type being pointed to.
4687 llvm::Constant *PointeeTypeInfo =
4688 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4689 Fields.push_back(Elt: PointeeTypeInfo);
4690
4691 // Itanium C++ ABI 2.9.5p9:
4692 // __context is a pointer to an abi::__class_type_info corresponding to the
4693 // class type containing the member pointed to
4694 // (e.g., the "A" in "int A::*").
4695 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
4696 Fields.push_back(Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: T));
4697}
4698
4699llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) {
4700 return ItaniumRTTIBuilder(*this).BuildTypeInfo(Ty);
4701}
4702
4703void ItaniumCXXABI::EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD) {
4704 // Types added here must also be added to TypeInfoIsInStandardLibrary.
4705 QualType FundamentalTypes[] = {
4706 getContext().VoidTy, getContext().NullPtrTy,
4707 getContext().BoolTy, getContext().WCharTy,
4708 getContext().CharTy, getContext().UnsignedCharTy,
4709 getContext().SignedCharTy, getContext().ShortTy,
4710 getContext().UnsignedShortTy, getContext().IntTy,
4711 getContext().UnsignedIntTy, getContext().LongTy,
4712 getContext().UnsignedLongTy, getContext().LongLongTy,
4713 getContext().UnsignedLongLongTy, getContext().Int128Ty,
4714 getContext().UnsignedInt128Ty, getContext().HalfTy,
4715 getContext().FloatTy, getContext().DoubleTy,
4716 getContext().LongDoubleTy, getContext().Float128Ty,
4717 getContext().Char8Ty, getContext().Char16Ty,
4718 getContext().Char32Ty
4719 };
4720 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
4721 RD->hasAttr<DLLExportAttr>() || CGM.shouldMapVisibilityToDLLExport(D: RD)
4722 ? llvm::GlobalValue::DLLExportStorageClass
4723 : llvm::GlobalValue::DefaultStorageClass;
4724 llvm::GlobalValue::VisibilityTypes Visibility =
4725 CodeGenModule::GetLLVMVisibility(V: RD->getVisibility());
4726 for (const QualType &FundamentalType : FundamentalTypes) {
4727 QualType PointerType = getContext().getPointerType(T: FundamentalType);
4728 QualType PointerTypeConst = getContext().getPointerType(
4729 T: FundamentalType.withConst());
4730 for (QualType Type : {FundamentalType, PointerType, PointerTypeConst})
4731 ItaniumRTTIBuilder(*this).BuildTypeInfo(
4732 Ty: Type, Linkage: llvm::GlobalValue::ExternalLinkage,
4733 Visibility, DLLStorageClass);
4734 }
4735}
4736
4737/// What sort of uniqueness rules should we use for the RTTI for the
4738/// given type?
4739ItaniumCXXABI::RTTIUniquenessKind ItaniumCXXABI::classifyRTTIUniqueness(
4740 QualType CanTy, llvm::GlobalValue::LinkageTypes Linkage) const {
4741 if (shouldRTTIBeUnique())
4742 return RUK_Unique;
4743
4744 // It's only necessary for linkonce_odr or weak_odr linkage.
4745 if (Linkage != llvm::GlobalValue::LinkOnceODRLinkage &&
4746 Linkage != llvm::GlobalValue::WeakODRLinkage)
4747 return RUK_Unique;
4748
4749 // It's only necessary with default visibility.
4750 if (CanTy->getVisibility() != DefaultVisibility)
4751 return RUK_Unique;
4752
4753 // If we're not required to publish this symbol, hide it.
4754 if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
4755 return RUK_NonUniqueHidden;
4756
4757 // If we're required to publish this symbol, as we might be under an
4758 // explicit instantiation, leave it with default visibility but
4759 // enable string-comparisons.
4760 assert(Linkage == llvm::GlobalValue::WeakODRLinkage);
4761 return RUK_NonUniqueVisible;
4762}
4763
4764// Find out how to codegen the complete destructor and constructor
4765namespace {
4766enum class StructorCodegen { Emit, RAUW, Alias, COMDAT };
4767}
4768static StructorCodegen getCodegenToUse(CodeGenModule &CGM,
4769 const CXXMethodDecl *MD) {
4770 if (!CGM.getCodeGenOpts().CXXCtorDtorAliases)
4771 return StructorCodegen::Emit;
4772
4773 // The complete and base structors are not equivalent if there are any virtual
4774 // bases, so emit separate functions.
4775 if (MD->getParent()->getNumVBases())
4776 return StructorCodegen::Emit;
4777
4778 GlobalDecl AliasDecl;
4779 if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
4780 AliasDecl = GlobalDecl(DD, Dtor_Complete);
4781 } else {
4782 const auto *CD = cast<CXXConstructorDecl>(Val: MD);
4783 AliasDecl = GlobalDecl(CD, Ctor_Complete);
4784 }
4785 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4786
4787 if (llvm::GlobalValue::isDiscardableIfUnused(Linkage))
4788 return StructorCodegen::RAUW;
4789
4790 // FIXME: Should we allow available_externally aliases?
4791 if (!llvm::GlobalAlias::isValidLinkage(L: Linkage))
4792 return StructorCodegen::RAUW;
4793
4794 if (llvm::GlobalValue::isWeakForLinker(Linkage)) {
4795 // Only ELF and wasm support COMDATs with arbitrary names (C5/D5).
4796 if (CGM.getTarget().getTriple().isOSBinFormatELF() ||
4797 CGM.getTarget().getTriple().isOSBinFormatWasm())
4798 return StructorCodegen::COMDAT;
4799 return StructorCodegen::Emit;
4800 }
4801
4802 return StructorCodegen::Alias;
4803}
4804
4805static void emitConstructorDestructorAlias(CodeGenModule &CGM,
4806 GlobalDecl AliasDecl,
4807 GlobalDecl TargetDecl) {
4808 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4809
4810 StringRef MangledName = CGM.getMangledName(GD: AliasDecl);
4811 llvm::GlobalValue *Entry = CGM.GetGlobalValue(Ref: MangledName);
4812 if (Entry && !Entry->isDeclaration())
4813 return;
4814
4815 auto *Aliasee = cast<llvm::GlobalValue>(Val: CGM.GetAddrOfGlobal(GD: TargetDecl));
4816
4817 // Create the alias with no name.
4818 auto *Alias = llvm::GlobalAlias::create(Linkage, Name: "", Aliasee);
4819
4820 // Constructors and destructors are always unnamed_addr.
4821 Alias->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4822
4823 // Switch any previous uses to the alias.
4824 if (Entry) {
4825 assert(Entry->getType() == Aliasee->getType() &&
4826 "declaration exists with different type");
4827 Alias->takeName(V: Entry);
4828 Entry->replaceAllUsesWith(V: Alias);
4829 Entry->eraseFromParent();
4830 } else {
4831 Alias->setName(MangledName);
4832 }
4833
4834 // Finally, set up the alias with its proper name and attributes.
4835 CGM.SetCommonAttributes(GD: AliasDecl, GV: Alias);
4836}
4837
4838void ItaniumCXXABI::emitCXXStructor(GlobalDecl GD) {
4839 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
4840 auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD);
4841 const CXXDestructorDecl *DD = CD ? nullptr : cast<CXXDestructorDecl>(Val: MD);
4842
4843 StructorCodegen CGType = getCodegenToUse(CGM, MD);
4844
4845 if (CD ? GD.getCtorType() == Ctor_Complete
4846 : GD.getDtorType() == Dtor_Complete) {
4847 GlobalDecl BaseDecl;
4848 if (CD)
4849 BaseDecl = GD.getWithCtorType(Type: Ctor_Base);
4850 else
4851 BaseDecl = GD.getWithDtorType(Type: Dtor_Base);
4852
4853 if (CGType == StructorCodegen::Alias || CGType == StructorCodegen::COMDAT) {
4854 emitConstructorDestructorAlias(CGM, AliasDecl: GD, TargetDecl: BaseDecl);
4855 return;
4856 }
4857
4858 if (CGType == StructorCodegen::RAUW) {
4859 StringRef MangledName = CGM.getMangledName(GD);
4860 auto *Aliasee = CGM.GetAddrOfGlobal(GD: BaseDecl);
4861 CGM.addReplacement(Name: MangledName, C: Aliasee);
4862 return;
4863 }
4864 }
4865
4866 // The base destructor is equivalent to the base destructor of its
4867 // base class if there is exactly one non-virtual base class with a
4868 // non-trivial destructor, there are no fields with a non-trivial
4869 // destructor, and the body of the destructor is trivial.
4870 if (DD && GD.getDtorType() == Dtor_Base &&
4871 CGType != StructorCodegen::COMDAT &&
4872 !CGM.TryEmitBaseDestructorAsAlias(D: DD))
4873 return;
4874
4875 // FIXME: The deleting destructor is equivalent to the selected operator
4876 // delete if:
4877 // * either the delete is a destroying operator delete or the destructor
4878 // would be trivial if it weren't virtual,
4879 // * the conversion from the 'this' parameter to the first parameter of the
4880 // destructor is equivalent to a bitcast,
4881 // * the destructor does not have an implicit "this" return, and
4882 // * the operator delete has the same calling convention and IR function type
4883 // as the destructor.
4884 // In such cases we should try to emit the deleting dtor as an alias to the
4885 // selected 'operator delete'.
4886
4887 llvm::Function *Fn = CGM.codegenCXXStructor(GD);
4888
4889 if (CGType == StructorCodegen::COMDAT) {
4890 SmallString<256> Buffer;
4891 llvm::raw_svector_ostream Out(Buffer);
4892 if (DD)
4893 getMangleContext().mangleCXXDtorComdat(D: DD, Out);
4894 else
4895 getMangleContext().mangleCXXCtorComdat(D: CD, Out);
4896 llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Name: Out.str());
4897 Fn->setComdat(C);
4898 } else {
4899 CGM.maybeSetTrivialComdat(D: *MD, GO&: *Fn);
4900 }
4901}
4902
4903static llvm::FunctionCallee getBeginCatchFn(CodeGenModule &CGM) {
4904 // void *__cxa_begin_catch(void*);
4905 llvm::FunctionType *FTy = llvm::FunctionType::get(
4906 Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4907
4908 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_begin_catch");
4909}
4910
4911static llvm::FunctionCallee getEndCatchFn(CodeGenModule &CGM) {
4912 // void __cxa_end_catch();
4913 llvm::FunctionType *FTy =
4914 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
4915
4916 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_end_catch");
4917}
4918
4919static llvm::FunctionCallee getGetExceptionPtrFn(CodeGenModule &CGM) {
4920 // void *__cxa_get_exception_ptr(void*);
4921 llvm::FunctionType *FTy = llvm::FunctionType::get(
4922 Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4923
4924 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_get_exception_ptr");
4925}
4926
4927namespace {
4928 /// A cleanup to call __cxa_end_catch. In many cases, the caught
4929 /// exception type lets us state definitively that the thrown exception
4930 /// type does not have a destructor. In particular:
4931 /// - Catch-alls tell us nothing, so we have to conservatively
4932 /// assume that the thrown exception might have a destructor.
4933 /// - Catches by reference behave according to their base types.
4934 /// - Catches of non-record types will only trigger for exceptions
4935 /// of non-record types, which never have destructors.
4936 /// - Catches of record types can trigger for arbitrary subclasses
4937 /// of the caught type, so we have to assume the actual thrown
4938 /// exception type might have a throwing destructor, even if the
4939 /// caught type's destructor is trivial or nothrow.
4940 struct CallEndCatch final : EHScopeStack::Cleanup {
4941 CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
4942 bool MightThrow;
4943
4944 void Emit(CodeGenFunction &CGF, Flags flags) override {
4945 if (!MightThrow) {
4946 CGF.EmitNounwindRuntimeCall(callee: getEndCatchFn(CGM&: CGF.CGM));
4947 return;
4948 }
4949
4950 CGF.EmitRuntimeCallOrInvoke(callee: getEndCatchFn(CGM&: CGF.CGM));
4951 }
4952 };
4953}
4954
4955/// Emits a call to __cxa_begin_catch and enters a cleanup to call
4956/// __cxa_end_catch. If -fassume-nothrow-exception-dtor is specified, we assume
4957/// that the exception object's dtor is nothrow, therefore the __cxa_end_catch
4958/// call can be marked as nounwind even if EndMightThrow is true.
4959///
4960/// \param EndMightThrow - true if __cxa_end_catch might throw
4961static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
4962 llvm::Value *Exn,
4963 bool EndMightThrow) {
4964 llvm::CallInst *call =
4965 CGF.EmitNounwindRuntimeCall(callee: getBeginCatchFn(CGM&: CGF.CGM), args: Exn);
4966
4967 CGF.EHStack.pushCleanup<CallEndCatch>(
4968 Kind: NormalAndEHCleanup,
4969 A: EndMightThrow && !CGF.CGM.getLangOpts().AssumeNothrowExceptionDtor);
4970
4971 return call;
4972}
4973
4974/// A "special initializer" callback for initializing a catch
4975/// parameter during catch initialization.
4976static void InitCatchParam(CodeGenFunction &CGF,
4977 const VarDecl &CatchParam,
4978 Address ParamAddr,
4979 SourceLocation Loc) {
4980 // Load the exception from where the landing pad saved it.
4981 llvm::Value *Exn = CGF.getExceptionFromSlot();
4982
4983 CanQualType CatchType =
4984 CGF.CGM.getContext().getCanonicalType(T: CatchParam.getType());
4985 llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(T: CatchType);
4986
4987 // If we're catching by reference, we can just cast the object
4988 // pointer to the appropriate pointer.
4989 if (isa<ReferenceType>(Val: CatchType)) {
4990 QualType CaughtType = cast<ReferenceType>(Val&: CatchType)->getPointeeType();
4991 bool EndCatchMightThrow = CaughtType->isRecordType();
4992
4993 // __cxa_begin_catch returns the adjusted object pointer.
4994 llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: EndCatchMightThrow);
4995
4996 // We have no way to tell the personality function that we're
4997 // catching by reference, so if we're catching a pointer,
4998 // __cxa_begin_catch will actually return that pointer by value.
4999 if (const PointerType *PT = dyn_cast<PointerType>(Val&: CaughtType)) {
5000 QualType PointeeType = PT->getPointeeType();
5001
5002 // When catching by reference, generally we should just ignore
5003 // this by-value pointer and use the exception object instead.
5004 if (!PointeeType->isRecordType()) {
5005
5006 // Exn points to the struct _Unwind_Exception header, which
5007 // we have to skip past in order to reach the exception data.
5008 unsigned HeaderSize =
5009 CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException();
5010 AdjustedExn =
5011 CGF.Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: Exn, Idx0: HeaderSize);
5012
5013 // However, if we're catching a pointer-to-record type that won't
5014 // work, because the personality function might have adjusted
5015 // the pointer. There's actually no way for us to fully satisfy
5016 // the language/ABI contract here: we can't use Exn because it
5017 // might have the wrong adjustment, but we can't use the by-value
5018 // pointer because it's off by a level of abstraction.
5019 //
5020 // The current solution is to dump the adjusted pointer into an
5021 // alloca, which breaks language semantics (because changing the
5022 // pointer doesn't change the exception) but at least works.
5023 // The better solution would be to filter out non-exact matches
5024 // and rethrow them, but this is tricky because the rethrow
5025 // really needs to be catchable by other sites at this landing
5026 // pad. The best solution is to fix the personality function.
5027 } else {
5028 // Pull the pointer for the reference type off.
5029 llvm::Type *PtrTy = CGF.ConvertTypeForMem(T: CaughtType);
5030
5031 // Create the temporary and write the adjusted pointer into it.
5032 Address ExnPtrTmp =
5033 CGF.CreateTempAlloca(Ty: PtrTy, align: CGF.getPointerAlign(), Name: "exn.byref.tmp");
5034 llvm::Value *Casted = CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: PtrTy);
5035 CGF.Builder.CreateStore(Val: Casted, Addr: ExnPtrTmp);
5036
5037 // Bind the reference to the temporary.
5038 AdjustedExn = ExnPtrTmp.emitRawPointer(CGF);
5039 }
5040 }
5041
5042 llvm::Value *ExnCast =
5043 CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.byref");
5044 CGF.Builder.CreateStore(Val: ExnCast, Addr: ParamAddr);
5045 return;
5046 }
5047
5048 // Scalars and complexes.
5049 TypeEvaluationKind TEK = CGF.getEvaluationKind(T: CatchType);
5050 if (TEK != TEK_Aggregate) {
5051 llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: false);
5052
5053 // If the catch type is a pointer type, __cxa_begin_catch returns
5054 // the pointer by value.
5055 if (CatchType->hasPointerRepresentation()) {
5056 llvm::Value *CastExn =
5057 CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.casted");
5058
5059 switch (CatchType.getQualifiers().getObjCLifetime()) {
5060 case Qualifiers::OCL_Strong:
5061 CastExn = CGF.EmitARCRetainNonBlock(value: CastExn);
5062 [[fallthrough]];
5063
5064 case Qualifiers::OCL_None:
5065 case Qualifiers::OCL_ExplicitNone:
5066 case Qualifiers::OCL_Autoreleasing:
5067 CGF.Builder.CreateStore(Val: CastExn, Addr: ParamAddr);
5068 return;
5069
5070 case Qualifiers::OCL_Weak:
5071 CGF.EmitARCInitWeak(addr: ParamAddr, value: CastExn);
5072 return;
5073 }
5074 llvm_unreachable("bad ownership qualifier!");
5075 }
5076
5077 // Otherwise, it returns a pointer into the exception object.
5078
5079 LValue srcLV = CGF.MakeNaturalAlignAddrLValue(V: AdjustedExn, T: CatchType);
5080 LValue destLV = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
5081 switch (TEK) {
5082 case TEK_Complex:
5083 CGF.EmitStoreOfComplex(V: CGF.EmitLoadOfComplex(src: srcLV, loc: Loc), dest: destLV,
5084 /*init*/ isInit: true);
5085 return;
5086 case TEK_Scalar: {
5087 llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc);
5088 CGF.EmitStoreOfScalar(value: ExnLoad, lvalue: destLV, /*init*/ isInit: true);
5089 return;
5090 }
5091 case TEK_Aggregate:
5092 llvm_unreachable("evaluation kind filtered out!");
5093 }
5094 llvm_unreachable("bad evaluation kind");
5095 }
5096
5097 assert(isa<RecordType>(CatchType) && "unexpected catch type!");
5098 auto catchRD = CatchType->getAsCXXRecordDecl();
5099 CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(CD: catchRD);
5100
5101 llvm::Type *PtrTy = CGF.DefaultPtrTy;
5102
5103 // Check for a copy expression. If we don't have a copy expression,
5104 // that means a trivial copy is okay.
5105 const Expr *copyExpr = CatchParam.getInit();
5106 if (!copyExpr) {
5107 llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: true);
5108 Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
5109 LLVMCatchTy, caughtExnAlignment);
5110 LValue Dest = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
5111 LValue Src = CGF.MakeAddrLValue(Addr: adjustedExn, T: CatchType);
5112 CGF.EmitAggregateCopy(Dest, Src, EltTy: CatchType, MayOverlap: AggValueSlot::DoesNotOverlap);
5113 return;
5114 }
5115
5116 // We have to call __cxa_get_exception_ptr to get the adjusted
5117 // pointer before copying.
5118 llvm::CallInst *rawAdjustedExn =
5119 CGF.EmitNounwindRuntimeCall(callee: getGetExceptionPtrFn(CGM&: CGF.CGM), args: Exn);
5120
5121 // Cast that to the appropriate type.
5122 Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
5123 LLVMCatchTy, caughtExnAlignment);
5124
5125 // The copy expression is defined in terms of an OpaqueValueExpr.
5126 // Find it and map it to the adjusted expression.
5127 CodeGenFunction::OpaqueValueMapping
5128 opaque(CGF, OpaqueValueExpr::findInCopyConstruct(expr: copyExpr),
5129 CGF.MakeAddrLValue(Addr: adjustedExn, T: CatchParam.getType()));
5130
5131 // Call the copy ctor in a terminate scope.
5132 CGF.EHStack.pushTerminate();
5133
5134 // Perform the copy construction.
5135 CGF.EmitAggExpr(E: copyExpr,
5136 AS: AggValueSlot::forAddr(addr: ParamAddr, quals: Qualifiers(),
5137 isDestructed: AggValueSlot::IsNotDestructed,
5138 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
5139 isAliased: AggValueSlot::IsNotAliased,
5140 mayOverlap: AggValueSlot::DoesNotOverlap));
5141
5142 // Leave the terminate scope.
5143 CGF.EHStack.popTerminate();
5144
5145 // Undo the opaque value mapping.
5146 opaque.pop();
5147
5148 // Finally we can call __cxa_begin_catch.
5149 CallBeginCatch(CGF, Exn, EndMightThrow: true);
5150}
5151
5152/// Begins a catch statement by initializing the catch variable and
5153/// calling __cxa_begin_catch.
5154void ItaniumCXXABI::emitBeginCatch(CodeGenFunction &CGF,
5155 const CXXCatchStmt *S) {
5156 // We have to be very careful with the ordering of cleanups here:
5157 // C++ [except.throw]p4:
5158 // The destruction [of the exception temporary] occurs
5159 // immediately after the destruction of the object declared in
5160 // the exception-declaration in the handler.
5161 //
5162 // So the precise ordering is:
5163 // 1. Construct catch variable.
5164 // 2. __cxa_begin_catch
5165 // 3. Enter __cxa_end_catch cleanup
5166 // 4. Enter dtor cleanup
5167 //
5168 // We do this by using a slightly abnormal initialization process.
5169 // Delegation sequence:
5170 // - ExitCXXTryStmt opens a RunCleanupsScope
5171 // - EmitAutoVarAlloca creates the variable and debug info
5172 // - InitCatchParam initializes the variable from the exception
5173 // - CallBeginCatch calls __cxa_begin_catch
5174 // - CallBeginCatch enters the __cxa_end_catch cleanup
5175 // - EmitAutoVarCleanups enters the variable destructor cleanup
5176 // - EmitCXXTryStmt emits the code for the catch body
5177 // - EmitCXXTryStmt close the RunCleanupsScope
5178
5179 VarDecl *CatchParam = S->getExceptionDecl();
5180 if (!CatchParam) {
5181 llvm::Value *Exn = CGF.getExceptionFromSlot();
5182 CallBeginCatch(CGF, Exn, EndMightThrow: true);
5183 return;
5184 }
5185
5186 // Emit the local.
5187 CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(var: *CatchParam);
5188 {
5189 ApplyAtomGroup Grp(CGF.getDebugInfo());
5190 InitCatchParam(CGF, CatchParam: *CatchParam, ParamAddr: var.getObjectAddress(CGF),
5191 Loc: S->getBeginLoc());
5192 }
5193 CGF.EmitAutoVarCleanups(emission: var);
5194}
5195
5196/// Get or define the following function:
5197/// void @__clang_call_terminate(i8* %exn) nounwind noreturn
5198/// This code is used only in C++.
5199static llvm::FunctionCallee getClangCallTerminateFn(CodeGenModule &CGM) {
5200 ASTContext &C = CGM.getContext();
5201 const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
5202 resultType: C.VoidTy, argTypes: {C.getPointerType(T: C.CharTy)});
5203 llvm::FunctionType *fnTy = CGM.getTypes().GetFunctionType(Info: FI);
5204 llvm::FunctionCallee fnRef = CGM.CreateRuntimeFunction(
5205 Ty: fnTy, Name: "__clang_call_terminate", ExtraAttrs: llvm::AttributeList(), /*Local=*/true);
5206 llvm::Function *fn =
5207 cast<llvm::Function>(Val: fnRef.getCallee()->stripPointerCasts());
5208 if (fn->empty()) {
5209 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: fn, /*IsThunk=*/false);
5210 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: fn);
5211 fn->setDoesNotThrow();
5212 fn->setDoesNotReturn();
5213
5214 // What we really want is to massively penalize inlining without
5215 // forbidding it completely. The difference between that and
5216 // 'noinline' is negligible.
5217 fn->addFnAttr(Kind: llvm::Attribute::NoInline);
5218
5219 // Allow this function to be shared across translation units, but
5220 // we don't want it to turn into an exported symbol.
5221 fn->setLinkage(llvm::Function::LinkOnceODRLinkage);
5222 fn->setVisibility(llvm::Function::HiddenVisibility);
5223 if (CGM.supportsCOMDAT())
5224 fn->setComdat(CGM.getModule().getOrInsertComdat(Name: fn->getName()));
5225
5226 // Set up the function.
5227 llvm::BasicBlock *entry =
5228 llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "", Parent: fn);
5229 CGBuilderTy builder(CGM, entry);
5230
5231 // Pull the exception pointer out of the parameter list.
5232 llvm::Value *exn = &*fn->arg_begin();
5233
5234 // Call __cxa_begin_catch(exn).
5235 llvm::CallInst *catchCall = builder.CreateCall(Callee: getBeginCatchFn(CGM), Args: exn);
5236 catchCall->setDoesNotThrow();
5237 catchCall->setCallingConv(CGM.getRuntimeCC());
5238
5239 // Call std::terminate().
5240 llvm::CallInst *termCall = builder.CreateCall(Callee: CGM.getTerminateFn());
5241 termCall->setDoesNotThrow();
5242 termCall->setDoesNotReturn();
5243 termCall->setCallingConv(CGM.getRuntimeCC());
5244
5245 // std::terminate cannot return.
5246 builder.CreateUnreachable();
5247 }
5248 return fnRef;
5249}
5250
5251llvm::CallInst *
5252ItaniumCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
5253 llvm::Value *Exn) {
5254 // In C++, we want to call __cxa_begin_catch() before terminating.
5255 if (Exn) {
5256 assert(CGF.CGM.getLangOpts().CPlusPlus);
5257 return CGF.EmitNounwindRuntimeCall(callee: getClangCallTerminateFn(CGM&: CGF.CGM), args: Exn);
5258 }
5259 return CGF.EmitNounwindRuntimeCall(callee: CGF.CGM.getTerminateFn());
5260}
5261
5262std::pair<llvm::Value *, const CXXRecordDecl *>
5263ItaniumCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
5264 const CXXRecordDecl *RD) {
5265 return {CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: RD), RD};
5266}
5267
5268llvm::Constant *
5269ItaniumCXXABI::getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD) {
5270 const CXXMethodDecl *origMD =
5271 cast<CXXMethodDecl>(Val: CGM.getItaniumVTableContext()
5272 .findOriginalMethod(GD: MD->getCanonicalDecl())
5273 .getDecl());
5274 llvm::Constant *thunk = getOrCreateVirtualFunctionPointerThunk(MD: origMD);
5275 QualType funcType = CGM.getContext().getMemberPointerType(
5276 T: MD->getType(), /*Qualifier=*/std::nullopt, Cls: MD->getParent());
5277 return CGM.getMemberFunctionPointer(Pointer: thunk, FT: funcType);
5278}
5279
5280void WebAssemblyCXXABI::emitBeginCatch(CodeGenFunction &CGF,
5281 const CXXCatchStmt *C) {
5282 if (CGF.getTarget().hasFeature(Feature: "exception-handling"))
5283 CGF.EHStack.pushCleanup<CatchRetScope>(
5284 Kind: NormalCleanup, A: cast<llvm::CatchPadInst>(Val: CGF.CurrentFuncletPad));
5285 ItaniumCXXABI::emitBeginCatch(CGF, S: C);
5286}
5287
5288llvm::CallInst *
5289WebAssemblyCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
5290 llvm::Value *Exn) {
5291 // Itanium ABI calls __clang_call_terminate(), which __cxa_begin_catch() on
5292 // the violating exception to mark it handled, but it is currently hard to do
5293 // with wasm EH instruction structure with catch/catch_all, we just call
5294 // std::terminate and ignore the violating exception as in CGCXXABI in Wasm EH
5295 // and call __clang_call_terminate only in Emscripten EH.
5296 // TODO Consider code transformation that makes calling __clang_call_terminate
5297 // in Wasm EH possible.
5298 if (Exn && !EHPersonality::get(CGF).isWasmPersonality()) {
5299 assert(CGF.CGM.getLangOpts().CPlusPlus);
5300 return CGF.EmitNounwindRuntimeCall(callee: getClangCallTerminateFn(CGM&: CGF.CGM), args: Exn);
5301 }
5302 return CGCXXABI::emitTerminateForUnexpectedException(CGF, Exn);
5303}
5304
5305/// Register a global destructor as best as we know how.
5306void XLCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
5307 llvm::FunctionCallee Dtor,
5308 llvm::Constant *Addr) {
5309 if (D.getTLSKind() != VarDecl::TLS_None) {
5310 llvm::PointerType *PtrTy = CGF.DefaultPtrTy;
5311
5312 // extern "C" int __pt_atexit_np(int flags, int(*)(int,...), ...);
5313 llvm::FunctionType *AtExitTy =
5314 llvm::FunctionType::get(Result: CGM.IntTy, Params: {CGM.IntTy, PtrTy}, isVarArg: true);
5315
5316 // Fetch the actual function.
5317 llvm::FunctionCallee AtExit =
5318 CGM.CreateRuntimeFunction(Ty: AtExitTy, Name: "__pt_atexit_np");
5319
5320 // Create __dtor function for the var decl.
5321 llvm::Function *DtorStub = CGF.createTLSAtExitStub(VD: D, Dtor, Addr, AtExit);
5322
5323 // Register above __dtor with atexit().
5324 // First param is flags and must be 0, second param is function ptr
5325 llvm::Value *NV = llvm::Constant::getNullValue(Ty: CGM.IntTy);
5326 CGF.EmitNounwindRuntimeCall(callee: AtExit, args: {NV, DtorStub});
5327
5328 // Cannot unregister TLS __dtor so done
5329 return;
5330 }
5331
5332 // Create __dtor function for the var decl.
5333 llvm::Function *DtorStub =
5334 cast<llvm::Function>(Val: CGF.createAtExitStub(VD: D, Dtor, Addr));
5335
5336 // Register above __dtor with atexit().
5337 CGF.registerGlobalDtorWithAtExit(dtorStub: DtorStub);
5338
5339 // Emit __finalize function to unregister __dtor and (as appropriate) call
5340 // __dtor.
5341 emitCXXStermFinalizer(D, dtorStub: DtorStub, addr: Addr);
5342}
5343
5344void XLCXXABI::emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
5345 llvm::Constant *addr) {
5346 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
5347 SmallString<256> FnName;
5348 {
5349 llvm::raw_svector_ostream Out(FnName);
5350 getMangleContext().mangleDynamicStermFinalizer(D: &D, Out);
5351 }
5352
5353 // Create the finalization action associated with a variable.
5354 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
5355 llvm::Function *StermFinalizer = CGM.CreateGlobalInitOrCleanUpFunction(
5356 ty: FTy, name: FnName.str(), FI, Loc: D.getLocation());
5357
5358 CodeGenFunction CGF(CGM);
5359
5360 CGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: StermFinalizer, FnInfo: FI,
5361 Args: FunctionArgList(), Loc: D.getLocation(),
5362 StartLoc: D.getInit()->getExprLoc());
5363
5364 // The unatexit subroutine unregisters __dtor functions that were previously
5365 // registered by the atexit subroutine. If the referenced function is found,
5366 // the unatexit returns a value of 0, meaning that the cleanup is still
5367 // pending (and we should call the __dtor function).
5368 llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub);
5369
5370 llvm::Value *NeedsDestruct = CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
5371
5372 llvm::BasicBlock *DestructCallBlock = CGF.createBasicBlock(name: "destruct.call");
5373 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "destruct.end");
5374
5375 // Check if unatexit returns a value of 0. If it does, jump to
5376 // DestructCallBlock, otherwise jump to EndBlock directly.
5377 CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
5378
5379 CGF.EmitBlock(BB: DestructCallBlock);
5380
5381 // Emit the call to dtorStub.
5382 llvm::CallInst *CI = CGF.Builder.CreateCall(Callee: dtorStub);
5383
5384 // Make sure the call and the callee agree on calling convention.
5385 CI->setCallingConv(dtorStub->getCallingConv());
5386
5387 CGF.EmitBlock(BB: EndBlock);
5388
5389 CGF.FinishFunction();
5390
5391 if (auto *IPA = D.getAttr<InitPriorityAttr>()) {
5392 CGM.AddCXXPrioritizedStermFinalizerEntry(StermFinalizer,
5393 Priority: IPA->getPriority());
5394 } else if (isTemplateInstantiation(Kind: D.getTemplateSpecializationKind()) ||
5395 getContext().GetGVALinkageForVariable(VD: &D) == GVA_DiscardableODR) {
5396 // According to C++ [basic.start.init]p2, class template static data
5397 // members (i.e., implicitly or explicitly instantiated specializations)
5398 // have unordered initialization. As a consequence, we can put them into
5399 // their own llvm.global_dtors entry.
5400 CGM.AddCXXStermFinalizerToGlobalDtor(StermFinalizer, Priority: 65535);
5401 } else {
5402 CGM.AddCXXStermFinalizerEntry(DtorFn: StermFinalizer);
5403 }
5404}
5405