1//===------- ItaniumCXXABI.cpp - Emit LLVM Code from ASTs for a Module ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides C++ code generation targeting the Itanium C++ ABI. The class
10// in this file generates structures that follow the Itanium C++ ABI, which is
11// documented at:
12// https://itanium-cxx-abi.github.io/cxx-abi/abi.html
13// https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
14//
15// It also supports the closely-related ARM ABI, documented at:
16// https://developer.arm.com/documentation/ihi0041/g/
17//
18//===----------------------------------------------------------------------===//
19
20#include "CGCXXABI.h"
21#include "CGCleanup.h"
22#include "CGDebugInfo.h"
23#include "CGRecordLayout.h"
24#include "CGVTables.h"
25#include "CodeGenFunction.h"
26#include "CodeGenModule.h"
27#include "TargetInfo.h"
28#include "clang/AST/Attr.h"
29#include "clang/AST/Mangle.h"
30#include "clang/AST/StmtCXX.h"
31#include "clang/AST/Type.h"
32#include "clang/Basic/PointerAuthOptions.h"
33#include "clang/CodeGen/ConstantInitBuilder.h"
34#include "clang/CodeGenUtils/ItaniumCXXABIUtils.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Intrinsics.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/ConvertEBCDIC.h"
41#include "llvm/Support/ScopedPrinter.h"
42
43#include <optional>
44
45using namespace clang;
46using namespace CodeGen;
47
48namespace {
49class ItaniumCXXABI : public CodeGen::CGCXXABI {
50 /// VTables - All the vtables which have been defined.
51 llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
52
53 /// All the thread wrapper functions that have been used.
54 llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
55 ThreadWrappers;
56
57protected:
58 bool UseARMMethodPtrABI;
59 bool UseARMGuardVarABI;
60 bool Use32BitVTableOffsetABI;
61
62 ItaniumMangleContext &getMangleContext() {
63 return cast<ItaniumMangleContext>(Val&: CodeGen::CGCXXABI::getMangleContext());
64 }
65
66public:
67 ItaniumCXXABI(CodeGen::CodeGenModule &CGM,
68 bool UseARMMethodPtrABI = false,
69 bool UseARMGuardVarABI = false) :
70 CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI),
71 UseARMGuardVarABI(UseARMGuardVarABI),
72 Use32BitVTableOffsetABI(false) { }
73
74 bool classifyReturnType(CGFunctionInfo &FI) const override;
75
76 RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override {
77 // If C++ prohibits us from making a copy, pass by address.
78 if (!RD->canPassInRegisters())
79 return RAA_Indirect;
80 return RAA_Default;
81 }
82
83 bool isThisCompleteObject(GlobalDecl GD) const override {
84 // The Itanium ABI has separate complete-object vs. base-object
85 // variants of both constructors and destructors.
86 if (isa<CXXDestructorDecl>(Val: GD.getDecl())) {
87 switch (GD.getDtorType()) {
88 case Dtor_Complete:
89 case Dtor_Deleting:
90 return true;
91
92 case Dtor_Base:
93 return false;
94
95 case Dtor_Comdat:
96 llvm_unreachable("emitting dtor comdat as function?");
97 case Dtor_Unified:
98 llvm_unreachable("emitting unified dtor as function?");
99 case Dtor_VectorDeleting:
100 llvm_unreachable("unexpected dtor kind for this ABI");
101 }
102 llvm_unreachable("bad dtor kind");
103 }
104 if (isa<CXXConstructorDecl>(Val: GD.getDecl())) {
105 switch (GD.getCtorType()) {
106 case Ctor_Complete:
107 return true;
108
109 case Ctor_Base:
110 return false;
111
112 case Ctor_CopyingClosure:
113 case Ctor_DefaultClosure:
114 llvm_unreachable("closure ctors in Itanium ABI?");
115
116 case Ctor_Comdat:
117 llvm_unreachable("emitting ctor comdat as function?");
118
119 case Ctor_Unified:
120 llvm_unreachable("emitting unified ctor as function?");
121 }
122 llvm_unreachable("bad dtor kind");
123 }
124
125 // No other kinds.
126 return false;
127 }
128
129 bool isZeroInitializable(const MemberPointerType *MPT) override;
130
131 llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
132
133 CGCallee
134 EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
135 const Expr *E,
136 Address This,
137 llvm::Value *&ThisPtrForCall,
138 llvm::Value *MemFnPtr,
139 const MemberPointerType *MPT) override;
140
141 llvm::Value *EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
142 Address Base, llvm::Value *MemPtr,
143 const MemberPointerType *MPT,
144 bool IsInBounds) override;
145
146 llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
147 const CastExpr *E,
148 llvm::Value *Src) override;
149 llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
150 llvm::Constant *Src) override;
151
152 llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
153
154 llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
155 llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
156 CharUnits offset) override;
157 llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
158 llvm::Constant *BuildMemberPointer(const CXXMethodDecl *MD,
159 CharUnits ThisAdjustment);
160
161 llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
162 llvm::Value *L, llvm::Value *R,
163 const MemberPointerType *MPT,
164 bool Inequality) override;
165
166 llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
167 llvm::Value *Addr,
168 const MemberPointerType *MPT) override;
169
170 void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
171 Address Ptr, QualType ElementType,
172 const CXXDestructorDecl *Dtor) override;
173
174 void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
175 void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
176
177 void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
178
179 llvm::CallInst *
180 emitTerminateForUnexpectedException(CodeGenFunction &CGF,
181 llvm::Value *Exn) override;
182
183 void EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD);
184 llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
185 CatchTypeInfo
186 getAddrOfCXXCatchHandlerType(QualType Ty,
187 QualType CatchHandlerType) override {
188 return CatchTypeInfo{.RTTI: getAddrOfRTTIDescriptor(Ty), .Flags: 0};
189 }
190
191 bool shouldTypeidBeNullChecked(QualType SrcRecordTy) override;
192 void EmitBadTypeidCall(CodeGenFunction &CGF) override;
193 llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
194 Address ThisPtr,
195 llvm::Type *StdTypeInfoPtrTy) override;
196
197 bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
198 QualType SrcRecordTy) override;
199
200 /// Determine whether we know that all instances of type RecordTy will have
201 /// the same vtable pointer values, that is distinct from all other vtable
202 /// pointers. While this is required by the Itanium ABI, it doesn't happen in
203 /// practice in some cases due to language extensions.
204 bool hasUniqueVTablePointer(QualType RecordTy) {
205 const CXXRecordDecl *RD = RecordTy->getAsCXXRecordDecl();
206
207 // The exact dynamic_cast optimization relies on the vtable having a unique
208 // address. -fno-assume-unique-vtables disables it, and under -fapple-kext
209 // multiple definitions of the same vtable may be emitted.
210 if (CGM.getCodeGenOpts().DisableExactDynamicCast ||
211 getContext().getLangOpts().AppleKext)
212 return false;
213
214 // If the type_info* would be null, the vtable might be merged with that of
215 // another type.
216 if (!CGM.shouldEmitRTTI())
217 return false;
218
219 // If there's only one definition of the vtable in the program, it has a
220 // unique address.
221 if (!llvm::GlobalValue::isWeakForLinker(Linkage: CGM.getVTableLinkage(RD)))
222 return true;
223
224 // Even if there are multiple definitions of the vtable, they are required
225 // by the ABI to use the same symbol name, so should be merged at load
226 // time. However, if the class has hidden visibility, there can be
227 // different versions of the class in different modules, and the ABI
228 // library might treat them as being the same.
229 if (CGM.GetLLVMVisibility(V: RD->getVisibility()) !=
230 llvm::GlobalValue::DefaultVisibility)
231 return false;
232
233 // A vague-linkage (weak) vtable on a target whose ABI may duplicate it can
234 // be emitted with a distinct address in more than one image, so its address
235 // cannot be assumed unique.
236 return !CGM.mayVTableBeDuplicated(Linkage: CGM.getVTableLinkage(RD));
237 }
238
239 bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
240 return hasUniqueVTablePointer(RecordTy: DestRecordTy);
241 }
242
243 std::optional<ExactDynamicCastInfo>
244 getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
245 QualType DestRecordTy) override;
246
247 llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
248 QualType SrcRecordTy, QualType DestTy,
249 QualType DestRecordTy,
250 llvm::BasicBlock *CastEnd) override;
251
252 llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address ThisAddr,
253 QualType SrcRecordTy, QualType DestTy,
254 QualType DestRecordTy,
255 const ExactDynamicCastInfo &CastInfo,
256 llvm::BasicBlock *CastSuccess,
257 llvm::BasicBlock *CastFail) override;
258
259 llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
260 QualType SrcRecordTy) override;
261
262 bool EmitBadCastCall(CodeGenFunction &CGF) override;
263
264 llvm::Value *
265 GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
266 const CXXRecordDecl *ClassDecl,
267 const CXXRecordDecl *BaseClassDecl) override;
268
269 void EmitCXXConstructors(const CXXConstructorDecl *D) override;
270
271 AddedStructorArgCounts
272 buildStructorSignature(GlobalDecl GD,
273 SmallVectorImpl<CanQualType> &ArgTys) override;
274
275 bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
276 CXXDtorType DT) const override {
277 // Itanium does not emit any destructor variant as an inline thunk.
278 // Delegating may occur as an optimization, but all variants are either
279 // emitted with external linkage or as linkonce if they are inline and used.
280 return false;
281 }
282
283 void EmitCXXDestructors(const CXXDestructorDecl *D) override;
284
285 void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
286 FunctionArgList &Params) override;
287
288 void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
289
290 AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
291 const CXXConstructorDecl *D,
292 CXXCtorType Type,
293 bool ForVirtualBase,
294 bool Delegating) override;
295
296 llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
297 const CXXDestructorDecl *DD,
298 CXXDtorType Type,
299 bool ForVirtualBase,
300 bool Delegating) override;
301
302 void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
303 CXXDtorType Type, bool ForVirtualBase,
304 bool Delegating, Address This,
305 QualType ThisTy) override;
306
307 void emitVTableDefinitions(CodeGenVTables &CGVT,
308 const CXXRecordDecl *RD) override;
309
310 bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
311 CodeGenFunction::VPtr Vptr) override;
312
313 bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
314 return true;
315 }
316
317 llvm::Constant *
318 getVTableAddressPoint(BaseSubobject Base,
319 const CXXRecordDecl *VTableClass) override;
320
321 llvm::Value *getVTableAddressPointInStructor(
322 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
323 BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
324
325 llvm::Value *getVTableAddressPointInStructorWithVTT(
326 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
327 BaseSubobject Base, const CXXRecordDecl *NearestVBase);
328
329 llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
330 CharUnits VPtrOffset) override;
331
332 CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
333 Address This, llvm::Type *Ty,
334 SourceLocation Loc) override;
335
336 llvm::Value *
337 EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor,
338 CXXDtorType DtorType, Address This,
339 DeleteOrMemberCallExpr E,
340 llvm::CallBase **CallOrInvoke) override;
341
342 void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
343
344 bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override;
345 bool canSpeculativelyEmitVTableAsBaseClass(const CXXRecordDecl *RD) const;
346
347 void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
348 bool ReturnAdjustment) override {
349 // Allow inlining of thunks by emitting them with available_externally
350 // linkage together with vtables when needed.
351 if (ForVTable && !Thunk->hasLocalLinkage())
352 Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
353 CGM.setGVProperties(GV: Thunk, GD);
354 }
355
356 bool exportThunk() override { return true; }
357
358 llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
359 const CXXRecordDecl *UnadjustedThisClass,
360 const ThunkInfo &TI) override;
361
362 llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
363 const CXXRecordDecl *UnadjustedRetClass,
364 const ReturnAdjustment &RA) override;
365
366 size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
367 FunctionArgList &Args) const override {
368 assert(!Args.empty() && "expected the arglist to not be empty!");
369 return Args.size() - 1;
370 }
371
372 StringRef GetPureVirtualCallName() override { return "__cxa_pure_virtual"; }
373 StringRef GetDeletedVirtualCallName() override
374 { return "__cxa_deleted_virtual"; }
375
376 CharUnits getArrayCookieSizeImpl(QualType elementType) override;
377 Address InitializeArrayCookie(CodeGenFunction &CGF,
378 Address NewPtr,
379 llvm::Value *NumElements,
380 const CXXNewExpr *expr,
381 QualType ElementType) override;
382 llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
383 Address allocPtr,
384 CharUnits cookieSize) override;
385
386 void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
387 llvm::GlobalVariable *DeclPtr,
388 bool PerformInit) override;
389 void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
390 llvm::FunctionCallee dtor,
391 llvm::Constant *addr) override;
392
393 llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD,
394 llvm::Value *Val);
395 void EmitThreadLocalInitFuncs(
396 CodeGenModule &CGM,
397 ArrayRef<const VarDecl *> CXXThreadLocals,
398 ArrayRef<llvm::Function *> CXXThreadLocalInits,
399 ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
400
401 bool usesThreadWrapperFunction(const VarDecl *VD) const override {
402 return !isEmittedWithConstantInitializer(VD) ||
403 mayNeedDestruction(VD);
404 }
405 LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
406 QualType LValType) override;
407
408 bool NeedsVTTParameter(GlobalDecl GD) override;
409
410 llvm::Constant *
411 getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD);
412
413 /**************************** RTTI Uniqueness ******************************/
414
415protected:
416 /// Returns true if the ABI requires RTTI type_info objects to be unique
417 /// across a program.
418 virtual bool shouldRTTIBeUnique() const { return true; }
419
420public:
421 /// What sort of unique-RTTI behavior should we use?
422 enum RTTIUniquenessKind {
423 /// We are guaranteeing, or need to guarantee, that the RTTI string
424 /// is unique.
425 RUK_Unique,
426
427 /// We are not guaranteeing uniqueness for the RTTI string, so we
428 /// can demote to hidden visibility but must use string comparisons.
429 RUK_NonUniqueHidden,
430
431 /// We are not guaranteeing uniqueness for the RTTI string, so we
432 /// have to use string comparisons, but we also have to emit it with
433 /// non-hidden visibility.
434 RUK_NonUniqueVisible
435 };
436
437 /// Return the required visibility status for the given type and linkage in
438 /// the current ABI.
439 RTTIUniquenessKind
440 classifyRTTIUniqueness(QualType CanTy,
441 llvm::GlobalValue::LinkageTypes Linkage) const;
442 friend class ItaniumRTTIBuilder;
443
444 void emitCXXStructor(GlobalDecl GD) override;
445
446 std::pair<llvm::Value *, const CXXRecordDecl *>
447 LoadVTablePtr(CodeGenFunction &CGF, Address This,
448 const CXXRecordDecl *RD) override;
449
450 private:
451 llvm::Constant *
452 getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD);
453
454 bool hasAnyUnusedVirtualInlineFunction(const CXXRecordDecl *RD) const {
455 const auto &VtableLayout =
456 CGM.getItaniumVTableContext().getVTableLayout(RD);
457
458 for (const auto &VtableComponent : VtableLayout.vtable_components()) {
459 // Skip empty slot.
460 if (!VtableComponent.isUsedFunctionPointerKind())
461 continue;
462
463 const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
464 const FunctionDecl *FD = Method->getDefinition();
465 const bool IsInlined =
466 Method->getCanonicalDecl()->isInlined() || (FD && FD->isInlined());
467 if (!IsInlined)
468 continue;
469
470 StringRef Name = CGM.getMangledName(
471 GD: VtableComponent.getGlobalDecl(/*HasVectorDeletingDtors=*/false));
472 auto *Entry = CGM.GetGlobalValue(Ref: Name);
473 // This checks if virtual inline function has already been emitted.
474 // Note that it is possible that this inline function would be emitted
475 // after trying to emit vtable speculatively. Because of this we do
476 // an extra pass after emitting all deferred vtables to find and emit
477 // these vtables opportunistically.
478 if (!Entry || Entry->isDeclaration())
479 return true;
480 }
481 return false;
482 }
483
484 bool isVTableHidden(const CXXRecordDecl *RD) const {
485 const auto &VtableLayout =
486 CGM.getItaniumVTableContext().getVTableLayout(RD);
487
488 for (const auto &VtableComponent : VtableLayout.vtable_components()) {
489 if (VtableComponent.isRTTIKind()) {
490 const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl();
491 if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility)
492 return true;
493 } else if (VtableComponent.isUsedFunctionPointerKind()) {
494 const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
495 if (Method->getVisibility() == Visibility::HiddenVisibility &&
496 !Method->isDefined())
497 return true;
498 }
499 }
500 return false;
501 }
502};
503
504class ARMCXXABI : public ItaniumCXXABI {
505public:
506 ARMCXXABI(CodeGen::CodeGenModule &CGM) :
507 ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
508 /*UseARMGuardVarABI=*/true) {}
509
510 bool constructorsAndDestructorsReturnThis() const override { return true; }
511
512 void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV,
513 QualType ResTy) override;
514
515 CharUnits getArrayCookieSizeImpl(QualType elementType) override;
516 Address InitializeArrayCookie(CodeGenFunction &CGF,
517 Address NewPtr,
518 llvm::Value *NumElements,
519 const CXXNewExpr *expr,
520 QualType ElementType) override;
521 llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
522 CharUnits cookieSize) override;
523};
524
525class AppleARM64CXXABI : public ARMCXXABI {
526public:
527 AppleARM64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
528 Use32BitVTableOffsetABI = true;
529 }
530
531 // ARM64 libraries are prepared for non-unique RTTI.
532 bool shouldRTTIBeUnique() const override { return false; }
533};
534
535class FuchsiaCXXABI final : public ItaniumCXXABI {
536public:
537 explicit FuchsiaCXXABI(CodeGen::CodeGenModule &CGM)
538 : ItaniumCXXABI(CGM) {}
539
540private:
541 bool constructorsAndDestructorsReturnThis() const override { return true; }
542};
543
544class WebAssemblyCXXABI final : public ItaniumCXXABI {
545public:
546 explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM)
547 : ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
548 /*UseARMGuardVarABI=*/true) {}
549 void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
550 llvm::CallInst *
551 emitTerminateForUnexpectedException(CodeGenFunction &CGF,
552 llvm::Value *Exn) override;
553
554private:
555 bool constructorsAndDestructorsReturnThis() const override { return true; }
556 bool canCallMismatchedFunctionType() const override { return false; }
557};
558
559class XLCXXABI final : public ItaniumCXXABI {
560public:
561 explicit XLCXXABI(CodeGen::CodeGenModule &CGM)
562 : ItaniumCXXABI(CGM) {}
563
564 void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
565 llvm::FunctionCallee dtor,
566 llvm::Constant *addr) override;
567
568 bool useSinitAndSterm() const override { return true; }
569
570private:
571 void emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
572 llvm::Constant *addr);
573};
574}
575
576CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
577 switch (CGM.getContext().getCXXABIKind()) {
578 // For IR-generation purposes, there's no significant difference
579 // between the ARM and iOS ABIs.
580 case TargetCXXABI::GenericARM:
581 case TargetCXXABI::iOS:
582 case TargetCXXABI::WatchOS:
583 return new ARMCXXABI(CGM);
584
585 case TargetCXXABI::AppleARM64:
586 return new AppleARM64CXXABI(CGM);
587
588 case TargetCXXABI::Fuchsia:
589 return new FuchsiaCXXABI(CGM);
590
591 // Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
592 // include the other 32-bit ARM oddities: constructor/destructor return values
593 // and array cookies.
594 case TargetCXXABI::GenericAArch64:
595 return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
596 /*UseARMGuardVarABI=*/true);
597
598 case TargetCXXABI::GenericMIPS:
599 return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
600
601 case TargetCXXABI::WebAssembly:
602 return new WebAssemblyCXXABI(CGM);
603
604 case TargetCXXABI::XL:
605 return new XLCXXABI(CGM);
606
607 case TargetCXXABI::GenericItanium:
608 return new ItaniumCXXABI(CGM);
609
610 case TargetCXXABI::Microsoft:
611 llvm_unreachable("Microsoft ABI is not Itanium-based");
612 }
613 llvm_unreachable("bad ABI kind");
614}
615
616llvm::Type *
617ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
618 if (MPT->isMemberDataPointer())
619 return CGM.PtrDiffTy;
620 return llvm::StructType::get(elt1: CGM.PtrDiffTy, elts: CGM.PtrDiffTy);
621}
622
623/// In the Itanium and ARM ABIs, method pointers have the form:
624/// struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
625///
626/// In the Itanium ABI:
627/// - method pointers are virtual if (memptr.ptr & 1) is nonzero
628/// - the this-adjustment is (memptr.adj)
629/// - the virtual offset is (memptr.ptr - 1)
630///
631/// In the ARM ABI:
632/// - method pointers are virtual if (memptr.adj & 1) is nonzero
633/// - the this-adjustment is (memptr.adj >> 1)
634/// - the virtual offset is (memptr.ptr)
635/// ARM uses 'adj' for the virtual flag because Thumb functions
636/// may be only single-byte aligned.
637///
638/// If the member is virtual, the adjusted 'this' pointer points
639/// to a vtable pointer from which the virtual offset is applied.
640///
641/// If the member is non-virtual, memptr.ptr is the address of
642/// the function to call.
643CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
644 CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
645 llvm::Value *&ThisPtrForCall,
646 llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
647 CGBuilderTy &Builder = CGF.Builder;
648
649 const FunctionProtoType *FPT =
650 MPT->getPointeeType()->castAs<FunctionProtoType>();
651 auto *RD = MPT->getMostRecentCXXRecordDecl();
652
653 llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 1);
654
655 llvm::BasicBlock *FnVirtual = CGF.createBasicBlock(name: "memptr.virtual");
656 llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock(name: "memptr.nonvirtual");
657 llvm::BasicBlock *FnEnd = CGF.createBasicBlock(name: "memptr.end");
658
659 // Extract memptr.adj, which is in the second field.
660 llvm::Value *RawAdj = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 1, Name: "memptr.adj");
661
662 // Compute the true adjustment.
663 llvm::Value *Adj = RawAdj;
664 if (UseARMMethodPtrABI)
665 Adj = Builder.CreateAShr(LHS: Adj, RHS: ptrdiff_1, Name: "memptr.adj.shifted");
666
667 // Apply the adjustment and cast back to the original struct type
668 // for consistency.
669 llvm::Value *This = ThisAddr.emitRawPointer(CGF);
670 This = Builder.CreateInBoundsGEP(Ty: Builder.getInt8Ty(), Ptr: This, IdxList: Adj);
671 ThisPtrForCall = This;
672
673 // Load the function pointer.
674 llvm::Value *FnAsInt = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 0, Name: "memptr.ptr");
675
676 // If the LSB in the function pointer is 1, the function pointer points to
677 // a virtual function.
678 llvm::Value *IsVirtual;
679 if (UseARMMethodPtrABI)
680 IsVirtual = Builder.CreateAnd(LHS: RawAdj, RHS: ptrdiff_1);
681 else
682 IsVirtual = Builder.CreateAnd(LHS: FnAsInt, RHS: ptrdiff_1);
683 IsVirtual = Builder.CreateIsNotNull(Arg: IsVirtual, Name: "memptr.isvirtual");
684 Builder.CreateCondBr(Cond: IsVirtual, True: FnVirtual, False: FnNonVirtual);
685
686 // In the virtual path, the adjustment left 'This' pointing to the
687 // vtable of the correct base subobject. The "function pointer" is an
688 // offset within the vtable (+1 for the virtual flag on non-ARM).
689 CGF.EmitBlock(BB: FnVirtual);
690
691 // Cast the adjusted this to a pointer to vtable pointer and load.
692 llvm::Type *VTableTy = CGF.CGM.GlobalsInt8PtrTy;
693 CharUnits VTablePtrAlign =
694 CGF.CGM.getDynamicOffsetAlignment(ActualAlign: ThisAddr.getAlignment(), Class: RD,
695 ExpectedTargetAlign: CGF.getPointerAlign());
696 llvm::Value *VTable = CGF.GetVTablePtr(
697 This: Address(This, ThisAddr.getElementType(), VTablePtrAlign), VTableTy, VTableClass: RD);
698
699 // Apply the offset.
700 // On ARM64, to reserve extra space in virtual member function pointers,
701 // we only pay attention to the low 32 bits of the offset.
702 llvm::Value *VTableOffset = FnAsInt;
703 if (!UseARMMethodPtrABI)
704 VTableOffset = Builder.CreateSub(LHS: VTableOffset, RHS: ptrdiff_1);
705 if (Use32BitVTableOffsetABI) {
706 VTableOffset = Builder.CreateTrunc(V: VTableOffset, DestTy: CGF.Int32Ty);
707 VTableOffset = Builder.CreateZExt(V: VTableOffset, DestTy: CGM.PtrDiffTy);
708 }
709
710 // Check the address of the function pointer if CFI on member function
711 // pointers is enabled.
712 llvm::Constant *CheckSourceLocation;
713 llvm::Constant *CheckTypeDesc;
714 bool ShouldEmitCFICheck = CGF.SanOpts.has(K: SanitizerKind::CFIMFCall) &&
715 CGM.HasHiddenLTOVisibility(RD);
716
717 if (ShouldEmitCFICheck) {
718 if (const auto *BinOp = dyn_cast<BinaryOperator>(Val: E)) {
719 if (BinOp->isPtrMemOp() &&
720 BinOp->getRHS()
721 ->getType()
722 ->hasPointeeToCFIUncheckedCalleeFunctionType())
723 ShouldEmitCFICheck = false;
724 }
725 }
726
727 bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
728 CGM.HasHiddenLTOVisibility(RD);
729 // TODO: Update this name not to be restricted to WPD only
730 // as we now emit the vtable info info for speculative devirtualization as
731 // well.
732 bool ShouldEmitWPDInfo =
733 (CGM.getCodeGenOpts().WholeProgramVTables &&
734 // Don't insert type tests if we are forcing public visibility.
735 !CGM.AlwaysHasLTOVisibilityPublic(RD)) ||
736 CGM.getCodeGenOpts().DevirtualizeSpeculatively;
737 llvm::Value *VirtualFn = nullptr;
738
739 {
740 auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
741 auto CheckHandler = SanitizerHandler::CFICheckFail;
742 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
743
744 llvm::Value *TypeId = nullptr;
745 llvm::Value *CheckResult = nullptr;
746
747 if (ShouldEmitCFICheck || ShouldEmitVFEInfo || ShouldEmitWPDInfo) {
748 // If doing CFI, VFE or WPD, we will need the metadata node to check
749 // against.
750 llvm::Metadata *MD =
751 CGM.CreateMetadataIdentifierForVirtualMemPtrType(T: QualType(MPT, 0));
752 TypeId = llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
753 }
754
755 if (ShouldEmitVFEInfo) {
756 llvm::Value *VFPAddr =
757 Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
758
759 // If doing VFE, load from the vtable with a type.checked.load intrinsic
760 // call. Note that we use the GEP to calculate the address to load from
761 // and pass 0 as the offset to the intrinsic. This is because every
762 // vtable slot of the correct type is marked with matching metadata, and
763 // we know that the load must be from one of these slots.
764 llvm::Value *CheckedLoad = Builder.CreateCall(
765 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_checked_load),
766 Args: {VFPAddr, llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0), TypeId});
767 CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
768 VirtualFn = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0);
769 } else {
770 // When not doing VFE, emit a normal load, as it allows more
771 // optimisations than type.checked.load.
772 if (ShouldEmitCFICheck || ShouldEmitWPDInfo) {
773 llvm::Value *VFPAddr =
774 Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
775 llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
776 ? llvm::Intrinsic::type_test
777 : llvm::Intrinsic::public_type_test;
778
779 CheckResult =
780 Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VFPAddr, TypeId});
781 }
782
783 if (CGM.getLangOpts().RelativeCXXABIVTables) {
784 VirtualFn = CGF.Builder.CreateCall(
785 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative,
786 Tys: {VTableOffset->getType()}),
787 Args: {VTable, VTableOffset});
788 } else {
789 llvm::Value *VFPAddr =
790 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
791 VirtualFn = CGF.Builder.CreateAlignedLoad(Ty: CGF.DefaultPtrTy, Addr: VFPAddr,
792 Align: CGF.getPointerAlign(),
793 Name: "memptr.virtualfn");
794 }
795 }
796 assert(VirtualFn && "Virtual fuction pointer not created!");
797 assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || !ShouldEmitWPDInfo ||
798 CheckResult) &&
799 "Check result required but not created!");
800
801 if (ShouldEmitCFICheck) {
802 // If doing CFI, emit the check.
803 CheckSourceLocation = CGF.EmitCheckSourceLocation(Loc: E->getBeginLoc());
804 CheckTypeDesc = CGF.EmitCheckTypeDescriptor(T: QualType(MPT, 0));
805 llvm::Constant *StaticData[] = {
806 llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_VMFCall),
807 CheckSourceLocation,
808 CheckTypeDesc,
809 };
810
811 if (CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIMFCall)) {
812 CGF.EmitTrapCheck(Checked: CheckResult, CheckHandlerID: CheckHandler);
813 } else {
814 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
815 Context&: CGM.getLLVMContext(),
816 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
817 llvm::Value *ValidVtable = Builder.CreateCall(
818 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {VTable, AllVtables});
819 CGF.EmitCheck(Checked: std::make_pair(x&: CheckResult, y&: CheckOrdinal), Check: CheckHandler,
820 StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
821 }
822
823 FnVirtual = Builder.GetInsertBlock();
824 }
825 } // End of sanitizer scope
826
827 CGF.EmitBranch(Block: FnEnd);
828
829 // In the non-virtual path, the function pointer is actually a
830 // function pointer.
831 CGF.EmitBlock(BB: FnNonVirtual);
832 llvm::Value *NonVirtualFn =
833 Builder.CreateIntToPtr(V: FnAsInt, DestTy: CGF.DefaultPtrTy, Name: "memptr.nonvirtualfn");
834
835 // Check the function pointer if CFI on member function pointers is enabled.
836 if (ShouldEmitCFICheck) {
837 CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
838 if (RD->hasDefinition()) {
839 auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
840 auto CheckHandler = SanitizerHandler::CFICheckFail;
841 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
842
843 llvm::Constant *StaticData[] = {
844 llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_NVMFCall),
845 CheckSourceLocation,
846 CheckTypeDesc,
847 };
848
849 llvm::Value *Bit = Builder.getFalse();
850 for (const CXXRecordDecl *Base : CGM.getMostBaseClasses(RD)) {
851 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(
852 T: getContext().getMemberPointerType(T: MPT->getPointeeType(),
853 /*Qualifier=*/std::nullopt,
854 Cls: Base->getCanonicalDecl()));
855 llvm::Value *TypeId =
856 llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
857
858 llvm::Value *TypeTest =
859 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
860 Args: {NonVirtualFn, TypeId});
861 Bit = Builder.CreateOr(LHS: Bit, RHS: TypeTest);
862 }
863
864 CGF.EmitCheck(Checked: std::make_pair(x&: Bit, y&: CheckOrdinal), Check: CheckHandler, StaticArgs: StaticData,
865 DynamicArgs: {NonVirtualFn, llvm::UndefValue::get(T: CGF.IntPtrTy)});
866
867 FnNonVirtual = Builder.GetInsertBlock();
868 }
869 }
870
871 // We're done.
872 CGF.EmitBlock(BB: FnEnd);
873 llvm::PHINode *CalleePtr = Builder.CreatePHI(Ty: CGF.DefaultPtrTy, NumReservedValues: 2);
874 CalleePtr->addIncoming(V: VirtualFn, BB: FnVirtual);
875 CalleePtr->addIncoming(V: NonVirtualFn, BB: FnNonVirtual);
876
877 CGPointerAuthInfo PointerAuth;
878
879 if (const auto &Schema =
880 CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers) {
881 llvm::PHINode *DiscriminatorPHI = Builder.CreatePHI(Ty: CGF.IntPtrTy, NumReservedValues: 2);
882 DiscriminatorPHI->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
883 BB: FnVirtual);
884 const auto &AuthInfo =
885 CGM.getMemberFunctionPointerAuthInfo(FT: QualType(MPT, 0));
886 assert(Schema.getKey() == AuthInfo.getKey() &&
887 "Keys for virtual and non-virtual member functions must match");
888 auto *NonVirtualDiscriminator = AuthInfo.getDiscriminator();
889 DiscriminatorPHI->addIncoming(V: NonVirtualDiscriminator, BB: FnNonVirtual);
890 PointerAuth = CGPointerAuthInfo(
891 Schema.getKey(), Schema.getAuthenticationMode(), Schema.isIsaPointer(),
892 Schema.authenticatesNullValues(), DiscriminatorPHI);
893 }
894
895 CGCallee Callee(FPT, CalleePtr, PointerAuth);
896 return Callee;
897}
898
899/// Compute an l-value by applying the given pointer-to-member to a
900/// base object.
901llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
902 CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
903 const MemberPointerType *MPT, bool IsInBounds) {
904 assert(MemPtr->getType() == CGM.PtrDiffTy);
905
906 CGBuilderTy &Builder = CGF.Builder;
907
908 // Apply the offset.
909 llvm::Value *BaseAddr = Base.emitRawPointer(CGF);
910 return Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: BaseAddr, IdxList: MemPtr, Name: "memptr.offset",
911 NW: IsInBounds ? llvm::GEPNoWrapFlags::inBounds()
912 : llvm::GEPNoWrapFlags::none());
913}
914
915// See if it's possible to return a constant signed pointer.
916static llvm::Constant *pointerAuthResignConstant(
917 llvm::Value *Ptr, const CGPointerAuthInfo &CurAuthInfo,
918 const CGPointerAuthInfo &NewAuthInfo, CodeGenModule &CGM) {
919 const auto *CPA = dyn_cast<llvm::ConstantPtrAuth>(Val: Ptr);
920
921 if (!CPA)
922 return nullptr;
923
924 assert(CPA->getKey()->getZExtValue() == CurAuthInfo.getKey() &&
925 CPA->getAddrDiscriminator()->isNullValue() &&
926 CPA->getDiscriminator() == CurAuthInfo.getDiscriminator() &&
927 "unexpected key or discriminators");
928
929 return CGM.getConstantSignedPointer(
930 Pointer: CPA->getPointer(), Key: NewAuthInfo.getKey(), StorageAddress: nullptr,
931 OtherDiscriminator: cast<llvm::ConstantInt>(Val: NewAuthInfo.getDiscriminator()));
932}
933
934/// Perform a bitcast, derived-to-base, or base-to-derived member pointer
935/// conversion.
936///
937/// Bitcast conversions are always a no-op under Itanium.
938///
939/// Obligatory offset/adjustment diagram:
940/// <-- offset --> <-- adjustment -->
941/// |--------------------------|----------------------|--------------------|
942/// ^Derived address point ^Base address point ^Member address point
943///
944/// So when converting a base member pointer to a derived member pointer,
945/// we add the offset to the adjustment because the address point has
946/// decreased; and conversely, when converting a derived MP to a base MP
947/// we subtract the offset from the adjustment because the address point
948/// has increased.
949///
950/// The standard forbids (at compile time) conversion to and from
951/// virtual bases, which is why we don't have to consider them here.
952///
953/// The standard forbids (at run time) casting a derived MP to a base
954/// MP when the derived MP does not point to a member of the base.
955/// This is why -1 is a reasonable choice for null data member
956/// pointers.
957llvm::Value *
958ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
959 const CastExpr *E,
960 llvm::Value *src) {
961 // Use constant emission if we can.
962 if (isa<llvm::Constant>(Val: src))
963 return EmitMemberPointerConversion(E, Src: cast<llvm::Constant>(Val: src));
964
965 assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
966 E->getCastKind() == CK_BaseToDerivedMemberPointer ||
967 E->getCastKind() == CK_ReinterpretMemberPointer);
968
969 CGBuilderTy &Builder = CGF.Builder;
970 QualType DstType = E->getType();
971
972 if (DstType->isMemberFunctionPointerType()) {
973 if (const auto &NewAuthInfo =
974 CGM.getMemberFunctionPointerAuthInfo(FT: DstType)) {
975 QualType SrcType = E->getSubExpr()->getType();
976 assert(SrcType->isMemberFunctionPointerType());
977 const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: SrcType);
978 llvm::Value *MemFnPtr = Builder.CreateExtractValue(Agg: src, Idxs: 0, Name: "memptr.ptr");
979 llvm::Type *OrigTy = MemFnPtr->getType();
980
981 llvm::BasicBlock *StartBB = Builder.GetInsertBlock();
982 llvm::BasicBlock *ResignBB = CGF.createBasicBlock(name: "resign");
983 llvm::BasicBlock *MergeBB = CGF.createBasicBlock(name: "merge");
984
985 // Check whether we have a virtual offset or a pointer to a function.
986 assert(UseARMMethodPtrABI && "ARM ABI expected");
987 llvm::Value *Adj = Builder.CreateExtractValue(Agg: src, Idxs: 1, Name: "memptr.adj");
988 llvm::Constant *Ptrdiff_1 = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 1);
989 llvm::Value *AndVal = Builder.CreateAnd(LHS: Adj, RHS: Ptrdiff_1);
990 llvm::Value *IsVirtualOffset =
991 Builder.CreateIsNotNull(Arg: AndVal, Name: "is.virtual.offset");
992 Builder.CreateCondBr(Cond: IsVirtualOffset, True: MergeBB, False: ResignBB);
993
994 CGF.EmitBlock(BB: ResignBB);
995 llvm::Type *PtrTy = llvm::PointerType::getUnqual(C&: CGM.getLLVMContext());
996 MemFnPtr = Builder.CreateIntToPtr(V: MemFnPtr, DestTy: PtrTy);
997 MemFnPtr =
998 CGF.emitPointerAuthResign(Pointer: MemFnPtr, PointerType: SrcType, CurAuthInfo, NewAuthInfo,
999 IsKnownNonNull: isa<llvm::Constant>(Val: src));
1000 MemFnPtr = Builder.CreatePtrToInt(V: MemFnPtr, DestTy: OrigTy);
1001 llvm::Value *ResignedVal = Builder.CreateInsertValue(Agg: src, Val: MemFnPtr, Idxs: 0);
1002 ResignBB = Builder.GetInsertBlock();
1003
1004 CGF.EmitBlock(BB: MergeBB);
1005 llvm::PHINode *NewSrc = Builder.CreatePHI(Ty: src->getType(), NumReservedValues: 2);
1006 NewSrc->addIncoming(V: src, BB: StartBB);
1007 NewSrc->addIncoming(V: ResignedVal, BB: ResignBB);
1008 src = NewSrc;
1009 }
1010 }
1011
1012 // Under Itanium, reinterprets don't require any additional processing.
1013 if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
1014
1015 llvm::Constant *adj = getMemberPointerAdjustment(E);
1016 if (!adj) return src;
1017
1018 bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
1019
1020 const MemberPointerType *destTy =
1021 E->getType()->castAs<MemberPointerType>();
1022
1023 // For member data pointers, this is just a matter of adding the
1024 // offset if the source is non-null.
1025 if (destTy->isMemberDataPointer()) {
1026 llvm::Value *dst;
1027 if (isDerivedToBase)
1028 dst = Builder.CreateNSWSub(LHS: src, RHS: adj, Name: "adj");
1029 else
1030 dst = Builder.CreateNSWAdd(LHS: src, RHS: adj, Name: "adj");
1031
1032 // Null check.
1033 llvm::Value *null = llvm::Constant::getAllOnesValue(Ty: src->getType());
1034 llvm::Value *isNull = Builder.CreateICmpEQ(LHS: src, RHS: null, Name: "memptr.isnull");
1035 return Builder.CreateSelect(C: isNull, True: src, False: dst);
1036 }
1037
1038 // The this-adjustment is left-shifted by 1 on ARM.
1039 if (UseARMMethodPtrABI) {
1040 uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
1041 offset <<= 1;
1042 adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
1043 }
1044
1045 llvm::Value *srcAdj = Builder.CreateExtractValue(Agg: src, Idxs: 1, Name: "src.adj");
1046 llvm::Value *dstAdj;
1047 if (isDerivedToBase)
1048 dstAdj = Builder.CreateNSWSub(LHS: srcAdj, RHS: adj, Name: "adj");
1049 else
1050 dstAdj = Builder.CreateNSWAdd(LHS: srcAdj, RHS: adj, Name: "adj");
1051
1052 return Builder.CreateInsertValue(Agg: src, Val: dstAdj, Idxs: 1);
1053}
1054
1055static llvm::Constant *
1056pointerAuthResignMemberFunctionPointer(llvm::Constant *Src, QualType DestType,
1057 QualType SrcType, CodeGenModule &CGM) {
1058 assert(DestType->isMemberFunctionPointerType() &&
1059 SrcType->isMemberFunctionPointerType() &&
1060 "member function pointers expected");
1061 if (DestType == SrcType)
1062 return Src;
1063
1064 const auto &NewAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: DestType);
1065 const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(FT: SrcType);
1066
1067 if (!NewAuthInfo && !CurAuthInfo)
1068 return Src;
1069
1070 llvm::Constant *MemFnPtr = Src->getAggregateElement(Elt: 0u);
1071 if (MemFnPtr->getNumOperands() == 0) {
1072 // src must be a pair of null pointers.
1073 assert(isa<llvm::ConstantInt>(MemFnPtr) && "constant int expected");
1074 return Src;
1075 }
1076
1077 llvm::Constant *ConstPtr = pointerAuthResignConstant(
1078 Ptr: cast<llvm::User>(Val: MemFnPtr)->getOperand(i: 0), CurAuthInfo, NewAuthInfo, CGM);
1079 ConstPtr = llvm::ConstantExpr::getPtrToInt(C: ConstPtr, Ty: MemFnPtr->getType());
1080 return ConstantFoldInsertValueInstruction(Agg: Src, Val: ConstPtr, Idxs: 0);
1081}
1082
1083llvm::Constant *
1084ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
1085 llvm::Constant *src) {
1086 assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
1087 E->getCastKind() == CK_BaseToDerivedMemberPointer ||
1088 E->getCastKind() == CK_ReinterpretMemberPointer);
1089
1090 QualType DstType = E->getType();
1091
1092 if (DstType->isMemberFunctionPointerType())
1093 src = pointerAuthResignMemberFunctionPointer(
1094 Src: src, DestType: DstType, SrcType: E->getSubExpr()->getType(), CGM);
1095
1096 // Under Itanium, reinterprets don't require any additional processing.
1097 if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
1098
1099 // If the adjustment is trivial, we don't need to do anything.
1100 llvm::Constant *adj = getMemberPointerAdjustment(E);
1101 if (!adj) return src;
1102
1103 bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
1104
1105 const MemberPointerType *destTy =
1106 E->getType()->castAs<MemberPointerType>();
1107
1108 // For member data pointers, this is just a matter of adding the
1109 // offset if the source is non-null.
1110 if (destTy->isMemberDataPointer()) {
1111 // null maps to null.
1112 if (src->isAllOnesValue()) return src;
1113
1114 if (isDerivedToBase)
1115 return llvm::ConstantExpr::getNSWSub(C1: src, C2: adj);
1116 else
1117 return llvm::ConstantExpr::getNSWAdd(C1: src, C2: adj);
1118 }
1119
1120 // The this-adjustment is left-shifted by 1 on ARM.
1121 if (UseARMMethodPtrABI) {
1122 uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
1123 offset <<= 1;
1124 adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
1125 }
1126
1127 llvm::Constant *srcAdj = src->getAggregateElement(Elt: 1);
1128 llvm::Constant *dstAdj;
1129 if (isDerivedToBase)
1130 dstAdj = llvm::ConstantExpr::getNSWSub(C1: srcAdj, C2: adj);
1131 else
1132 dstAdj = llvm::ConstantExpr::getNSWAdd(C1: srcAdj, C2: adj);
1133
1134 llvm::Constant *res = ConstantFoldInsertValueInstruction(Agg: src, Val: dstAdj, Idxs: 1);
1135 assert(res != nullptr && "Folding must succeed");
1136 return res;
1137}
1138
1139llvm::Constant *
1140ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
1141 // Itanium C++ ABI 2.3:
1142 // A NULL pointer is represented as -1.
1143 if (MPT->isMemberDataPointer())
1144 return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: -1ULL, /*isSigned=*/IsSigned: true);
1145
1146 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 0);
1147 llvm::Constant *Values[2] = { Zero, Zero };
1148 return llvm::ConstantStruct::getAnon(V: Values);
1149}
1150
1151llvm::Constant *
1152ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
1153 CharUnits offset) {
1154 // Itanium C++ ABI 2.3:
1155 // A pointer to data member is an offset from the base address of
1156 // the class object containing it, represented as a ptrdiff_t
1157 return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: offset.getQuantity());
1158}
1159
1160llvm::Constant *
1161ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
1162 return BuildMemberPointer(MD, ThisAdjustment: CharUnits::Zero());
1163}
1164
1165llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD,
1166 CharUnits ThisAdjustment) {
1167 assert(MD->isInstance() && "Member function must not be static!");
1168
1169 CodeGenTypes &Types = CGM.getTypes();
1170
1171 // Get the function pointer (or index if this is a virtual function).
1172 llvm::Constant *MemPtr[2];
1173 if (MD->isVirtual()) {
1174 uint64_t Index = CGM.getItaniumVTableContext().getMethodVTableIndex(GD: MD);
1175 uint64_t VTableOffset;
1176 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1177 // Multiply by 4-byte relative offsets.
1178 VTableOffset = Index * 4;
1179 } else {
1180 const ASTContext &Context = getContext();
1181 CharUnits PointerWidth = Context.toCharUnitsFromBits(
1182 BitSize: Context.getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default));
1183 VTableOffset = Index * PointerWidth.getQuantity();
1184 }
1185
1186 if (UseARMMethodPtrABI) {
1187 // ARM C++ ABI 3.2.1:
1188 // This ABI specifies that adj contains twice the this
1189 // adjustment, plus 1 if the member function is virtual. The
1190 // least significant bit of adj then makes exactly the same
1191 // discrimination as the least significant bit of ptr does for
1192 // Itanium.
1193
1194 // We cannot use the Itanium ABI's representation for virtual member
1195 // function pointers under pointer authentication because it would
1196 // require us to store both the virtual offset and the constant
1197 // discriminator in the pointer, which would be immediately vulnerable
1198 // to attack. Instead we introduce a thunk that does the virtual dispatch
1199 // and store it as if it were a non-virtual member function. This means
1200 // that virtual function pointers may not compare equal anymore, but
1201 // fortunately they aren't required to by the standard, and we do make
1202 // a best-effort attempt to re-use the thunk.
1203 //
1204 // To support interoperation with code in which pointer authentication
1205 // is disabled, derefencing a member function pointer must still handle
1206 // the virtual case, but it can use a discriminator which should never
1207 // be valid.
1208 const auto &Schema =
1209 CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers;
1210 if (Schema)
1211 MemPtr[0] = llvm::ConstantExpr::getPtrToInt(
1212 C: getSignedVirtualMemberFunctionPointer(MD), Ty: CGM.PtrDiffTy);
1213 else
1214 MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset);
1215 // Don't set the LSB of adj to 1 if pointer authentication for member
1216 // function pointers is enabled.
1217 MemPtr[1] = llvm::ConstantInt::get(
1218 Ty: CGM.PtrDiffTy, V: 2 * ThisAdjustment.getQuantity() + !Schema);
1219 } else {
1220 // Itanium C++ ABI 2.3:
1221 // For a virtual function, [the pointer field] is 1 plus the
1222 // virtual table offset (in bytes) of the function,
1223 // represented as a ptrdiff_t.
1224 MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset + 1);
1225 MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1226 V: ThisAdjustment.getQuantity());
1227 }
1228 } else {
1229 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1230 llvm::Type *Ty;
1231 // Check whether the function has a computable LLVM signature.
1232 if (Types.isFuncTypeConvertible(FT: FPT)) {
1233 // The function has a computable LLVM signature; use the correct type.
1234 Ty = Types.GetFunctionType(Info: Types.arrangeCXXMethodDeclaration(MD));
1235 } else {
1236 // Use an arbitrary non-function type to tell GetAddrOfFunction that the
1237 // function type is incomplete.
1238 Ty = CGM.PtrDiffTy;
1239 }
1240 llvm::Constant *addr = CGM.getMemberFunctionPointer(FD: MD, Ty);
1241
1242 MemPtr[0] = llvm::ConstantExpr::getPtrToInt(C: addr, Ty: CGM.PtrDiffTy);
1243 MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1244 V: (UseARMMethodPtrABI ? 2 : 1) *
1245 ThisAdjustment.getQuantity());
1246 }
1247
1248 return llvm::ConstantStruct::getAnon(V: MemPtr);
1249}
1250
1251llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
1252 QualType MPType) {
1253 const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
1254 const ValueDecl *MPD = MP.getMemberPointerDecl();
1255 if (!MPD)
1256 return EmitNullMemberPointer(MPT);
1257
1258 CharUnits ThisAdjustment = getContext().getMemberPointerPathAdjustment(MP);
1259
1260 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MPD)) {
1261 llvm::Constant *Src = BuildMemberPointer(MD, ThisAdjustment);
1262 QualType SrcType = getContext().getMemberPointerType(
1263 T: MD->getType(), /*Qualifier=*/std::nullopt, Cls: MD->getParent());
1264 return pointerAuthResignMemberFunctionPointer(Src, DestType: MPType, SrcType, CGM);
1265 }
1266
1267 getContext().recordMemberDataPointerEvaluation(VD: MPD);
1268 CharUnits FieldOffset =
1269 getContext().toCharUnitsFromBits(BitSize: getContext().getFieldOffset(FD: MPD));
1270 return EmitMemberDataPointer(MPT, offset: ThisAdjustment + FieldOffset);
1271}
1272
1273/// The comparison algorithm is pretty easy: the member pointers are
1274/// the same if they're either bitwise identical *or* both null.
1275///
1276/// ARM is different here only because null-ness is more complicated.
1277llvm::Value *
1278ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
1279 llvm::Value *L,
1280 llvm::Value *R,
1281 const MemberPointerType *MPT,
1282 bool Inequality) {
1283 CGBuilderTy &Builder = CGF.Builder;
1284
1285 llvm::ICmpInst::Predicate Eq;
1286 llvm::Instruction::BinaryOps And, Or;
1287 if (Inequality) {
1288 Eq = llvm::ICmpInst::ICMP_NE;
1289 And = llvm::Instruction::Or;
1290 Or = llvm::Instruction::And;
1291 } else {
1292 Eq = llvm::ICmpInst::ICMP_EQ;
1293 And = llvm::Instruction::And;
1294 Or = llvm::Instruction::Or;
1295 }
1296
1297 // Member data pointers are easy because there's a unique null
1298 // value, so it just comes down to bitwise equality.
1299 if (MPT->isMemberDataPointer())
1300 return Builder.CreateICmp(P: Eq, LHS: L, RHS: R);
1301
1302 // For member function pointers, the tautologies are more complex.
1303 // The Itanium tautology is:
1304 // (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
1305 // The ARM tautology is:
1306 // (L == R) <==> (L.ptr == R.ptr &&
1307 // (L.adj == R.adj ||
1308 // (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
1309 // The inequality tautologies have exactly the same structure, except
1310 // applying De Morgan's laws.
1311
1312 llvm::Value *LPtr = Builder.CreateExtractValue(Agg: L, Idxs: 0, Name: "lhs.memptr.ptr");
1313 llvm::Value *RPtr = Builder.CreateExtractValue(Agg: R, Idxs: 0, Name: "rhs.memptr.ptr");
1314
1315 // This condition tests whether L.ptr == R.ptr. This must always be
1316 // true for equality to hold.
1317 llvm::Value *PtrEq = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: RPtr, Name: "cmp.ptr");
1318
1319 // This condition, together with the assumption that L.ptr == R.ptr,
1320 // tests whether the pointers are both null. ARM imposes an extra
1321 // condition.
1322 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: LPtr->getType());
1323 llvm::Value *EqZero = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: Zero, Name: "cmp.ptr.null");
1324
1325 // This condition tests whether L.adj == R.adj. If this isn't
1326 // true, the pointers are unequal unless they're both null.
1327 llvm::Value *LAdj = Builder.CreateExtractValue(Agg: L, Idxs: 1, Name: "lhs.memptr.adj");
1328 llvm::Value *RAdj = Builder.CreateExtractValue(Agg: R, Idxs: 1, Name: "rhs.memptr.adj");
1329 llvm::Value *AdjEq = Builder.CreateICmp(P: Eq, LHS: LAdj, RHS: RAdj, Name: "cmp.adj");
1330
1331 // Null member function pointers on ARM clear the low bit of Adj,
1332 // so the zero condition has to check that neither low bit is set.
1333 if (UseARMMethodPtrABI) {
1334 llvm::Value *One = llvm::ConstantInt::get(Ty: LPtr->getType(), V: 1);
1335
1336 // Compute (l.adj | r.adj) & 1 and test it against zero.
1337 llvm::Value *OrAdj = Builder.CreateOr(LHS: LAdj, RHS: RAdj, Name: "or.adj");
1338 llvm::Value *OrAdjAnd1 = Builder.CreateAnd(LHS: OrAdj, RHS: One);
1339 llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(P: Eq, LHS: OrAdjAnd1, RHS: Zero,
1340 Name: "cmp.or.adj");
1341 EqZero = Builder.CreateBinOp(Opc: And, LHS: EqZero, RHS: OrAdjAnd1EqZero);
1342 }
1343
1344 // Tie together all our conditions.
1345 llvm::Value *Result = Builder.CreateBinOp(Opc: Or, LHS: EqZero, RHS: AdjEq);
1346 Result = Builder.CreateBinOp(Opc: And, LHS: PtrEq, RHS: Result,
1347 Name: Inequality ? "memptr.ne" : "memptr.eq");
1348 return Result;
1349}
1350
1351llvm::Value *
1352ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
1353 llvm::Value *MemPtr,
1354 const MemberPointerType *MPT) {
1355 CGBuilderTy &Builder = CGF.Builder;
1356
1357 /// For member data pointers, this is just a check against -1.
1358 if (MPT->isMemberDataPointer()) {
1359 assert(MemPtr->getType() == CGM.PtrDiffTy);
1360 llvm::Value *NegativeOne =
1361 llvm::Constant::getAllOnesValue(Ty: MemPtr->getType());
1362 return Builder.CreateICmpNE(LHS: MemPtr, RHS: NegativeOne, Name: "memptr.tobool");
1363 }
1364
1365 // In Itanium, a member function pointer is not null if 'ptr' is not null.
1366 llvm::Value *Ptr = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 0, Name: "memptr.ptr");
1367
1368 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 0);
1369 llvm::Value *Result = Builder.CreateICmpNE(LHS: Ptr, RHS: Zero, Name: "memptr.tobool");
1370
1371 // On ARM, a member function pointer is also non-null if the low bit of 'adj'
1372 // (the virtual bit) is set.
1373 if (UseARMMethodPtrABI) {
1374 llvm::Constant *One = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 1);
1375 llvm::Value *Adj = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 1, Name: "memptr.adj");
1376 llvm::Value *VirtualBit = Builder.CreateAnd(LHS: Adj, RHS: One, Name: "memptr.virtualbit");
1377 llvm::Value *IsVirtual = Builder.CreateICmpNE(LHS: VirtualBit, RHS: Zero,
1378 Name: "memptr.isvirtual");
1379 Result = Builder.CreateOr(LHS: Result, RHS: IsVirtual);
1380 }
1381
1382 return Result;
1383}
1384
1385bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
1386 const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
1387 if (!RD)
1388 return false;
1389
1390 // If C++ prohibits us from making a copy, return by address using the target
1391 // hook getSRetAddrSpace to decide the AS.
1392 if (!RD->canPassInRegisters()) {
1393 auto Align = CGM.getContext().getTypeAlignInChars(T: FI.getReturnType());
1394 LangAS SRetAS = CGM.getTargetCodeGenInfo().getSRetAddrSpace(RD);
1395 unsigned AS = CGM.getContext().getTargetAddressSpace(AS: SRetAS);
1396 FI.getReturnInfo() =
1397 ABIArgInfo::getIndirect(Alignment: Align, /*AddrSpace=*/AS, /*ByVal=*/false);
1398 return true;
1399 }
1400 return false;
1401}
1402
1403/// The Itanium ABI requires non-zero initialization only for data
1404/// member pointers, for which '0' is a valid offset.
1405bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
1406 return MPT->isMemberFunctionPointer();
1407}
1408
1409/// The Itanium ABI always places an offset to the complete object
1410/// at entry -2 in the vtable.
1411void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
1412 const CXXDeleteExpr *DE,
1413 Address Ptr,
1414 QualType ElementType,
1415 const CXXDestructorDecl *Dtor) {
1416 bool UseGlobalDelete = DE->isGlobalDelete();
1417 if (UseGlobalDelete) {
1418 // Derive the complete-object pointer, which is what we need
1419 // to pass to the deallocation function.
1420
1421 // Grab the vtable pointer as an intptr_t*.
1422 auto *ClassDecl = ElementType->castAsCXXRecordDecl();
1423 llvm::Value *VTable = CGF.GetVTablePtr(This: Ptr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1424
1425 // Track back to entry -2 and pull out the offset there.
1426 llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
1427 Ty: CGF.IntPtrTy, Ptr: VTable, Idx0: -2, Name: "complete-offset.ptr");
1428 llvm::Value *Offset = CGF.Builder.CreateAlignedLoad(Ty: CGF.IntPtrTy, Addr: OffsetPtr,
1429 Align: CGF.getPointerAlign());
1430
1431 // Apply the offset.
1432 llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
1433 CompletePtr =
1434 CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: CompletePtr, IdxList: Offset);
1435
1436 // If we're supposed to call the global delete, make sure we do so
1437 // even if the destructor throws.
1438 CGF.pushCallObjectDeleteCleanup(OperatorDelete: DE->getOperatorDelete(), CompletePtr,
1439 ElementType);
1440 }
1441
1442 // FIXME: Provide a source location here even though there's no
1443 // CXXMemberCallExpr for dtor call.
1444 CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
1445 EmitVirtualDestructorCall(CGF, Dtor, DtorType, This: Ptr, E: DE,
1446 /*CallOrInvoke=*/nullptr);
1447
1448 if (UseGlobalDelete)
1449 CGF.PopCleanupBlock();
1450}
1451
1452void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
1453 // void __cxa_rethrow();
1454
1455 llvm::FunctionType *FTy =
1456 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
1457
1458 llvm::FunctionCallee Fn = CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_rethrow");
1459
1460 if (isNoReturn)
1461 CGF.EmitNoreturnRuntimeCallOrInvoke(callee: Fn, args: {});
1462 else
1463 CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1464}
1465
1466static llvm::FunctionCallee getAllocateExceptionFn(CodeGenModule &CGM) {
1467 // void *__cxa_allocate_exception(size_t thrown_size);
1468
1469 llvm::FunctionType *FTy =
1470 llvm::FunctionType::get(Result: CGM.Int8PtrTy, Params: CGM.SizeTy, /*isVarArg=*/false);
1471
1472 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_allocate_exception");
1473}
1474
1475static llvm::FunctionCallee getThrowFn(CodeGenModule &CGM) {
1476 // void __cxa_throw(void *thrown_exception, std::type_info *tinfo,
1477 // void (*dest) (void *));
1478
1479 llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.GlobalsInt8PtrTy, CGM.Int8PtrTy };
1480 llvm::FunctionType *FTy =
1481 llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, /*isVarArg=*/false);
1482
1483 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_throw");
1484}
1485
1486void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
1487 QualType ThrowType = E->getSubExpr()->getType();
1488 // Now allocate the exception object.
1489 llvm::Type *SizeTy = CGF.ConvertType(T: getContext().getSizeType());
1490 uint64_t TypeSize = getContext().getTypeSizeInChars(T: ThrowType).getQuantity();
1491
1492 llvm::FunctionCallee AllocExceptionFn = getAllocateExceptionFn(CGM);
1493 llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
1494 callee: AllocExceptionFn, args: llvm::ConstantInt::get(Ty: SizeTy, V: TypeSize), name: "exception");
1495
1496 CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment();
1497 CGF.EmitAnyExprToExn(
1498 E: E->getSubExpr(), Addr: Address(ExceptionPtr, CGM.Int8Ty, ExnAlign));
1499
1500 // Now throw the exception.
1501 llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(Ty: ThrowType,
1502 /*ForEH=*/true);
1503
1504 // The address of the destructor. If the exception type has a
1505 // trivial destructor (or isn't a record), we just pass null.
1506 llvm::Constant *Dtor = nullptr;
1507 if (const auto *Record = ThrowType->getAsCXXRecordDecl();
1508 Record && !Record->hasTrivialDestructor()) {
1509 // __cxa_throw is declared to take its destructor as void (*)(void *). We
1510 // must match that if function pointers can be authenticated with a
1511 // discriminator based on their type.
1512 const ASTContext &Ctx = getContext();
1513 QualType DtorTy = Ctx.getFunctionType(ResultTy: Ctx.VoidTy, Args: {Ctx.VoidPtrTy},
1514 EPI: FunctionProtoType::ExtProtoInfo());
1515
1516 CXXDestructorDecl *DtorD = Record->getDestructor();
1517 Dtor = CGM.getAddrOfCXXStructor(GD: GlobalDecl(DtorD, Dtor_Complete));
1518 Dtor = CGM.getFunctionPointer(Pointer: Dtor, FunctionType: DtorTy);
1519 }
1520 if (!Dtor) Dtor = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
1521
1522 llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor };
1523 CGF.EmitNoreturnRuntimeCallOrInvoke(callee: getThrowFn(CGM), args);
1524}
1525
1526static llvm::FunctionCallee getItaniumDynamicCastFn(CodeGenFunction &CGF) {
1527 // void *__dynamic_cast(const void *sub,
1528 // global_as const abi::__class_type_info *src,
1529 // global_as const abi::__class_type_info *dst,
1530 // std::ptrdiff_t src2dst_offset);
1531
1532 llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
1533 llvm::Type *GlobInt8PtrTy = CGF.GlobalsInt8PtrTy;
1534 llvm::Type *PtrDiffTy =
1535 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1536
1537 llvm::Type *Args[4] = { Int8PtrTy, GlobInt8PtrTy, GlobInt8PtrTy, PtrDiffTy };
1538
1539 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int8PtrTy, Params: Args, isVarArg: false);
1540
1541 // Mark the function as nounwind willreturn readonly.
1542 llvm::AttrBuilder FuncAttrs(CGF.getLLVMContext());
1543 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1544 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
1545 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
1546 llvm::AttributeList Attrs = llvm::AttributeList::get(
1547 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: FuncAttrs);
1548
1549 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__dynamic_cast", ExtraAttrs: Attrs);
1550}
1551
1552static llvm::FunctionCallee getBadCastFn(CodeGenFunction &CGF) {
1553 // void __cxa_bad_cast();
1554 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1555 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_cast");
1556}
1557
1558static llvm::FunctionCallee getBadTypeidFn(CodeGenFunction &CGF) {
1559 // void __cxa_bad_typeid();
1560 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1561
1562 return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_typeid");
1563}
1564
1565bool ItaniumCXXABI::shouldTypeidBeNullChecked(QualType SrcRecordTy) {
1566 return true;
1567}
1568
1569void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
1570 llvm::FunctionCallee Fn = getBadTypeidFn(CGF);
1571 llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1572 Call->setDoesNotReturn();
1573 CGF.Builder.CreateUnreachable();
1574}
1575
1576llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
1577 QualType SrcRecordTy,
1578 Address ThisPtr,
1579 llvm::Type *StdTypeInfoPtrTy) {
1580 auto *ClassDecl = SrcRecordTy->castAsCXXRecordDecl();
1581 llvm::Value *Value = CGF.GetVTablePtr(This: ThisPtr, VTableTy: CGM.GlobalsInt8PtrTy,
1582 VTableClass: ClassDecl);
1583
1584 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1585 // Load the type info.
1586 Value = CGF.Builder.CreateCall(
1587 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative, Tys: {CGM.Int32Ty}),
1588 Args: {Value, llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: -4)});
1589 } else {
1590 // Load the type info.
1591 Value =
1592 CGF.Builder.CreateConstInBoundsGEP1_64(Ty: StdTypeInfoPtrTy, Ptr: Value, Idx0: -1ULL);
1593 }
1594 return CGF.Builder.CreateAlignedLoad(Ty: StdTypeInfoPtrTy, Addr: Value,
1595 Align: CGF.getPointerAlign());
1596}
1597
1598bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1599 QualType SrcRecordTy) {
1600 return SrcIsPtr;
1601}
1602
1603llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
1604 CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1605 QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1606 llvm::Type *PtrDiffLTy =
1607 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1608
1609 llvm::Value *SrcRTTI =
1610 CGF.CGM.GetAddrOfRTTIDescriptor(Ty: SrcRecordTy.getUnqualifiedType());
1611 llvm::Value *DestRTTI =
1612 CGF.CGM.GetAddrOfRTTIDescriptor(Ty: DestRecordTy.getUnqualifiedType());
1613
1614 // Compute the offset hint.
1615 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1616 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1617 llvm::Value *OffsetHint = llvm::ConstantInt::getSigned(
1618 Ty: PtrDiffLTy,
1619 V: CodeGenUtils::computeOffsetHint(Ctx&: CGF.getContext(), Src: SrcDecl, Dst: DestDecl)
1620 .getQuantity());
1621
1622 // Emit the call to __dynamic_cast.
1623 llvm::Value *Value = ThisAddr.emitRawPointer(CGF);
1624 if (CGM.getCodeGenOpts().PointerAuth.CXXVTablePointers) {
1625 // We perform a no-op load of the vtable pointer here to force an
1626 // authentication. In environments that do not support pointer
1627 // authentication this is a an actual no-op that will be elided. When
1628 // pointer authentication is supported and enforced on vtable pointers this
1629 // load can trap.
1630 llvm::Value *Vtable =
1631 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGM.Int8PtrTy, VTableClass: SrcDecl,
1632 AuthMode: CodeGenFunction::VTableAuthMode::MustTrap);
1633 assert(Vtable);
1634 (void)Vtable;
1635 }
1636
1637 llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
1638 Value = CGF.EmitNounwindRuntimeCall(callee: getItaniumDynamicCastFn(CGF), args);
1639
1640 /// C++ [expr.dynamic.cast]p9:
1641 /// A failed cast to reference type throws std::bad_cast
1642 if (DestTy->isReferenceType()) {
1643 llvm::BasicBlock *BadCastBlock =
1644 CGF.createBasicBlock(name: "dynamic_cast.bad_cast");
1645
1646 llvm::Value *IsNull = CGF.Builder.CreateIsNull(Arg: Value);
1647 CGF.Builder.CreateCondBr(Cond: IsNull, True: BadCastBlock, False: CastEnd);
1648
1649 CGF.EmitBlock(BB: BadCastBlock);
1650 EmitBadCastCall(CGF);
1651 }
1652
1653 return Value;
1654}
1655
1656std::optional<CGCXXABI::ExactDynamicCastInfo>
1657ItaniumCXXABI::getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
1658 QualType DestRecordTy) {
1659 assert(shouldEmitExactDynamicCast(DestRecordTy));
1660
1661 ASTContext &Context = getContext();
1662
1663 // Find all the inheritance paths.
1664 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1665 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1666 CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
1667 /*DetectVirtual=*/false);
1668 (void)DestDecl->isDerivedFrom(Base: SrcDecl, Paths);
1669
1670 // Find an offset within `DestDecl` where a `SrcDecl` instance and its vptr
1671 // might appear.
1672 std::optional<CharUnits> Offset;
1673 for (const CXXBasePath &Path : Paths) {
1674 // dynamic_cast only finds public inheritance paths.
1675 if (Path.Access != AS_public)
1676 continue;
1677
1678 CharUnits PathOffset;
1679 for (const CXXBasePathElement &PathElement : Path) {
1680 // Find the offset along this inheritance step.
1681 const CXXRecordDecl *Base =
1682 PathElement.Base->getType()->getAsCXXRecordDecl();
1683 if (PathElement.Base->isVirtual()) {
1684 // For a virtual base class, we know that the derived class is exactly
1685 // DestDecl, so we can use the vbase offset from its layout.
1686 const ASTRecordLayout &L = Context.getASTRecordLayout(D: DestDecl);
1687 PathOffset = L.getVBaseClassOffset(VBase: Base);
1688 } else {
1689 const ASTRecordLayout &L =
1690 Context.getASTRecordLayout(D: PathElement.Class);
1691 PathOffset += L.getBaseClassOffset(Base);
1692 }
1693 }
1694
1695 if (!Offset)
1696 Offset = PathOffset;
1697 else if (Offset != PathOffset) {
1698 // Base appears in at least two different places.
1699 return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/true,
1700 .Offset: CharUnits::Zero()};
1701 }
1702 }
1703 if (!Offset)
1704 return std::nullopt;
1705 return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/false, .Offset: *Offset};
1706}
1707
1708llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
1709 CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1710 QualType DestTy, QualType DestRecordTy,
1711 const ExactDynamicCastInfo &ExactCastInfo, llvm::BasicBlock *CastSuccess,
1712 llvm::BasicBlock *CastFail) {
1713 const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1714 const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1715 auto AuthenticateVTable = [&](Address ThisAddr, const CXXRecordDecl *Decl) {
1716 if (!CGF.getLangOpts().PointerAuthCalls)
1717 return;
1718 (void)CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: Decl,
1719 AuthMode: CodeGenFunction::VTableAuthMode::MustTrap);
1720 };
1721
1722 bool PerformPostCastAuthentication = false;
1723 llvm::Value *VTable = nullptr;
1724 if (ExactCastInfo.RequiresCastToPrimaryBase) {
1725 // Base appears in at least two different places. Find the most-derived
1726 // object and see if it's a DestDecl. Note that the most-derived object
1727 // must be at least as aligned as this base class subobject, and must
1728 // have a vptr at offset 0.
1729 llvm::Value *PrimaryBase =
1730 emitDynamicCastToVoid(CGF, Value: ThisAddr, SrcRecordTy);
1731 ThisAddr = Address(PrimaryBase, CGF.VoidPtrTy, ThisAddr.getAlignment());
1732 SrcDecl = DestDecl;
1733 // This unauthenticated load is unavoidable, so we're relying on the
1734 // authenticated load in the dynamic cast to void, and we'll manually
1735 // authenticate the resulting v-table at the end of the cast check.
1736 PerformPostCastAuthentication = CGF.getLangOpts().PointerAuthCalls;
1737 CGPointerAuthInfo StrippingAuthInfo(0, PointerAuthenticationMode::Strip,
1738 false, false, nullptr);
1739 Address VTablePtrPtr = ThisAddr.withElementType(ElemTy: CGM.GlobalsInt8PtrTy);
1740 VTable = CGF.Builder.CreateLoad(Addr: VTablePtrPtr, Name: "vtable");
1741 if (PerformPostCastAuthentication)
1742 VTable = CGF.EmitPointerAuthAuth(Info: StrippingAuthInfo, Pointer: VTable);
1743 } else
1744 VTable = CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGM.GlobalsInt8PtrTy, VTableClass: SrcDecl);
1745
1746 // Compare the vptr against the expected vptr for the destination type at
1747 // this offset.
1748 llvm::Constant *ExpectedVTable = getVTableAddressPoint(
1749 Base: BaseSubobject(SrcDecl, ExactCastInfo.Offset), VTableClass: DestDecl);
1750 llvm::Value *Success = CGF.Builder.CreateICmpEQ(LHS: VTable, RHS: ExpectedVTable);
1751 llvm::Value *AdjustedThisPtr = ThisAddr.emitRawPointer(CGF);
1752
1753 if (!ExactCastInfo.Offset.isZero()) {
1754 CharUnits::QuantityType Offset = ExactCastInfo.Offset.getQuantity();
1755 llvm::Constant *OffsetConstant =
1756 llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: -Offset);
1757 AdjustedThisPtr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.CharTy, Ptr: AdjustedThisPtr,
1758 IdxList: OffsetConstant);
1759 PerformPostCastAuthentication = CGF.getLangOpts().PointerAuthCalls;
1760 }
1761
1762 if (PerformPostCastAuthentication) {
1763 // If we've changed the object pointer we authenticate the vtable pointer
1764 // of the resulting object.
1765 llvm::BasicBlock *NonNullBlock = CGF.Builder.GetInsertBlock();
1766 llvm::BasicBlock *PostCastAuthSuccess =
1767 CGF.createBasicBlock(name: "dynamic_cast.postauth.success");
1768 llvm::BasicBlock *PostCastAuthComplete =
1769 CGF.createBasicBlock(name: "dynamic_cast.postauth.complete");
1770 CGF.Builder.CreateCondBr(Cond: Success, True: PostCastAuthSuccess,
1771 False: PostCastAuthComplete);
1772 CGF.EmitBlock(BB: PostCastAuthSuccess);
1773 Address AdjustedThisAddr =
1774 Address(AdjustedThisPtr, CGF.IntPtrTy, CGF.getPointerAlign());
1775 AuthenticateVTable(AdjustedThisAddr, DestDecl);
1776 CGF.EmitBranch(Block: PostCastAuthComplete);
1777 CGF.EmitBlock(BB: PostCastAuthComplete);
1778 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: AdjustedThisPtr->getType(), NumReservedValues: 2);
1779 PHI->addIncoming(V: AdjustedThisPtr, BB: PostCastAuthSuccess);
1780 llvm::Value *NullValue =
1781 llvm::Constant::getNullValue(Ty: AdjustedThisPtr->getType());
1782 PHI->addIncoming(V: NullValue, BB: NonNullBlock);
1783 AdjustedThisPtr = PHI;
1784 }
1785 CGF.Builder.CreateCondBr(Cond: Success, True: CastSuccess, False: CastFail);
1786 return AdjustedThisPtr;
1787}
1788
1789llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
1790 Address ThisAddr,
1791 QualType SrcRecordTy) {
1792 auto *ClassDecl = SrcRecordTy->castAsCXXRecordDecl();
1793 llvm::Value *OffsetToTop;
1794 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1795 // Get the vtable pointer.
1796 llvm::Value *VTable =
1797 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1798
1799 // Get the offset-to-top from the vtable.
1800 OffsetToTop =
1801 CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGM.Int32Ty, Ptr: VTable, Idx0: -2U);
1802 OffsetToTop = CGF.Builder.CreateAlignedLoad(
1803 Ty: CGM.Int32Ty, Addr: OffsetToTop, Align: CharUnits::fromQuantity(Quantity: 4), Name: "offset.to.top");
1804 } else {
1805 llvm::Type *PtrDiffLTy =
1806 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1807
1808 // Get the vtable pointer.
1809 llvm::Value *VTable =
1810 CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.DefaultPtrTy, VTableClass: ClassDecl);
1811
1812 // Get the offset-to-top from the vtable.
1813 OffsetToTop =
1814 CGF.Builder.CreateConstInBoundsGEP1_64(Ty: PtrDiffLTy, Ptr: VTable, Idx0: -2ULL);
1815 OffsetToTop = CGF.Builder.CreateAlignedLoad(
1816 Ty: PtrDiffLTy, Addr: OffsetToTop, Align: CGF.getPointerAlign(), Name: "offset.to.top");
1817 }
1818 // Finally, add the offset to the pointer.
1819 return CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ThisAddr.emitRawPointer(CGF),
1820 IdxList: OffsetToTop);
1821}
1822
1823bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1824 llvm::FunctionCallee Fn = getBadCastFn(CGF);
1825 llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1826 Call->setDoesNotReturn();
1827 CGF.Builder.CreateUnreachable();
1828 return true;
1829}
1830
1831llvm::Value *
1832ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
1833 Address This,
1834 const CXXRecordDecl *ClassDecl,
1835 const CXXRecordDecl *BaseClassDecl) {
1836 llvm::Value *VTablePtr = CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: ClassDecl);
1837 CharUnits VBaseOffsetOffset =
1838 CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD: ClassDecl,
1839 VBase: BaseClassDecl);
1840 llvm::Value *VBaseOffsetPtr =
1841 CGF.Builder.CreateConstGEP1_64(
1842 Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VBaseOffsetOffset.getQuantity(),
1843 Name: "vbase.offset.ptr");
1844
1845 llvm::Value *VBaseOffset;
1846 if (CGM.getLangOpts().RelativeCXXABIVTables) {
1847 VBaseOffset = CGF.Builder.CreateAlignedLoad(
1848 Ty: CGF.Int32Ty, Addr: VBaseOffsetPtr, Align: CharUnits::fromQuantity(Quantity: 4),
1849 Name: "vbase.offset");
1850 } else {
1851 VBaseOffset = CGF.Builder.CreateAlignedLoad(
1852 Ty: CGM.PtrDiffTy, Addr: VBaseOffsetPtr, Align: CGF.getPointerAlign(), Name: "vbase.offset");
1853 }
1854 return VBaseOffset;
1855}
1856
1857void ItaniumCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1858 // Just make sure we're in sync with TargetCXXABI.
1859 assert(CGM.getTarget().getCXXABI().hasConstructorVariants());
1860
1861 // The constructor used for constructing this as a base class;
1862 // ignores virtual bases.
1863 CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Base));
1864
1865 // The constructor used for constructing this as a complete class;
1866 // constructs the virtual bases, then calls the base constructor.
1867 if (!D->getParent()->isAbstract()) {
1868 // We don't need to emit the complete ctor if the class is abstract.
1869 CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Complete));
1870 }
1871}
1872
1873CGCXXABI::AddedStructorArgCounts
1874ItaniumCXXABI::buildStructorSignature(GlobalDecl GD,
1875 SmallVectorImpl<CanQualType> &ArgTys) {
1876 ASTContext &Context = getContext();
1877
1878 // All parameters are already in place except VTT, which goes after 'this'.
1879 // These are Clang types, so we don't need to worry about sret yet.
1880
1881 // Check if we need to add a VTT parameter (which has type global void **).
1882 if ((isa<CXXConstructorDecl>(Val: GD.getDecl()) ? GD.getCtorType() == Ctor_Base
1883 : GD.getDtorType() == Dtor_Base) &&
1884 cast<CXXMethodDecl>(Val: GD.getDecl())->getParent()->getNumVBases() != 0) {
1885 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1886 QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1887 ArgTys.insert(I: ArgTys.begin() + 1,
1888 Elt: Context.getPointerType(T: CanQualType::CreateUnsafe(Other: Q)));
1889 return AddedStructorArgCounts::prefix(N: 1);
1890 }
1891 return AddedStructorArgCounts{};
1892}
1893
1894void ItaniumCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1895 // The destructor used for destructing this as a base class; ignores
1896 // virtual bases.
1897 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Base));
1898
1899 // The destructor used for destructing this as a most-derived class;
1900 // call the base destructor and then destructs any virtual bases.
1901 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Complete));
1902
1903 // The destructor in a virtual table is always a 'deleting'
1904 // destructor, which calls the complete destructor and then uses the
1905 // appropriate operator delete.
1906 if (D->isVirtual())
1907 CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Deleting));
1908}
1909
1910void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1911 QualType &ResTy,
1912 FunctionArgList &Params) {
1913 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1914 assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1915
1916 // Check if we need a VTT parameter as well.
1917 if (NeedsVTTParameter(GD: CGF.CurGD)) {
1918 ASTContext &Context = getContext();
1919
1920 // FIXME: avoid the fake decl
1921 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1922 QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1923 QualType T = Context.getPointerType(T: Q);
1924 auto *VTTDecl = ImplicitParamDecl::Create(
1925 C&: Context, /*DC=*/nullptr, IdLoc: MD->getLocation(), Id: &Context.Idents.get(Name: "vtt"),
1926 T, ParamKind: ImplicitParamKind::CXXVTT);
1927 Params.insert(I: Params.begin() + 1, Elt: VTTDecl);
1928 getStructorImplicitParamDecl(CGF) = VTTDecl;
1929 }
1930}
1931
1932void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1933 // Naked functions have no prolog.
1934 if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
1935 return;
1936
1937 /// Initialize the 'this' slot. In the Itanium C++ ABI, no prologue
1938 /// adjustments are required, because they are all handled by thunks.
1939 setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
1940
1941 /// Initialize the 'vtt' slot if needed.
1942 if (getStructorImplicitParamDecl(CGF)) {
1943 getStructorImplicitParamValue(CGF) = CGF.Builder.CreateLoad(
1944 Addr: CGF.GetAddrOfLocalVar(VD: getStructorImplicitParamDecl(CGF)), Name: "vtt");
1945 }
1946
1947 /// If this is a function that the ABI specifies returns 'this', initialize
1948 /// the return slot to 'this' at the start of the function.
1949 ///
1950 /// Unlike the setting of return types, this is done within the ABI
1951 /// implementation instead of by clients of CGCXXABI because:
1952 /// 1) getThisValue is currently protected
1953 /// 2) in theory, an ABI could implement 'this' returns some other way;
1954 /// HasThisReturn only specifies a contract, not the implementation
1955 if (HasThisReturn(GD: CGF.CurGD))
1956 CGF.Builder.CreateStore(Val: getThisValue(CGF), Addr: CGF.ReturnValue);
1957}
1958
1959CGCXXABI::AddedStructorArgs ItaniumCXXABI::getImplicitConstructorArgs(
1960 CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
1961 bool ForVirtualBase, bool Delegating) {
1962 if (!NeedsVTTParameter(GD: GlobalDecl(D, Type)))
1963 return AddedStructorArgs{};
1964
1965 // Insert the implicit 'vtt' argument as the second argument. Make sure to
1966 // correctly reflect its address space, which can differ from generic on
1967 // some targets.
1968 llvm::Value *VTT =
1969 CGF.GetVTTParameter(GD: GlobalDecl(D, Type), ForVirtualBase, Delegating);
1970 LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1971 QualType Q = getContext().getAddrSpaceQualType(T: getContext().VoidPtrTy, AddressSpace: AS);
1972 QualType VTTTy = getContext().getPointerType(T: Q);
1973 return AddedStructorArgs::prefix(Args: {{.Value: VTT, .Type: VTTTy}});
1974}
1975
1976llvm::Value *ItaniumCXXABI::getCXXDestructorImplicitParam(
1977 CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
1978 bool ForVirtualBase, bool Delegating) {
1979 GlobalDecl GD(DD, Type);
1980 return CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
1981}
1982
1983void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
1984 const CXXDestructorDecl *DD,
1985 CXXDtorType Type, bool ForVirtualBase,
1986 bool Delegating, Address This,
1987 QualType ThisTy) {
1988 GlobalDecl GD(DD, Type);
1989 llvm::Value *VTT =
1990 getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating);
1991 QualType VTTTy = getContext().getPointerType(T: getContext().VoidPtrTy);
1992
1993 CGCallee Callee;
1994 if (getContext().getLangOpts().AppleKext &&
1995 Type != Dtor_Base && DD->isVirtual())
1996 Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, RD: DD->getParent());
1997 else
1998 Callee = CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD), abstractInfo: GD);
1999
2000 CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: CGF.getAsNaturalPointerTo(Addr: This, PointeeType: ThisTy),
2001 ThisTy, ImplicitParam: VTT, ImplicitParamTy: VTTTy, E: nullptr);
2002}
2003
2004// Check if any non-inline method has the specified attribute.
2005template <typename T>
2006static bool CXXRecordNonInlineHasAttr(const CXXRecordDecl *RD) {
2007 for (const auto *D : RD->noload_decls()) {
2008 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2009 if (FD->isInlined() || FD->doesThisDeclarationHaveABody() ||
2010 FD->isPureVirtual())
2011 continue;
2012 if (D->hasAttr<T>())
2013 return true;
2014 }
2015 }
2016
2017 return false;
2018}
2019
2020static void setVTableSelectiveDLLImportExport(CodeGenModule &CGM,
2021 llvm::GlobalVariable *VTable,
2022 const CXXRecordDecl *RD) {
2023 if (VTable->getDLLStorageClass() !=
2024 llvm::GlobalVariable::DefaultStorageClass ||
2025 RD->hasAttr<DLLImportAttr>() || RD->hasAttr<DLLExportAttr>())
2026 return;
2027
2028 if (CGM.getVTables().isVTableExternal(RD)) {
2029 if (CXXRecordNonInlineHasAttr<DLLImportAttr>(RD))
2030 VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2031 } else if (CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
2032 VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2033}
2034
2035void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
2036 const CXXRecordDecl *RD) {
2037 llvm::GlobalVariable *VTable = getAddrOfVTable(RD, VPtrOffset: CharUnits());
2038 if (VTable->hasInitializer())
2039 return;
2040
2041 ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
2042 const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
2043 llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
2044 llvm::Constant *RTTI =
2045 CGM.GetAddrOfRTTIDescriptor(Ty: CGM.getContext().getCanonicalTagType(TD: RD));
2046
2047 // Create and set the initializer.
2048 ConstantInitBuilder builder(CGM);
2049 auto components = builder.beginStruct();
2050 CGVT.createVTableInitializer(builder&: components, layout: VTLayout, rtti: RTTI,
2051 vtableHasLocalLinkage: llvm::GlobalValue::isLocalLinkage(Linkage));
2052 components.finishAndSetAsInitializer(global: VTable);
2053
2054 // Set the correct linkage.
2055 VTable->setLinkage(Linkage);
2056
2057 // On a target that may duplicate vtables, a weak vtable does not have a
2058 // unique address, so its address is insignificant and it can be marked
2059 // unnamed_addr.
2060 if (CGM.mayVTableBeDuplicated(Linkage: VTable->getLinkage()))
2061 VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2062
2063 if (CGM.supportsCOMDAT() && VTable->isWeakForLinker())
2064 VTable->setComdat(CGM.getModule().getOrInsertComdat(Name: VTable->getName()));
2065
2066 if (CGM.getTarget().hasPS4DLLImportExport())
2067 setVTableSelectiveDLLImportExport(CGM, VTable, RD);
2068
2069 // Set the right visibility.
2070 CGM.setGVProperties(GV: VTable, D: RD);
2071
2072 // If this is the magic class __cxxabiv1::__fundamental_type_info,
2073 // we will emit the typeinfo for the fundamental types. This is the
2074 // same behaviour as GCC.
2075 const DeclContext *DC = RD->getDeclContext();
2076 if (RD->getIdentifier() &&
2077 RD->getIdentifier()->isStr(Str: "__fundamental_type_info") &&
2078 isa<NamespaceDecl>(Val: DC) && cast<NamespaceDecl>(Val: DC)->getIdentifier() &&
2079 cast<NamespaceDecl>(Val: DC)->getIdentifier()->isStr(Str: "__cxxabiv1") &&
2080 DC->getParent()->isTranslationUnit())
2081 EmitFundamentalRTTIDescriptors(RD);
2082
2083 // Always emit type metadata on non-available_externally definitions, and on
2084 // available_externally definitions if we are performing whole program
2085 // devirtualization or speculative devirtualization. We need the type metadata
2086 // on all vtable definitions to ensure we associate derived classes with base
2087 // classes defined in headers but with a strong definition only in a shared
2088 // library.
2089 if (!VTable->isDeclarationForLinker() ||
2090 CGM.getCodeGenOpts().WholeProgramVTables ||
2091 CGM.getCodeGenOpts().DevirtualizeSpeculatively) {
2092 CGM.EmitVTableTypeMetadata(RD, VTable, VTLayout);
2093 // For available_externally definitions, add the vtable to
2094 // @llvm.compiler.used so that it isn't deleted before whole program
2095 // analysis.
2096 if (VTable->isDeclarationForLinker()) {
2097 assert(CGM.getCodeGenOpts().WholeProgramVTables ||
2098 CGM.getCodeGenOpts().DevirtualizeSpeculatively);
2099 CGM.addCompilerUsedGlobal(GV: VTable);
2100 }
2101 }
2102
2103 if (CGM.getLangOpts().RelativeCXXABIVTables) {
2104 CGVT.RemoveHwasanMetadata(GV: VTable);
2105 if (!VTable->isDSOLocal())
2106 CGVT.GenerateRelativeVTableAlias(VTable, AliasNameRef: VTable->getName());
2107 }
2108
2109 // Emit symbol for debugger only if requested debug info.
2110 if (CGDebugInfo *DI = CGM.getModuleDebugInfo())
2111 DI->emitVTableSymbol(VTable, RD);
2112}
2113
2114bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
2115 CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
2116 if (Vptr.NearestVBase == nullptr)
2117 return false;
2118 return NeedsVTTParameter(GD: CGF.CurGD);
2119}
2120
2121llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
2122 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
2123 const CXXRecordDecl *NearestVBase) {
2124
2125 if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
2126 NeedsVTTParameter(GD: CGF.CurGD)) {
2127 return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base,
2128 NearestVBase);
2129 }
2130 return getVTableAddressPoint(Base, VTableClass);
2131}
2132
2133llvm::Constant *
2134ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base,
2135 const CXXRecordDecl *VTableClass) {
2136 llvm::GlobalValue *VTable = getAddrOfVTable(RD: VTableClass, VPtrOffset: CharUnits());
2137
2138 // Find the appropriate vtable within the vtable group, and the address point
2139 // within that vtable.
2140 const VTableLayout &Layout =
2141 CGM.getItaniumVTableContext().getVTableLayout(RD: VTableClass);
2142 VTableLayout::AddressPointLocation AddressPoint =
2143 Layout.getAddressPoint(Base);
2144 llvm::Constant *Indices[] = {
2145 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
2146 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.VTableIndex),
2147 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.AddressPointIndex),
2148 };
2149
2150 // Add inrange attribute to indicate that only the VTableIndex can be
2151 // accessed.
2152 unsigned ComponentSize =
2153 CGM.getDataLayout().getTypeAllocSize(Ty: CGM.getVTableComponentType());
2154 unsigned VTableSize =
2155 ComponentSize * Layout.getVTableSize(i: AddressPoint.VTableIndex);
2156 unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex;
2157 llvm::ConstantRange InRange(
2158 llvm::APInt(32, (int)-Offset, true),
2159 llvm::APInt(32, (int)(VTableSize - Offset), true));
2160 return llvm::ConstantExpr::getGetElementPtr(
2161 DL: CGM.getDataLayout(), Ty: VTable->getValueType(), C: VTable, IdxList: Indices,
2162 NW: llvm::GEPNoWrapFlags::inBounds(), InRange);
2163}
2164
2165llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
2166 CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
2167 const CXXRecordDecl *NearestVBase) {
2168 assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
2169 NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT");
2170
2171 // Get the secondary vpointer index.
2172 uint64_t VirtualPointerIndex =
2173 CGM.getVTables().getSecondaryVirtualPointerIndex(RD: VTableClass, Base);
2174
2175 /// Load the VTT.
2176 llvm::Value *VTT = CGF.LoadCXXVTT();
2177 if (VirtualPointerIndex)
2178 VTT = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.GlobalsVoidPtrTy, Ptr: VTT,
2179 Idx0: VirtualPointerIndex);
2180
2181 // And load the address point from the VTT.
2182 llvm::Value *AP =
2183 CGF.Builder.CreateAlignedLoad(Ty: CGF.GlobalsVoidPtrTy, Addr: VTT,
2184 Align: CGF.getPointerAlign());
2185
2186 if (auto PointerAuth = CGM.getVTablePointerAuthInfo(Context: &CGF, Record: VTableClass, StorageAddress: VTT,
2187 /*IsVTTEntry=*/true))
2188 AP = CGF.EmitPointerAuthAuth(Info: *PointerAuth, Pointer: AP);
2189
2190 return AP;
2191}
2192
2193llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
2194 CharUnits VPtrOffset) {
2195 assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
2196
2197 llvm::GlobalVariable *&VTable = VTables[RD];
2198 if (VTable)
2199 return VTable;
2200
2201 // Queue up this vtable for possible deferred emission.
2202 CGM.addDeferredVTable(RD);
2203
2204 SmallString<256> Name;
2205 llvm::raw_svector_ostream Out(Name);
2206 getMangleContext().mangleCXXVTable(RD, Out);
2207
2208 const VTableLayout &VTLayout =
2209 CGM.getItaniumVTableContext().getVTableLayout(RD);
2210 llvm::Type *VTableType = CGM.getVTables().getVTableType(layout: VTLayout);
2211
2212 // Use pointer to global alignment for the vtable. Otherwise we would align
2213 // them based on the size of the initializer which doesn't make sense as only
2214 // single values are read.
2215 unsigned PAlign = CGM.getVtableGlobalVarAlignment();
2216
2217 VTable = CGM.CreateOrReplaceCXXRuntimeVariable(
2218 Name, Ty: VTableType, Linkage: llvm::GlobalValue::ExternalLinkage,
2219 Alignment: getContext().toCharUnitsFromBits(BitSize: PAlign).getAsAlign());
2220 if (!CGM.shouldEmitRTTI())
2221 VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2222
2223 if (CGM.getTarget().hasPS4DLLImportExport())
2224 setVTableSelectiveDLLImportExport(CGM, VTable, RD);
2225
2226 CGM.setGVProperties(GV: VTable, D: RD);
2227 return VTable;
2228}
2229
2230CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
2231 GlobalDecl GD,
2232 Address This,
2233 llvm::Type *Ty,
2234 SourceLocation Loc) {
2235 llvm::Type *PtrTy = CGM.GlobalsInt8PtrTy;
2236 auto *MethodDecl = cast<CXXMethodDecl>(Val: GD.getDecl());
2237 llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy: PtrTy, VTableClass: MethodDecl->getParent());
2238
2239 // For the translation of virtual functions, we need to map the (potential)
2240 // host vtable to the device vtable. This is done by calling the runtime
2241 // function
2242 // __llvm_omp_indirect_call_lookup.
2243 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
2244 auto *NewPtrTy = CGM.VoidPtrTy;
2245 llvm::Type *RtlFnArgs[] = {NewPtrTy};
2246 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
2247 Ty: llvm::FunctionType::get(Result: NewPtrTy, Params: RtlFnArgs, isVarArg: false),
2248 Name: "__llvm_omp_indirect_call_lookup");
2249 auto *BackupTy = VTable->getType();
2250 // Need to convert to generic address space
2251 VTable = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: VTable, DestTy: NewPtrTy);
2252 VTable = CGF.EmitRuntimeCall(callee: DeviceRtlFn, args: {VTable});
2253 // convert to original address space
2254 VTable = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: VTable, DestTy: BackupTy);
2255 }
2256
2257 uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
2258 llvm::Value *VFunc, *VTableSlotPtr = nullptr;
2259 auto &Schema = CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers;
2260
2261 llvm::Type *ComponentTy = CGM.getVTables().getVTableComponentType();
2262 uint64_t ByteOffset =
2263 VTableIndex * CGM.getDataLayout().getTypeSizeInBits(Ty: ComponentTy) / 8;
2264
2265 if (!Schema && CGF.ShouldEmitVTableTypeCheckedLoad(RD: MethodDecl->getParent())) {
2266 VFunc = CGF.EmitVTableTypeCheckedLoad(RD: MethodDecl->getParent(), VTable,
2267 VTableTy: PtrTy, VTableByteOffset: ByteOffset);
2268 } else {
2269 CGF.EmitTypeMetadataCodeForVCall(RD: MethodDecl->getParent(), VTable, Loc);
2270
2271 llvm::Value *VFuncLoad;
2272 if (CGM.getLangOpts().RelativeCXXABIVTables) {
2273 VFuncLoad = CGF.Builder.CreateCall(
2274 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::load_relative, Tys: {CGM.Int32Ty}),
2275 Args: {VTable, llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: ByteOffset)});
2276 } else {
2277 VTableSlotPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2278 Ty: PtrTy, Ptr: VTable, Idx0: VTableIndex, Name: "vfn");
2279 VFuncLoad = CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Addr: VTableSlotPtr,
2280 Align: CGF.getPointerAlign());
2281 }
2282
2283 // Add !invariant.load md to virtual function load to indicate that
2284 // function didn't change inside vtable.
2285 // It's safe to add it without -fstrict-vtable-pointers, but it would not
2286 // help in devirtualization because it will only matter if we will have 2
2287 // the same virtual function loads from the same vtable load, which won't
2288 // happen without enabled devirtualization with -fstrict-vtable-pointers.
2289 if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2290 CGM.getCodeGenOpts().StrictVTablePointers) {
2291 if (auto *VFuncLoadInstr = dyn_cast<llvm::Instruction>(Val: VFuncLoad)) {
2292 VFuncLoadInstr->setMetadata(
2293 KindID: llvm::LLVMContext::MD_invariant_load,
2294 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(),
2295 MDs: llvm::ArrayRef<llvm::Metadata *>()));
2296 }
2297 }
2298 VFunc = VFuncLoad;
2299 }
2300
2301 CGPointerAuthInfo PointerAuth;
2302 if (Schema) {
2303 assert(VTableSlotPtr && "virtual function pointer not set");
2304 GD = CGM.getItaniumVTableContext().findOriginalMethod(GD: GD.getCanonicalDecl());
2305 PointerAuth = CGF.EmitPointerAuthInfo(Schema, StorageAddress: VTableSlotPtr, SchemaDecl: GD, SchemaType: QualType());
2306 }
2307 CGCallee Callee(GD, VFunc, PointerAuth);
2308 return Callee;
2309}
2310
2311llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
2312 CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
2313 Address This, DeleteOrMemberCallExpr E, llvm::CallBase **CallOrInvoke) {
2314 auto *CE = dyn_cast<const CXXMemberCallExpr *>(Val&: E);
2315 auto *D = dyn_cast<const CXXDeleteExpr *>(Val&: E);
2316 assert((CE != nullptr) ^ (D != nullptr));
2317 assert(CE == nullptr || CE->arguments().empty());
2318 assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
2319
2320 GlobalDecl GD(Dtor, DtorType);
2321 const CGFunctionInfo *FInfo =
2322 &CGM.getTypes().arrangeCXXStructorDeclaration(GD);
2323 llvm::FunctionType *Ty = CGF.CGM.getTypes().GetFunctionType(Info: *FInfo);
2324 CGCallee Callee = CGCallee::forVirtual(CE, MD: GD, Addr: This, FTy: Ty);
2325
2326 QualType ThisTy;
2327 if (CE) {
2328 ThisTy = CE->getObjectType();
2329 } else {
2330 ThisTy = D->getDestroyedType();
2331 }
2332
2333 CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: This.emitRawPointer(CGF), ThisTy,
2334 ImplicitParam: nullptr, ImplicitParamTy: QualType(), E: nullptr, CallOrInvoke);
2335 return nullptr;
2336}
2337
2338void ItaniumCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
2339 CodeGenVTables &VTables = CGM.getVTables();
2340 llvm::GlobalVariable *VTT = VTables.GetAddrOfVTT(RD);
2341 VTables.EmitVTTDefinition(VTT, Linkage: CGM.getVTableLinkage(RD), RD);
2342}
2343
2344bool ItaniumCXXABI::canSpeculativelyEmitVTableAsBaseClass(
2345 const CXXRecordDecl *RD) const {
2346 // We don't emit available_externally vtables if we are in -fapple-kext mode
2347 // because kext mode does not permit devirtualization.
2348 if (CGM.getLangOpts().AppleKext)
2349 return false;
2350
2351 // If the vtable is hidden then it is not safe to emit an available_externally
2352 // copy of vtable.
2353 if (isVTableHidden(RD))
2354 return false;
2355
2356 if (CGM.getCodeGenOpts().ForceEmitVTables)
2357 return true;
2358
2359 // A speculative vtable can only be generated if all virtual inline functions
2360 // defined by this class are emitted. The vtable in the final program contains
2361 // for each virtual inline function not used in the current TU a function that
2362 // is equivalent to the unused function. The function in the actual vtable
2363 // does not have to be declared under the same symbol (e.g., a virtual
2364 // destructor that can be substituted with its base class's destructor). Since
2365 // inline functions are emitted lazily and this emissions does not account for
2366 // speculative emission of a vtable, we might generate a speculative vtable
2367 // with references to inline functions that are not emitted under that name.
2368 // This can lead to problems when devirtualizing a call to such a function,
2369 // that result in linking errors. Hence, if there are any unused virtual
2370 // inline function, we cannot emit the speculative vtable.
2371 // FIXME we can still emit a copy of the vtable if we
2372 // can emit definition of the inline functions.
2373 if (hasAnyUnusedVirtualInlineFunction(RD))
2374 return false;
2375
2376 // For a class with virtual bases, we must also be able to speculatively
2377 // emit the VTT, because CodeGen doesn't have separate notions of "can emit
2378 // the vtable" and "can emit the VTT". For a base subobject, this means we
2379 // need to be able to emit non-virtual base vtables.
2380 if (RD->getNumVBases()) {
2381 for (const auto &B : RD->bases()) {
2382 auto *BRD = B.getType()->getAsCXXRecordDecl();
2383 assert(BRD && "no class for base specifier");
2384 if (B.isVirtual() || !BRD->isDynamicClass())
2385 continue;
2386 if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2387 return false;
2388 }
2389 }
2390
2391 return true;
2392}
2393
2394bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const {
2395 if (!canSpeculativelyEmitVTableAsBaseClass(RD))
2396 return false;
2397
2398 if (RD->shouldEmitInExternalSource())
2399 return false;
2400
2401 // For a complete-object vtable (or more specifically, for the VTT), we need
2402 // to be able to speculatively emit the vtables of all dynamic virtual bases.
2403 for (const auto &B : RD->vbases()) {
2404 auto *BRD = B.getType()->getAsCXXRecordDecl();
2405 assert(BRD && "no class for base specifier");
2406 if (!BRD->isDynamicClass())
2407 continue;
2408 if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2409 return false;
2410 }
2411
2412 return true;
2413}
2414static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
2415 Address InitialPtr,
2416 const CXXRecordDecl *UnadjustedClass,
2417 int64_t NonVirtualAdjustment,
2418 int64_t VirtualAdjustment,
2419 bool IsReturnAdjustment) {
2420 if (!NonVirtualAdjustment && !VirtualAdjustment)
2421 return InitialPtr.emitRawPointer(CGF);
2422
2423 Address V = InitialPtr.withElementType(ElemTy: CGF.Int8Ty);
2424
2425 // In a base-to-derived cast, the non-virtual adjustment is applied first.
2426 if (NonVirtualAdjustment && !IsReturnAdjustment) {
2427 V = CGF.Builder.CreateConstInBoundsByteGEP(Addr: V,
2428 Offset: CharUnits::fromQuantity(Quantity: NonVirtualAdjustment));
2429 }
2430
2431 // Perform the virtual adjustment if we have one.
2432 llvm::Value *ResultPtr;
2433 if (VirtualAdjustment) {
2434 llvm::Value *VTablePtr =
2435 CGF.GetVTablePtr(This: V, VTableTy: CGF.Int8PtrTy, VTableClass: UnadjustedClass);
2436
2437 llvm::Value *Offset;
2438 llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2439 Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VirtualAdjustment);
2440 if (CGF.CGM.getLangOpts().RelativeCXXABIVTables) {
2441 // Load the adjustment offset from the vtable as a 32-bit int.
2442 Offset =
2443 CGF.Builder.CreateAlignedLoad(Ty: CGF.Int32Ty, Addr: OffsetPtr,
2444 Align: CharUnits::fromQuantity(Quantity: 4));
2445 } else {
2446 llvm::Type *PtrDiffTy =
2447 CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
2448
2449 // Load the adjustment offset from the vtable.
2450 Offset = CGF.Builder.CreateAlignedLoad(Ty: PtrDiffTy, Addr: OffsetPtr,
2451 Align: CGF.getPointerAlign());
2452 }
2453 // Adjust our pointer.
2454 ResultPtr = CGF.Builder.CreateInBoundsGEP(Ty: V.getElementType(),
2455 Ptr: V.emitRawPointer(CGF), IdxList: Offset);
2456 } else {
2457 ResultPtr = V.emitRawPointer(CGF);
2458 }
2459
2460 // In a derived-to-base conversion, the non-virtual adjustment is
2461 // applied second.
2462 if (NonVirtualAdjustment && IsReturnAdjustment) {
2463 ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.Int8Ty, Ptr: ResultPtr,
2464 Idx0: NonVirtualAdjustment);
2465 }
2466
2467 return ResultPtr;
2468}
2469
2470llvm::Value *
2471ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This,
2472 const CXXRecordDecl *UnadjustedClass,
2473 const ThunkInfo &TI) {
2474 return performTypeAdjustment(CGF, InitialPtr: This, UnadjustedClass, NonVirtualAdjustment: TI.This.NonVirtual,
2475 VirtualAdjustment: TI.This.Virtual.Itanium.VCallOffsetOffset,
2476 /*IsReturnAdjustment=*/false);
2477}
2478
2479llvm::Value *
2480ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
2481 const CXXRecordDecl *UnadjustedClass,
2482 const ReturnAdjustment &RA) {
2483 return performTypeAdjustment(CGF, InitialPtr: Ret, UnadjustedClass, NonVirtualAdjustment: RA.NonVirtual,
2484 VirtualAdjustment: RA.Virtual.Itanium.VBaseOffsetOffset,
2485 /*IsReturnAdjustment=*/true);
2486}
2487
2488void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
2489 RValue RV, QualType ResultType) {
2490 if (!isa<CXXDestructorDecl>(Val: CGF.CurGD.getDecl()))
2491 return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
2492
2493 // Destructor thunks in the ARM ABI have indeterminate results.
2494 llvm::Type *T = CGF.ReturnValue.getElementType();
2495 RValue Undef = RValue::get(V: llvm::UndefValue::get(T));
2496 return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV: Undef, ResultType);
2497}
2498
2499/************************** Array allocation cookies **************************/
2500
2501CharUnits ItaniumCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2502 // The array cookie is a size_t; pad that up to the element alignment.
2503 // The cookie is actually right-justified in that space.
2504 return std::max(a: CharUnits::fromQuantity(Quantity: CGM.SizeSizeInBytes),
2505 b: CGM.getContext().getPreferredTypeAlignInChars(T: elementType));
2506}
2507
2508Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2509 Address NewPtr,
2510 llvm::Value *NumElements,
2511 const CXXNewExpr *expr,
2512 QualType ElementType) {
2513 assert(requiresArrayCookie(expr));
2514
2515 unsigned AS = NewPtr.getAddressSpace();
2516
2517 ASTContext &Ctx = getContext();
2518 CharUnits SizeSize = CGF.getSizeSize();
2519
2520 // The size of the cookie.
2521 CharUnits CookieSize =
2522 std::max(a: SizeSize, b: Ctx.getPreferredTypeAlignInChars(T: ElementType));
2523 assert(CookieSize == getArrayCookieSizeImpl(ElementType));
2524
2525 // Compute an offset to the cookie.
2526 Address CookiePtr = NewPtr;
2527 CharUnits CookieOffset = CookieSize - SizeSize;
2528 if (!CookieOffset.isZero())
2529 CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: CookiePtr, Offset: CookieOffset);
2530
2531 // Write the number of elements into the appropriate slot.
2532 Address NumElementsPtr = CookiePtr.withElementType(ElemTy: CGF.SizeTy);
2533 llvm::Instruction *SI = CGF.Builder.CreateStore(Val: NumElements, Addr: NumElementsPtr);
2534
2535 // Handle the array cookie specially in ASan.
2536 if (CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) && AS == 0 &&
2537 (expr->getOperatorNew()->isReplaceableGlobalAllocationFunction() ||
2538 CGM.getCodeGenOpts().SanitizeAddressPoisonCustomArrayCookie)) {
2539 // The store to the CookiePtr does not need to be instrumented.
2540 SI->setNoSanitizeMetadata();
2541 llvm::FunctionType *FTy =
2542 llvm::FunctionType::get(Result: CGM.VoidTy, Params: NumElementsPtr.getType(), isVarArg: false);
2543 llvm::FunctionCallee F =
2544 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_poison_cxx_array_cookie");
2545 CGF.Builder.CreateCall(Callee: F, Args: NumElementsPtr.emitRawPointer(CGF));
2546 }
2547
2548 // Finally, compute a pointer to the actual data buffer by skipping
2549 // over the cookie completely.
2550 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: NewPtr, Offset: CookieSize);
2551}
2552
2553llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2554 Address allocPtr,
2555 CharUnits cookieSize) {
2556 // The element size is right-justified in the cookie.
2557 Address numElementsPtr = allocPtr;
2558 CharUnits numElementsOffset = cookieSize - CGF.getSizeSize();
2559 if (!numElementsOffset.isZero())
2560 numElementsPtr =
2561 CGF.Builder.CreateConstInBoundsByteGEP(Addr: numElementsPtr, Offset: numElementsOffset);
2562
2563 unsigned AS = allocPtr.getAddressSpace();
2564 numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2565 if (!CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) || AS != 0)
2566 return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2567 // In asan mode emit a function call instead of a regular load and let the
2568 // run-time deal with it: if the shadow is properly poisoned return the
2569 // cookie, otherwise return 0 to avoid an infinite loop calling DTORs.
2570 // We can't simply ignore this load using nosanitize metadata because
2571 // the metadata may be lost.
2572 llvm::FunctionType *FTy =
2573 llvm::FunctionType::get(Result: CGF.SizeTy, Params: CGF.DefaultPtrTy, isVarArg: false);
2574 llvm::FunctionCallee F =
2575 CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_load_cxx_array_cookie");
2576 return CGF.Builder.CreateCall(Callee: F, Args: numElementsPtr.emitRawPointer(CGF));
2577}
2578
2579CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2580 // ARM says that the cookie is always:
2581 // struct array_cookie {
2582 // std::size_t element_size; // element_size != 0
2583 // std::size_t element_count;
2584 // };
2585 // But the base ABI doesn't give anything an alignment greater than
2586 // 8, so we can dismiss this as typical ABI-author blindness to
2587 // actual language complexity and round up to the element alignment.
2588 return std::max(a: CharUnits::fromQuantity(Quantity: 2 * CGM.SizeSizeInBytes),
2589 b: CGM.getContext().getTypeAlignInChars(T: elementType));
2590}
2591
2592Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2593 Address newPtr,
2594 llvm::Value *numElements,
2595 const CXXNewExpr *expr,
2596 QualType elementType) {
2597 assert(requiresArrayCookie(expr));
2598
2599 // The cookie is always at the start of the buffer.
2600 Address cookie = newPtr;
2601
2602 // The first element is the element size.
2603 cookie = cookie.withElementType(ElemTy: CGF.SizeTy);
2604 llvm::Value *elementSize = llvm::ConstantInt::get(Ty: CGF.SizeTy,
2605 V: getContext().getTypeSizeInChars(T: elementType).getQuantity());
2606 CGF.Builder.CreateStore(Val: elementSize, Addr: cookie);
2607
2608 // The second element is the element count.
2609 cookie = CGF.Builder.CreateConstInBoundsGEP(Addr: cookie, Index: 1);
2610 CGF.Builder.CreateStore(Val: numElements, Addr: cookie);
2611
2612 // Finally, compute a pointer to the actual data buffer by skipping
2613 // over the cookie completely.
2614 CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
2615 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: newPtr, Offset: cookieSize);
2616}
2617
2618llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2619 Address allocPtr,
2620 CharUnits cookieSize) {
2621 // The number of elements is at offset sizeof(size_t) relative to
2622 // the allocated pointer.
2623 Address numElementsPtr
2624 = CGF.Builder.CreateConstInBoundsByteGEP(Addr: allocPtr, Offset: CGF.getSizeSize());
2625
2626 numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2627 return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2628}
2629
2630/*********************** Static local initialization **************************/
2631
2632static llvm::FunctionCallee getGuardAcquireFn(CodeGenModule &CGM,
2633 llvm::PointerType *GuardPtrTy) {
2634 // int __cxa_guard_acquire(__guard *guard_object);
2635 llvm::FunctionType *FTy =
2636 llvm::FunctionType::get(Result: CGM.getTypes().ConvertType(T: CGM.getContext().IntTy),
2637 Params: GuardPtrTy, /*isVarArg=*/false);
2638 return CGM.CreateRuntimeFunction(
2639 Ty: FTy, Name: "__cxa_guard_acquire",
2640 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2641 Index: llvm::AttributeList::FunctionIndex,
2642 Kinds: llvm::Attribute::NoUnwind));
2643}
2644
2645static llvm::FunctionCallee getGuardReleaseFn(CodeGenModule &CGM,
2646 llvm::PointerType *GuardPtrTy) {
2647 // void __cxa_guard_release(__guard *guard_object);
2648 llvm::FunctionType *FTy =
2649 llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2650 return CGM.CreateRuntimeFunction(
2651 Ty: FTy, Name: "__cxa_guard_release",
2652 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2653 Index: llvm::AttributeList::FunctionIndex,
2654 Kinds: llvm::Attribute::NoUnwind));
2655}
2656
2657static llvm::FunctionCallee getGuardAbortFn(CodeGenModule &CGM,
2658 llvm::PointerType *GuardPtrTy) {
2659 // void __cxa_guard_abort(__guard *guard_object);
2660 llvm::FunctionType *FTy =
2661 llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2662 return CGM.CreateRuntimeFunction(
2663 Ty: FTy, Name: "__cxa_guard_abort",
2664 ExtraAttrs: llvm::AttributeList::get(C&: CGM.getLLVMContext(),
2665 Index: llvm::AttributeList::FunctionIndex,
2666 Kinds: llvm::Attribute::NoUnwind));
2667}
2668
2669namespace {
2670 struct CallGuardAbort final : EHScopeStack::Cleanup {
2671 llvm::GlobalVariable *Guard;
2672 CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
2673
2674 void Emit(CodeGenFunction &CGF, Flags flags) override {
2675 CGF.EmitNounwindRuntimeCall(callee: getGuardAbortFn(CGM&: CGF.CGM, GuardPtrTy: Guard->getType()),
2676 args: Guard);
2677 }
2678 };
2679}
2680
2681/// The ARM code here follows the Itanium code closely enough that we
2682/// just special-case it at particular places.
2683void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
2684 const VarDecl &D,
2685 llvm::GlobalVariable *var,
2686 bool shouldPerformInit) {
2687 CGBuilderTy &Builder = CGF.Builder;
2688
2689 // Inline variables that weren't instantiated from variable templates have
2690 // partially-ordered initialization within their translation unit.
2691 bool NonTemplateInline =
2692 D.isInline() &&
2693 !isTemplateInstantiation(Kind: D.getTemplateSpecializationKind());
2694
2695 // We only need to use thread-safe statics for local non-TLS variables and
2696 // inline variables; other global initialization is always single-threaded
2697 // or (through lazy dynamic loading in multiple threads) unsequenced.
2698 bool threadsafe = getContext().getLangOpts().ThreadsafeStatics &&
2699 (D.isLocalVarDecl() || NonTemplateInline) &&
2700 !D.getTLSKind();
2701
2702 // If we have a global variable with internal linkage and thread-safe statics
2703 // are disabled, we can just let the guard variable be of type i8.
2704 bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
2705
2706 llvm::IntegerType *guardTy;
2707 CharUnits guardAlignment;
2708 if (useInt8GuardVariable) {
2709 guardTy = CGF.Int8Ty;
2710 guardAlignment = CharUnits::One();
2711 } else {
2712 // Guard variables are 64 bits in the generic ABI and size width on ARM
2713 // (i.e. 32-bit on AArch32, 64-bit on AArch64).
2714 if (UseARMGuardVarABI) {
2715 guardTy = CGF.SizeTy;
2716 guardAlignment = CGF.getSizeAlign();
2717 } else {
2718 guardTy = CGF.Int64Ty;
2719 guardAlignment =
2720 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getABITypeAlign(Ty: guardTy));
2721 }
2722 }
2723 llvm::PointerType *guardPtrTy = llvm::PointerType::get(
2724 C&: CGF.CGM.getLLVMContext(),
2725 AddressSpace: CGF.CGM.getDataLayout().getDefaultGlobalsAddressSpace());
2726
2727 // Create the guard variable if we don't already have it (as we
2728 // might if we're double-emitting this function body).
2729 llvm::GlobalVariable *guard = CGM.getStaticLocalDeclGuardAddress(D: &D);
2730 if (!guard) {
2731 // Mangle the name for the guard.
2732 SmallString<256> guardName;
2733 {
2734 llvm::raw_svector_ostream out(guardName);
2735 getMangleContext().mangleStaticGuardVariable(D: &D, out);
2736 }
2737
2738 // Create the guard variable with a zero-initializer.
2739 // Just absorb linkage, visibility and dll storage class from the guarded
2740 // variable.
2741 guard = new llvm::GlobalVariable(CGM.getModule(), guardTy,
2742 false, var->getLinkage(),
2743 llvm::ConstantInt::get(Ty: guardTy, V: 0),
2744 guardName.str());
2745 guard->setDSOLocal(var->isDSOLocal());
2746 guard->setVisibility(var->getVisibility());
2747 guard->setDLLStorageClass(var->getDLLStorageClass());
2748 // If the variable is thread-local, so is its guard variable.
2749 guard->setThreadLocalMode(var->getThreadLocalMode());
2750 guard->setAlignment(guardAlignment.getAsAlign());
2751
2752 // The ABI says: "It is suggested that it be emitted in the same COMDAT
2753 // group as the associated data object." In practice, this doesn't work for
2754 // non-ELF and non-Wasm object formats, so only do it for ELF and Wasm.
2755 llvm::Comdat *C = var->getComdat();
2756 if (!D.isLocalVarDecl() && C &&
2757 (CGM.getTarget().getTriple().isOSBinFormatELF() ||
2758 CGM.getTarget().getTriple().isOSBinFormatWasm())) {
2759 guard->setComdat(C);
2760 } else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) {
2761 guard->setComdat(CGM.getModule().getOrInsertComdat(Name: guard->getName()));
2762 }
2763
2764 CGM.setStaticLocalDeclGuardAddress(D: &D, C: guard);
2765 }
2766
2767 Address guardAddr = Address(guard, guard->getValueType(), guardAlignment);
2768
2769 // Test whether the variable has completed initialization.
2770 //
2771 // Itanium C++ ABI 3.3.2:
2772 // The following is pseudo-code showing how these functions can be used:
2773 // if (obj_guard.first_byte == 0) {
2774 // if ( __cxa_guard_acquire (&obj_guard) ) {
2775 // try {
2776 // ... initialize the object ...;
2777 // } catch (...) {
2778 // __cxa_guard_abort (&obj_guard);
2779 // throw;
2780 // }
2781 // ... queue object destructor with __cxa_atexit() ...;
2782 // __cxa_guard_release (&obj_guard);
2783 // }
2784 // }
2785 //
2786 // If threadsafe statics are enabled, but we don't have inline atomics, just
2787 // call __cxa_guard_acquire unconditionally. The "inline" check isn't
2788 // actually inline, and the user might not expect calls to __atomic libcalls.
2789
2790 unsigned MaxInlineWidthInBits = CGF.getTarget().getMaxAtomicInlineWidth();
2791 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2792 if (!threadsafe || MaxInlineWidthInBits) {
2793 // Load the first byte of the guard variable.
2794 llvm::LoadInst *LI =
2795 Builder.CreateLoad(Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2796
2797 // Itanium ABI:
2798 // An implementation supporting thread-safety on multiprocessor
2799 // systems must also guarantee that references to the initialized
2800 // object do not occur before the load of the initialization flag.
2801 //
2802 // In LLVM, we do this by marking the load Acquire.
2803 if (threadsafe)
2804 LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
2805
2806 // For ARM, we should only check the first bit, rather than the entire byte:
2807 //
2808 // ARM C++ ABI 3.2.3.1:
2809 // To support the potential use of initialization guard variables
2810 // as semaphores that are the target of ARM SWP and LDREX/STREX
2811 // synchronizing instructions we define a static initialization
2812 // guard variable to be a 4-byte aligned, 4-byte word with the
2813 // following inline access protocol.
2814 // #define INITIALIZED 1
2815 // if ((obj_guard & INITIALIZED) != INITIALIZED) {
2816 // if (__cxa_guard_acquire(&obj_guard))
2817 // ...
2818 // }
2819 //
2820 // and similarly for ARM64:
2821 //
2822 // ARM64 C++ ABI 3.2.2:
2823 // This ABI instead only specifies the value bit 0 of the static guard
2824 // variable; all other bits are platform defined. Bit 0 shall be 0 when the
2825 // variable is not initialized and 1 when it is.
2826 llvm::Value *V =
2827 (UseARMGuardVarABI && !useInt8GuardVariable)
2828 ? Builder.CreateAnd(LHS: LI, RHS: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1))
2829 : LI;
2830 llvm::Value *NeedsInit = Builder.CreateIsNull(Arg: V, Name: "guard.uninitialized");
2831
2832 llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock(name: "init.check");
2833
2834 // Check if the first byte of the guard variable is zero.
2835 CGF.EmitCXXGuardedInitBranch(NeedsInit, InitBlock: InitCheckBlock, NoInitBlock: EndBlock,
2836 Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2837
2838 CGF.EmitBlock(BB: InitCheckBlock);
2839 }
2840
2841 // The semantics of dynamic initialization of variables with static or thread
2842 // storage duration depends on whether they are declared at block-scope. The
2843 // initialization of such variables at block-scope can be aborted with an
2844 // exception and later retried (per C++20 [stmt.dcl]p4), and recursive entry
2845 // to their initialization has undefined behavior (also per C++20
2846 // [stmt.dcl]p4). For such variables declared at non-block scope, exceptions
2847 // lead to termination (per C++20 [except.terminate]p1), and recursive
2848 // references to the variables are governed only by the lifetime rules (per
2849 // C++20 [class.cdtor]p2), which means such references are perfectly fine as
2850 // long as they avoid touching memory. As a result, block-scope variables must
2851 // not be marked as initialized until after initialization completes (unless
2852 // the mark is reverted following an exception), but non-block-scope variables
2853 // must be marked prior to initialization so that recursive accesses during
2854 // initialization do not restart initialization.
2855
2856 // Variables used when coping with thread-safe statics and exceptions.
2857 if (threadsafe) {
2858 // Call __cxa_guard_acquire.
2859 llvm::Value *V
2860 = CGF.EmitNounwindRuntimeCall(callee: getGuardAcquireFn(CGM, GuardPtrTy: guardPtrTy), args: guard);
2861
2862 llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2863
2864 Builder.CreateCondBr(Cond: Builder.CreateIsNotNull(Arg: V, Name: "tobool"),
2865 True: InitBlock, False: EndBlock);
2866
2867 // Call __cxa_guard_abort along the exceptional edge.
2868 CGF.EHStack.pushCleanup<CallGuardAbort>(Kind: EHCleanup, A: guard);
2869
2870 CGF.EmitBlock(BB: InitBlock);
2871 } else if (!D.isLocalVarDecl()) {
2872 // For non-local variables, store 1 into the first byte of the guard
2873 // variable before the object initialization begins so that references
2874 // to the variable during initialization don't restart initialization.
2875 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2876 Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2877 }
2878
2879 // Emit the initializer and add a global destructor if appropriate.
2880 CGF.EmitCXXGlobalVarDeclInit(D, GV: var, PerformInit: shouldPerformInit);
2881
2882 if (threadsafe) {
2883 // Pop the guard-abort cleanup if we pushed one.
2884 CGF.PopCleanupBlock();
2885
2886 // Call __cxa_guard_release. This cannot throw.
2887 CGF.EmitNounwindRuntimeCall(callee: getGuardReleaseFn(CGM, GuardPtrTy: guardPtrTy),
2888 args: guardAddr.emitRawPointer(CGF));
2889 } else if (D.isLocalVarDecl()) {
2890 // For local variables, store 1 into the first byte of the guard variable
2891 // after the object initialization completes so that initialization is
2892 // retried if initialization is interrupted by an exception.
2893 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2894 Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2895 }
2896
2897 CGF.EmitBlock(BB: EndBlock);
2898}
2899
2900/// Register a global destructor using __cxa_atexit.
2901static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF,
2902 llvm::FunctionCallee dtor,
2903 llvm::Constant *addr, bool TLS) {
2904 assert(!CGF.getTarget().getTriple().isOSAIX() &&
2905 "unexpected call to emitGlobalDtorWithCXAAtExit");
2906 assert((TLS || CGF.getTypes().getCodeGenOpts().CXAAtExit) &&
2907 "__cxa_atexit is disabled");
2908 const char *Name = "__cxa_atexit";
2909 if (TLS) {
2910 const llvm::Triple &T = CGF.getTarget().getTriple();
2911 Name = T.isOSDarwin() ? "_tlv_atexit" : "__cxa_thread_atexit";
2912 }
2913
2914 // We're assuming that the destructor function is something we can
2915 // reasonably call with the default CC.
2916 llvm::Type *dtorTy = CGF.DefaultPtrTy;
2917
2918 // Preserve address space of addr.
2919 auto AddrAS = addr ? addr->getType()->getPointerAddressSpace() : 0;
2920 auto AddrPtrTy = AddrAS ? llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: AddrAS)
2921 : CGF.Int8PtrTy;
2922
2923 // Create a variable that binds the atexit to this shared object.
2924 llvm::Constant *handle =
2925 CGF.CGM.CreateRuntimeVariable(Ty: CGF.Int8Ty, Name: "__dso_handle");
2926 auto *GV = cast<llvm::GlobalValue>(Val: handle->stripPointerCasts());
2927 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
2928
2929 // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
2930 llvm::Type *paramTys[] = {dtorTy, AddrPtrTy, handle->getType()};
2931 llvm::FunctionType *atexitTy =
2932 llvm::FunctionType::get(Result: CGF.IntTy, Params: paramTys, isVarArg: false);
2933
2934 // Fetch the actual function.
2935 llvm::FunctionCallee atexit = CGF.CGM.CreateRuntimeFunction(Ty: atexitTy, Name);
2936 if (llvm::Function *fn = dyn_cast<llvm::Function>(Val: atexit.getCallee()))
2937 fn->setDoesNotThrow();
2938
2939 const auto &Context = CGF.CGM.getContext();
2940 FunctionProtoType::ExtProtoInfo EPI(Context.getDefaultCallingConvention(
2941 /*IsVariadic=*/false, /*IsCXXMethod=*/false));
2942 QualType fnType =
2943 Context.getFunctionType(ResultTy: Context.VoidTy, Args: {Context.VoidPtrTy}, EPI);
2944 llvm::Value *dtorCallee = dtor.getCallee();
2945 dtorCallee =
2946 CGF.CGM.getFunctionPointer(Pointer: cast<llvm::Constant>(Val: dtorCallee), FunctionType: fnType);
2947
2948 if (dtorCallee->getType()->getPointerAddressSpace() != AddrAS)
2949 dtorCallee = CGF.performAddrSpaceCast(Src: dtorCallee, DestTy: AddrPtrTy);
2950
2951 if (!addr)
2952 // addr is null when we are trying to register a dtor annotated with
2953 // __attribute__((destructor)) in a constructor function. Using null here is
2954 // okay because this argument is just passed back to the destructor
2955 // function.
2956 addr = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
2957
2958 llvm::Value *args[] = {dtorCallee, addr, handle};
2959 CGF.EmitNounwindRuntimeCall(callee: atexit, args);
2960}
2961
2962static llvm::Function *createGlobalInitOrCleanupFn(CodeGen::CodeGenModule &CGM,
2963 StringRef FnName) {
2964 // Create a function that registers/unregisters destructors that have the same
2965 // priority.
2966 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
2967 llvm::Function *GlobalInitOrCleanupFn = CGM.CreateGlobalInitOrCleanUpFunction(
2968 ty: FTy, name: FnName, FI: CGM.getTypes().arrangeNullaryFunction(), Loc: SourceLocation());
2969
2970 return GlobalInitOrCleanupFn;
2971}
2972
2973void CodeGenModule::unregisterGlobalDtorsWithUnAtExit() {
2974 for (const auto &I : DtorsUsingAtExit) {
2975 int Priority = I.first;
2976 std::string GlobalCleanupFnName =
2977 std::string("__GLOBAL_cleanup_") + llvm::to_string(Value: Priority);
2978
2979 llvm::Function *GlobalCleanupFn =
2980 createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalCleanupFnName);
2981
2982 CodeGenFunction CGF(*this);
2983 CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalCleanupFn,
2984 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
2985 Loc: SourceLocation(), StartLoc: SourceLocation());
2986 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2987
2988 // Get the destructor function type, void(*)(void).
2989 llvm::FunctionType *dtorFuncTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
2990
2991 // Destructor functions are run/unregistered in non-ascending
2992 // order of their priorities.
2993 const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
2994 auto itv = Dtors.rbegin();
2995 while (itv != Dtors.rend()) {
2996 llvm::Function *Dtor = *itv;
2997
2998 // We're assuming that the destructor function is something we can
2999 // reasonably call with the correct CC.
3000 llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub: Dtor);
3001 llvm::Value *NeedsDestruct =
3002 CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
3003
3004 llvm::BasicBlock *DestructCallBlock =
3005 CGF.createBasicBlock(name: "destruct.call");
3006 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(
3007 name: (itv + 1) != Dtors.rend() ? "unatexit.call" : "destruct.end");
3008 // Check if unatexit returns a value of 0. If it does, jump to
3009 // DestructCallBlock, otherwise jump to EndBlock directly.
3010 CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
3011
3012 CGF.EmitBlock(BB: DestructCallBlock);
3013
3014 // Emit the call to casted Dtor.
3015 llvm::CallInst *CI = CGF.Builder.CreateCall(FTy: dtorFuncTy, Callee: Dtor);
3016 // Make sure the call and the callee agree on calling convention.
3017 CI->setCallingConv(Dtor->getCallingConv());
3018
3019 CGF.EmitBlock(BB: EndBlock);
3020
3021 itv++;
3022 }
3023
3024 CGF.FinishFunction();
3025 AddGlobalDtor(Dtor: GlobalCleanupFn, Priority);
3026 }
3027}
3028
3029void CodeGenModule::registerGlobalDtorsWithAtExit() {
3030 for (const auto &I : DtorsUsingAtExit) {
3031 int Priority = I.first;
3032 std::string GlobalInitFnName =
3033 std::string("__GLOBAL_init_") + llvm::to_string(Value: Priority);
3034 llvm::Function *GlobalInitFn =
3035 createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalInitFnName);
3036
3037 CodeGenFunction CGF(*this);
3038 CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalInitFn,
3039 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
3040 Loc: SourceLocation(), StartLoc: SourceLocation());
3041 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
3042
3043 // Since constructor functions are run in non-descending order of their
3044 // priorities, destructors are registered in non-descending order of their
3045 // priorities, and since destructor functions are run in the reverse order
3046 // of their registration, destructor functions are run in non-ascending
3047 // order of their priorities.
3048 const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
3049 for (auto *Dtor : Dtors) {
3050 // Register the destructor function calling __cxa_atexit if it is
3051 // available. Otherwise fall back on calling atexit.
3052 if (getCodeGenOpts().CXAAtExit) {
3053 emitGlobalDtorWithCXAAtExit(CGF, dtor: Dtor, addr: nullptr, TLS: false);
3054 } else {
3055 // We're assuming that the destructor function is something we can
3056 // reasonably call with the correct CC.
3057 CGF.registerGlobalDtorWithAtExit(dtorStub: Dtor);
3058 }
3059 }
3060
3061 CGF.FinishFunction();
3062 AddGlobalCtor(Ctor: GlobalInitFn, Priority);
3063 }
3064
3065 if (getCXXABI().useSinitAndSterm())
3066 unregisterGlobalDtorsWithUnAtExit();
3067}
3068
3069/// Register a global destructor as best as we know how.
3070void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
3071 llvm::FunctionCallee dtor,
3072 llvm::Constant *addr) {
3073 if (D.isNoDestroy(CGM.getContext()))
3074 return;
3075
3076 // HLSL doesn't support atexit.
3077 if (CGM.getLangOpts().HLSL)
3078 return CGM.AddCXXDtorEntry(DtorFn: dtor, Object: addr);
3079
3080 // OpenMP offloading supports C++ constructors and destructors but we do not
3081 // always have 'atexit' available. Instead lower these to use the LLVM global
3082 // destructors which we can handle directly in the runtime. Note that this is
3083 // not strictly 1-to-1 with using `atexit` because we no longer tear down
3084 // globals in reverse order of when they were constructed.
3085 if (!CGM.getLangOpts().hasAtExit() && !D.isStaticLocal())
3086 return CGF.registerGlobalDtorWithLLVM(D, fn: dtor, addr);
3087
3088 // emitGlobalDtorWithCXAAtExit will emit a call to either __cxa_thread_atexit
3089 // or __cxa_atexit depending on whether this VarDecl is a thread-local storage
3090 // or not. CXAAtExit controls only __cxa_atexit, so use it if it is enabled.
3091 // We can always use __cxa_thread_atexit.
3092 if (CGM.getCodeGenOpts().CXAAtExit || D.getTLSKind())
3093 return emitGlobalDtorWithCXAAtExit(CGF, dtor, addr, TLS: D.getTLSKind());
3094
3095 // In Apple kexts, we want to add a global destructor entry.
3096 // FIXME: shouldn't this be guarded by some variable?
3097 if (CGM.getLangOpts().AppleKext) {
3098 // Generate a global destructor entry.
3099 return CGM.AddCXXDtorEntry(DtorFn: dtor, Object: addr);
3100 }
3101
3102 CGF.registerGlobalDtorWithAtExit(D, fn: dtor, addr);
3103}
3104
3105static bool isThreadWrapperReplaceable(const VarDecl *VD,
3106 CodeGen::CodeGenModule &CGM) {
3107 assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!");
3108 // Darwin prefers to have references to thread local variables to go through
3109 // the thread wrapper instead of directly referencing the backing variable.
3110 return VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3111 CGM.getTarget().getTriple().isOSDarwin();
3112}
3113
3114/// Get the appropriate linkage for the wrapper function. This is essentially
3115/// the weak form of the variable's linkage; every translation unit which needs
3116/// the wrapper emits a copy, and we want the linker to merge them.
3117static llvm::GlobalValue::LinkageTypes
3118getThreadLocalWrapperLinkage(const VarDecl *VD, CodeGen::CodeGenModule &CGM) {
3119 llvm::GlobalValue::LinkageTypes VarLinkage =
3120 CGM.getLLVMLinkageVarDefinition(VD);
3121
3122 // For internal linkage variables, we don't need an external or weak wrapper.
3123 if (llvm::GlobalValue::isLocalLinkage(Linkage: VarLinkage))
3124 return VarLinkage;
3125
3126 // If the thread wrapper is replaceable, give it appropriate linkage.
3127 if (isThreadWrapperReplaceable(VD, CGM))
3128 if (!llvm::GlobalVariable::isLinkOnceLinkage(Linkage: VarLinkage) &&
3129 !llvm::GlobalVariable::isWeakODRLinkage(Linkage: VarLinkage))
3130 return VarLinkage;
3131 return llvm::GlobalValue::WeakODRLinkage;
3132}
3133
3134llvm::Function *
3135ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
3136 llvm::Value *Val) {
3137 // Mangle the name for the thread_local wrapper function.
3138 SmallString<256> WrapperName;
3139 {
3140 llvm::raw_svector_ostream Out(WrapperName);
3141 getMangleContext().mangleItaniumThreadLocalWrapper(D: VD, Out);
3142 }
3143
3144 // FIXME: If VD is a definition, we should regenerate the function attributes
3145 // before returning.
3146 if (llvm::Value *V = CGM.getModule().getNamedValue(Name: WrapperName))
3147 return cast<llvm::Function>(Val: V);
3148
3149 QualType RetQT = VD->getType();
3150 if (RetQT->isReferenceType())
3151 RetQT = RetQT.getNonReferenceType();
3152
3153 const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
3154 resultType: getContext().getPointerType(T: RetQT), args: FunctionArgList());
3155
3156 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FI);
3157 llvm::Function *Wrapper =
3158 llvm::Function::Create(Ty: FnTy, Linkage: getThreadLocalWrapperLinkage(VD, CGM),
3159 N: WrapperName.str(), M: &CGM.getModule());
3160
3161 if (CGM.supportsCOMDAT() && Wrapper->isWeakForLinker())
3162 Wrapper->setComdat(CGM.getModule().getOrInsertComdat(Name: Wrapper->getName()));
3163
3164 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Wrapper, /*IsThunk=*/false);
3165
3166 // Always resolve references to the wrapper at link time.
3167 if (!Wrapper->hasLocalLinkage())
3168 if (!isThreadWrapperReplaceable(VD, CGM) ||
3169 llvm::GlobalVariable::isLinkOnceLinkage(Linkage: Wrapper->getLinkage()) ||
3170 llvm::GlobalVariable::isWeakODRLinkage(Linkage: Wrapper->getLinkage()) ||
3171 VD->getVisibility() == HiddenVisibility)
3172 Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility);
3173
3174 if (isThreadWrapperReplaceable(VD, CGM)) {
3175 Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3176 Wrapper->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3177 }
3178
3179 ThreadWrappers.push_back(Elt: {VD, Wrapper});
3180 return Wrapper;
3181}
3182
3183void ItaniumCXXABI::EmitThreadLocalInitFuncs(
3184 CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
3185 ArrayRef<llvm::Function *> CXXThreadLocalInits,
3186 ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
3187 llvm::Function *InitFunc = nullptr;
3188
3189 // Separate initializers into those with ordered (or partially-ordered)
3190 // initialization and those with unordered initialization.
3191 llvm::SmallVector<llvm::Function *, 8> OrderedInits;
3192 llvm::SmallDenseMap<const VarDecl *, llvm::Function *> UnorderedInits;
3193 for (unsigned I = 0; I != CXXThreadLocalInits.size(); ++I) {
3194 if (isTemplateInstantiation(
3195 Kind: CXXThreadLocalInitVars[I]->getTemplateSpecializationKind()))
3196 UnorderedInits[CXXThreadLocalInitVars[I]->getCanonicalDecl()] =
3197 CXXThreadLocalInits[I];
3198 else
3199 OrderedInits.push_back(Elt: CXXThreadLocalInits[I]);
3200 }
3201
3202 if (!OrderedInits.empty()) {
3203 // Generate a guarded initialization function.
3204 llvm::FunctionType *FTy =
3205 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
3206 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3207 InitFunc = CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: "__tls_init", FI,
3208 Loc: SourceLocation(),
3209 /*TLS=*/true);
3210 llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
3211 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
3212 llvm::GlobalVariable::InternalLinkage,
3213 llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 0), "__tls_guard");
3214 Guard->setThreadLocal(true);
3215 Guard->setThreadLocalMode(CGM.GetDefaultLLVMTLSModel());
3216
3217 CharUnits GuardAlign = CharUnits::One();
3218 Guard->setAlignment(GuardAlign.getAsAlign());
3219
3220 CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(
3221 Fn: InitFunc, CXXThreadLocals: OrderedInits, Guard: ConstantAddress(Guard, CGM.Int8Ty, GuardAlign));
3222 // On Darwin platforms, use CXX_FAST_TLS calling convention.
3223 if (CGM.getTarget().getTriple().isOSDarwin()) {
3224 InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3225 InitFunc->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3226 }
3227 }
3228
3229 // Create declarations for thread wrappers for all thread-local variables
3230 // with non-discardable definitions in this translation unit.
3231 for (const VarDecl *VD : CXXThreadLocals) {
3232 if (VD->hasDefinition() &&
3233 !isDiscardableGVALinkage(L: getContext().GetGVALinkageForVariable(VD))) {
3234 llvm::GlobalValue *GV = CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD));
3235 getOrCreateThreadLocalWrapper(VD, Val: GV);
3236 }
3237 }
3238
3239 // Emit all referenced thread wrappers.
3240 for (auto VDAndWrapper : ThreadWrappers) {
3241 const VarDecl *VD = VDAndWrapper.first;
3242 llvm::GlobalVariable *Var =
3243 cast<llvm::GlobalVariable>(Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD)));
3244 llvm::Function *Wrapper = VDAndWrapper.second;
3245
3246 // Some targets require that all access to thread local variables go through
3247 // the thread wrapper. This means that we cannot attempt to create a thread
3248 // wrapper or a thread helper.
3249 if (!VD->hasDefinition()) {
3250 if (isThreadWrapperReplaceable(VD, CGM)) {
3251 Wrapper->setLinkage(llvm::Function::ExternalLinkage);
3252 continue;
3253 }
3254
3255 // If this isn't a TU in which this variable is defined, the thread
3256 // wrapper is discardable.
3257 if (Wrapper->getLinkage() == llvm::Function::WeakODRLinkage)
3258 Wrapper->setLinkage(llvm::Function::LinkOnceODRLinkage);
3259 }
3260
3261 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Wrapper);
3262
3263 // Mangle the name for the thread_local initialization function.
3264 SmallString<256> InitFnName;
3265 {
3266 llvm::raw_svector_ostream Out(InitFnName);
3267 getMangleContext().mangleItaniumThreadLocalInit(D: VD, Out);
3268 }
3269
3270 llvm::FunctionType *InitFnTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
3271
3272 // If we have a definition for the variable, emit the initialization
3273 // function as an alias to the global Init function (if any). Otherwise,
3274 // produce a declaration of the initialization function.
3275 llvm::GlobalValue *Init = nullptr;
3276 bool InitIsInitFunc = false;
3277 bool HasConstantInitialization = false;
3278 if (!usesThreadWrapperFunction(VD)) {
3279 HasConstantInitialization = true;
3280 } else if (VD->hasDefinition()) {
3281 InitIsInitFunc = true;
3282 llvm::Function *InitFuncToUse = InitFunc;
3283 if (isTemplateInstantiation(Kind: VD->getTemplateSpecializationKind()))
3284 InitFuncToUse = UnorderedInits.lookup(Val: VD->getCanonicalDecl());
3285 if (InitFuncToUse)
3286 Init = llvm::GlobalAlias::create(Linkage: Var->getLinkage(), Name: InitFnName.str(),
3287 Aliasee: InitFuncToUse);
3288 } else {
3289 // Emit a weak global function referring to the initialization function.
3290 // This function will not exist if the TU defining the thread_local
3291 // variable in question does not need any dynamic initialization for
3292 // its thread_local variables.
3293 Init = llvm::Function::Create(Ty: InitFnTy,
3294 Linkage: llvm::GlobalVariable::ExternalWeakLinkage,
3295 N: InitFnName.str(), M: &CGM.getModule());
3296 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3297 CGM.SetLLVMFunctionAttributes(
3298 GD: GlobalDecl(), Info: FI, F: cast<llvm::Function>(Val: Init), /*IsThunk=*/false);
3299 }
3300
3301 if (Init) {
3302 Init->setVisibility(Var->getVisibility());
3303 // Don't mark an extern_weak function DSO local on windows.
3304 if (!CGM.getTriple().isOSWindows() || !Init->hasExternalWeakLinkage())
3305 Init->setDSOLocal(Var->isDSOLocal());
3306 }
3307
3308 llvm::LLVMContext &Context = CGM.getModule().getContext();
3309
3310 // The linker on AIX is not happy with missing weak symbols. However,
3311 // other TUs will not know whether the initialization routine exists
3312 // so create an empty, init function to satisfy the linker.
3313 // This is needed whenever a thread wrapper function is not used, and
3314 // also when the symbol is weak.
3315 if (CGM.getTriple().isOSAIX() && VD->hasDefinition() &&
3316 isEmittedWithConstantInitializer(VD, InspectInitForWeakDef: true) &&
3317 !mayNeedDestruction(VD)) {
3318 // Init should be null. If it were non-null, then the logic above would
3319 // either be defining the function to be an alias or declaring the
3320 // function with the expectation that the definition of the variable
3321 // is elsewhere.
3322 assert(Init == nullptr && "Expected Init to be null.");
3323
3324 llvm::Function *Func = llvm::Function::Create(
3325 Ty: InitFnTy, Linkage: Var->getLinkage(), N: InitFnName.str(), M: &CGM.getModule());
3326 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3327 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI,
3328 F: cast<llvm::Function>(Val: Func),
3329 /*IsThunk=*/false);
3330 // Create a function body that just returns
3331 llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Func);
3332 CGBuilderTy Builder(CGM, Entry);
3333 Builder.CreateRetVoid();
3334 }
3335
3336 llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3337 CGBuilderTy Builder(CGM, Entry);
3338 if (HasConstantInitialization) {
3339 // No dynamic initialization to invoke.
3340 } else if (InitIsInitFunc) {
3341 if (Init) {
3342 llvm::CallInst *CallVal = Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3343 if (isThreadWrapperReplaceable(VD, CGM)) {
3344 CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3345 llvm::Function *Fn =
3346 cast<llvm::Function>(Val: cast<llvm::GlobalAlias>(Val: Init)->getAliasee());
3347 Fn->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3348 }
3349 }
3350 } else if (CGM.getTriple().isOSAIX()) {
3351 // On AIX, except if constinit and also neither of class type or of
3352 // (possibly multi-dimensional) array of class type, thread_local vars
3353 // will have init routines regardless of whether they are
3354 // const-initialized. Since the routine is guaranteed to exist, we can
3355 // unconditionally call it without testing for its existance. This
3356 // avoids potentially unresolved weak symbols which the AIX linker
3357 // isn't happy with.
3358 Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3359 } else {
3360 // Don't know whether we have an init function. Call it if it exists.
3361 llvm::Value *Have = Builder.CreateIsNotNull(Arg: Init);
3362 llvm::BasicBlock *InitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3363 llvm::BasicBlock *ExitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3364 Builder.CreateCondBr(Cond: Have, True: InitBB, False: ExitBB);
3365
3366 Builder.SetInsertPoint(InitBB);
3367 Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3368 Builder.CreateBr(Dest: ExitBB);
3369
3370 Builder.SetInsertPoint(ExitBB);
3371 }
3372
3373 // For a reference, the result of the wrapper function is a pointer to
3374 // the referenced object.
3375 llvm::Value *Val = Builder.CreateThreadLocalAddress(Ptr: Var);
3376
3377 if (VD->getType()->isReferenceType()) {
3378 CharUnits Align = CGM.getContext().getDeclAlign(D: VD);
3379 Val = Builder.CreateAlignedLoad(Ty: Var->getValueType(), Addr: Val, Align);
3380 }
3381 Val = Builder.CreateAddrSpaceCast(V: Val, DestTy: Wrapper->getReturnType());
3382
3383 Builder.CreateRet(V: Val);
3384 }
3385}
3386
3387LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
3388 const VarDecl *VD,
3389 QualType LValType) {
3390 llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3391 llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val);
3392
3393 llvm::CallInst *CallVal =
3394 CGF.Builder.CreateCall(Callee: Wrapper, Args: {}, OpBundles: CGF.getBundlesForFunclet(Callee: Wrapper));
3395 CallVal->setCallingConv(Wrapper->getCallingConv());
3396
3397 LValue LV;
3398 if (VD->getType()->isReferenceType())
3399 LV = CGF.MakeNaturalAlignRawAddrLValue(V: CallVal, T: LValType);
3400 else
3401 LV = CGF.MakeRawAddrLValue(V: CallVal, T: LValType,
3402 Alignment: CGF.getContext().getDeclAlign(D: VD));
3403 // FIXME: need setObjCGCLValueClass?
3404 return LV;
3405}
3406
3407/// Return whether the given global decl needs a VTT parameter, which it does
3408/// if it's a base constructor or destructor with virtual bases.
3409bool ItaniumCXXABI::NeedsVTTParameter(GlobalDecl GD) {
3410 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
3411
3412 // We don't have any virtual bases, just return early.
3413 if (!MD->getParent()->getNumVBases())
3414 return false;
3415
3416 // Check if we have a base constructor.
3417 if (isa<CXXConstructorDecl>(Val: MD) && GD.getCtorType() == Ctor_Base)
3418 return true;
3419
3420 // Check if we have a base destructor.
3421 if (isa<CXXDestructorDecl>(Val: MD) && GD.getDtorType() == Dtor_Base)
3422 return true;
3423
3424 return false;
3425}
3426
3427llvm::Constant *
3428ItaniumCXXABI::getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD) {
3429 SmallString<256> MethodName;
3430 llvm::raw_svector_ostream Out(MethodName);
3431 getMangleContext().mangleCXXName(GD: MD, Out);
3432 MethodName += "_vfpthunk_";
3433 StringRef ThunkName = MethodName.str();
3434 llvm::Function *ThunkFn;
3435 if ((ThunkFn = cast_or_null<llvm::Function>(
3436 Val: CGM.getModule().getNamedValue(Name: ThunkName))))
3437 return ThunkFn;
3438
3439 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeCXXMethodDeclaration(MD);
3440 llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
3441 llvm::GlobalValue::LinkageTypes Linkage =
3442 MD->isExternallyVisible() ? llvm::GlobalValue::LinkOnceODRLinkage
3443 : llvm::GlobalValue::InternalLinkage;
3444 ThunkFn =
3445 llvm::Function::Create(Ty: ThunkTy, Linkage, N: ThunkName, M: &CGM.getModule());
3446 if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
3447 ThunkFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3448 assert(ThunkFn->getName() == ThunkName && "name was uniqued!");
3449
3450 CGM.SetLLVMFunctionAttributes(GD: MD, Info: FnInfo, F: ThunkFn, /*IsThunk=*/true);
3451 CGM.SetLLVMFunctionAttributesForDefinition(D: MD, F: ThunkFn);
3452
3453 // Stack protection sometimes gets inserted after the musttail call.
3454 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtect);
3455 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtectStrong);
3456 ThunkFn->removeFnAttr(Kind: llvm::Attribute::StackProtectReq);
3457
3458 // Start codegen.
3459 CodeGenFunction CGF(CGM);
3460 CGF.CurGD = GlobalDecl(MD);
3461 CGF.CurFuncIsThunk = true;
3462
3463 // Build FunctionArgs.
3464 FunctionArgList FunctionArgs;
3465 CGF.BuildFunctionArgList(GD: CGF.CurGD, Args&: FunctionArgs);
3466
3467 CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
3468 Args: FunctionArgs, Loc: MD->getLocation(), StartLoc: SourceLocation());
3469
3470 // Emit an artificial location for this function.
3471 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
3472
3473 llvm::Value *ThisVal = loadIncomingCXXThis(CGF);
3474 setCXXABIThisValue(CGF, ThisPtr: ThisVal);
3475
3476 CallArgList CallArgs;
3477 for (const VarDecl *VD : FunctionArgs)
3478 CGF.EmitDelegateCallArg(args&: CallArgs, param: VD, loc: SourceLocation());
3479
3480 const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
3481 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FPT, /*this*/ additional: 1);
3482 const CGFunctionInfo &CallInfo =
3483 CGM.getTypes().arrangeCXXMethodCall(args: CallArgs, type: FPT, required: Required, numPrefixArgs: 0, ABIInfoFD: MD);
3484 CGCallee Callee = CGCallee::forVirtual(CE: nullptr, MD: GlobalDecl(MD),
3485 Addr: getThisAddress(CGF), FTy: ThunkTy);
3486 llvm::CallBase *CallOrInvoke;
3487 CGF.EmitCall(CallInfo, Callee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: &CallOrInvoke,
3488 /*IsMustTail=*/true, Loc: SourceLocation(), IsVirtualFunctionPointerThunk: true);
3489 auto *Call = cast<llvm::CallInst>(Val: CallOrInvoke);
3490 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
3491 if (Call->getType()->isVoidTy())
3492 CGF.Builder.CreateRetVoid();
3493 else
3494 CGF.Builder.CreateRet(V: Call);
3495
3496 // Finish the function to maintain CodeGenFunction invariants.
3497 // FIXME: Don't emit unreachable code.
3498 CGF.EmitBlock(BB: CGF.createBasicBlock());
3499 CGF.FinishFunction();
3500 return ThunkFn;
3501}
3502
3503namespace {
3504class ItaniumRTTIBuilder {
3505 CodeGenModule &CGM; // Per-module state.
3506 llvm::LLVMContext &VMContext;
3507 const ItaniumCXXABI &CXXABI; // Per-module state.
3508
3509 /// Fields - The fields of the RTTI descriptor currently being built.
3510 SmallVector<llvm::Constant *, 16> Fields;
3511
3512 /// GetAddrOfTypeName - Returns the mangled type name of the given type.
3513 llvm::GlobalVariable *
3514 GetAddrOfTypeName(QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage);
3515
3516 /// GetAddrOfExternalRTTIDescriptor - Returns the constant for the RTTI
3517 /// descriptor of the given type.
3518 llvm::Constant *GetAddrOfExternalRTTIDescriptor(QualType Ty);
3519
3520 /// BuildVTablePointer - Build the vtable pointer for the given type.
3521 void BuildVTablePointer(const Type *Ty, llvm::Constant *StorageAddress);
3522
3523 /// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
3524 /// inheritance, according to the Itanium C++ ABI, 2.9.5p6b.
3525 void BuildSIClassTypeInfo(const CXXRecordDecl *RD);
3526
3527 /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
3528 /// classes with bases that do not satisfy the abi::__si_class_type_info
3529 /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
3530 void BuildVMIClassTypeInfo(const CXXRecordDecl *RD);
3531
3532 /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, used
3533 /// for pointer types.
3534 void BuildPointerTypeInfo(QualType PointeeTy);
3535
3536 /// BuildObjCObjectTypeInfo - Build the appropriate kind of
3537 /// type_info for an object type.
3538 void BuildObjCObjectTypeInfo(const ObjCObjectType *Ty);
3539
3540 /// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
3541 /// struct, used for member pointer types.
3542 void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty);
3543
3544public:
3545 ItaniumRTTIBuilder(const ItaniumCXXABI &ABI)
3546 : CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {}
3547
3548 /// BuildTypeInfo - Build the RTTI type info struct for the given type, or
3549 /// link to an existing RTTI descriptor if one already exists.
3550 llvm::Constant *BuildTypeInfo(QualType Ty);
3551
3552 /// BuildTypeInfo - Build the RTTI type info struct for the given type. The
3553 /// TypeInfo* properties apply to the type info, the others to the type name.
3554 llvm::Constant *BuildTypeInfo(
3555 QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage,
3556 llvm::GlobalValue::VisibilityTypes Visibility,
3557 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass,
3558 llvm::GlobalValue::VisibilityTypes TypeInfoVisibility,
3559 llvm::GlobalValue::DLLStorageClassTypes TypeInfoDLLStorageClass);
3560};
3561}
3562
3563llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName(
3564 QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) {
3565 SmallString<256> Name;
3566 llvm::raw_svector_ostream Out(Name);
3567 CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(T: Ty, Out);
3568
3569 // We know that the mangled name of the type starts at index 4 of the
3570 // mangled name of the typename, so we can just index into it in order to
3571 // get the mangled name of the type.
3572 llvm::Constant *Init;
3573 if (CGM.getTriple().isOSzOS()) {
3574 // On z/OS, typename is stored as 2 encodings: EBCDIC followed by ASCII.
3575 SmallString<256> DualEncodedName;
3576 llvm::ConverterEBCDIC::convertToEBCDIC(Source: Name.substr(Start: 4), Result&: DualEncodedName);
3577 DualEncodedName += '\0';
3578 DualEncodedName += Name.substr(Start: 4);
3579 Init = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: DualEncodedName);
3580 } else
3581 Init = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Name.substr(Start: 4));
3582
3583 auto Align = CGM.getContext().getTypeAlignInChars(T: CGM.getContext().CharTy);
3584
3585 llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
3586 Name, Ty: Init->getType(), Linkage, Alignment: Align.getAsAlign());
3587
3588 GV->setInitializer(Init);
3589
3590 return GV;
3591}
3592
3593llvm::Constant *
3594ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
3595 // Mangle the RTTI name.
3596 SmallString<256> Name;
3597 llvm::raw_svector_ostream Out(Name);
3598 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
3599
3600 // Look for an existing global.
3601 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
3602
3603 if (!GV) {
3604 // Create a new global variable.
3605 // Note for the future: If we would ever like to do deferred emission of
3606 // RTTI, check if emitting vtables opportunistically need any adjustment.
3607
3608 GV = new llvm::GlobalVariable(
3609 CGM.getModule(), CGM.GlobalsInt8PtrTy,
3610 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, Name);
3611 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
3612 CGM.setGVProperties(GV, D: RD);
3613 // Import the typeinfo symbol when all non-inline virtual methods are
3614 // imported.
3615 if (CGM.getTarget().hasPS4DLLImportExport()) {
3616 if (RD && CXXRecordNonInlineHasAttr<DLLImportAttr>(RD)) {
3617 GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
3618 CGM.setDSOLocal(GV);
3619 }
3620 }
3621 }
3622
3623 return GV;
3624}
3625
3626/// TypeInfoIsInStandardLibrary - Given a builtin type, returns whether the type
3627/// info for that type is defined in the standard library.
3628static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
3629 // Itanium C++ ABI 2.9.2:
3630 // Basic type information (e.g. for "int", "bool", etc.) will be kept in
3631 // the run-time support library. Specifically, the run-time support
3632 // library should contain type_info objects for the types X, X* and
3633 // X const*, for every X in: void, std::nullptr_t, bool, wchar_t, char,
3634 // unsigned char, signed char, short, unsigned short, int, unsigned int,
3635 // long, unsigned long, long long, unsigned long long, float, double,
3636 // long double, char16_t, char32_t, and the IEEE 754r decimal and
3637 // half-precision floating point types.
3638 //
3639 // GCC also emits RTTI for __int128.
3640 // FIXME: We do not emit RTTI information for decimal types here.
3641
3642 // Types added here must also be added to EmitFundamentalRTTIDescriptors.
3643 switch (Ty->getKind()) {
3644 case BuiltinType::Void:
3645 case BuiltinType::NullPtr:
3646 case BuiltinType::Bool:
3647 case BuiltinType::WChar_S:
3648 case BuiltinType::WChar_U:
3649 case BuiltinType::Char_U:
3650 case BuiltinType::Char_S:
3651 case BuiltinType::UChar:
3652 case BuiltinType::SChar:
3653 case BuiltinType::Short:
3654 case BuiltinType::UShort:
3655 case BuiltinType::Int:
3656 case BuiltinType::UInt:
3657 case BuiltinType::Long:
3658 case BuiltinType::ULong:
3659 case BuiltinType::LongLong:
3660 case BuiltinType::ULongLong:
3661 case BuiltinType::Half:
3662 case BuiltinType::Float:
3663 case BuiltinType::Double:
3664 case BuiltinType::LongDouble:
3665 case BuiltinType::Float16:
3666 case BuiltinType::Float128:
3667 case BuiltinType::Ibm128:
3668 case BuiltinType::Char8:
3669 case BuiltinType::Char16:
3670 case BuiltinType::Char32:
3671 case BuiltinType::Int128:
3672 case BuiltinType::UInt128:
3673 return true;
3674
3675#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
3676 case BuiltinType::Id:
3677#include "clang/Basic/OpenCLImageTypes.def"
3678#define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
3679 case BuiltinType::Id:
3680#include "clang/Basic/OpenCLExtensionTypes.def"
3681 case BuiltinType::OCLSampler:
3682 case BuiltinType::OCLEvent:
3683 case BuiltinType::OCLClkEvent:
3684 case BuiltinType::OCLQueue:
3685 case BuiltinType::OCLReserveID:
3686#define SVE_TYPE(Name, Id, SingletonId) \
3687 case BuiltinType::Id:
3688#include "clang/Basic/AArch64ACLETypes.def"
3689#define PPC_VECTOR_TYPE(Name, Id, Size) \
3690 case BuiltinType::Id:
3691#include "clang/Basic/PPCTypes.def"
3692#define RVV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3693#include "clang/Basic/RISCVVTypes.def"
3694#define WASM_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3695#include "clang/Basic/WebAssemblyReferenceTypes.def"
3696#define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align) case BuiltinType::Id:
3697#include "clang/Basic/AMDGPUTypes.def"
3698#define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3699#include "clang/Basic/HLSLIntangibleTypes.def"
3700#define HLSL_PACKED_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3701#include "clang/Basic/HLSLPackedTypes.def"
3702#define SPIRV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3703#include "clang/Basic/SPIRVTypes.def"
3704 case BuiltinType::ShortAccum:
3705 case BuiltinType::Accum:
3706 case BuiltinType::LongAccum:
3707 case BuiltinType::UShortAccum:
3708 case BuiltinType::UAccum:
3709 case BuiltinType::ULongAccum:
3710 case BuiltinType::ShortFract:
3711 case BuiltinType::Fract:
3712 case BuiltinType::LongFract:
3713 case BuiltinType::UShortFract:
3714 case BuiltinType::UFract:
3715 case BuiltinType::ULongFract:
3716 case BuiltinType::SatShortAccum:
3717 case BuiltinType::SatAccum:
3718 case BuiltinType::SatLongAccum:
3719 case BuiltinType::SatUShortAccum:
3720 case BuiltinType::SatUAccum:
3721 case BuiltinType::SatULongAccum:
3722 case BuiltinType::SatShortFract:
3723 case BuiltinType::SatFract:
3724 case BuiltinType::SatLongFract:
3725 case BuiltinType::SatUShortFract:
3726 case BuiltinType::SatUFract:
3727 case BuiltinType::SatULongFract:
3728 case BuiltinType::BFloat16:
3729 case BuiltinType::MetaInfo:
3730 return false;
3731
3732 case BuiltinType::Dependent:
3733#define BUILTIN_TYPE(Id, SingletonId)
3734#define PLACEHOLDER_TYPE(Id, SingletonId) \
3735 case BuiltinType::Id:
3736#include "clang/AST/BuiltinTypes.def"
3737 llvm_unreachable("asking for RRTI for a placeholder type!");
3738
3739 case BuiltinType::ObjCId:
3740 case BuiltinType::ObjCClass:
3741 case BuiltinType::ObjCSel:
3742 llvm_unreachable("FIXME: Objective-C types are unsupported!");
3743 }
3744
3745 llvm_unreachable("Invalid BuiltinType Kind!");
3746}
3747
3748static bool TypeInfoIsInStandardLibrary(const PointerType *PointerTy) {
3749 QualType PointeeTy = PointerTy->getPointeeType();
3750 const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: PointeeTy);
3751 if (!BuiltinTy)
3752 return false;
3753
3754 // Check the qualifiers.
3755 Qualifiers Quals = PointeeTy.getQualifiers();
3756 Quals.removeConst();
3757
3758 if (!Quals.empty())
3759 return false;
3760
3761 return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3762}
3763
3764/// IsStandardLibraryRTTIDescriptor - Returns whether the type
3765/// information for the given type exists in the standard library.
3766static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
3767 // Type info for builtin types is defined in the standard library.
3768 if (const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: Ty))
3769 return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3770
3771 // Type info for some pointer types to builtin types is defined in the
3772 // standard library.
3773 if (const PointerType *PointerTy = dyn_cast<PointerType>(Val&: Ty))
3774 return TypeInfoIsInStandardLibrary(PointerTy);
3775
3776 return false;
3777}
3778
3779/// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
3780/// the given type exists somewhere else, and that we should not emit the type
3781/// information in this translation unit. Assumes that it is not a
3782/// standard-library type.
3783static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
3784 QualType Ty) {
3785 ASTContext &Context = CGM.getContext();
3786
3787 // If RTTI is disabled, assume it might be disabled in the
3788 // translation unit that defines any potential key function, too.
3789 if (!Context.getLangOpts().RTTI) return false;
3790
3791 if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
3792 const CXXRecordDecl *RD =
3793 cast<CXXRecordDecl>(Val: RecordTy->getDecl())->getDefinitionOrSelf();
3794 if (!RD->hasDefinition())
3795 return false;
3796
3797 if (!RD->isDynamicClass())
3798 return false;
3799
3800 // FIXME: this may need to be reconsidered if the key function
3801 // changes.
3802 // N.B. We must always emit the RTTI data ourselves if there exists a key
3803 // function.
3804 bool IsDLLImport = RD->hasAttr<DLLImportAttr>();
3805
3806 // Don't import the RTTI but emit it locally.
3807 if (CGM.getTriple().isOSCygMing())
3808 return false;
3809
3810 if (CGM.getVTables().isVTableExternal(RD)) {
3811 if (CGM.getTarget().hasPS4DLLImportExport())
3812 return true;
3813
3814 return IsDLLImport && !CGM.getTriple().isWindowsItaniumEnvironment()
3815 ? false
3816 : true;
3817 }
3818 if (IsDLLImport)
3819 return true;
3820 }
3821
3822 return false;
3823}
3824
3825void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
3826 llvm::Constant *StorageAddress) {
3827 // abi::__class_type_info.
3828 static const char * const ClassTypeInfo =
3829 "_ZTVN10__cxxabiv117__class_type_infoE";
3830 // abi::__si_class_type_info.
3831 static const char * const SIClassTypeInfo =
3832 "_ZTVN10__cxxabiv120__si_class_type_infoE";
3833 // abi::__vmi_class_type_info.
3834 static const char * const VMIClassTypeInfo =
3835 "_ZTVN10__cxxabiv121__vmi_class_type_infoE";
3836
3837 const char *VTableName = nullptr;
3838
3839 switch (Ty->getTypeClass()) {
3840#define TYPE(Class, Base)
3841#define ABSTRACT_TYPE(Class, Base)
3842#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
3843#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
3844#define DEPENDENT_TYPE(Class, Base) case Type::Class:
3845#include "clang/AST/TypeNodes.inc"
3846 llvm_unreachable("Non-canonical and dependent types shouldn't get here");
3847
3848 case Type::LValueReference:
3849 case Type::RValueReference:
3850 llvm_unreachable("References shouldn't get here");
3851
3852 case Type::Auto:
3853 case Type::DeducedTemplateSpecialization:
3854 llvm_unreachable("Undeduced type shouldn't get here");
3855
3856 case Type::Pipe:
3857 llvm_unreachable("Pipe types shouldn't get here");
3858
3859 case Type::ArrayParameter:
3860 llvm_unreachable("Array Parameter types should not get here.");
3861
3862 case Type::Builtin:
3863 case Type::BitInt:
3864 case Type::OverflowBehavior:
3865 // GCC treats vector and complex types as fundamental types.
3866 case Type::Vector:
3867 case Type::ExtVector:
3868 case Type::ConstantMatrix:
3869 case Type::Complex:
3870 case Type::Atomic:
3871 // FIXME: GCC treats block pointers as fundamental types?!
3872 case Type::BlockPointer:
3873 // abi::__fundamental_type_info.
3874 VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE";
3875 break;
3876
3877 case Type::ConstantArray:
3878 case Type::IncompleteArray:
3879 case Type::VariableArray:
3880 // abi::__array_type_info.
3881 VTableName = "_ZTVN10__cxxabiv117__array_type_infoE";
3882 break;
3883
3884 case Type::FunctionNoProto:
3885 case Type::FunctionProto:
3886 // abi::__function_type_info.
3887 VTableName = "_ZTVN10__cxxabiv120__function_type_infoE";
3888 break;
3889
3890 case Type::Enum:
3891 // abi::__enum_type_info.
3892 VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE";
3893 break;
3894
3895 case Type::Record: {
3896 const auto *RD = cast<CXXRecordDecl>(Val: cast<RecordType>(Val: Ty)->getDecl())
3897 ->getDefinitionOrSelf();
3898
3899 if (!RD->hasDefinition() || !RD->getNumBases()) {
3900 VTableName = ClassTypeInfo;
3901 } else if (CodeGenUtils::canUseSingleInheritance(RD)) {
3902 VTableName = SIClassTypeInfo;
3903 } else {
3904 VTableName = VMIClassTypeInfo;
3905 }
3906
3907 break;
3908 }
3909
3910 case Type::ObjCObject:
3911 // Ignore protocol qualifiers.
3912 Ty = cast<ObjCObjectType>(Val: Ty)->getBaseType().getTypePtr();
3913
3914 // Handle id and Class.
3915 if (isa<BuiltinType>(Val: Ty)) {
3916 VTableName = ClassTypeInfo;
3917 break;
3918 }
3919
3920 assert(isa<ObjCInterfaceType>(Ty));
3921 [[fallthrough]];
3922
3923 case Type::ObjCInterface:
3924 if (cast<ObjCInterfaceType>(Val: Ty)->getDecl()->getSuperClass()) {
3925 VTableName = SIClassTypeInfo;
3926 } else {
3927 VTableName = ClassTypeInfo;
3928 }
3929 break;
3930
3931 case Type::ObjCObjectPointer:
3932 case Type::Pointer:
3933 // abi::__pointer_type_info.
3934 VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE";
3935 break;
3936
3937 case Type::MemberPointer:
3938 // abi::__pointer_to_member_type_info.
3939 VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE";
3940 break;
3941
3942 case Type::HLSLAttributedResource:
3943 case Type::HLSLInlineSpirv:
3944 llvm_unreachable("HLSL doesn't support virtual functions");
3945 }
3946
3947 llvm::Constant *VTable = nullptr;
3948
3949 // Check if the alias exists. If it doesn't, then get or create the global.
3950 if (CGM.getLangOpts().RelativeCXXABIVTables)
3951 VTable = CGM.getModule().getNamedAlias(Name: VTableName);
3952 if (!VTable) {
3953 llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.GlobalsInt8PtrTy, NumElements: 0);
3954 VTable = CGM.getModule().getOrInsertGlobal(Name: VTableName, Ty);
3955 }
3956
3957 CGM.setDSOLocal(cast<llvm::GlobalValue>(Val: VTable->stripPointerCasts()));
3958
3959 llvm::Type *PtrDiffTy =
3960 CGM.getTypes().ConvertType(T: CGM.getContext().getPointerDiffType());
3961
3962 // The vtable address point is 2.
3963 if (CGM.getLangOpts().RelativeCXXABIVTables) {
3964 // The vtable address point is 8 bytes after its start:
3965 // 4 for the offset to top + 4 for the relative offset to rtti.
3966 llvm::Constant *Eight = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 8);
3967 VTable = llvm::ConstantExpr::getInBoundsPtrAdd(Ptr: VTable, Offset: Eight);
3968 } else {
3969 llvm::Constant *Two = llvm::ConstantInt::get(Ty: PtrDiffTy, V: 2);
3970 VTable = llvm::ConstantExpr::getGetElementPtr(
3971 DL: CGM.getDataLayout(), Ty: CGM.GlobalsInt8PtrTy, C: VTable, IdxList: Two,
3972 NW: llvm::GEPNoWrapFlags::inBounds());
3973 }
3974
3975 if (const auto &Schema =
3976 CGM.getCodeGenOpts().PointerAuth.CXXTypeInfoVTablePointer)
3977 VTable = CGM.getConstantSignedPointer(
3978 Pointer: VTable, Schema,
3979 StorageAddress: Schema.isAddressDiscriminated() ? StorageAddress : nullptr,
3980 SchemaDecl: GlobalDecl(), SchemaType: QualType(Ty, 0));
3981
3982 Fields.push_back(Elt: VTable);
3983}
3984
3985/// Return the linkage that the type info and type info name constants
3986/// should have for the given type.
3987static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM,
3988 QualType Ty) {
3989 // Itanium C++ ABI 2.9.5p7:
3990 // In addition, it and all of the intermediate abi::__pointer_type_info
3991 // structs in the chain down to the abi::__class_type_info for the
3992 // incomplete class type must be prevented from resolving to the
3993 // corresponding type_info structs for the complete class type, possibly
3994 // by making them local static objects. Finally, a dummy class RTTI is
3995 // generated for the incomplete type that will not resolve to the final
3996 // complete class RTTI (because the latter need not exist), possibly by
3997 // making it a local static object.
3998 if (CodeGenUtils::containsIncompleteClassType(Ty))
3999 return llvm::GlobalValue::InternalLinkage;
4000
4001 switch (Ty->getLinkage()) {
4002 case Linkage::Invalid:
4003 llvm_unreachable("Linkage hasn't been computed!");
4004
4005 case Linkage::None:
4006 case Linkage::Internal:
4007 case Linkage::UniqueExternal:
4008 return llvm::GlobalValue::InternalLinkage;
4009
4010 case Linkage::VisibleNone:
4011 case Linkage::Module:
4012 case Linkage::External:
4013 // RTTI is not enabled, which means that this type info struct is going
4014 // to be used for exception handling. Give it linkonce_odr linkage.
4015 if (!CGM.getLangOpts().RTTI)
4016 return llvm::GlobalValue::LinkOnceODRLinkage;
4017
4018 if (const RecordType *Record = dyn_cast<RecordType>(Val&: Ty)) {
4019 const auto *RD =
4020 cast<CXXRecordDecl>(Val: Record->getDecl())->getDefinitionOrSelf();
4021 if (RD->hasAttr<WeakAttr>())
4022 return llvm::GlobalValue::WeakODRLinkage;
4023 if (CGM.getTriple().isWindowsItaniumEnvironment())
4024 if (RD->hasAttr<DLLImportAttr>() &&
4025 ShouldUseExternalRTTIDescriptor(CGM, Ty))
4026 return llvm::GlobalValue::ExternalLinkage;
4027 // MinGW always uses LinkOnceODRLinkage for type info.
4028 if (RD->isDynamicClass() &&
4029 !CGM.getContext().getTargetInfo().getTriple().isOSCygMing())
4030 return CGM.getVTableLinkage(RD);
4031 }
4032
4033 return llvm::GlobalValue::LinkOnceODRLinkage;
4034 }
4035
4036 llvm_unreachable("Invalid linkage!");
4037}
4038
4039/// A dllexported global must not be hidden, so dllexport takes precedence over
4040/// the visibility implied by -fvisibility=hidden, as elsewhere in CodeGen.
4041static llvm::GlobalValue::VisibilityTypes
4042getRTTIVisibility(llvm::GlobalValue::VisibilityTypes Visibility,
4043 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass) {
4044 if (DLLStorageClass == llvm::GlobalValue::DLLExportStorageClass &&
4045 Visibility == llvm::GlobalValue::HiddenVisibility)
4046 return llvm::GlobalValue::DefaultVisibility;
4047 return Visibility;
4048}
4049
4050llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty) {
4051 // We want to operate on the canonical type.
4052 Ty = Ty.getCanonicalType();
4053
4054 // Check if we've already emitted an RTTI descriptor for this type.
4055 SmallString<256> Name;
4056 llvm::raw_svector_ostream Out(Name);
4057 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
4058
4059 llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
4060 if (OldGV && !OldGV->isDeclaration()) {
4061 assert(!OldGV->hasAvailableExternallyLinkage() &&
4062 "available_externally typeinfos not yet implemented");
4063
4064 return OldGV;
4065 }
4066
4067 // Check if there is already an external RTTI descriptor for this type.
4068 if (IsStandardLibraryRTTIDescriptor(Ty) ||
4069 ShouldUseExternalRTTIDescriptor(CGM, Ty))
4070 return GetAddrOfExternalRTTIDescriptor(Ty);
4071
4072 // Emit the standard library with external linkage.
4073 llvm::GlobalVariable::LinkageTypes Linkage = getTypeInfoLinkage(CGM, Ty);
4074
4075 // Give the type_info object and name the formal visibility of the
4076 // type itself.
4077 llvm::GlobalValue::VisibilityTypes llvmVisibility;
4078 if (llvm::GlobalValue::isLocalLinkage(Linkage))
4079 // If the linkage is local, only default visibility makes sense.
4080 llvmVisibility = llvm::GlobalValue::DefaultVisibility;
4081 else if (CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage) ==
4082 ItaniumCXXABI::RUK_NonUniqueHidden)
4083 llvmVisibility = llvm::GlobalValue::HiddenVisibility;
4084 else
4085 llvmVisibility = CodeGenModule::GetLLVMVisibility(V: Ty->getVisibility());
4086
4087 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
4088 llvm::GlobalValue::DefaultStorageClass;
4089 if (auto RD = Ty->getAsCXXRecordDecl()) {
4090 if ((CGM.getTriple().isWindowsItaniumEnvironment() &&
4091 RD->hasAttr<DLLExportAttr>()) ||
4092 (CGM.shouldMapVisibilityToDLLExport(D: RD) &&
4093 !llvm::GlobalValue::isLocalLinkage(Linkage) &&
4094 llvmVisibility == llvm::GlobalValue::DefaultVisibility))
4095 DLLStorageClass = llvm::GlobalValue::DLLExportStorageClass;
4096 }
4097 llvmVisibility = getRTTIVisibility(Visibility: llvmVisibility, DLLStorageClass);
4098
4099 // Export the typeinfo in the same circumstances as the vtable is exported.
4100 llvm::GlobalValue::DLLStorageClassTypes TypeInfoDLLStorageClass =
4101 DLLStorageClass;
4102 if (CGM.getTarget().hasPS4DLLImportExport() &&
4103 TypeInfoDLLStorageClass != llvm::GlobalValue::DLLExportStorageClass) {
4104 if (auto RD = Ty->getAsCXXRecordDecl()) {
4105 if (RD->hasAttr<DLLExportAttr>() ||
4106 CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
4107 TypeInfoDLLStorageClass = llvm::GlobalValue::DLLExportStorageClass;
4108 }
4109 }
4110 llvm::GlobalValue::VisibilityTypes TypeInfoVisibility =
4111 getRTTIVisibility(Visibility: llvmVisibility, DLLStorageClass: TypeInfoDLLStorageClass);
4112
4113 return BuildTypeInfo(Ty, Linkage, Visibility: llvmVisibility, DLLStorageClass,
4114 TypeInfoVisibility, TypeInfoDLLStorageClass);
4115}
4116
4117llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
4118 QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage,
4119 llvm::GlobalValue::VisibilityTypes Visibility,
4120 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass,
4121 llvm::GlobalValue::VisibilityTypes TypeInfoVisibility,
4122 llvm::GlobalValue::DLLStorageClassTypes TypeInfoDLLStorageClass) {
4123 SmallString<256> Name;
4124 llvm::raw_svector_ostream Out(Name);
4125 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
4126 llvm::Module &M = CGM.getModule();
4127 llvm::GlobalVariable *OldGV = M.getNamedGlobal(Name);
4128 // int8 is an arbitrary type to be replaced later with replaceInitializer.
4129 llvm::GlobalVariable *GV =
4130 new llvm::GlobalVariable(M, CGM.Int8Ty, /*isConstant=*/true, Linkage,
4131 /*Initializer=*/nullptr, Name);
4132
4133 // Add the vtable pointer.
4134 BuildVTablePointer(Ty: cast<Type>(Val&: Ty), StorageAddress: GV);
4135
4136 // And the name.
4137 llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
4138 llvm::Constant *TypeNameField;
4139
4140 // If we're supposed to demote the visibility, be sure to set a flag
4141 // to use a string comparison for type_info comparisons.
4142 ItaniumCXXABI::RTTIUniquenessKind RTTIUniqueness =
4143 CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage);
4144 if (RTTIUniqueness != ItaniumCXXABI::RUK_Unique) {
4145 // The flag is the sign bit, which on ARM64 is defined to be clear
4146 // for global pointers. This is very ARM64-specific.
4147 TypeNameField = llvm::ConstantExpr::getPtrToInt(C: TypeName, Ty: CGM.Int64Ty);
4148 llvm::Constant *flag =
4149 llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: ((uint64_t)1) << 63);
4150 TypeNameField = llvm::ConstantExpr::getAdd(C1: TypeNameField, C2: flag);
4151 TypeNameField =
4152 llvm::ConstantExpr::getIntToPtr(C: TypeNameField, Ty: CGM.GlobalsInt8PtrTy);
4153 } else {
4154 TypeNameField = TypeName;
4155 }
4156 Fields.push_back(Elt: TypeNameField);
4157
4158 switch (Ty->getTypeClass()) {
4159#define TYPE(Class, Base)
4160#define ABSTRACT_TYPE(Class, Base)
4161#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
4162#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
4163#define DEPENDENT_TYPE(Class, Base) case Type::Class:
4164#include "clang/AST/TypeNodes.inc"
4165 llvm_unreachable("Non-canonical and dependent types shouldn't get here");
4166
4167 // GCC treats vector types as fundamental types.
4168 case Type::Builtin:
4169 case Type::Vector:
4170 case Type::ExtVector:
4171 case Type::ConstantMatrix:
4172 case Type::Complex:
4173 case Type::BlockPointer:
4174 // Itanium C++ ABI 2.9.5p4:
4175 // abi::__fundamental_type_info adds no data members to std::type_info.
4176 break;
4177
4178 case Type::LValueReference:
4179 case Type::RValueReference:
4180 llvm_unreachable("References shouldn't get here");
4181
4182 case Type::Auto:
4183 case Type::DeducedTemplateSpecialization:
4184 llvm_unreachable("Undeduced type shouldn't get here");
4185
4186 case Type::Pipe:
4187 break;
4188
4189 case Type::BitInt:
4190 break;
4191
4192 case Type::ConstantArray:
4193 case Type::IncompleteArray:
4194 case Type::VariableArray:
4195 case Type::ArrayParameter:
4196 // Itanium C++ ABI 2.9.5p5:
4197 // abi::__array_type_info adds no data members to std::type_info.
4198 break;
4199
4200 case Type::FunctionNoProto:
4201 case Type::FunctionProto:
4202 // Itanium C++ ABI 2.9.5p5:
4203 // abi::__function_type_info adds no data members to std::type_info.
4204 break;
4205
4206 case Type::Enum:
4207 // Itanium C++ ABI 2.9.5p5:
4208 // abi::__enum_type_info adds no data members to std::type_info.
4209 break;
4210
4211 case Type::Record: {
4212 const auto *RD = cast<CXXRecordDecl>(Val: cast<RecordType>(Val&: Ty)->getDecl())
4213 ->getDefinitionOrSelf();
4214 if (!RD->hasDefinition() || !RD->getNumBases()) {
4215 // We don't need to emit any fields.
4216 break;
4217 }
4218
4219 if (CodeGenUtils::canUseSingleInheritance(RD))
4220 BuildSIClassTypeInfo(RD);
4221 else
4222 BuildVMIClassTypeInfo(RD);
4223
4224 break;
4225 }
4226
4227 case Type::ObjCObject:
4228 case Type::ObjCInterface:
4229 BuildObjCObjectTypeInfo(Ty: cast<ObjCObjectType>(Val&: Ty));
4230 break;
4231
4232 case Type::ObjCObjectPointer:
4233 BuildPointerTypeInfo(PointeeTy: cast<ObjCObjectPointerType>(Val&: Ty)->getPointeeType());
4234 break;
4235
4236 case Type::Pointer:
4237 BuildPointerTypeInfo(PointeeTy: cast<PointerType>(Val&: Ty)->getPointeeType());
4238 break;
4239
4240 case Type::MemberPointer:
4241 BuildPointerToMemberTypeInfo(Ty: cast<MemberPointerType>(Val&: Ty));
4242 break;
4243
4244 case Type::Atomic:
4245 // No fields, at least for the moment.
4246 break;
4247
4248 case Type::OverflowBehavior:
4249 break;
4250
4251 case Type::HLSLAttributedResource:
4252 case Type::HLSLInlineSpirv:
4253 llvm_unreachable("HLSL doesn't support RTTI");
4254 }
4255
4256 GV->replaceInitializer(InitVal: llvm::ConstantStruct::getAnon(V: Fields));
4257
4258 // If there's already an old global variable, replace it with the new one.
4259 if (OldGV) {
4260 GV->takeName(V: OldGV);
4261 OldGV->replaceAllUsesWith(V: GV);
4262 OldGV->eraseFromParent();
4263 }
4264
4265 if (CGM.supportsCOMDAT() && GV->isWeakForLinker())
4266 GV->setComdat(M.getOrInsertComdat(Name: GV->getName()));
4267
4268 CharUnits Align = CGM.getContext().toCharUnitsFromBits(
4269 BitSize: CGM.getTarget().getPointerAlign(AddrSpace: CGM.GetGlobalVarAddressSpace(D: nullptr)));
4270 GV->setAlignment(Align.getAsAlign());
4271
4272 // The Itanium ABI specifies that type_info objects must be globally
4273 // unique, with one exception: if the type is an incomplete class
4274 // type or a (possibly indirect) pointer to one. That exception
4275 // affects the general case of comparing type_info objects produced
4276 // by the typeid operator, which is why the comparison operators on
4277 // std::type_info generally use the type_info name pointers instead
4278 // of the object addresses. However, the language's built-in uses
4279 // of RTTI generally require class types to be complete, even when
4280 // manipulating pointers to those class types. This allows the
4281 // implementation of dynamic_cast to rely on address equality tests,
4282 // which is much faster.
4283
4284 // All of this is to say that it's important that both the type_info
4285 // object and the type_info name be uniqued when weakly emitted.
4286
4287 TypeName->setVisibility(Visibility);
4288 CGM.setDSOLocal(TypeName);
4289
4290 GV->setVisibility(TypeInfoVisibility);
4291 CGM.setDSOLocal(GV);
4292
4293 TypeName->setDLLStorageClass(DLLStorageClass);
4294 GV->setDLLStorageClass(TypeInfoDLLStorageClass);
4295
4296 TypeName->setPartition(CGM.getCodeGenOpts().SymbolPartition);
4297 GV->setPartition(CGM.getCodeGenOpts().SymbolPartition);
4298
4299 return GV;
4300}
4301
4302/// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info
4303/// for the given Objective-C object type.
4304void ItaniumRTTIBuilder::BuildObjCObjectTypeInfo(const ObjCObjectType *OT) {
4305 // Drop qualifiers.
4306 const Type *T = OT->getBaseType().getTypePtr();
4307 assert(isa<BuiltinType>(T) || isa<ObjCInterfaceType>(T));
4308
4309 // The builtin types are abi::__class_type_infos and don't require
4310 // extra fields.
4311 if (isa<BuiltinType>(Val: T)) return;
4312
4313 ObjCInterfaceDecl *Class = cast<ObjCInterfaceType>(Val: T)->getDecl();
4314 ObjCInterfaceDecl *Super = Class->getSuperClass();
4315
4316 // Root classes are also __class_type_info.
4317 if (!Super) return;
4318
4319 QualType SuperTy = CGM.getContext().getObjCInterfaceType(Decl: Super);
4320
4321 // Everything else is single inheritance.
4322 llvm::Constant *BaseTypeInfo =
4323 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: SuperTy);
4324 Fields.push_back(Elt: BaseTypeInfo);
4325}
4326
4327/// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
4328/// inheritance, according to the Itanium C++ ABI, 2.95p6b.
4329void ItaniumRTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
4330 // Itanium C++ ABI 2.9.5p6b:
4331 // It adds to abi::__class_type_info a single member pointing to the
4332 // type_info structure for the base type,
4333 llvm::Constant *BaseTypeInfo =
4334 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: RD->bases_begin()->getType());
4335 Fields.push_back(Elt: BaseTypeInfo);
4336}
4337
4338/// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
4339/// classes with bases that do not satisfy the abi::__si_class_type_info
4340/// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
4341void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
4342 llvm::Type *UnsignedIntLTy =
4343 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4344
4345 // Itanium C++ ABI 2.9.5p6c:
4346 // __flags is a word with flags describing details about the class
4347 // structure, which may be referenced by using the __flags_masks
4348 // enumeration. These flags refer to both direct and indirect bases.
4349 unsigned Flags = CodeGenUtils::computeVMIClassTypeInfoFlags(RD);
4350 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4351
4352 // Itanium C++ ABI 2.9.5p6c:
4353 // __base_count is a word with the number of direct proper base class
4354 // descriptions that follow.
4355 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: RD->getNumBases()));
4356
4357 if (!RD->getNumBases())
4358 return;
4359
4360 // Now add the base class descriptions.
4361
4362 // Itanium C++ ABI 2.9.5p6c:
4363 // __base_info[] is an array of base class descriptions -- one for every
4364 // direct proper base. Each description is of the type:
4365 //
4366 // struct abi::__base_class_type_info {
4367 // public:
4368 // const __class_type_info *__base_type;
4369 // long __offset_flags;
4370 //
4371 // enum __offset_flags_masks {
4372 // __virtual_mask = 0x1,
4373 // __public_mask = 0x2,
4374 // __offset_shift = 8
4375 // };
4376 // };
4377
4378 // If we're in mingw and 'long' isn't wide enough for a pointer, use 'long
4379 // long' instead of 'long' for __offset_flags. libstdc++abi uses long long on
4380 // LLP64 platforms.
4381 // FIXME: Consider updating libc++abi to match, and extend this logic to all
4382 // LLP64 platforms.
4383 QualType OffsetFlagsTy = CGM.getContext().LongTy;
4384 const TargetInfo &TI = CGM.getContext().getTargetInfo();
4385 if (TI.getTriple().isOSCygMing() &&
4386 TI.getPointerWidth(AddrSpace: LangAS::Default) > TI.getLongWidth())
4387 OffsetFlagsTy = CGM.getContext().LongLongTy;
4388 llvm::Type *OffsetFlagsLTy =
4389 CGM.getTypes().ConvertType(T: OffsetFlagsTy);
4390
4391 for (const auto &Base : RD->bases()) {
4392 // The __base_type member points to the RTTI for the base type.
4393 Fields.push_back(Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: Base.getType()));
4394
4395 auto *BaseDecl = Base.getType()->castAsCXXRecordDecl();
4396 int64_t OffsetFlags = 0;
4397
4398 // All but the lower 8 bits of __offset_flags are a signed offset.
4399 // For a non-virtual base, this is the offset in the object of the base
4400 // subobject. For a virtual base, this is the offset in the virtual table of
4401 // the virtual base offset for the virtual base referenced (negative).
4402 CharUnits Offset;
4403 if (Base.isVirtual())
4404 Offset =
4405 CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD, VBase: BaseDecl);
4406 else {
4407 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
4408 Offset = Layout.getBaseClassOffset(Base: BaseDecl);
4409 };
4410
4411 OffsetFlags = uint64_t(Offset.getQuantity()) << 8;
4412
4413 // The low-order byte of __offset_flags contains flags, as given by the
4414 // masks from the enumeration __offset_flags_masks.
4415 if (Base.isVirtual())
4416 OffsetFlags |= CodeGenUtils::BCTI_Virtual;
4417 if (Base.getAccessSpecifier() == AS_public)
4418 OffsetFlags |= CodeGenUtils::BCTI_Public;
4419
4420 Fields.push_back(Elt: llvm::ConstantInt::getSigned(Ty: OffsetFlagsLTy, V: OffsetFlags));
4421 }
4422}
4423
4424/// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct,
4425/// used for pointer types.
4426void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {
4427 // Itanium C++ ABI 2.9.5p7:
4428 // __flags is a flag word describing the cv-qualification and other
4429 // attributes of the type pointed to
4430 unsigned Flags = CodeGenUtils::extractPBaseFlags(Ctx: CGM.getContext(), Type&: PointeeTy);
4431
4432 llvm::Type *UnsignedIntLTy =
4433 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4434 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4435
4436 // Itanium C++ ABI 2.9.5p7:
4437 // __pointee is a pointer to the std::type_info derivation for the
4438 // unqualified type being pointed to.
4439 llvm::Constant *PointeeTypeInfo =
4440 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4441 Fields.push_back(Elt: PointeeTypeInfo);
4442}
4443
4444/// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
4445/// struct, used for member pointer types.
4446void
4447ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
4448 QualType PointeeTy = Ty->getPointeeType();
4449
4450 // Itanium C++ ABI 2.9.5p7:
4451 // __flags is a flag word describing the cv-qualification and other
4452 // attributes of the type pointed to.
4453 unsigned Flags = CodeGenUtils::extractPBaseFlags(Ctx: CGM.getContext(), Type&: PointeeTy);
4454
4455 const auto *RD = Ty->getMostRecentCXXRecordDecl();
4456 if (!RD->hasDefinition())
4457 Flags |= CodeGenUtils::PTI_ContainingClassIncomplete;
4458
4459 llvm::Type *UnsignedIntLTy =
4460 CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4461 Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4462
4463 // Itanium C++ ABI 2.9.5p7:
4464 // __pointee is a pointer to the std::type_info derivation for the
4465 // unqualified type being pointed to.
4466 llvm::Constant *PointeeTypeInfo =
4467 ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4468 Fields.push_back(Elt: PointeeTypeInfo);
4469
4470 // Itanium C++ ABI 2.9.5p9:
4471 // __context is a pointer to an abi::__class_type_info corresponding to the
4472 // class type containing the member pointed to
4473 // (e.g., the "A" in "int A::*").
4474 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
4475 Fields.push_back(Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: T));
4476}
4477
4478llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) {
4479 return ItaniumRTTIBuilder(*this).BuildTypeInfo(Ty);
4480}
4481
4482void ItaniumCXXABI::EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD) {
4483 // Types added here must also be added to TypeInfoIsInStandardLibrary.
4484 QualType FundamentalTypes[] = {
4485 getContext().VoidTy, getContext().NullPtrTy,
4486 getContext().BoolTy, getContext().WCharTy,
4487 getContext().CharTy, getContext().UnsignedCharTy,
4488 getContext().SignedCharTy, getContext().ShortTy,
4489 getContext().UnsignedShortTy, getContext().IntTy,
4490 getContext().UnsignedIntTy, getContext().LongTy,
4491 getContext().UnsignedLongTy, getContext().LongLongTy,
4492 getContext().UnsignedLongLongTy, getContext().Int128Ty,
4493 getContext().UnsignedInt128Ty, getContext().HalfTy,
4494 getContext().FloatTy, getContext().DoubleTy,
4495 getContext().LongDoubleTy, getContext().Float128Ty,
4496 getContext().Char8Ty, getContext().Char16Ty,
4497 getContext().Char32Ty
4498 };
4499 llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
4500 RD->hasAttr<DLLExportAttr>() || CGM.shouldMapVisibilityToDLLExport(D: RD)
4501 ? llvm::GlobalValue::DLLExportStorageClass
4502 : llvm::GlobalValue::DefaultStorageClass;
4503 llvm::GlobalValue::VisibilityTypes Visibility = getRTTIVisibility(
4504 Visibility: CodeGenModule::GetLLVMVisibility(V: RD->getVisibility()), DLLStorageClass);
4505 for (const QualType &FundamentalType : FundamentalTypes) {
4506 QualType PointerType = getContext().getPointerType(T: FundamentalType);
4507 QualType PointerTypeConst = getContext().getPointerType(
4508 T: FundamentalType.withConst());
4509 for (QualType Type : {FundamentalType, PointerType, PointerTypeConst})
4510 ItaniumRTTIBuilder(*this).BuildTypeInfo(
4511 Ty: Type, Linkage: llvm::GlobalValue::ExternalLinkage, Visibility, DLLStorageClass,
4512 TypeInfoVisibility: Visibility, TypeInfoDLLStorageClass: DLLStorageClass);
4513 }
4514}
4515
4516/// What sort of uniqueness rules should we use for the RTTI for the
4517/// given type?
4518ItaniumCXXABI::RTTIUniquenessKind ItaniumCXXABI::classifyRTTIUniqueness(
4519 QualType CanTy, llvm::GlobalValue::LinkageTypes Linkage) const {
4520 if (shouldRTTIBeUnique())
4521 return RUK_Unique;
4522
4523 // It's only necessary for linkonce_odr or weak_odr linkage.
4524 if (Linkage != llvm::GlobalValue::LinkOnceODRLinkage &&
4525 Linkage != llvm::GlobalValue::WeakODRLinkage)
4526 return RUK_Unique;
4527
4528 // It's only necessary with default visibility.
4529 if (CanTy->getVisibility() != DefaultVisibility)
4530 return RUK_Unique;
4531
4532 // If we're not required to publish this symbol, hide it.
4533 if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
4534 return RUK_NonUniqueHidden;
4535
4536 // If we're required to publish this symbol, as we might be under an
4537 // explicit instantiation, leave it with default visibility but
4538 // enable string-comparisons.
4539 assert(Linkage == llvm::GlobalValue::WeakODRLinkage);
4540 return RUK_NonUniqueVisible;
4541}
4542
4543// Find out how to codegen the complete destructor and constructor
4544namespace {
4545enum class StructorCodegen { Emit, RAUW, Alias, COMDAT };
4546} // namespace
4547
4548// Returns true if the complete constructor/destructor variant must be retained
4549// as a distinct symbol rather than being silently replaced in the IR (RAUW).
4550static bool
4551structorSymbolMustBeRetained(CodeGenModule &CGM, const CXXMethodDecl *MD,
4552 llvm::GlobalValue::LinkageTypes Linkage) {
4553 if (MD->hasAttr<UsedAttr>())
4554 return true;
4555 return CGM.getCodeGenOpts().KeepInlineFunctions && MD->isInlined() &&
4556 Linkage != llvm::GlobalValue::AvailableExternallyLinkage;
4557}
4558
4559static StructorCodegen getCodegenToUse(CodeGenModule &CGM,
4560 const CXXMethodDecl *MD) {
4561 if (!CGM.getCodeGenOpts().CXXCtorDtorAliases)
4562 return StructorCodegen::Emit;
4563
4564 // The complete and base structors are not equivalent if there are any virtual
4565 // bases, so emit separate functions.
4566 if (MD->getParent()->getNumVBases())
4567 return StructorCodegen::Emit;
4568
4569 GlobalDecl AliasDecl;
4570 if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
4571 AliasDecl = GlobalDecl(DD, Dtor_Complete);
4572 } else {
4573 const auto *CD = cast<CXXConstructorDecl>(Val: MD);
4574 AliasDecl = GlobalDecl(CD, Ctor_Complete);
4575 }
4576 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4577
4578 if (llvm::GlobalValue::isDiscardableIfUnused(Linkage) &&
4579 !structorSymbolMustBeRetained(CGM, MD, Linkage))
4580 return StructorCodegen::RAUW;
4581
4582 // FIXME: Should we allow available_externally aliases?
4583 if (!llvm::GlobalAlias::isValidLinkage(L: Linkage))
4584 return StructorCodegen::RAUW;
4585
4586 if (llvm::GlobalValue::isWeakForLinker(Linkage)) {
4587 // Only ELF and wasm support COMDATs with arbitrary names (C5/D5).
4588 if (CGM.getTarget().getTriple().isOSBinFormatELF() ||
4589 CGM.getTarget().getTriple().isOSBinFormatWasm())
4590 return StructorCodegen::COMDAT;
4591 return StructorCodegen::Emit;
4592 }
4593
4594 return StructorCodegen::Alias;
4595}
4596
4597static void emitConstructorDestructorAlias(CodeGenModule &CGM,
4598 GlobalDecl AliasDecl,
4599 GlobalDecl TargetDecl) {
4600 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4601
4602 StringRef MangledName = CGM.getMangledName(GD: AliasDecl);
4603 llvm::GlobalValue *Entry = CGM.GetGlobalValue(Ref: MangledName);
4604 if (Entry && !Entry->isDeclaration())
4605 return;
4606
4607 auto *Aliasee = cast<llvm::GlobalValue>(Val: CGM.GetAddrOfGlobal(GD: TargetDecl));
4608
4609 // Create the alias with no name.
4610 auto *Alias = llvm::GlobalAlias::create(Linkage, Name: "", Aliasee);
4611
4612 // Constructors and destructors are always unnamed_addr.
4613 Alias->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4614
4615 // Switch any previous uses to the alias.
4616 if (Entry) {
4617 assert(Entry->getType() == Aliasee->getType() &&
4618 "declaration exists with different type");
4619 Alias->takeName(V: Entry);
4620 Entry->replaceAllUsesWith(V: Alias);
4621 Entry->eraseFromParent();
4622 } else {
4623 Alias->setName(MangledName);
4624 }
4625
4626 // Finally, set up the alias with its proper name and attributes.
4627 CGM.SetCommonAttributes(GD: AliasDecl, GV: Alias);
4628}
4629
4630void ItaniumCXXABI::emitCXXStructor(GlobalDecl GD) {
4631 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
4632 auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD);
4633 const CXXDestructorDecl *DD = CD ? nullptr : cast<CXXDestructorDecl>(Val: MD);
4634
4635 StructorCodegen CGType = getCodegenToUse(CGM, MD);
4636
4637 if (CD ? GD.getCtorType() == Ctor_Complete
4638 : GD.getDtorType() == Dtor_Complete) {
4639 GlobalDecl BaseDecl;
4640 if (CD)
4641 BaseDecl = GD.getWithCtorType(Type: Ctor_Base);
4642 else
4643 BaseDecl = GD.getWithDtorType(Type: Dtor_Base);
4644
4645 if (CGType == StructorCodegen::Alias || CGType == StructorCodegen::COMDAT) {
4646 emitConstructorDestructorAlias(CGM, AliasDecl: GD, TargetDecl: BaseDecl);
4647 return;
4648 }
4649
4650 if (CGType == StructorCodegen::RAUW) {
4651 StringRef MangledName = CGM.getMangledName(GD);
4652 auto *Aliasee = CGM.GetAddrOfGlobal(GD: BaseDecl);
4653 CGM.addReplacement(Name: MangledName, C: Aliasee);
4654 return;
4655 }
4656 }
4657
4658 // The base destructor is equivalent to the base destructor of its
4659 // base class if there is exactly one non-virtual base class with a
4660 // non-trivial destructor, there are no fields with a non-trivial
4661 // destructor, and the body of the destructor is trivial.
4662 if (DD && GD.getDtorType() == Dtor_Base &&
4663 CGType != StructorCodegen::COMDAT &&
4664 !CGM.TryEmitBaseDestructorAsAlias(D: DD))
4665 return;
4666
4667 // FIXME: The deleting destructor is equivalent to the selected operator
4668 // delete if:
4669 // * either the delete is a destroying operator delete or the destructor
4670 // would be trivial if it weren't virtual,
4671 // * the conversion from the 'this' parameter to the first parameter of the
4672 // destructor is equivalent to a bitcast,
4673 // * the destructor does not have an implicit "this" return, and
4674 // * the operator delete has the same calling convention and IR function type
4675 // as the destructor.
4676 // In such cases we should try to emit the deleting dtor as an alias to the
4677 // selected 'operator delete'.
4678
4679 llvm::Function *Fn = CGM.codegenCXXStructor(GD);
4680
4681 if (CGType == StructorCodegen::COMDAT) {
4682 SmallString<256> Buffer;
4683 llvm::raw_svector_ostream Out(Buffer);
4684 if (DD)
4685 getMangleContext().mangleCXXDtorComdat(D: DD, Out);
4686 else
4687 getMangleContext().mangleCXXCtorComdat(D: CD, Out);
4688 llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Name: Out.str());
4689 Fn->setComdat(C);
4690 } else {
4691 CGM.maybeSetTrivialComdat(D: *MD, GO&: *Fn);
4692 }
4693}
4694
4695static llvm::FunctionCallee getBeginCatchFn(CodeGenModule &CGM) {
4696 // void *__cxa_begin_catch(void*);
4697 llvm::FunctionType *FTy = llvm::FunctionType::get(
4698 Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4699
4700 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_begin_catch");
4701}
4702
4703static llvm::FunctionCallee getEndCatchFn(CodeGenModule &CGM) {
4704 // void __cxa_end_catch();
4705 llvm::FunctionType *FTy =
4706 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
4707
4708 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_end_catch");
4709}
4710
4711static llvm::FunctionCallee getGetExceptionPtrFn(CodeGenModule &CGM) {
4712 // void *__cxa_get_exception_ptr(void*);
4713 llvm::FunctionType *FTy = llvm::FunctionType::get(
4714 Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4715
4716 return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_get_exception_ptr");
4717}
4718
4719namespace {
4720 /// A cleanup to call __cxa_end_catch. In many cases, the caught
4721 /// exception type lets us state definitively that the thrown exception
4722 /// type does not have a destructor. In particular:
4723 /// - Catch-alls tell us nothing, so we have to conservatively
4724 /// assume that the thrown exception might have a destructor.
4725 /// - Catches by reference behave according to their base types.
4726 /// - Catches of non-record types will only trigger for exceptions
4727 /// of non-record types, which never have destructors.
4728 /// - Catches of record types can trigger for arbitrary subclasses
4729 /// of the caught type, so we have to assume the actual thrown
4730 /// exception type might have a throwing destructor, even if the
4731 /// caught type's destructor is trivial or nothrow.
4732 struct CallEndCatch final : EHScopeStack::Cleanup {
4733 CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
4734 bool MightThrow;
4735
4736 void Emit(CodeGenFunction &CGF, Flags flags) override {
4737 if (!MightThrow) {
4738 CGF.EmitNounwindRuntimeCall(callee: getEndCatchFn(CGM&: CGF.CGM));
4739 return;
4740 }
4741
4742 CGF.EmitRuntimeCallOrInvoke(callee: getEndCatchFn(CGM&: CGF.CGM));
4743 }
4744 };
4745}
4746
4747/// Emits a call to __cxa_begin_catch and enters a cleanup to call
4748/// __cxa_end_catch. If -fassume-nothrow-exception-dtor is specified, we assume
4749/// that the exception object's dtor is nothrow, therefore the __cxa_end_catch
4750/// call can be marked as nounwind even if EndMightThrow is true.
4751///
4752/// \param EndMightThrow - true if __cxa_end_catch might throw
4753static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
4754 llvm::Value *Exn,
4755 bool EndMightThrow) {
4756 llvm::CallInst *call =
4757 CGF.EmitNounwindRuntimeCall(callee: getBeginCatchFn(CGM&: CGF.CGM), args: Exn);
4758
4759 CGF.EHStack.pushCleanup<CallEndCatch>(
4760 Kind: NormalAndEHCleanup,
4761 A: EndMightThrow && !CGF.CGM.getLangOpts().AssumeNothrowExceptionDtor);
4762
4763 return call;
4764}
4765
4766/// A "special initializer" callback for initializing a catch
4767/// parameter during catch initialization.
4768static void InitCatchParam(CodeGenFunction &CGF,
4769 const VarDecl &CatchParam,
4770 Address ParamAddr,
4771 SourceLocation Loc) {
4772 // Load the exception from where the landing pad saved it.
4773 llvm::Value *Exn = CGF.getExceptionFromSlot();
4774
4775 CanQualType CatchType =
4776 CGF.CGM.getContext().getCanonicalType(T: CatchParam.getType());
4777 llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(T: CatchType);
4778
4779 // If we're catching by reference, we can just cast the object
4780 // pointer to the appropriate pointer.
4781 if (isa<ReferenceType>(Val: CatchType)) {
4782 QualType CaughtType = cast<ReferenceType>(Val&: CatchType)->getPointeeType();
4783 bool EndCatchMightThrow = CaughtType->isRecordType();
4784
4785 // __cxa_begin_catch returns the adjusted object pointer.
4786 llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: EndCatchMightThrow);
4787
4788 // We have no way to tell the personality function that we're
4789 // catching by reference, so if we're catching a pointer,
4790 // __cxa_begin_catch will actually return that pointer by value.
4791 if (const PointerType *PT = dyn_cast<PointerType>(Val&: CaughtType)) {
4792 QualType PointeeType = PT->getPointeeType();
4793
4794 // When catching by reference, generally we should just ignore
4795 // this by-value pointer and use the exception object instead.
4796 if (!PointeeType->isRecordType()) {
4797
4798 // Exn points to the struct _Unwind_Exception header, which
4799 // we have to skip past in order to reach the exception data.
4800 unsigned HeaderSize =
4801 CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException();
4802 AdjustedExn =
4803 CGF.Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: Exn, Idx0: HeaderSize);
4804
4805 // However, if we're catching a pointer-to-record type that won't
4806 // work, because the personality function might have adjusted
4807 // the pointer. There's actually no way for us to fully satisfy
4808 // the language/ABI contract here: we can't use Exn because it
4809 // might have the wrong adjustment, but we can't use the by-value
4810 // pointer because it's off by a level of abstraction.
4811 //
4812 // The current solution is to dump the adjusted pointer into an
4813 // alloca, which breaks language semantics (because changing the
4814 // pointer doesn't change the exception) but at least works.
4815 // The better solution would be to filter out non-exact matches
4816 // and rethrow them, but this is tricky because the rethrow
4817 // really needs to be catchable by other sites at this landing
4818 // pad. The best solution is to fix the personality function.
4819 } else {
4820 // Pull the pointer for the reference type off.
4821 llvm::Type *PtrTy = CGF.ConvertTypeForMem(T: CaughtType);
4822
4823 // Create the temporary and write the adjusted pointer into it.
4824 Address ExnPtrTmp =
4825 CGF.CreateTempAlloca(Ty: PtrTy, align: CGF.getPointerAlign(), Name: "exn.byref.tmp");
4826 llvm::Value *Casted = CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: PtrTy);
4827 CGF.Builder.CreateStore(Val: Casted, Addr: ExnPtrTmp);
4828
4829 // Bind the reference to the temporary.
4830 AdjustedExn = ExnPtrTmp.emitRawPointer(CGF);
4831 }
4832 }
4833
4834 llvm::Value *ExnCast =
4835 CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.byref");
4836 CGF.Builder.CreateStore(Val: ExnCast, Addr: ParamAddr);
4837 return;
4838 }
4839
4840 // Scalars and complexes.
4841 TypeEvaluationKind TEK = CGF.getEvaluationKind(T: CatchType);
4842 if (TEK != TEK_Aggregate) {
4843 llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: false);
4844
4845 // If the catch type is a pointer type, __cxa_begin_catch returns
4846 // the pointer by value.
4847 if (CatchType->hasPointerRepresentation()) {
4848 llvm::Value *CastExn =
4849 CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.casted");
4850
4851 switch (CatchType.getQualifiers().getObjCLifetime()) {
4852 case Qualifiers::OCL_Strong:
4853 CastExn = CGF.EmitARCRetainNonBlock(value: CastExn);
4854 [[fallthrough]];
4855
4856 case Qualifiers::OCL_None:
4857 case Qualifiers::OCL_ExplicitNone:
4858 case Qualifiers::OCL_Autoreleasing:
4859 CGF.Builder.CreateStore(Val: CastExn, Addr: ParamAddr);
4860 return;
4861
4862 case Qualifiers::OCL_Weak:
4863 CGF.EmitARCInitWeak(addr: ParamAddr, value: CastExn);
4864 return;
4865 }
4866 llvm_unreachable("bad ownership qualifier!");
4867 }
4868
4869 // Otherwise, it returns a pointer into the exception object.
4870
4871 LValue srcLV = CGF.MakeNaturalAlignAddrLValue(V: AdjustedExn, T: CatchType);
4872 LValue destLV = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
4873 switch (TEK) {
4874 case TEK_Complex:
4875 CGF.EmitStoreOfComplex(V: CGF.EmitLoadOfComplex(src: srcLV, loc: Loc), dest: destLV,
4876 /*init*/ isInit: true);
4877 return;
4878 case TEK_Scalar: {
4879 llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc);
4880 CGF.EmitStoreOfScalar(value: ExnLoad, lvalue: destLV, /*init*/ isInit: true);
4881 return;
4882 }
4883 case TEK_Aggregate:
4884 llvm_unreachable("evaluation kind filtered out!");
4885 }
4886 llvm_unreachable("bad evaluation kind");
4887 }
4888
4889 assert(isa<RecordType>(CatchType) && "unexpected catch type!");
4890 auto catchRD = CatchType->getAsCXXRecordDecl();
4891 CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(CD: catchRD);
4892
4893 llvm::Type *PtrTy = CGF.DefaultPtrTy;
4894
4895 // Check for a copy expression. If we don't have a copy expression,
4896 // that means a trivial copy is okay.
4897 const Expr *copyExpr = CatchParam.getInit();
4898 if (!copyExpr) {
4899 llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: true);
4900 Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
4901 LLVMCatchTy, caughtExnAlignment);
4902 LValue Dest = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
4903 LValue Src = CGF.MakeAddrLValue(Addr: adjustedExn, T: CatchType);
4904 CGF.EmitAggregateCopy(Dest, Src, EltTy: CatchType, MayOverlap: AggValueSlot::DoesNotOverlap);
4905 return;
4906 }
4907
4908 // We have to call __cxa_get_exception_ptr to get the adjusted
4909 // pointer before copying.
4910 llvm::CallInst *rawAdjustedExn =
4911 CGF.EmitNounwindRuntimeCall(callee: getGetExceptionPtrFn(CGM&: CGF.CGM), args: Exn);
4912
4913 // Cast that to the appropriate type.
4914 Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
4915 LLVMCatchTy, caughtExnAlignment);
4916
4917 // The copy expression is defined in terms of an OpaqueValueExpr.
4918 // Find it and map it to the adjusted expression.
4919 CodeGenFunction::OpaqueValueMapping
4920 opaque(CGF, OpaqueValueExpr::findInCopyConstruct(expr: copyExpr),
4921 CGF.MakeAddrLValue(Addr: adjustedExn, T: CatchParam.getType()));
4922
4923 // Call the copy ctor in a terminate scope.
4924 CGF.EHStack.pushTerminate();
4925
4926 // Perform the copy construction.
4927 CGF.EmitAggExpr(E: copyExpr,
4928 AS: AggValueSlot::forAddr(addr: ParamAddr, quals: Qualifiers(),
4929 isDestructed: AggValueSlot::IsNotDestructed,
4930 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4931 isAliased: AggValueSlot::IsNotAliased,
4932 mayOverlap: AggValueSlot::DoesNotOverlap));
4933
4934 // Leave the terminate scope.
4935 CGF.EHStack.popTerminate();
4936
4937 // Undo the opaque value mapping.
4938 opaque.pop();
4939
4940 // Finally we can call __cxa_begin_catch.
4941 CallBeginCatch(CGF, Exn, EndMightThrow: true);
4942}
4943
4944/// Begins a catch statement by initializing the catch variable and
4945/// calling __cxa_begin_catch.
4946void ItaniumCXXABI::emitBeginCatch(CodeGenFunction &CGF,
4947 const CXXCatchStmt *S) {
4948 // We have to be very careful with the ordering of cleanups here:
4949 // C++ [except.throw]p4:
4950 // The destruction [of the exception temporary] occurs
4951 // immediately after the destruction of the object declared in
4952 // the exception-declaration in the handler.
4953 //
4954 // So the precise ordering is:
4955 // 1. Construct catch variable.
4956 // 2. __cxa_begin_catch
4957 // 3. Enter __cxa_end_catch cleanup
4958 // 4. Enter dtor cleanup
4959 //
4960 // We do this by using a slightly abnormal initialization process.
4961 // Delegation sequence:
4962 // - ExitCXXTryStmt opens a RunCleanupsScope
4963 // - EmitAutoVarAlloca creates the variable and debug info
4964 // - InitCatchParam initializes the variable from the exception
4965 // - CallBeginCatch calls __cxa_begin_catch
4966 // - CallBeginCatch enters the __cxa_end_catch cleanup
4967 // - EmitAutoVarCleanups enters the variable destructor cleanup
4968 // - EmitCXXTryStmt emits the code for the catch body
4969 // - EmitCXXTryStmt close the RunCleanupsScope
4970
4971 VarDecl *CatchParam = S->getExceptionDecl();
4972 if (!CatchParam) {
4973 llvm::Value *Exn = CGF.getExceptionFromSlot();
4974 CallBeginCatch(CGF, Exn, EndMightThrow: true);
4975 return;
4976 }
4977
4978 // Emit the local.
4979 CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(var: *CatchParam);
4980 {
4981 ApplyAtomGroup Grp(CGF.getDebugInfo());
4982 InitCatchParam(CGF, CatchParam: *CatchParam, ParamAddr: var.getObjectAddress(CGF),
4983 Loc: S->getBeginLoc());
4984 }
4985 CGF.EmitAutoVarCleanups(emission: var);
4986}
4987
4988/// Get or define the following function:
4989/// void @__clang_call_terminate(i8* %exn) nounwind noreturn
4990/// This code is used only in C++.
4991static llvm::FunctionCallee getClangCallTerminateFn(CodeGenModule &CGM) {
4992 ASTContext &C = CGM.getContext();
4993 const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
4994 resultType: C.VoidTy, argTypes: {C.getPointerType(T: C.CharTy)});
4995 llvm::FunctionType *fnTy = CGM.getTypes().GetFunctionType(Info: FI);
4996 llvm::FunctionCallee fnRef = CGM.CreateRuntimeFunction(
4997 Ty: fnTy, Name: "__clang_call_terminate", ExtraAttrs: llvm::AttributeList(), /*Local=*/true);
4998 llvm::Function *fn =
4999 cast<llvm::Function>(Val: fnRef.getCallee()->stripPointerCasts());
5000 if (fn->empty()) {
5001 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: fn, /*IsThunk=*/false);
5002 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: fn);
5003 fn->setDoesNotThrow();
5004 fn->setDoesNotReturn();
5005
5006 // What we really want is to massively penalize inlining without
5007 // forbidding it completely. The difference between that and
5008 // 'noinline' is negligible.
5009 fn->addFnAttr(Kind: llvm::Attribute::NoInline);
5010
5011 // Allow this function to be shared across translation units, but
5012 // we don't want it to turn into an exported symbol.
5013 fn->setLinkage(llvm::Function::LinkOnceODRLinkage);
5014 fn->setVisibility(llvm::Function::HiddenVisibility);
5015 if (CGM.supportsCOMDAT())
5016 fn->setComdat(CGM.getModule().getOrInsertComdat(Name: fn->getName()));
5017
5018 // Set up the function.
5019 llvm::BasicBlock *entry =
5020 llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "", Parent: fn);
5021 CGBuilderTy builder(CGM, entry);
5022
5023 // Pull the exception pointer out of the parameter list.
5024 llvm::Value *exn = &*fn->arg_begin();
5025
5026 // Call __cxa_begin_catch(exn).
5027 llvm::CallInst *catchCall = builder.CreateCall(Callee: getBeginCatchFn(CGM), Args: exn);
5028 catchCall->setDoesNotThrow();
5029 catchCall->setCallingConv(CGM.getRuntimeCC());
5030
5031 // Call std::terminate().
5032 llvm::CallInst *termCall = builder.CreateCall(Callee: CGM.getTerminateFn());
5033 termCall->setDoesNotThrow();
5034 termCall->setDoesNotReturn();
5035 termCall->setCallingConv(CGM.getRuntimeCC());
5036
5037 // std::terminate cannot return.
5038 builder.CreateUnreachable();
5039 }
5040 return fnRef;
5041}
5042
5043llvm::CallInst *
5044ItaniumCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
5045 llvm::Value *Exn) {
5046 // In C++, we want to call __cxa_begin_catch() before terminating.
5047 if (Exn) {
5048 assert(CGF.CGM.getLangOpts().CPlusPlus);
5049 return CGF.EmitNounwindRuntimeCall(callee: getClangCallTerminateFn(CGM&: CGF.CGM), args: Exn);
5050 }
5051 return CGF.EmitNounwindRuntimeCall(callee: CGF.CGM.getTerminateFn());
5052}
5053
5054std::pair<llvm::Value *, const CXXRecordDecl *>
5055ItaniumCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
5056 const CXXRecordDecl *RD) {
5057 return {CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: RD), RD};
5058}
5059
5060llvm::Constant *
5061ItaniumCXXABI::getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD) {
5062 const CXXMethodDecl *origMD =
5063 cast<CXXMethodDecl>(Val: CGM.getItaniumVTableContext()
5064 .findOriginalMethod(GD: MD->getCanonicalDecl())
5065 .getDecl());
5066 llvm::Constant *thunk = getOrCreateVirtualFunctionPointerThunk(MD: origMD);
5067 QualType funcType = CGM.getContext().getMemberPointerType(
5068 T: MD->getType(), /*Qualifier=*/std::nullopt, Cls: MD->getParent());
5069 return CGM.getMemberFunctionPointer(Pointer: thunk, FT: funcType);
5070}
5071
5072void WebAssemblyCXXABI::emitBeginCatch(CodeGenFunction &CGF,
5073 const CXXCatchStmt *C) {
5074 if (CGF.getTarget().hasFeature(Feature: "exception-handling"))
5075 CGF.EHStack.pushCleanup<CatchRetScope>(
5076 Kind: NormalCleanup, A: cast<llvm::CatchPadInst>(Val: CGF.CurrentFuncletPad));
5077 ItaniumCXXABI::emitBeginCatch(CGF, S: C);
5078}
5079
5080llvm::CallInst *
5081WebAssemblyCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
5082 llvm::Value *Exn) {
5083 // Itanium ABI calls __clang_call_terminate(), which __cxa_begin_catch() on
5084 // the violating exception to mark it handled, but it is currently hard to do
5085 // with wasm EH instruction structure with catch/catch_all, we just call
5086 // std::terminate and ignore the violating exception as in CGCXXABI in Wasm EH
5087 // and call __clang_call_terminate only in Emscripten EH.
5088 // TODO Consider code transformation that makes calling __clang_call_terminate
5089 // in Wasm EH possible.
5090 if (Exn && !getEHPersonality(CGF).isWasmPersonality()) {
5091 assert(CGF.CGM.getLangOpts().CPlusPlus);
5092 return CGF.EmitNounwindRuntimeCall(callee: getClangCallTerminateFn(CGM&: CGF.CGM), args: Exn);
5093 }
5094 return CGCXXABI::emitTerminateForUnexpectedException(CGF, Exn);
5095}
5096
5097/// Register a global destructor as best as we know how.
5098void XLCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
5099 llvm::FunctionCallee Dtor,
5100 llvm::Constant *Addr) {
5101 if (D.getTLSKind() != VarDecl::TLS_None) {
5102 llvm::PointerType *PtrTy = CGF.DefaultPtrTy;
5103
5104 // extern "C" int __pt_atexit_np(int flags, int(*)(int,...), ...);
5105 llvm::FunctionType *AtExitTy =
5106 llvm::FunctionType::get(Result: CGM.IntTy, Params: {CGM.IntTy, PtrTy}, isVarArg: true);
5107
5108 // Fetch the actual function.
5109 llvm::FunctionCallee AtExit =
5110 CGM.CreateRuntimeFunction(Ty: AtExitTy, Name: "__pt_atexit_np");
5111
5112 // Create __dtor function for the var decl.
5113 llvm::Function *DtorStub = CGF.createTLSAtExitStub(VD: D, Dtor, Addr, AtExit);
5114
5115 // Register above __dtor with atexit().
5116 // First param is flags and must be 0, second param is function ptr
5117 llvm::Value *NV = llvm::Constant::getNullValue(Ty: CGM.IntTy);
5118 CGF.EmitNounwindRuntimeCall(callee: AtExit, args: {NV, DtorStub});
5119
5120 // Cannot unregister TLS __dtor so done
5121 return;
5122 }
5123
5124 // Create __dtor function for the var decl.
5125 llvm::Function *DtorStub =
5126 cast<llvm::Function>(Val: CGF.createAtExitStub(VD: D, Dtor, Addr));
5127
5128 // Register above __dtor with atexit().
5129 CGF.registerGlobalDtorWithAtExit(dtorStub: DtorStub);
5130
5131 // Emit __finalize function to unregister __dtor and (as appropriate) call
5132 // __dtor.
5133 emitCXXStermFinalizer(D, dtorStub: DtorStub, addr: Addr);
5134}
5135
5136void XLCXXABI::emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
5137 llvm::Constant *addr) {
5138 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
5139 SmallString<256> FnName;
5140 {
5141 llvm::raw_svector_ostream Out(FnName);
5142 getMangleContext().mangleDynamicStermFinalizer(D: &D, Out);
5143 }
5144
5145 // Create the finalization action associated with a variable.
5146 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
5147 llvm::Function *StermFinalizer = CGM.CreateGlobalInitOrCleanUpFunction(
5148 ty: FTy, name: FnName.str(), FI, Loc: D.getLocation());
5149
5150 CodeGenFunction CGF(CGM);
5151
5152 CGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: StermFinalizer, FnInfo: FI,
5153 Args: FunctionArgList(), Loc: D.getLocation(),
5154 StartLoc: D.getInit()->getExprLoc());
5155
5156 // The unatexit subroutine unregisters __dtor functions that were previously
5157 // registered by the atexit subroutine. If the referenced function is found,
5158 // the unatexit returns a value of 0, meaning that the cleanup is still
5159 // pending (and we should call the __dtor function).
5160 llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub);
5161
5162 llvm::Value *NeedsDestruct = CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
5163
5164 llvm::BasicBlock *DestructCallBlock = CGF.createBasicBlock(name: "destruct.call");
5165 llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "destruct.end");
5166
5167 // Check if unatexit returns a value of 0. If it does, jump to
5168 // DestructCallBlock, otherwise jump to EndBlock directly.
5169 CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
5170
5171 CGF.EmitBlock(BB: DestructCallBlock);
5172
5173 // Emit the call to dtorStub.
5174 llvm::CallInst *CI = CGF.Builder.CreateCall(Callee: dtorStub);
5175
5176 // Make sure the call and the callee agree on calling convention.
5177 CI->setCallingConv(dtorStub->getCallingConv());
5178
5179 CGF.EmitBlock(BB: EndBlock);
5180
5181 CGF.FinishFunction();
5182
5183 if (auto *IPA = D.getAttr<InitPriorityAttr>()) {
5184 CGM.AddCXXPrioritizedStermFinalizerEntry(StermFinalizer,
5185 Priority: IPA->getPriority());
5186 } else if (isTemplateInstantiation(Kind: D.getTemplateSpecializationKind()) ||
5187 getContext().GetGVALinkageForVariable(VD: &D) == GVA_DiscardableODR) {
5188 // According to C++ [basic.start.init]p2, class template static data
5189 // members (i.e., implicitly or explicitly instantiated specializations)
5190 // have unordered initialization. As a consequence, we can put them into
5191 // their own llvm.global_dtors entry.
5192 CGM.AddCXXStermFinalizerToGlobalDtor(StermFinalizer, Priority: 65535);
5193 } else {
5194 CGM.AddCXXStermFinalizerEntry(DtorFn: StermFinalizer);
5195 }
5196}
5197